incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [20/51] [abbrv] git commit: Refactoring some code.
Date Tue, 11 Dec 2012 02:20:59 GMT
Refactoring some code.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/7821c8f4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/7821c8f4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/7821c8f4

Branch: refs/heads/0.2-dev
Commit: 7821c8f4b976b0a084a721c75c4957cb6a66886b
Parents: 64943aa
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Sat Nov 24 20:52:41 2012 -0500
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Sat Nov 24 20:52:41 2012 -0500

----------------------------------------------------------------------
 .../blur/manager/writer/TransactionRecorder.java   |    2 +-
 .../org/apache/blur/thrift/BlurShardServer.java    |   12 +-
 .../org/apache/blur/thrift/lucene/Convert.java     |  444 ---------------
 .../apache/blur/utils/ThriftLuceneConversion.java  |  443 ++++++++++++++
 .../org/apache/blur/manager/IndexManagerTest.java  |   13 -
 .../org/apache/blur/thrift/ITBlurClusterTest.java  |    6 +-
 .../blur/testsuite/SearchLuceneApiTable.java       |    6 +-
 7 files changed, 456 insertions(+), 470 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7821c8f4/src/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java
b/src/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java
index 72d19d1..4901be3 100644
--- a/src/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java
+++ b/src/blur-core/src/main/java/org/apache/blur/manager/writer/TransactionRecorder.java
@@ -16,7 +16,7 @@ package org.apache.blur.manager.writer;
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import static org.apache.blur.thrift.lucene.Convert.toLucene;
+import static org.apache.blur.utils.ThriftLuceneConversion.toLucene;
 
 import java.io.ByteArrayInputStream;
 import java.io.DataInputStream;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7821c8f4/src/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java b/src/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java
index 088370f..032aa3c 100644
--- a/src/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java
+++ b/src/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java
@@ -16,10 +16,10 @@ package org.apache.blur.thrift;
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import static org.apache.blur.thrift.lucene.Convert.toThrift;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_CACHE_MAX_QUERYCACHE_ELEMENTS;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_CACHE_MAX_TIMETOLIVE;
 import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_DATA_FETCH_THREAD_COUNT;
+import static org.apache.blur.utils.ThriftLuceneConversion.toThrift;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
@@ -61,9 +61,9 @@ import org.apache.blur.thrift.generated.TableStats;
 import org.apache.blur.thrift.generated.Term;
 import org.apache.blur.thrift.generated.TopFieldDocs;
 import org.apache.blur.thrift.generated.UpdatePackage;
-import org.apache.blur.thrift.lucene.Convert;
 import org.apache.blur.utils.BlurConstants;
 import org.apache.blur.utils.BlurUtil;
+import org.apache.blur.utils.ThriftLuceneConversion;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.search.Filter;
 import org.apache.lucene.search.IndexSearcher;
@@ -216,10 +216,10 @@ public class BlurShardServer extends TableAdmin implements Iface {
       Collection<SearchAction> searchersToSearch = getSearchActions(tableDescriptor,
shardIndexes, searchers);
 
       List<Future<TopFieldDocs>> futures = new ArrayList<Future<TopFieldDocs>>(searchersToSearch.size());
-      Query query = Convert.toLuceneQuery(queryArgs.query);
-      Filter filter = Convert.toLuceneFilter(queryArgs);
-      Sort sort = Convert.toLuceneSort(queryArgs);
-      ScoreDoc after = Convert.toLucene(queryArgs.getAfter());
+      Query query = ThriftLuceneConversion.toLuceneQuery(queryArgs.query);
+      Filter filter = ThriftLuceneConversion.toLuceneFilter(queryArgs);
+      Sort sort = ThriftLuceneConversion.toLuceneSort(queryArgs);
+      ScoreDoc after = ThriftLuceneConversion.toLucene(queryArgs.getAfter());
       boolean doDocScores = queryArgs.isDoDocScores();
       boolean doMaxScore = queryArgs.isDoMaxScore();
       int numberToFetch = queryArgs.getNumberToFetch();

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7821c8f4/src/blur-core/src/main/java/org/apache/blur/thrift/lucene/Convert.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/thrift/lucene/Convert.java b/src/blur-core/src/main/java/org/apache/blur/thrift/lucene/Convert.java
deleted file mode 100644
index 2a3ce3e..0000000
--- a/src/blur-core/src/main/java/org/apache/blur/thrift/lucene/Convert.java
+++ /dev/null
@@ -1,444 +0,0 @@
-package org.apache.blur.thrift.lucene;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.blur.log.Log;
-import org.apache.blur.log.LogFactory;
-import org.apache.blur.lucene.serializer.QueryWritable;
-import org.apache.blur.thrift.generated.Field;
-import org.apache.blur.thrift.generated.QueryArgs;
-import org.apache.blur.thrift.generated.ScoreDoc;
-import org.apache.blur.thrift.generated.SortField;
-import org.apache.blur.thrift.generated.SortType;
-import org.apache.blur.thrift.generated.TYPE;
-import org.apache.blur.thrift.generated.TopFieldDocs;
-import org.apache.blur.utils.BlurUtil;
-import org.apache.hadoop.io.DataInputBuffer;
-import org.apache.hadoop.io.DataOutputBuffer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.DoubleField;
-import org.apache.lucene.document.Field.Store;
-import org.apache.lucene.document.FieldType;
-import org.apache.lucene.document.FloatField;
-import org.apache.lucene.document.IntField;
-import org.apache.lucene.document.LongField;
-import org.apache.lucene.document.StoredField;
-import org.apache.lucene.document.StringField;
-import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.IndexableField;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.Filter;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.Sort;
-import org.apache.lucene.search.SortField.Type;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.util.BytesRef;
-
-public class Convert {
-
-  private static final Log LOG = LogFactory.getLog(Convert.class);
-
-  public static TopFieldDocs toThrift(TopDocs topDocs) {
-    return toThrift(new org.apache.lucene.search.TopFieldDocs(topDocs.totalHits, topDocs.scoreDocs,
null, Float.NaN));
-  }
-
-  public static org.apache.lucene.search.ScoreDoc toLucene(org.apache.blur.thrift.generated.ScoreDoc
scoreDoc) {
-    if (scoreDoc == null) {
-      return null;
-    }
-    long docLocation = scoreDoc.getDocLocation();
-    return new org.apache.lucene.search.ScoreDoc(BlurUtil.getDocumentId(docLocation), (float)
scoreDoc.score, BlurUtil.getShardIndex(docLocation));
-  }
-
-  public static org.apache.blur.thrift.generated.Document toThrift(org.apache.lucene.document.Document
document) {
-    org.apache.blur.thrift.generated.Document result = new org.apache.blur.thrift.generated.Document();
-    List<IndexableField> fields = document.getFields();
-    for (IndexableField field : fields) {
-      result.addToFields(toThrift(field));
-    }
-    return result;
-  }
-
-  public static Field toThrift(IndexableField field) {
-    Field result = new Field();
-    result.setBoost(field.boost());
-    result.setName(field.name());
-    Number number = field.numericValue();
-    if (number != null) {
-      if (number instanceof Byte || number instanceof Short || number instanceof Integer)
{
-        result.setType(TYPE.INT);
-        result.setValue(toByteBuffer((Integer) number));
-      } else if (number instanceof Long) {
-        result.setType(TYPE.LONG);
-        result.setValue(toByteBuffer((Long) number));
-      } else if (number instanceof Float) {
-        result.setType(TYPE.FLOAT);
-        result.setValue(toByteBuffer((Float) number));
-      } else if (number instanceof Double) {
-        result.setType(TYPE.DOUBLE);
-        result.setValue(toByteBuffer((Double) number));
-      } else {
-        throw new IllegalArgumentException("cannot store numeric type " + number.getClass());
-      }
-    } else {
-      BytesRef bytes = field.binaryValue();
-      if (bytes != null) {
-        result.setType(TYPE.BINARY);
-        result.setValue(toByteBuffer(bytes));
-      } else {
-        result.setType(TYPE.STRING);
-        result.setValue(toByteBuffer(field.stringValue()));
-      }
-    }
-    return result;
-  }
-
-  public static ByteBuffer toByteBuffer(String val) {
-    if (val == null) {
-      return null;
-    }
-    return toByteBuffer(new BytesRef(val));
-  }
-
-  public static ByteBuffer toByteBuffer(BytesRef val) {
-    if (val == null) {
-      return null;
-    }
-    return flip(ByteBuffer.allocate(val.length).put(val.bytes, val.offset, val.length));
-  }
-
-  private static ByteBuffer flip(ByteBuffer buffer) {
-    buffer.flip();
-    return buffer;
-  }
-
-  public static ByteBuffer toByteBuffer(Double val) {
-    if (val == null) {
-      return null;
-    }
-    return flip(ByteBuffer.allocate(8).putDouble(val));
-  }
-
-  public static ByteBuffer toByteBuffer(Long val) {
-    if (val == null) {
-      return null;
-    }
-    return flip(ByteBuffer.allocate(8).putLong(val));
-  }
-
-  public static ByteBuffer toByteBuffer(Float val) {
-    if (val == null) {
-      return null;
-    }
-    return flip(ByteBuffer.allocate(4).putFloat(val));
-  }
-
-  public static ByteBuffer toByteBuffer(Integer val) {
-    if (val == null) {
-      return null;
-    }
-    return flip(ByteBuffer.allocate(4).putInt(val));
-  }
-
-  public static String toString(org.apache.blur.thrift.generated.Document document) {
-    StringBuilder builder = new StringBuilder();
-    builder.append('{');
-    for (Field f : document.getFields()) {
-      if (builder.length() != 0) {
-        builder.append(',');
-      }
-      builder.append(toString(f));
-    }
-    builder.append('}');
-    return builder.toString();
-  }
-
-  public static String toString(Field f) {
-    StringBuilder builder = new StringBuilder();
-    builder.append('{');
-    String name = f.getName();
-    builder.append(name);
-    builder.append(',');
-
-    TYPE type = f.getType();
-    builder.append(type.name());
-    builder.append(',');
-
-    byte[] value = f.getValue();
-    builder.append(toString(type, value));
-    builder.append(',');
-
-    double boost = f.getBoost();
-    builder.append(boost);
-    builder.append('}');
-    return builder.toString();
-  }
-
-  public static String toString(TYPE type, byte[] value) {
-    switch (type) {
-    case STRING:
-      return new BytesRef(value).utf8ToString();
-    case INT:
-      return Integer.toString(ByteBuffer.wrap(value).getInt());
-    case LONG:
-      return Long.toString(ByteBuffer.wrap(value).getLong());
-    case FLOAT:
-      return Float.toString(ByteBuffer.wrap(value).getFloat());
-    case DOUBLE:
-      return Double.toString(ByteBuffer.wrap(value).getDouble());
-    case BINARY:
-      return ByteBuffer.wrap(value).toString();
-    default:
-      throw new RuntimeException("Type unknown.");
-    }
-  }
-
-  public static Document toLucene(org.apache.blur.thrift.generated.Document document) throws
IOException {
-    Document result = new Document();
-    List<Field> fields = document.getFields();
-    for (Field field : fields) {
-      result.add(toLucene(field));
-    }
-    return result;
-  }
-
-  public static IndexableField toLucene(Field field) throws IOException {
-    TYPE type = field.getType();
-    org.apache.lucene.document.Field result;
-    switch (type) {
-    case STRING:
-      result = new StringField(field.getName(), toString(field.getValue()), Store.YES);
-      break;
-    case TEXT:
-      result = new TextField(field.getName(), toString(field.getValue()), Store.YES);
-      break;
-    case INT:
-      result = new IntField(field.getName(), toInteger(field.getValue()), Store.YES);
-      break;
-    case FLOAT:
-      result = new FloatField(field.getName(), toFloat(field.getValue()), Store.YES);
-      break;
-    case LONG:
-      result = new LongField(field.getName(), toLong(field.getValue()), Store.YES);
-      break;
-    case DOUBLE:
-      result = new DoubleField(field.getName(), toDouble(field.getValue()), Store.YES);
-      break;
-    case BINARY:
-      result = new StoredField(field.getName(), field.getValue());
-      break;
-    default:
-      throw new RuntimeException("Not supported");
-    }
-    FieldType fieldType = result.fieldType();
-    if (fieldType.indexed() && !fieldType.omitNorms()) {
-      result.setBoost((float) field.getBoost());
-    }
-    return result;
-  }
-
-  public static double toDouble(byte[] value) {
-    throw new RuntimeException("Not implemented");
-  }
-
-  public static long toLong(byte[] value) {
-    throw new RuntimeException("Not implemented");
-  }
-
-  public static float toFloat(byte[] value) {
-    throw new RuntimeException("Not implemented");
-  }
-
-  public static int toInteger(byte[] value) {
-    throw new RuntimeException("Not implemented");
-  }
-
-  public static String toString(byte[] value) throws IOException {
-    return new String(value, "UTF-8");
-  }
-
-  public static Term toLucene(org.apache.blur.thrift.generated.Term term) {
-    return new Term(term.getField(), new BytesRef(term.getBytes()));
-  }
-
-  public static Term[] toLucene(org.apache.blur.thrift.generated.Term[] terms) {
-    Term[] result = new Term[terms.length];
-    for (int i = 0; i < terms.length; i++) {
-      result[i] = toLucene(terms[i]);
-    }
-    return result;
-  }
-
-  public static List<Document> toLucene(List<org.apache.blur.thrift.generated.Document>
documents) throws IOException {
-    List<Document> docs = new ArrayList<Document>();
-    for (org.apache.blur.thrift.generated.Document doc : documents) {
-      docs.add(toLucene(doc));
-    }
-    return docs;
-  }
-
-  public static Query[] toLucene(ByteBuffer[] queries) throws IOException {
-    Query[] result = new Query[queries.length];
-    for (int i = 0; i < queries.length; i++) {
-      result[i] = toLuceneQuery(queries[i]);
-    }
-    return result;
-  }
-
-  public static Query toLuceneQuery(ByteBuffer byteBuffer) throws IOException {
-    DataInputBuffer in = new DataInputBuffer();
-    in.reset(byteBuffer.array(),byteBuffer.arrayOffset() + byteBuffer.position() , byteBuffer.remaining());
-    QueryWritable qw = new QueryWritable();
-    qw.readFields(in);
-    return qw.getQuery();
-  }
-
-  public static Query toLucene(QueryArgs queryArgs) throws IOException {
-    return toLuceneQuery(queryArgs.getQuery());
-  }
-
-  public static Query toLuceneQuery(byte[] bs) throws IOException {
-    DataInputBuffer in = new DataInputBuffer();
-    in.reset(bs, bs.length);
-    QueryWritable qw = new QueryWritable();
-    qw.readFields(in);
-    return qw.getQuery();
-  }
-
-  public static byte[] toBytes(Query query) throws IOException {
-    DataOutputBuffer out = new DataOutputBuffer();
-    QueryWritable qw = new QueryWritable();
-    qw.setQuery(query);
-    qw.write(out);
-    out.close();
-
-    byte[] data = out.getData();
-    int length = out.getLength();
-    byte[] buf = new byte[length];
-    System.arraycopy(data, 0, buf, 0, length);
-    return buf;
-  }
-
-  public static Filter toLuceneFilter(QueryArgs queryArgs) {
-    byte[] filter = queryArgs.getFilter();
-    return null;
-  }
-
-  public static Sort toLuceneSort(QueryArgs queryArgs) {
-    return null;
-  }
-
-  public static TopFieldDocs toThrift(org.apache.lucene.search.TopFieldDocs topFieldDocs)
{
-    TopFieldDocs result = new TopFieldDocs();
-    result.setFields(toThrift(topFieldDocs.fields));
-    result.setMaxScore(topFieldDocs.getMaxScore());
-    result.setScoreDocs(toThrift(topFieldDocs.scoreDocs));
-    result.setTotalHits(topFieldDocs.totalHits);
-    return result;
-  }
-
-  public static List<ScoreDoc> toThrift(org.apache.lucene.search.ScoreDoc[] scoreDocs)
{
-    List<ScoreDoc> result = new ArrayList<ScoreDoc>(scoreDocs.length);
-    for (int i = 0; i < scoreDocs.length; i++) {
-      org.apache.lucene.search.ScoreDoc scoreDoc = scoreDocs[i];
-      if (scoreDoc instanceof org.apache.lucene.search.FieldDoc) {
-        result.add(toThrift((org.apache.lucene.search.FieldDoc) scoreDoc));
-      } else {
-        result.add(toThrift(scoreDoc));
-      }
-    }
-    return result;
-  }
-
-  public List<ScoreDoc> toThrift(org.apache.lucene.search.FieldDoc[] fieldDocs) {
-    List<ScoreDoc> result = new ArrayList<ScoreDoc>(fieldDocs.length);
-    for (int i = 0; i < fieldDocs.length; i++) {
-      result.add(toThrift(fieldDocs[i]));
-    }
-    return result;
-  }
-
-  public static List<SortField> toThrift(org.apache.lucene.search.SortField[] fields)
{
-    if (fields == null) {
-      return null;
-    }
-    List<SortField> result = new ArrayList<SortField>(fields.length);
-    for (int i = 0; i < fields.length; i++) {
-      result.add(toThrift(fields[i]));
-    }
-    return result;
-  }
-
-  public static SortField toThrift(org.apache.lucene.search.SortField sortField) {
-    if (sortField == null) {
-      return null;
-    }
-    SortField result = new SortField();
-    result.setField(sortField.getField());
-    result.setReverse(sortField.getReverse());
-    result.setType(toThrift(sortField.getType()));
-    return result;
-  }
-
-  public static ScoreDoc toThrift(org.apache.lucene.search.ScoreDoc scoreDoc) {
-    if (scoreDoc == null) {
-      return null;
-    }
-    ScoreDoc result = new ScoreDoc();
-    result.setDocLocation(BlurUtil.getDocLocation(scoreDoc.shardIndex, scoreDoc.doc));
-    result.setScore(scoreDoc.score);
-    return result;
-  }
-
-  public static ScoreDoc toThrift(org.apache.lucene.search.FieldDoc fieldDoc) {
-    if (fieldDoc == null) {
-      return null;
-    }
-    ScoreDoc result = new ScoreDoc();
-    result.setDocLocation(BlurUtil.getDocLocation(fieldDoc.shardIndex, fieldDoc.doc));
-    result.setFields(convert(fieldDoc.fields));
-    result.setScore(fieldDoc.score);
-    return result;
-  }
-
-  public static SortType toThrift(Type type) {
-    if (type == null) {
-      return null;
-    }
-    switch (type) {
-    case BYTE:
-      return SortType.BYTE;
-    case BYTES:
-      return SortType.BYTES;
-    case SCORE:
-      return SortType.SCORE;
-    case DOC:
-      return SortType.DOC;
-    case STRING:
-      return SortType.STRING;
-    case INT:
-      return SortType.INT;
-    case FLOAT:
-      return SortType.FLOAT;
-    case LONG:
-      return SortType.LONG;
-    case DOUBLE:
-      return SortType.DOUBLE;
-    case SHORT:
-      return SortType.SHORT;
-    case STRING_VAL:
-      return SortType.STRING_VAL;
-    default:
-      throw new RuntimeException("Not supported");
-    }
-  }
-
-  public static List<ByteBuffer> convert(Object[] fields) {
-    // throw new RuntimeException("Not supported YET!");
-    LOG.error("Not supported YET!");
-    return null;
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7821c8f4/src/blur-core/src/main/java/org/apache/blur/utils/ThriftLuceneConversion.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/utils/ThriftLuceneConversion.java
b/src/blur-core/src/main/java/org/apache/blur/utils/ThriftLuceneConversion.java
new file mode 100644
index 0000000..543c82c
--- /dev/null
+++ b/src/blur-core/src/main/java/org/apache/blur/utils/ThriftLuceneConversion.java
@@ -0,0 +1,443 @@
+package org.apache.blur.utils;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.blur.log.Log;
+import org.apache.blur.log.LogFactory;
+import org.apache.blur.lucene.serializer.QueryWritable;
+import org.apache.blur.thrift.generated.Field;
+import org.apache.blur.thrift.generated.QueryArgs;
+import org.apache.blur.thrift.generated.ScoreDoc;
+import org.apache.blur.thrift.generated.SortField;
+import org.apache.blur.thrift.generated.SortType;
+import org.apache.blur.thrift.generated.TYPE;
+import org.apache.blur.thrift.generated.TopFieldDocs;
+import org.apache.hadoop.io.DataInputBuffer;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.DoubleField;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.FloatField;
+import org.apache.lucene.document.IntField;
+import org.apache.lucene.document.LongField;
+import org.apache.lucene.document.StoredField;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.Filter;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField.Type;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.util.BytesRef;
+
+public class ThriftLuceneConversion {
+
+  private static final Log LOG = LogFactory.getLog(ThriftLuceneConversion.class);
+
+  public static TopFieldDocs toThrift(TopDocs topDocs) {
+    return toThrift(new org.apache.lucene.search.TopFieldDocs(topDocs.totalHits, topDocs.scoreDocs,
null, Float.NaN));
+  }
+
+  public static org.apache.lucene.search.ScoreDoc toLucene(org.apache.blur.thrift.generated.ScoreDoc
scoreDoc) {
+    if (scoreDoc == null) {
+      return null;
+    }
+    long docLocation = scoreDoc.getDocLocation();
+    return new org.apache.lucene.search.ScoreDoc(BlurUtil.getDocumentId(docLocation), (float)
scoreDoc.score, BlurUtil.getShardIndex(docLocation));
+  }
+
+  public static org.apache.blur.thrift.generated.Document toThrift(org.apache.lucene.document.Document
document) {
+    org.apache.blur.thrift.generated.Document result = new org.apache.blur.thrift.generated.Document();
+    List<IndexableField> fields = document.getFields();
+    for (IndexableField field : fields) {
+      result.addToFields(toThrift(field));
+    }
+    return result;
+  }
+
+  public static Field toThrift(IndexableField field) {
+    Field result = new Field();
+    result.setBoost(field.boost());
+    result.setName(field.name());
+    Number number = field.numericValue();
+    if (number != null) {
+      if (number instanceof Byte || number instanceof Short || number instanceof Integer)
{
+        result.setType(TYPE.INT);
+        result.setValue(toByteBuffer((Integer) number));
+      } else if (number instanceof Long) {
+        result.setType(TYPE.LONG);
+        result.setValue(toByteBuffer((Long) number));
+      } else if (number instanceof Float) {
+        result.setType(TYPE.FLOAT);
+        result.setValue(toByteBuffer((Float) number));
+      } else if (number instanceof Double) {
+        result.setType(TYPE.DOUBLE);
+        result.setValue(toByteBuffer((Double) number));
+      } else {
+        throw new IllegalArgumentException("cannot store numeric type " + number.getClass());
+      }
+    } else {
+      BytesRef bytes = field.binaryValue();
+      if (bytes != null) {
+        result.setType(TYPE.BINARY);
+        result.setValue(toByteBuffer(bytes));
+      } else {
+        result.setType(TYPE.STRING);
+        result.setValue(toByteBuffer(field.stringValue()));
+      }
+    }
+    return result;
+  }
+
+  public static ByteBuffer toByteBuffer(String val) {
+    if (val == null) {
+      return null;
+    }
+    return toByteBuffer(new BytesRef(val));
+  }
+
+  public static ByteBuffer toByteBuffer(BytesRef val) {
+    if (val == null) {
+      return null;
+    }
+    return flip(ByteBuffer.allocate(val.length).put(val.bytes, val.offset, val.length));
+  }
+
+  private static ByteBuffer flip(ByteBuffer buffer) {
+    buffer.flip();
+    return buffer;
+  }
+
+  public static ByteBuffer toByteBuffer(Double val) {
+    if (val == null) {
+      return null;
+    }
+    return flip(ByteBuffer.allocate(8).putDouble(val));
+  }
+
+  public static ByteBuffer toByteBuffer(Long val) {
+    if (val == null) {
+      return null;
+    }
+    return flip(ByteBuffer.allocate(8).putLong(val));
+  }
+
+  public static ByteBuffer toByteBuffer(Float val) {
+    if (val == null) {
+      return null;
+    }
+    return flip(ByteBuffer.allocate(4).putFloat(val));
+  }
+
+  public static ByteBuffer toByteBuffer(Integer val) {
+    if (val == null) {
+      return null;
+    }
+    return flip(ByteBuffer.allocate(4).putInt(val));
+  }
+
+  public static String toString(org.apache.blur.thrift.generated.Document document) {
+    StringBuilder builder = new StringBuilder();
+    builder.append('{');
+    for (Field f : document.getFields()) {
+      if (builder.length() != 0) {
+        builder.append(',');
+      }
+      builder.append(toString(f));
+    }
+    builder.append('}');
+    return builder.toString();
+  }
+
+  public static String toString(Field f) {
+    StringBuilder builder = new StringBuilder();
+    builder.append('{');
+    String name = f.getName();
+    builder.append(name);
+    builder.append(',');
+
+    TYPE type = f.getType();
+    builder.append(type.name());
+    builder.append(',');
+
+    byte[] value = f.getValue();
+    builder.append(toString(type, value));
+    builder.append(',');
+
+    double boost = f.getBoost();
+    builder.append(boost);
+    builder.append('}');
+    return builder.toString();
+  }
+
+  public static String toString(TYPE type, byte[] value) {
+    switch (type) {
+    case STRING:
+      return new BytesRef(value).utf8ToString();
+    case INT:
+      return Integer.toString(ByteBuffer.wrap(value).getInt());
+    case LONG:
+      return Long.toString(ByteBuffer.wrap(value).getLong());
+    case FLOAT:
+      return Float.toString(ByteBuffer.wrap(value).getFloat());
+    case DOUBLE:
+      return Double.toString(ByteBuffer.wrap(value).getDouble());
+    case BINARY:
+      return ByteBuffer.wrap(value).toString();
+    default:
+      throw new RuntimeException("Type unknown.");
+    }
+  }
+
+  public static Document toLucene(org.apache.blur.thrift.generated.Document document) throws
IOException {
+    Document result = new Document();
+    List<Field> fields = document.getFields();
+    for (Field field : fields) {
+      result.add(toLucene(field));
+    }
+    return result;
+  }
+
+  public static IndexableField toLucene(Field field) throws IOException {
+    TYPE type = field.getType();
+    org.apache.lucene.document.Field result;
+    switch (type) {
+    case STRING:
+      result = new StringField(field.getName(), toString(field.getValue()), Store.YES);
+      break;
+    case TEXT:
+      result = new TextField(field.getName(), toString(field.getValue()), Store.YES);
+      break;
+    case INT:
+      result = new IntField(field.getName(), toInteger(field.getValue()), Store.YES);
+      break;
+    case FLOAT:
+      result = new FloatField(field.getName(), toFloat(field.getValue()), Store.YES);
+      break;
+    case LONG:
+      result = new LongField(field.getName(), toLong(field.getValue()), Store.YES);
+      break;
+    case DOUBLE:
+      result = new DoubleField(field.getName(), toDouble(field.getValue()), Store.YES);
+      break;
+    case BINARY:
+      result = new StoredField(field.getName(), field.getValue());
+      break;
+    default:
+      throw new RuntimeException("Not supported");
+    }
+    FieldType fieldType = result.fieldType();
+    if (fieldType.indexed() && !fieldType.omitNorms()) {
+      result.setBoost((float) field.getBoost());
+    }
+    return result;
+  }
+
+  public static double toDouble(byte[] value) {
+    throw new RuntimeException("Not implemented");
+  }
+
+  public static long toLong(byte[] value) {
+    throw new RuntimeException("Not implemented");
+  }
+
+  public static float toFloat(byte[] value) {
+    throw new RuntimeException("Not implemented");
+  }
+
+  public static int toInteger(byte[] value) {
+    throw new RuntimeException("Not implemented");
+  }
+
+  public static String toString(byte[] value) throws IOException {
+    return new String(value, "UTF-8");
+  }
+
+  public static Term toLucene(org.apache.blur.thrift.generated.Term term) {
+    return new Term(term.getField(), new BytesRef(term.getBytes()));
+  }
+
+  public static Term[] toLucene(org.apache.blur.thrift.generated.Term[] terms) {
+    Term[] result = new Term[terms.length];
+    for (int i = 0; i < terms.length; i++) {
+      result[i] = toLucene(terms[i]);
+    }
+    return result;
+  }
+
+  public static List<Document> toLucene(List<org.apache.blur.thrift.generated.Document>
documents) throws IOException {
+    List<Document> docs = new ArrayList<Document>();
+    for (org.apache.blur.thrift.generated.Document doc : documents) {
+      docs.add(toLucene(doc));
+    }
+    return docs;
+  }
+
+  public static Query[] toLucene(ByteBuffer[] queries) throws IOException {
+    Query[] result = new Query[queries.length];
+    for (int i = 0; i < queries.length; i++) {
+      result[i] = toLuceneQuery(queries[i]);
+    }
+    return result;
+  }
+
+  public static Query toLuceneQuery(ByteBuffer byteBuffer) throws IOException {
+    DataInputBuffer in = new DataInputBuffer();
+    in.reset(byteBuffer.array(),byteBuffer.arrayOffset() + byteBuffer.position() , byteBuffer.remaining());
+    QueryWritable qw = new QueryWritable();
+    qw.readFields(in);
+    return qw.getQuery();
+  }
+
+  public static Query toLucene(QueryArgs queryArgs) throws IOException {
+    return toLuceneQuery(queryArgs.getQuery());
+  }
+
+  public static Query toLuceneQuery(byte[] bs) throws IOException {
+    DataInputBuffer in = new DataInputBuffer();
+    in.reset(bs, bs.length);
+    QueryWritable qw = new QueryWritable();
+    qw.readFields(in);
+    return qw.getQuery();
+  }
+
+  public static byte[] toBytes(Query query) throws IOException {
+    DataOutputBuffer out = new DataOutputBuffer();
+    QueryWritable qw = new QueryWritable();
+    qw.setQuery(query);
+    qw.write(out);
+    out.close();
+
+    byte[] data = out.getData();
+    int length = out.getLength();
+    byte[] buf = new byte[length];
+    System.arraycopy(data, 0, buf, 0, length);
+    return buf;
+  }
+
+  public static Filter toLuceneFilter(QueryArgs queryArgs) {
+    byte[] filter = queryArgs.getFilter();
+    return null;
+  }
+
+  public static Sort toLuceneSort(QueryArgs queryArgs) {
+    return null;
+  }
+
+  public static TopFieldDocs toThrift(org.apache.lucene.search.TopFieldDocs topFieldDocs)
{
+    TopFieldDocs result = new TopFieldDocs();
+    result.setFields(toThrift(topFieldDocs.fields));
+    result.setMaxScore(topFieldDocs.getMaxScore());
+    result.setScoreDocs(toThrift(topFieldDocs.scoreDocs));
+    result.setTotalHits(topFieldDocs.totalHits);
+    return result;
+  }
+
+  public static List<ScoreDoc> toThrift(org.apache.lucene.search.ScoreDoc[] scoreDocs)
{
+    List<ScoreDoc> result = new ArrayList<ScoreDoc>(scoreDocs.length);
+    for (int i = 0; i < scoreDocs.length; i++) {
+      org.apache.lucene.search.ScoreDoc scoreDoc = scoreDocs[i];
+      if (scoreDoc instanceof org.apache.lucene.search.FieldDoc) {
+        result.add(toThrift((org.apache.lucene.search.FieldDoc) scoreDoc));
+      } else {
+        result.add(toThrift(scoreDoc));
+      }
+    }
+    return result;
+  }
+
+  public List<ScoreDoc> toThrift(org.apache.lucene.search.FieldDoc[] fieldDocs) {
+    List<ScoreDoc> result = new ArrayList<ScoreDoc>(fieldDocs.length);
+    for (int i = 0; i < fieldDocs.length; i++) {
+      result.add(toThrift(fieldDocs[i]));
+    }
+    return result;
+  }
+
+  public static List<SortField> toThrift(org.apache.lucene.search.SortField[] fields)
{
+    if (fields == null) {
+      return null;
+    }
+    List<SortField> result = new ArrayList<SortField>(fields.length);
+    for (int i = 0; i < fields.length; i++) {
+      result.add(toThrift(fields[i]));
+    }
+    return result;
+  }
+
+  public static SortField toThrift(org.apache.lucene.search.SortField sortField) {
+    if (sortField == null) {
+      return null;
+    }
+    SortField result = new SortField();
+    result.setField(sortField.getField());
+    result.setReverse(sortField.getReverse());
+    result.setType(toThrift(sortField.getType()));
+    return result;
+  }
+
+  public static ScoreDoc toThrift(org.apache.lucene.search.ScoreDoc scoreDoc) {
+    if (scoreDoc == null) {
+      return null;
+    }
+    ScoreDoc result = new ScoreDoc();
+    result.setDocLocation(BlurUtil.getDocLocation(scoreDoc.shardIndex, scoreDoc.doc));
+    result.setScore(scoreDoc.score);
+    return result;
+  }
+
+  public static ScoreDoc toThrift(org.apache.lucene.search.FieldDoc fieldDoc) {
+    if (fieldDoc == null) {
+      return null;
+    }
+    ScoreDoc result = new ScoreDoc();
+    result.setDocLocation(BlurUtil.getDocLocation(fieldDoc.shardIndex, fieldDoc.doc));
+    result.setFields(convert(fieldDoc.fields));
+    result.setScore(fieldDoc.score);
+    return result;
+  }
+
+  public static SortType toThrift(Type type) {
+    if (type == null) {
+      return null;
+    }
+    switch (type) {
+    case BYTE:
+      return SortType.BYTE;
+    case BYTES:
+      return SortType.BYTES;
+    case SCORE:
+      return SortType.SCORE;
+    case DOC:
+      return SortType.DOC;
+    case STRING:
+      return SortType.STRING;
+    case INT:
+      return SortType.INT;
+    case FLOAT:
+      return SortType.FLOAT;
+    case LONG:
+      return SortType.LONG;
+    case DOUBLE:
+      return SortType.DOUBLE;
+    case SHORT:
+      return SortType.SHORT;
+    case STRING_VAL:
+      return SortType.STRING_VAL;
+    default:
+      throw new RuntimeException("Not supported");
+    }
+  }
+
+  public static List<ByteBuffer> convert(Object[] fields) {
+    // throw new RuntimeException("Not supported YET!");
+    LOG.error("Not supported YET!");
+    return null;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7821c8f4/src/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java b/src/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java
index 5fb73e6..82e3d92 100644
--- a/src/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java
+++ b/src/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java
@@ -23,14 +23,10 @@ import java.io.File;
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
 
 import org.apache.blur.manager.indexserver.LocalIndexServer;
 import org.apache.blur.metrics.BlurMetrics;
 import org.apache.blur.thrift.generated.BlurException;
-import org.apache.blur.thrift.generated.Schema;
 import org.apache.blur.utils.BlurConstants;
 import org.apache.blur.utils.BlurUtil;
 import org.apache.hadoop.conf.Configuration;
@@ -94,15 +90,6 @@ public class IndexManagerTest {
     assertEquals(0, indexManager.recordFrequency(TABLE, FAMILY, "testcol1", "NO VALUE"));
   }
 
-  @Test
-  public void testSchema() throws Exception {
-    Schema schema = indexManager.schema(TABLE);
-    assertEquals(TABLE, schema.table);
-    Map<String, Set<String>> columnFamilies = schema.columnFamilies;
-    assertEquals(new TreeSet<String>(Arrays.asList(FAMILY, FAMILY2)), new TreeSet<String>(columnFamilies.keySet()));
-    assertEquals(new TreeSet<String>(Arrays.asList("testcol1", "testcol2", "testcol3",
"testcol12", "testcol13")), new TreeSet<String>(columnFamilies.get(FAMILY)));
-    assertEquals(new TreeSet<String>(Arrays.asList("testcol18")), new TreeSet<String>(columnFamilies.get(FAMILY2)));
-  }
 
   @Test
   public void testTerms() throws Exception {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7821c8f4/src/blur-core/src/test/java/org/apache/blur/thrift/ITBlurClusterTest.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/test/java/org/apache/blur/thrift/ITBlurClusterTest.java b/src/blur-core/src/test/java/org/apache/blur/thrift/ITBlurClusterTest.java
index dd0e4b1..35c9fea 100644
--- a/src/blur-core/src/test/java/org/apache/blur/thrift/ITBlurClusterTest.java
+++ b/src/blur-core/src/test/java/org/apache/blur/thrift/ITBlurClusterTest.java
@@ -40,7 +40,7 @@ import org.apache.blur.thrift.generated.TYPE;
 import org.apache.blur.thrift.generated.TableDescriptor;
 import org.apache.blur.thrift.generated.TopFieldDocs;
 import org.apache.blur.thrift.generated.UpdatePackage;
-import org.apache.blur.thrift.lucene.Convert;
+import org.apache.blur.utils.ThriftLuceneConversion;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.LocalFileSystem;
@@ -133,7 +133,7 @@ public class ITBlurClusterTest {
     QueryArgs queryArgs = new QueryArgs();
     Term term = new Term("value", "test");
     TermQuery query = new TermQuery(term);
-    queryArgs.setQuery(Convert.toBytes(query));
+    queryArgs.setQuery(ThriftLuceneConversion.toBytes(query));
     List<TopFieldDocs> results = client.search(session, queryArgs);
     long totalHits = 0;
     for (TopFieldDocs fieldDocs : results) {
@@ -175,7 +175,7 @@ public class ITBlurClusterTest {
     QueryArgs queryArgs = new QueryArgs();
     Term term = new Term("value", "test");
     TermQuery query = new TermQuery(term);
-    queryArgs.setQuery(Convert.toBytes(query));
+    queryArgs.setQuery(ThriftLuceneConversion.toBytes(query));
     List<TopFieldDocs> results = client.search(session, queryArgs);
     long totalHits = 0;
     for (TopFieldDocs fieldDocs : results) {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7821c8f4/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/SearchLuceneApiTable.java
----------------------------------------------------------------------
diff --git a/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/SearchLuceneApiTable.java
b/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/SearchLuceneApiTable.java
index 3c344e3..5e03669 100644
--- a/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/SearchLuceneApiTable.java
+++ b/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/SearchLuceneApiTable.java
@@ -28,7 +28,7 @@ import org.apache.blur.thrift.generated.ShardServer.Client;
 import org.apache.blur.thrift.generated.QueryArgs;
 import org.apache.blur.thrift.generated.Session;
 import org.apache.blur.thrift.generated.TopFieldDocs;
-import org.apache.blur.thrift.lucene.Convert;
+import org.apache.blur.utils.ThriftLuceneConversion;
 import org.apache.hadoop.io.DataOutputBuffer;
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.queryparser.classic.ParseException;
@@ -85,9 +85,9 @@ public class SearchLuceneApiTable {
       System.out.println("================");
       for (TopFieldDocs topFieldDocs : topDocs) {
         System.out.println(topFieldDocs);
-        List<Document> docs = client.doc(session, Convert.toThrift(topFieldDocs.getScoreDocs()),
null);
+        List<Document> docs = client.doc(session, ThriftLuceneConversion.toThrift(topFieldDocs.getScoreDocs()),
null);
         for (Document document : docs) {
-          System.out.println(Convert.toString(document));
+          System.out.println(ThriftLuceneConversion.toString(document));
         }
       }
 


Mime
View raw message