incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [1/2] git commit: BLUR-ID:18 New Writables
Date Fri, 09 Nov 2012 03:30:29 GMT
Updated Branches:
  refs/heads/0.2-dev-mr-formats b0893e13f -> 996ed6a65


BLUR-ID:18 New Writables

Signed-off-by: Aaron McCurry <amccurry@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/996ed6a6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/996ed6a6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/996ed6a6

Branch: refs/heads/0.2-dev-mr-formats
Commit: 996ed6a65120ff9252c00db30d2ec96413eba0cd
Parents: 95bd2a0
Author: Gagan <gagandeepjuneja@gmail.com>
Authored: Mon Nov 5 18:57:43 2012 +0530
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Thu Nov 8 22:20:40 2012 -0500

----------------------------------------------------------------------
 .../java/org/apache/blur/mr/BlurDocLocation.java   |   85 ++++++++++
 .../main/java/org/apache/blur/mr/BlurDocument.java |   90 ++++++++++
 .../main/java/org/apache/blur/mr/BlurField.java    |  127 +++++++++++++++
 .../org/apache/blur/mr/BlurDocLocationTest.java    |   43 +++++
 .../java/org/apache/blur/mr/BlurDocumentTest.java  |   55 +++++++
 .../java/org/apache/blur/mr/BlurFieldTest.java     |   96 +++++++++++
 6 files changed, 496 insertions(+), 0 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/996ed6a6/src/blur-mapred/src/main/java/org/apache/blur/mr/BlurDocLocation.java
----------------------------------------------------------------------
diff --git a/src/blur-mapred/src/main/java/org/apache/blur/mr/BlurDocLocation.java b/src/blur-mapred/src/main/java/org/apache/blur/mr/BlurDocLocation.java
new file mode 100644
index 0000000..59b7d94
--- /dev/null
+++ b/src/blur-mapred/src/main/java/org/apache/blur/mr/BlurDocLocation.java
@@ -0,0 +1,85 @@
+package org.apache.blur.mr;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.blur.mapreduce.IOUtil;
+import org.apache.blur.thrift.generated.DocLocation;
+import org.apache.hadoop.io.Writable;
+
+public class BlurDocLocation implements Writable{
+  private int docId;
+  private int shardIndexId;
+
+  public BlurDocLocation(){
+    
+  }
+  
+  public BlurDocLocation(DocLocation docLocation){
+    docId = docLocation.getDoc();
+    shardIndexId = docLocation.getShardIndex();
+  }
+
+  
+  @Override
+  public void readFields(DataInput in) throws IOException {
+    docId = IOUtil.readVInt(in);
+    shardIndexId = IOUtil.readVInt(in);
+  }
+
+  @Override
+  public void write(DataOutput out) throws IOException {
+    IOUtil.writeVInt(out, docId);
+    IOUtil.writeVInt(out, shardIndexId);
+  }
+
+  public int getDocId() {
+    return docId;
+  }
+
+  public void setDocId(int docId) {
+    this.docId = docId;
+  }
+
+  public int getShardIndexId() {
+    return shardIndexId;
+  }
+
+  public void setShardIndexId(int shardIndexId) {
+    this.shardIndexId = shardIndexId;
+  }
+
+  @Override
+  public int hashCode() {
+    final int prime = 31;
+    int result = 1;
+    result = prime * result + docId;
+    result = prime * result + shardIndexId;
+    return result;
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj)
+      return true;
+    if (obj == null)
+      return false;
+    if (getClass() != obj.getClass())
+      return false;
+    BlurDocLocation other = (BlurDocLocation) obj;
+    if (docId != other.docId)
+      return false;
+    if (shardIndexId != other.shardIndexId)
+      return false;
+    return true;
+  }
+
+  @Override
+  public String toString() {
+    return "BlurDocLocation [docId=" + docId + ", shardIndexId=" + shardIndexId
+        + "]";
+  }
+
+   
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/996ed6a6/src/blur-mapred/src/main/java/org/apache/blur/mr/BlurDocument.java
----------------------------------------------------------------------
diff --git a/src/blur-mapred/src/main/java/org/apache/blur/mr/BlurDocument.java b/src/blur-mapred/src/main/java/org/apache/blur/mr/BlurDocument.java
new file mode 100644
index 0000000..31cc547
--- /dev/null
+++ b/src/blur-mapred/src/main/java/org/apache/blur/mr/BlurDocument.java
@@ -0,0 +1,90 @@
+package org.apache.blur.mr;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.blur.mapreduce.IOUtil;
+import org.apache.blur.thrift.generated.Document;
+import org.apache.blur.thrift.generated.Field;
+import org.apache.hadoop.io.Writable;
+
+public class BlurDocument implements Writable {
+  private List<BlurField> blurFields;
+
+  public BlurDocument() {
+    blurFields = new ArrayList<BlurField>();
+  }
+
+  public BlurDocument(Document document) {
+    blurFields = new ArrayList<BlurField>();
+    loadDocumentToBlurDocument(document);
+  }
+
+  private void loadDocumentToBlurDocument(Document document) {
+    for (Field field : document.getFields()) {
+      blurFields.add(new BlurField(field));
+    }
+  }
+
+  @Override
+  public void readFields(DataInput in) throws IOException {
+    int size = IOUtil.readVInt(in);
+    for (int i = 0; i < size; i++) {
+      BlurField blurField = new BlurField();
+      blurField.readFields(in);
+      blurFields.add(blurField);
+    }
+  }
+
+  @Override
+  public void write(DataOutput out) throws IOException {
+    IOUtil.writeVInt(out, blurFields.size());
+    for (BlurField blurField : blurFields) {
+      blurField.write(out);
+    }
+  }
+
+  public List<BlurField> getBlurFields() {
+    return blurFields;
+  }
+
+  public void setBlurFields(List<BlurField> blurFields) {
+    this.blurFields = blurFields;
+  }
+
+  @Override
+  public int hashCode() {
+    final int prime = 31;
+    int result = 1;
+    result = prime * result
+        + ((blurFields == null) ? 0 : blurFields.hashCode());
+    return result;
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj)
+      return true;
+    if (obj == null)
+      return false;
+    if (getClass() != obj.getClass())
+      return false;
+    BlurDocument other = (BlurDocument) obj;
+    if (blurFields == null) {
+      if (other.blurFields != null)
+        return false;
+    } else if (!blurFields.equals(other.blurFields))
+      return false;
+    return true;
+  }
+
+  @Override
+  public String toString() {
+    return "BlurDocument [blurFields=" + blurFields.toString() + "]";
+  }
+
+  
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/996ed6a6/src/blur-mapred/src/main/java/org/apache/blur/mr/BlurField.java
----------------------------------------------------------------------
diff --git a/src/blur-mapred/src/main/java/org/apache/blur/mr/BlurField.java b/src/blur-mapred/src/main/java/org/apache/blur/mr/BlurField.java
new file mode 100644
index 0000000..30a543d
--- /dev/null
+++ b/src/blur-mapred/src/main/java/org/apache/blur/mr/BlurField.java
@@ -0,0 +1,127 @@
+package org.apache.blur.mr;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.Arrays;
+
+import org.apache.blur.mapreduce.IOUtil;
+import org.apache.blur.thrift.generated.Field;
+import org.apache.blur.thrift.generated.TYPE;
+import org.apache.hadoop.io.Writable;
+
+public class BlurField implements Writable{
+  private String fieldName;
+  private TYPE fieldType;
+  private double fieldBoost = 1.0;
+  private byte[] fieldValue;
+
+  public BlurField(){
+    
+  }
+  
+  public BlurField(Field field){
+    fieldName = field.getName();
+    fieldType = field.getType();
+    fieldBoost = field.getBoost();
+    fieldValue = field.getValue();
+  }
+
+  
+  @Override
+  public void readFields(DataInput in) throws IOException {
+    fieldName = IOUtil.readString(in);
+    fieldType =  TYPE.findByValue(IOUtil.readVInt(in));
+    fieldBoost = in.readDouble();
+    int length = IOUtil.readVInt(in);
+    fieldValue = new byte[length];
+    in.readFully(fieldValue);
+  }
+
+  @Override
+  public void write(DataOutput out) throws IOException {
+    IOUtil.writeString(out, fieldName);
+    IOUtil.writeVInt(out, fieldType.getValue());
+    out.writeDouble(fieldBoost);
+    IOUtil.writeVInt(out,fieldValue.length);
+    out.write(fieldValue);
+  }
+
+  public String getFieldName() {
+    return fieldName;
+  }
+
+  public void setFieldName(String fieldName) {
+    this.fieldName = fieldName;
+  }
+
+  public TYPE getFieldTypeValue() {
+    return fieldType;
+  }
+
+  public void setFieldTypeValue(TYPE fieldType) {
+    this.fieldType = fieldType;
+  }
+
+  public double getFieldBoost() {
+    return fieldBoost;
+  }
+
+  public void setFieldBoost(double fieldBoost) {
+    this.fieldBoost = fieldBoost;
+  }
+
+  public byte[] getFieldValue() {
+    return fieldValue;
+  }
+
+  public void setFieldValue(byte[] fieldValue) {
+    this.fieldValue = fieldValue;
+  }
+
+  @Override
+  public int hashCode() {
+    final int prime = 31;
+    int result = 1;
+    long temp;
+    temp = Double.doubleToLongBits(fieldBoost);
+    result = prime * result + (int) (temp ^ (temp >>> 32));
+    result = prime * result + ((fieldName == null) ? 0 : fieldName.hashCode());
+    result = prime * result + fieldType.getValue();
+    result = prime * result + Arrays.hashCode(fieldValue);
+    return result;
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj)
+      return true;
+    if (obj == null)
+      return false;
+    if (getClass() != obj.getClass())
+      return false;
+    BlurField other = (BlurField) obj;
+    if (Double.doubleToLongBits(fieldBoost) != Double
+        .doubleToLongBits(other.fieldBoost))
+      return false;
+    if (fieldName == null) {
+      if (other.fieldName != null)
+        return false;
+    } else if (!fieldName.equals(other.fieldName))
+      return false;
+    if (fieldType.getValue() != other.fieldType.getValue())
+      return false;
+    if (!Arrays.equals(fieldValue, other.fieldValue))
+      return false;
+    return true;
+  }
+
+  @Override
+  public String toString() {
+    return "BlurField [fieldName=" + fieldName + ", fieldTypeValue="
+        + fieldType.getValue() + ", fieldBoost=" + fieldBoost + ", fieldValue="
+        + Arrays.toString(fieldValue) + "]";
+  }
+
+  
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/996ed6a6/src/blur-mapred/src/test/java/org/apache/blur/mr/BlurDocLocationTest.java
----------------------------------------------------------------------
diff --git a/src/blur-mapred/src/test/java/org/apache/blur/mr/BlurDocLocationTest.java b/src/blur-mapred/src/test/java/org/apache/blur/mr/BlurDocLocationTest.java
new file mode 100644
index 0000000..c27c95a
--- /dev/null
+++ b/src/blur-mapred/src/test/java/org/apache/blur/mr/BlurDocLocationTest.java
@@ -0,0 +1,43 @@
+package org.apache.blur.mr;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+
+import org.apache.blur.thrift.generated.DocLocation;
+import org.apache.hadoop.io.DataInputBuffer;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.junit.Before;
+import org.junit.Test;
+
+public class BlurDocLocationTest {
+
+  private BlurDocLocation blurDocLocationBefore;
+  private BlurDocLocation blurDocLocationAfter;
+  
+  @Before
+  public void setUp(){
+    DocLocation docLocation = new DocLocation();
+    docLocation.setDoc(0001);
+    docLocation.setShardIndex(001);
+    blurDocLocationBefore = new BlurDocLocation(docLocation);
+  }
+  
+  @Test
+  public void testBlurDocLocationWritable() throws IOException{
+    DataOutputBuffer dob = new DataOutputBuffer();
+    blurDocLocationBefore.write(dob);
+
+    DataInputBuffer dib = new DataInputBuffer();
+    dib.reset(dob.getData(), dob.getLength());
+    
+    blurDocLocationAfter = new BlurDocLocation();
+    blurDocLocationAfter.readFields(dib);
+    
+    assertTrue(blurDocLocationBefore.equals(blurDocLocationAfter));
+    assertEquals(blurDocLocationBefore.getDocId(), blurDocLocationAfter.getDocId());
+    assertEquals(blurDocLocationBefore.getShardIndexId(), blurDocLocationAfter.getShardIndexId());
+
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/996ed6a6/src/blur-mapred/src/test/java/org/apache/blur/mr/BlurDocumentTest.java
----------------------------------------------------------------------
diff --git a/src/blur-mapred/src/test/java/org/apache/blur/mr/BlurDocumentTest.java b/src/blur-mapred/src/test/java/org/apache/blur/mr/BlurDocumentTest.java
new file mode 100644
index 0000000..6f7e872
--- /dev/null
+++ b/src/blur-mapred/src/test/java/org/apache/blur/mr/BlurDocumentTest.java
@@ -0,0 +1,55 @@
+package org.apache.blur.mr;
+
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.blur.thrift.generated.Document;
+import org.apache.blur.thrift.generated.Field;
+import org.apache.blur.thrift.generated.TYPE;
+import org.apache.hadoop.io.DataInputBuffer;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.junit.Before;
+import org.junit.Test;
+
+public class BlurDocumentTest {
+  private BlurDocument blurDocumentBefore;
+  private BlurDocument blurDocumentAfter;
+  
+  @Before
+  public void setup(){
+    Field field1 = new Field();
+    field1.setName("Field1");
+    field1.setBoost(2.0);
+    field1.setType(TYPE.INT);
+    field1.setValue(ByteBuffer.allocate((Integer.SIZE)/8).putInt(5).array());
+    
+    Field field2 = new Field();
+    field2.setName("Field2");
+    field2.setBoost(2.0);
+    field2.setType(TYPE.STRING);
+    String value = "test";
+    field2.setValue(value.getBytes());
+    
+    Document d = new Document();
+    d.addToFields(field1);
+    d.addToFields(field2);
+    
+    blurDocumentBefore = new BlurDocument(d);
+  }
+  
+  @Test
+  public void testBlurDocumentWritable() throws IOException{
+    DataOutputBuffer dob = new DataOutputBuffer();
+    blurDocumentBefore.write(dob);
+
+    DataInputBuffer dib = new DataInputBuffer();
+    dib.reset(dob.getData(), dob.getLength());
+    
+    blurDocumentAfter = new BlurDocument();
+    blurDocumentAfter.readFields(dib);
+    
+    assertTrue(blurDocumentBefore.equals(blurDocumentAfter));
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/996ed6a6/src/blur-mapred/src/test/java/org/apache/blur/mr/BlurFieldTest.java
----------------------------------------------------------------------
diff --git a/src/blur-mapred/src/test/java/org/apache/blur/mr/BlurFieldTest.java b/src/blur-mapred/src/test/java/org/apache/blur/mr/BlurFieldTest.java
new file mode 100644
index 0000000..0d1e95c
--- /dev/null
+++ b/src/blur-mapred/src/test/java/org/apache/blur/mr/BlurFieldTest.java
@@ -0,0 +1,96 @@
+package org.apache.blur.mr;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.blur.thrift.generated.Field;
+import org.apache.blur.thrift.generated.TYPE;
+import org.apache.hadoop.io.DataInputBuffer;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.junit.Test;
+
+public class BlurFieldTest {
+  
+ 
+  @Test
+  public void testBlurFieldWritableforIntValue() throws IOException{
+    BlurField blurFieldBefore;
+    BlurField blurFieldAfter;
+    Field field = new Field();
+    field.setName("Field1");
+    field.setBoost(2.0);
+    field.setType(TYPE.INT);
+    field.setValue(ByteBuffer.allocate((Integer.SIZE)/8).putInt(5).array());
+    blurFieldBefore = new BlurField(field);
+    
+    DataOutputBuffer dob = new DataOutputBuffer();
+    blurFieldBefore.write(dob);
+
+    DataInputBuffer dib = new DataInputBuffer();
+    dib.reset(dob.getData(), dob.getLength());
+    
+    blurFieldAfter = new BlurField();
+    blurFieldAfter.readFields(dib);
+    
+    assertTrue(blurFieldBefore.equals(blurFieldAfter));
+    assertEquals(blurFieldBefore.getFieldName(),blurFieldAfter.getFieldName());
+    assertEquals((ByteBuffer.wrap(blurFieldBefore.getFieldValue()).getInt()),(ByteBuffer.wrap(blurFieldAfter.getFieldValue()).getInt()));
+    
+  }
+  
+  @Test
+  public void testBlurFieldWritableforDoubleValue() throws IOException{
+    BlurField blurFieldBefore;
+    BlurField blurFieldAfter;
+    Field field = new Field();
+    field.setName("Field1");
+    field.setBoost(2.0);
+    field.setType(TYPE.DOUBLE);
+    field.setValue(ByteBuffer.allocate((Double.SIZE)/8).putDouble(10.0).array());
+    blurFieldBefore = new BlurField(field);
+    
+    DataOutputBuffer dob = new DataOutputBuffer();
+    blurFieldBefore.write(dob);
+
+    DataInputBuffer dib = new DataInputBuffer();
+    dib.reset(dob.getData(), dob.getLength());
+    
+    blurFieldAfter = new BlurField();
+    blurFieldAfter.readFields(dib);
+    
+    assertTrue(blurFieldBefore.equals(blurFieldAfter));
+    assertEquals(blurFieldBefore.getFieldName(),blurFieldAfter.getFieldName());
+    assertEquals(String.valueOf((ByteBuffer.wrap(blurFieldBefore.getFieldValue()).getDouble())),String.valueOf((ByteBuffer.wrap(blurFieldAfter.getFieldValue()).getDouble())));
+    
+  }
+  
+  @Test
+  public void testBlurFieldWritableforStringValue() throws IOException{
+    BlurField blurFieldBefore;
+    BlurField blurFieldAfter;
+    Field field = new Field();
+    field.setName("Field1");
+    field.setBoost(2.0);
+    field.setType(TYPE.STRING);
+    field.setValue(new String("Test").getBytes());
+    blurFieldBefore = new BlurField(field);
+    
+    DataOutputBuffer dob = new DataOutputBuffer();
+    blurFieldBefore.write(dob);
+
+    DataInputBuffer dib = new DataInputBuffer();
+    dib.reset(dob.getData(), dob.getLength());
+    
+    blurFieldAfter = new BlurField();
+    blurFieldAfter.readFields(dib);
+    
+    assertTrue(blurFieldBefore.equals(blurFieldAfter));
+    assertEquals(blurFieldBefore.getFieldName(),blurFieldAfter.getFieldName());
+
+    assertEquals(new String(blurFieldBefore.getFieldValue()), new String(blurFieldAfter.getFieldValue()));
+    
+  }
+}


Mime
View raw message