drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From s..@apache.org
Subject [3/4] drill git commit: DRILL-1764: return max value capacity if vector has no children
Date Tue, 02 Dec 2014 13:53:35 GMT
DRILL-1764: return max value capacity if vector has no children


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/98a90d6e
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/98a90d6e
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/98a90d6e

Branch: refs/heads/master
Commit: 98a90d6e8aa261a6fbcc574b6dac2f70a1638938
Parents: 3581a32
Author: Hanifi Gunes <hgunes@maprtech.com>
Authored: Mon Dec 1 20:52:01 2014 -0800
Committer: Steven Phillips <sphillips@maprtech.com>
Committed: Tue Dec 2 04:55:11 2014 -0800

----------------------------------------------------------------------
 .../exec/store/mongo/MongoRecordReader.java     |  2 +-
 .../exec/store/easy/json/JSONRecordReader.java  |  9 +++--
 .../drill/exec/vector/complex/MapVector.java    |  2 +-
 .../exec/vector/complex/fn/JsonReader.java      | 38 ++++++++++----------
 .../complex/impl/VectorContainerWriter.java     |  8 ++---
 .../exec/store/json/TestJsonRecordReader.java   |  6 ++++
 .../resources/store/json/value-capacity.json    |  2 ++
 7 files changed, 39 insertions(+), 28 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/98a90d6e/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoRecordReader.java
----------------------------------------------------------------------
diff --git a/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoRecordReader.java
b/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoRecordReader.java
index 95c4b64..79abe60 100644
--- a/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoRecordReader.java
+++ b/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoRecordReader.java
@@ -195,7 +195,7 @@ public class MongoRecordReader extends AbstractRecordReader {
         writer.setPosition(docCount);
         String doc = cursor.next().toString();
         jsonReader.setSource(doc.getBytes(Charsets.UTF_8));
-        if(jsonReader.write(writer)) {
+        if(jsonReader.write(writer)== JsonReader.ReadState.WRITE_SUCCEED) {
           docCount++;
           break;
         }else{

http://git-wip-us.apache.org/repos/asf/drill/blob/98a90d6e/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java
index 557592b..2582e34 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java
@@ -33,6 +33,7 @@ import org.apache.drill.exec.physical.impl.OutputMutator;
 import org.apache.drill.exec.store.AbstractRecordReader;
 import org.apache.drill.exec.vector.BaseValueVector;
 import org.apache.drill.exec.vector.complex.fn.JsonReader;
+import org.apache.drill.exec.vector.complex.fn.JsonReader.ReadState;
 import org.apache.drill.exec.vector.complex.impl.VectorContainerWriter;
 import org.apache.drill.exec.vector.complex.writer.BaseWriter;
 import org.apache.hadoop.fs.FSDataInputStream;
@@ -104,13 +105,14 @@ public class JSONRecordReader extends AbstractRecordReader {
     writer.reset();
 
     recordCount = 0;
+    ReadState write = null;
 //    Stopwatch p = new Stopwatch().start();
     try{
       outside: while(recordCount < BaseValueVector.INITIAL_VALUE_ALLOCATION){
         writer.setPosition(recordCount);
-        boolean write = jsonReader.write(writer);
+        write = jsonReader.write(writer);
 
-        if(write){
+        if(write == ReadState.WRITE_SUCCEED){
 //          logger.debug("Wrote record.");
           recordCount++;
         }else{
@@ -125,6 +127,9 @@ public class JSONRecordReader extends AbstractRecordReader {
       writer.setValueCount(recordCount);
 //      p.stop();
 //      System.out.println(String.format("Wrote %d records in %dms.", recordCount, p.elapsed(TimeUnit.MILLISECONDS)));
+      if (recordCount == 0 && write == ReadState.WRITE_FAILURE) {
+        throw new IOException("Record was too large to copy into vector.");
+      }
 
       return recordCount;
 

http://git-wip-us.apache.org/repos/asf/drill/blob/98a90d6e/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
index 47b8252..6e26547 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
@@ -292,7 +292,7 @@ public class MapVector extends AbstractContainerVector {
   @Override
   public int getValueCapacity() {
     if (this.vectors.isEmpty()) {
-      return 0;
+      return Integer.MAX_VALUE;
     }
 
     final Ordering<ValueVector> natural = new Ordering<ValueVector>() {

http://git-wip-us.apache.org/repos/asf/drill/blob/98a90d6e/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
index cc06b05..e209917 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
@@ -70,7 +70,11 @@ public class JsonReader {
    */
   private boolean onReset = false;
 
-  public static enum ReadState {WRITE_FAILURE, END_OF_STREAM, WRITE_SUCCEED}
+  public static enum ReadState {
+    WRITE_FAILURE,
+    END_OF_STREAM,
+    WRITE_SUCCEED
+  }
 
   public JsonReader() throws IOException {
     this(null, false);
@@ -131,8 +135,7 @@ public class JsonReader {
   }
 
 
-  public boolean write(ComplexWriter writer) throws JsonParseException, IOException {
-
+  public ReadState write(ComplexWriter writer) throws IOException {
     JsonToken t = onReset ? parser.getCurrentToken() : parser.nextToken();
 
     while (!parser.hasCurrentToken() && parser.hasDataAvailable()) {
@@ -140,7 +143,7 @@ public class JsonReader {
     }
 
     if(!parser.hasCurrentToken()){
-      return false;
+      return ReadState.END_OF_STREAM;
     }
 
     if(onReset){
@@ -153,27 +156,26 @@ public class JsonReader {
 
     switch(readState){
     case END_OF_STREAM:
-      return false;
+      break;
     case WRITE_FAILURE:
       logger.debug("Ran out of space while writing object, rewinding to object start.");
       parser.resetToMark();
       onReset = true;
-      return false;
-
+      break;
     case WRITE_SUCCEED:
-      return true;
-
+      break;
+    default:
+      throw new IllegalStateException();
     }
-    throw new IllegalStateException();
 
+    return readState;
   }
 
-  private ReadState writeToVector(ComplexWriter writer, JsonToken t) throws JsonParseException,
IOException {
+  private ReadState writeToVector(ComplexWriter writer, JsonToken t) throws IOException {
     if (!writer.ok()) {
       return ReadState.WRITE_FAILURE;
     }
 
-
     switch (t) {
       case START_OBJECT:
         writeDataSwitch(writer.rootAsMap());
@@ -198,7 +200,7 @@ public class JsonReader {
       }
   }
 
-  private void writeDataSwitch(MapWriter w) throws JsonParseException, IOException{
+  private void writeDataSwitch(MapWriter w) throws IOException{
     if(this.allTextMode){
       writeDataAllText(w, this.selection);
     }else{
@@ -206,7 +208,7 @@ public class JsonReader {
     }
   }
 
-  private void writeDataSwitch(ListWriter w) throws JsonParseException, IOException{
+  private void writeDataSwitch(ListWriter w) throws IOException{
     if(this.allTextMode){
       writeDataAllText(w);
     }else{
@@ -227,7 +229,7 @@ public class JsonReader {
     }
   }
 
-  private void writeData(MapWriter map, FieldSelection selection) throws JsonParseException,
IOException {
+  private void writeData(MapWriter map, FieldSelection selection) throws IOException {
     //
     map.start();
     outside: while(true) {
@@ -299,7 +301,7 @@ public class JsonReader {
   }
 
 
-  private void writeDataAllText(MapWriter map, FieldSelection selection) throws JsonParseException,
IOException {
+  private void writeDataAllText(MapWriter map, FieldSelection selection) throws IOException
{
     //
     map.start();
     outside: while(true) {
@@ -378,7 +380,7 @@ public class JsonReader {
     writer.varChar().writeVarChar(0, prepareVarCharHolder(parser.getText()), workBuf);
   }
 
-  private void writeData(ListWriter list) throws JsonParseException, IOException {
+  private void writeData(ListWriter list) throws IOException {
     list.start();
     outside: while (true) {
       if (!list.ok()) {
@@ -431,7 +433,7 @@ public class JsonReader {
 
   }
 
-  private void writeDataAllText(ListWriter list) throws JsonParseException, IOException {
+  private void writeDataAllText(ListWriter list) throws IOException {
     list.start();
     outside: while (true) {
       if (!list.ok()) {

http://git-wip-us.apache.org/repos/asf/drill/blob/98a90d6e/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/VectorContainerWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/VectorContainerWriter.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/VectorContainerWriter.java
index 6b8a523..36184a7 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/VectorContainerWriter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/impl/VectorContainerWriter.java
@@ -45,11 +45,11 @@ public class VectorContainerWriter extends AbstractFieldWriter implements
Comple
 
   @Override
   public int getValueCapacity() {
-    return mapVector.getValueCapacity();
+    return mapRoot.getValueCapacity();
   }
 
   public void checkValueCapacity(){
-    inform(mapVector.getValueCapacity() > idx());
+    inform(getValueCapacity() > idx());
   }
 
   public MapVector getMapVector() {
@@ -79,10 +79,6 @@ public class VectorContainerWriter extends AbstractFieldWriter implements
Comple
     mapRoot.setPosition(index);
   }
 
-  public MapWriter directMap() {
-    return mapRoot;
-  }
-
   @Override
   public void allocate() {
     mapRoot.allocate();

http://git-wip-us.apache.org/repos/asf/drill/blob/98a90d6e/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/TestJsonRecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/TestJsonRecordReader.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/TestJsonRecordReader.java
index 4bc1ee7..449f091 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/TestJsonRecordReader.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/TestJsonRecordReader.java
@@ -51,4 +51,10 @@ public class TestJsonRecordReader extends BaseTestQuery{
     test("select a[0].b[1] from cp.`jsoninput/nestedArray.json`");
     test("select a[1].b[1] from cp.`jsoninput/nestedArray.json`");  // index out of the range.
Should return empty list.
   }
+
+  @Test
+  public void testEmptyMapDoesNotFailValueCapacityCheck() throws Exception {
+    final String sql = "select * from cp.`store/json/value-capacity.json`";
+    test(sql);
+  }
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/98a90d6e/exec/java-exec/src/test/resources/store/json/value-capacity.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/store/json/value-capacity.json b/exec/java-exec/src/test/resources/store/json/value-capacity.json
new file mode 100644
index 0000000..96ff644
--- /dev/null
+++ b/exec/java-exec/src/test/resources/store/json/value-capacity.json
@@ -0,0 +1,2 @@
+{"a":{}}
+{"a":{"b":1}}


Mime
View raw message