drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amansi...@apache.org
Subject [1/2] drill git commit: DRILL-5709: Provide a value vector method to convert a vector to nullable
Date Mon, 04 Sep 2017 17:29:43 GMT
Repository: drill
Updated Branches:
  refs/heads/master 789b83d77 -> e57514aad


DRILL-5709: Provide a value vector method to convert a vector to nullable

Please see the DRILL-5709 for an explanation and example.

close apache/drill#901


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/6829af09
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/6829af09
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/6829af09

Branch: refs/heads/master
Commit: 6829af095ac5bb78ed3df874f078891e61ef7721
Parents: 789b83d
Author: Paul Rogers <progers@maprtech.com>
Authored: Tue Aug 8 20:04:24 2017 -0700
Committer: Aman Sinha <asinha@maprtech.com>
Committed: Sun Sep 3 23:15:07 2017 -0700

----------------------------------------------------------------------
 .../org/apache/drill/vector/TestToNullable.java | 135 +++++++++++++++++++
 .../codegen/templates/FixedValueVectors.java    |   6 +
 .../codegen/templates/NullableValueVectors.java |  38 +++++-
 .../src/main/codegen/templates/UnionVector.java |   5 +
 .../templates/VariableLengthVectors.java        |   6 +
 .../drill/exec/vector/BaseValueVector.java      |  20 ++-
 .../org/apache/drill/exec/vector/BitVector.java |   6 +
 .../drill/exec/vector/NullableVector.java       |   3 +-
 .../apache/drill/exec/vector/ObjectVector.java  |   5 +
 .../apache/drill/exec/vector/ValueVector.java   |  13 ++
 .../apache/drill/exec/vector/ZeroVector.java    |   6 +
 .../drill/exec/vector/complex/MapVector.java    |   7 +-
 .../exec/vector/complex/RepeatedListVector.java |   5 +
 .../exec/vector/complex/RepeatedMapVector.java  |   5 +
 14 files changed, 250 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/6829af09/exec/java-exec/src/test/java/org/apache/drill/vector/TestToNullable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/vector/TestToNullable.java b/exec/java-exec/src/test/java/org/apache/drill/vector/TestToNullable.java
new file mode 100644
index 0000000..234ad88
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/vector/TestToNullable.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.vector;
+
+import static org.junit.Assert.*;
+
+import org.apache.drill.common.types.TypeProtos.DataMode;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.vector.IntVector;
+import org.apache.drill.exec.vector.NullableIntVector;
+import org.apache.drill.exec.vector.NullableVarCharVector;
+import org.apache.drill.exec.vector.VarCharVector;
+import org.apache.drill.test.SubOperatorTest;
+import org.apache.drill.test.rowSet.SchemaBuilder;
+import org.bouncycastle.util.Arrays;
+import org.junit.Test;
+
+public class TestToNullable extends SubOperatorTest {
+
+  @SuppressWarnings("resource")
+  @Test
+  public void testFixedWidth() {
+    MaterializedField intSchema =
+        SchemaBuilder.columnSchema("a", MinorType.INT, DataMode.REQUIRED);
+    IntVector intVector = new IntVector(intSchema, fixture.allocator());
+    IntVector.Mutator intMutator = intVector.getMutator();
+    intVector.allocateNew(100);
+    for (int i = 0; i < 100; i++) {
+      intMutator.set(i, i * 10);
+    }
+    intMutator.setValueCount(100);
+
+    MaterializedField nullableIntSchema =
+        SchemaBuilder.columnSchema("a", MinorType.INT, DataMode.OPTIONAL);
+    NullableIntVector nullableIntVector = new NullableIntVector(nullableIntSchema, fixture.allocator());
+
+    intVector.toNullable(nullableIntVector);
+
+    assertEquals(0, intVector.getAccessor().getValueCount());
+    NullableIntVector.Accessor niAccessor = nullableIntVector.getAccessor();
+    assertEquals(100, niAccessor.getValueCount());
+    for (int i = 0; i < 100; i++) {
+      assertFalse(niAccessor.isNull(i));
+      assertEquals(i * 10, niAccessor.get(i));
+    }
+
+    nullableIntVector.clear();
+
+    // Don't clear the intVector, it should be empty.
+    // If it is not, the test will fail with a memory leak error.
+  }
+
+  @SuppressWarnings("resource")
+  @Test
+  public void testNullable() {
+    MaterializedField nullableIntSchema =
+        SchemaBuilder.columnSchema("a", MinorType.INT, DataMode.OPTIONAL);
+    NullableIntVector sourceVector = new NullableIntVector(nullableIntSchema, fixture.allocator());
+    NullableIntVector.Mutator sourceMutator = sourceVector.getMutator();
+    sourceVector.allocateNew(100);
+    for (int i = 0; i < 100; i++) {
+      sourceMutator.set(i, i * 10);
+    }
+    sourceMutator.setValueCount(100);
+
+    NullableIntVector destVector = new NullableIntVector(nullableIntSchema, fixture.allocator());
+
+    sourceVector.toNullable(destVector);
+
+    assertEquals(0, sourceVector.getAccessor().getValueCount());
+    NullableIntVector.Accessor destAccessor = destVector.getAccessor();
+    assertEquals(100, destAccessor.getValueCount());
+    for (int i = 0; i < 100; i++) {
+      assertFalse(destAccessor.isNull(i));
+      assertEquals(i * 10, destAccessor.get(i));
+    }
+
+    destVector.clear();
+
+    // Don't clear the intVector, it should be empty.
+    // If it is not, the test will fail with a memory leak error.
+  }
+
+  @SuppressWarnings("resource")
+  @Test
+  public void testVariableWidth() {
+    MaterializedField nonNullableSchema =
+        SchemaBuilder.columnSchema("a", MinorType.VARCHAR, DataMode.REQUIRED);
+    VarCharVector nonNullableVector = new VarCharVector(nonNullableSchema, fixture.allocator());
+    VarCharVector.Mutator mutator = nonNullableVector.getMutator();
+    nonNullableVector.allocateNew(100, 20);
+    byte value[] = new byte[20];
+    for (int i = 0; i < 100; i++) {
+      Arrays.fill(value, (byte)('A' + i % 26));
+      mutator.setSafe(i, value);
+    }
+    mutator.setValueCount(100);
+
+    MaterializedField nullableVarCharSchema =
+        SchemaBuilder.columnSchema("a", MinorType.VARCHAR, DataMode.OPTIONAL);
+    NullableVarCharVector nullableVector = new NullableVarCharVector(nullableVarCharSchema,
fixture.allocator());
+
+    nonNullableVector.toNullable(nullableVector);
+
+    assertEquals(0, nonNullableVector.getAccessor().getValueCount());
+    NullableVarCharVector.Accessor nullableAccessor = nullableVector.getAccessor();
+    assertEquals(100, nullableAccessor.getValueCount());
+    for (int i = 0; i < 100; i++) {
+      assertFalse(nullableAccessor.isNull(i));
+      Arrays.fill(value, (byte)('A' + i % 26));
+      assertTrue(Arrays.areEqual(value, nullableAccessor.get(i)));
+    }
+
+    nullableVector.clear();
+
+    // Don't clear the nonNullableVector, it should be empty.
+    // If it is not, the test will fail with a memory leak error.
+  }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/6829af09/exec/vector/src/main/codegen/templates/FixedValueVectors.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/FixedValueVectors.java b/exec/vector/src/main/codegen/templates/FixedValueVectors.java
index 5a53e21..e8cdcf8 100644
--- a/exec/vector/src/main/codegen/templates/FixedValueVectors.java
+++ b/exec/vector/src/main/codegen/templates/FixedValueVectors.java
@@ -335,6 +335,12 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements
F
     ++allocationMonitor;
   }
 
+  @Override
+  public void toNullable(ValueVector nullableVector) {
+    Nullable${minor.class}Vector dest = (Nullable${minor.class}Vector) nullableVector;
+    dest.getMutator().fromNotNullable(this);
+  }
+
   public final class Accessor extends BaseDataValueVector.BaseAccessor {
     @Override
     public int getValueCount() {

http://git-wip-us.apache.org/repos/asf/drill/blob/6829af09/exec/vector/src/main/codegen/templates/NullableValueVectors.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/NullableValueVectors.java b/exec/vector/src/main/codegen/templates/NullableValueVectors.java
index a2c0deb..4418212 100644
--- a/exec/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/exec/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -133,9 +133,10 @@ public final class ${className} extends BaseDataValueVector implements
<#if type
   }
 
   @Override
-  public ${valuesName} getValuesVector() {
-    return values;
-  }
+  public ${valuesName} getValuesVector() { return values; }
+
+  @Override
+  public UInt1Vector getBitsVector() { return bits; }
 
   @Override
   public void setInitialCapacity(int numRecords) {
@@ -398,6 +399,14 @@ public final class ${className} extends BaseDataValueVector implements
<#if type
     mutator.exchange(other.getMutator());
   }
 
+  <#if type.major != "VarLen">
+  @Override
+  public void toNullable(ValueVector nullableVector) {
+    exchange(nullableVector);
+    clear();
+  }
+
+  </#if>
   public final class Accessor extends BaseDataValueVector.BaseAccessor <#if type.major
= "VarLen">implements VariableWidthVector.VariableWidthAccessor</#if> {
     final UInt1Vector.Accessor bAccessor = bits.getAccessor();
     final ${valuesName}.Accessor vAccessor = values.getAccessor();
@@ -449,8 +458,8 @@ public final class ${className} extends BaseDataValueVector implements
<#if type
     @Override
     public ${friendlyType} getObject(int index) {
       if (isNull(index)) {
-          return null;
-      }else{
+        return null;
+      } else {
         return vAccessor.getObject(index);
       }
     }
@@ -458,8 +467,8 @@ public final class ${className} extends BaseDataValueVector implements
<#if type
     <#if minor.class == "Interval" || minor.class == "IntervalDay" || minor.class == "IntervalYear">
     public StringBuilder getAsStringBuilder(int index) {
       if (isNull(index)) {
-          return null;
-      }else{
+        return null;
+      } else {
         return vAccessor.getAsStringBuilder(index);
       }
     }
@@ -778,6 +787,21 @@ public final class ${className} extends BaseDataValueVector implements
<#if type
       setCount = target.setCount;
       target.setCount = temp;
     }
+
+    public void fromNotNullable(${minor.class}Vector srce) {
+      clear();
+      final int valueCount = srce.getAccessor().getValueCount();
+
+      // Create a new bits vector, all values non-null
+
+      fillBitsVector(getBitsVector(), valueCount);
+
+      // Swap the data portion
+
+      getValuesVector().exchange(srce);
+      <#if type.major = "VarLen">lastSet = valueCount;</#if>
+      setValueCount(valueCount);
+    }
   }
 }
 </#list>

http://git-wip-us.apache.org/repos/asf/drill/blob/6829af09/exec/vector/src/main/codegen/templates/UnionVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/UnionVector.java b/exec/vector/src/main/codegen/templates/UnionVector.java
index 2c732f4..c198544 100644
--- a/exec/vector/src/main/codegen/templates/UnionVector.java
+++ b/exec/vector/src/main/codegen/templates/UnionVector.java
@@ -267,6 +267,11 @@ public class UnionVector implements ValueVector {
     return newVector;
   }
 
+  @Override
+  public void toNullable(ValueVector nullableVector) {
+    throw new UnsupportedOperationException();
+  }
+
   private class TransferImpl implements TransferPair {
 
     UnionVector to;

http://git-wip-us.apache.org/repos/asf/drill/blob/6829af09/exec/vector/src/main/codegen/templates/VariableLengthVectors.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/VariableLengthVectors.java b/exec/vector/src/main/codegen/templates/VariableLengthVectors.java
index 0eb8906..4527da8 100644
--- a/exec/vector/src/main/codegen/templates/VariableLengthVectors.java
+++ b/exec/vector/src/main/codegen/templates/VariableLengthVectors.java
@@ -422,6 +422,12 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements
V
     offsetVector.exchange(target.offsetVector);
   }
 
+  @Override
+  public void toNullable(ValueVector nullableVector) {
+    Nullable${minor.class}Vector dest = (Nullable${minor.class}Vector) nullableVector;
+    dest.getMutator().fromNotNullable(this);
+  }
+
   public final class Accessor extends BaseValueVector.BaseAccessor implements VariableWidthAccessor
{
     final UInt${type.width}Vector.Accessor oAccessor = offsetVector.getAccessor();
     public long getStartEnd(int index){

http://git-wip-us.apache.org/repos/asf/drill/blob/6829af09/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
index f4a5847..2179829 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
@@ -19,6 +19,7 @@ package org.apache.drill.exec.vector;
 
 import io.netty.buffer.DrillBuf;
 
+import java.util.Collections;
 import java.util.Iterator;
 
 import com.google.common.base.Preconditions;
@@ -116,7 +117,7 @@ public abstract class BaseValueVector implements ValueVector {
 
   @Override
   public Iterator<ValueVector> iterator() {
-    return Iterators.emptyIterator();
+    return Collections.emptyIterator();
   }
 
   public static boolean checkBufRefs(final ValueVector vv) {
@@ -133,5 +134,22 @@ public abstract class BaseValueVector implements ValueVector {
   public BufferAllocator getAllocator() {
     return allocator;
   }
+
+  public static void fillBitsVector(UInt1Vector bits, int valueCount) {
+
+    // Create a new bits vector, all values non-null
+
+    bits.allocateNew(valueCount);
+    UInt1Vector.Mutator bitsMutator = bits.getMutator();
+    for (int i = 0; i < valueCount; i++) {
+      bitsMutator.set(i, 1);
+    }
+    bitsMutator.setValueCount(valueCount);
+  }
+
+  @Override
+  public void toNullable(ValueVector nullableVector) {
+    throw new UnsupportedOperationException();
+  }
 }
 

http://git-wip-us.apache.org/repos/asf/drill/blob/6829af09/exec/vector/src/main/java/org/apache/drill/exec/vector/BitVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/BitVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/BitVector.java
index a7c81de..4d7098b 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/BitVector.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/BitVector.java
@@ -531,4 +531,10 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe
   public int getPayloadByteCount(int valueCount) {
     return getSizeFromCount(valueCount);
   }
+
+  @Override
+  public void toNullable(ValueVector nullableVector) {
+    NullableBitVector dest = (NullableBitVector) nullableVector;
+    dest.getMutator().fromNotNullable(this);
+  }
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/6829af09/exec/vector/src/main/java/org/apache/drill/exec/vector/NullableVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/NullableVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/NullableVector.java
index 8091c4c..51b5e0c 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/NullableVector.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/NullableVector.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -19,5 +19,6 @@ package org.apache.drill.exec.vector;
 
 public interface NullableVector extends ValueVector{
 
+  ValueVector getBitsVector();
   ValueVector getValuesVector();
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/6829af09/exec/vector/src/main/java/org/apache/drill/exec/vector/ObjectVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/ObjectVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/ObjectVector.java
index 3136e32..c9edeb0 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/ObjectVector.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/ObjectVector.java
@@ -206,6 +206,11 @@ public class ObjectVector extends BaseValueVector {
     throw new UnsupportedOperationException("ObjectVector does not support this");
   }
 
+  @Override
+  public void toNullable(ValueVector nullableVector) {
+    throw new UnsupportedOperationException();
+  }
+
   public final class Accessor extends BaseAccessor {
     @Override
     public Object getObject(int index) {

http://git-wip-us.apache.org/repos/asf/drill/blob/6829af09/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueVector.java
index 2926862..5125626 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueVector.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueVector.java
@@ -228,6 +228,19 @@ public interface ValueVector extends Closeable, Iterable<ValueVector>
{
   void exchange(ValueVector other);
 
   /**
+   * Convert a non-nullable vector to nullable by shuffling the data from
+   * one to the other. Avoids the need to generate copy code just to change
+   * mode. If this vector is non-nullable, accepts a nullable dual (same
+   * minor type, different mode.) If the vector is non-nullable, or non-scalar,
+   * then throws an exception.
+   *
+   * @param nullableVector nullable vector of the same minor type as
+   * this vector
+   */
+
+  void toNullable(ValueVector nullableVector);
+
+  /**
    * An abstraction that is used to read from this vector instance.
    */
   interface Accessor {

http://git-wip-us.apache.org/repos/asf/drill/blob/6829af09/exec/vector/src/main/java/org/apache/drill/exec/vector/ZeroVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/ZeroVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/ZeroVector.java
index 5786487..e6f0544 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/ZeroVector.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/ZeroVector.java
@@ -162,8 +162,14 @@ public class ZeroVector implements ValueVector {
   @Override
   public void exchange(ValueVector other) { }
 
+  @Override
   public void collectLedgers(Set<BufferLedger> ledgers) {}
 
   @Override
   public int getPayloadByteCount(int valueCount) { return 0; }
+
+  @Override
+  public void toNullable(ValueVector nullableVector) {
+    throw new UnsupportedOperationException();
+  }
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/6829af09/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
index f755081..f9ff58d 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
@@ -388,5 +388,10 @@ public class MapVector extends AbstractMapVector {
     valueCount = 0;
 
     super.close();
- }
+  }
+
+  @Override
+  public void toNullable(ValueVector nullableVector) {
+    throw new UnsupportedOperationException();
+  }
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/6829af09/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
index ab2c3d8..be9ebee 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
@@ -452,4 +452,9 @@ public class RepeatedListVector extends AbstractContainerVector
     // TODO: Figure out how to test this scenario, then what to do...
     throw new UnsupportedOperationException("Exchange() not yet supported for repeated lists");
   }
+
+  @Override
+  public void toNullable(ValueVector nullableVector) {
+    throw new UnsupportedOperationException();
+  }
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/6829af09/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
index be73fc8..999e29e 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
@@ -604,4 +604,9 @@ public class RepeatedMapVector extends AbstractMapVector
     super.collectLedgers(ledgers);
     offsets.collectLedgers(ledgers);
   }
+
+  @Override
+  public void toNullable(ValueVector nullableVector) {
+    throw new UnsupportedOperationException();
+  }
 }


Mime
View raw message