drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From j..@apache.org
Subject [5/9] drill git commit: DRILL-5351: Minimize bounds checking in var len vectors for Parquet reader
Date Mon, 03 Apr 2017 05:12:05 GMT
DRILL-5351: Minimize bounds checking in var len vectors for Parquet reader

close #781


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/152c87aa
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/152c87aa
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/152c87aa

Branch: refs/heads/master
Commit: 152c87aa6cad84c8752b4a87967c7826cc90dbaa
Parents: 0ded1d0
Author: Parth Chandra <pchandra@maprtech.com>
Authored: Fri Feb 10 17:40:25 2017 -0800
Committer: Jinfeng Ni <jni@apache.org>
Committed: Fri Mar 31 15:32:13 2017 -0700

----------------------------------------------------------------------
 .../codegen/templates/NullableValueVectors.java | 24 +++++--
 .../templates/VariableLengthVectors.java        | 72 +++++++++++++-------
 2 files changed, 65 insertions(+), 31 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/152c87aa/exec/vector/src/main/codegen/templates/NullableValueVectors.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/NullableValueVectors.java b/exec/vector/src/main/codegen/templates/NullableValueVectors.java
index b242728..fcbe79a 100644
--- a/exec/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/exec/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -509,7 +509,9 @@ public final class ${className} extends BaseDataValueVector implements
<#if type
       <#if type.major != "VarLen">
       throw new UnsupportedOperationException();
       <#else>
-      fillEmpties(index);
+      if (index > lastSet + 1) {
+        fillEmpties(index);
+      }
 
       bits.getMutator().setSafe(index, 1);
       values.getMutator().setSafe(index, value, start, length);
@@ -522,7 +524,9 @@ public final class ${className} extends BaseDataValueVector implements
<#if type
       <#if type.major != "VarLen">
       throw new UnsupportedOperationException();
       <#else>
-      fillEmpties(index);
+      if (index > lastSet + 1) {
+        fillEmpties(index);
+      }
 
       bits.getMutator().setSafe(index, 1);
       values.getMutator().setSafe(index, value, start, length);
@@ -587,7 +591,9 @@ public final class ${className} extends BaseDataValueVector implements
<#if type
 
     public void setSafe(int index, int isSet<#list fields as field><#if field.include!true
>, ${field.type} ${field.name}Field</#if></#list> ) {
       <#if type.major == "VarLen">
-      fillEmpties(index);
+      if (index > lastSet + 1) {
+        fillEmpties(index);
+      }
       </#if>
 
       bits.getMutator().setSafe(index, isSet);
@@ -600,7 +606,9 @@ public final class ${className} extends BaseDataValueVector implements
<#if type
     public void setSafe(int index, Nullable${minor.class}Holder value) {
 
       <#if type.major == "VarLen">
-      fillEmpties(index);
+      if (index > lastSet + 1) {
+        fillEmpties(index);
+      }
       </#if>
       bits.getMutator().setSafe(index, value.isSet);
       values.getMutator().setSafe(index, value);
@@ -611,7 +619,9 @@ public final class ${className} extends BaseDataValueVector implements
<#if type
     public void setSafe(int index, ${minor.class}Holder value) {
 
       <#if type.major == "VarLen">
-      fillEmpties(index);
+      if (index > lastSet + 1) {
+        fillEmpties(index);
+      }
       </#if>
       bits.getMutator().setSafe(index, 1);
       values.getMutator().setSafe(index, value);
@@ -622,7 +632,9 @@ public final class ${className} extends BaseDataValueVector implements
<#if type
     <#if !(type.major == "VarLen" || minor.class == "Decimal28Sparse" || minor.class ==
"Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense" ||
minor.class == "Interval" || minor.class == "IntervalDay")>
       public void setSafe(int index, ${minor.javaType!type.javaType} value) {
         <#if type.major == "VarLen">
-        fillEmpties(index);
+        if (index > lastSet + 1) {
+          fillEmpties(index);
+        }
         </#if>
         bits.getMutator().setSafe(index, 1);
         values.getMutator().setSafe(index, value);

http://git-wip-us.apache.org/repos/asf/drill/blob/152c87aa/exec/vector/src/main/codegen/templates/VariableLengthVectors.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/VariableLengthVectors.java b/exec/vector/src/main/codegen/templates/VariableLengthVectors.java
index bb1d4fb..e970271 100644
--- a/exec/vector/src/main/codegen/templates/VariableLengthVectors.java
+++ b/exec/vector/src/main/codegen/templates/VariableLengthVectors.java
@@ -507,11 +507,15 @@ public final class ${minor.class}Vector extends BaseDataValueVector
implements V
       assert index >= 0;
 
       final int currentOffset = offsetVector.getAccessor().get(index);
-      while (data.capacity() < currentOffset + bytes.length) {
-        reAlloc();
-      }
       offsetVector.getMutator().setSafe(index + 1, currentOffset + bytes.length);
-      data.setBytes(currentOffset, bytes, 0, bytes.length);
+      try {
+        data.setBytes(currentOffset, bytes, 0, bytes.length);
+      } catch (IndexOutOfBoundsException e) {
+        while (data.capacity() < currentOffset + bytes.length) {
+          reAlloc();
+        }
+        data.setBytes(currentOffset, bytes, 0, bytes.length);
+      }
     }
 
     /**
@@ -533,12 +537,15 @@ public final class ${minor.class}Vector extends BaseDataValueVector
implements V
       assert index >= 0;
 
       int currentOffset = offsetVector.getAccessor().get(index);
-
-      while (data.capacity() < currentOffset + length) {
-        reAlloc();
-      }
       offsetVector.getMutator().setSafe(index + 1, currentOffset + length);
-      data.setBytes(currentOffset, bytes, start, length);
+      try {
+        data.setBytes(currentOffset, bytes, start, length);
+      } catch (IndexOutOfBoundsException e) {
+        while (data.capacity() < currentOffset + length) {
+          reAlloc();
+        }
+        data.setBytes(currentOffset, bytes, start, length);
+      }
     }
 
     public void setSafe(int index, byte[] bytes, int start, int length) {
@@ -546,11 +553,15 @@ public final class ${minor.class}Vector extends BaseDataValueVector
implements V
 
       final int currentOffset = offsetVector.getAccessor().get(index);
 
-      while (data.capacity() < currentOffset + length) {
-        reAlloc();
-      }
       offsetVector.getMutator().setSafe(index + 1, currentOffset + length);
-      data.setBytes(currentOffset, bytes, start, length);
+      try {
+        data.setBytes(currentOffset, bytes, start, length);
+      } catch (IndexOutOfBoundsException e) {
+        while (data.capacity() < currentOffset + length) {
+          reAlloc();
+        }
+        data.setBytes(currentOffset, bytes, start, length);
+      }
     }
 
     @Override
@@ -567,12 +578,16 @@ public final class ${minor.class}Vector extends BaseDataValueVector
implements V
       final int len = end - start;
       final int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index
* ${type.width});
 
-      while(data.capacity() < outputStart + len) {
-        reAlloc();
+      offsetVector.getMutator().setSafe( index+1,  outputStart + len);
+      try{
+        buffer.getBytes(start, data, outputStart, len);
+      } catch (IndexOutOfBoundsException e) {
+        while (data.capacity() < outputStart + len) {
+          reAlloc();
+        }
+        buffer.getBytes(start, data, outputStart, len);
       }
 
-      offsetVector.getMutator().setSafe( index+1,  outputStart + len);
-      buffer.getBytes(start, data, outputStart, len);
     }
 
     public void setSafe(int index, Nullable${minor.class}Holder holder){
@@ -584,11 +599,14 @@ public final class ${minor.class}Vector extends BaseDataValueVector
implements V
 
       int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index
* ${type.width});
 
-      while(data.capacity() < outputStart + len) {
-        reAlloc();
+      try {
+        holder.buffer.getBytes(start, data, outputStart, len);
+      } catch (IndexOutOfBoundsException e) {
+        while (data.capacity() < outputStart + len) {
+          reAlloc();
+        }
+        holder.buffer.getBytes(start, data, outputStart, len);
       }
-
-      holder.buffer.getBytes(start, data, outputStart, len);
       offsetVector.getMutator().setSafe( index+1,  outputStart + len);
     }
 
@@ -598,11 +616,15 @@ public final class ${minor.class}Vector extends BaseDataValueVector
implements V
       final int len = end - start;
       final int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index
* ${type.width});
 
-      while(data.capacity() < outputStart + len) {
-        reAlloc();
-      }
 
-      holder.buffer.getBytes(start, data, outputStart, len);
+      try {
+        holder.buffer.getBytes(start, data, outputStart, len);
+      } catch (IndexOutOfBoundsException e) {
+        while(data.capacity() < outputStart + len) {
+          reAlloc();
+        }
+        holder.buffer.getBytes(start, data, outputStart, len);
+      }
       offsetVector.getMutator().setSafe( index+1,  outputStart + len);
     }
 


Mime
View raw message