hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject [1/3] hive git commit: HIVE-12054. Create vectorized ORC write method. (omalley reviewed by prasanthj)
Date Wed, 18 Nov 2015 04:37:01 GMT
Repository: hive
Updated Branches:
  refs/heads/master 255b2bdd9 -> 7f65e36d9


http://git-wip-us.apache.org/repos/asf/hive/blob/7f65e36d/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/BytesColumnVector.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/BytesColumnVector.java
b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/BytesColumnVector.java
index 8fa388b..99744cd 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/BytesColumnVector.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/BytesColumnVector.java
@@ -157,6 +157,19 @@ public class BytesColumnVector extends ColumnVector {
   }
 
   /**
+   * Set a field by actually copying in to a local buffer.
+   * If you must actually copy data in to the array, use this method.
+   * DO NOT USE this method unless it's not practical to set data by reference with setRef().
+   * Setting data by reference tends to run a lot faster than copying data in.
+   *
+   * @param elementNum index within column vector to set
+   * @param sourceBuf container of source data
+   */
+  public void setVal(int elementNum, byte[] sourceBuf) {
+    setVal(elementNum, sourceBuf, 0, sourceBuf.length);
+  }
+
+  /**
    * Set a field to the concatenation of two string values. Result data is copied
    * into the internal buffer.
    *

http://git-wip-us.apache.org/repos/asf/hive/blob/7f65e36d/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java
b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java
index f18b911..fcb1ae9 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java
@@ -83,22 +83,31 @@ public abstract class ColumnVector {
   }
 
   /**
-     * Resets the column to default state
-     *  - fills the isNull array with false
-     *  - sets noNulls to true
-     *  - sets isRepeating to false
-     */
-    public void reset() {
-      if (!noNulls) {
-        Arrays.fill(isNull, false);
-      }
-      noNulls = true;
-      isRepeating = false;
-      preFlattenNoNulls = true;
-      preFlattenIsRepeating = false;
+   * Resets the column to default state
+   *  - fills the isNull array with false
+   *  - sets noNulls to true
+   *  - sets isRepeating to false
+   */
+  public void reset() {
+    if (!noNulls) {
+      Arrays.fill(isNull, false);
     }
+    noNulls = true;
+    isRepeating = false;
+    preFlattenNoNulls = true;
+    preFlattenIsRepeating = false;
+  }
+
+  /**
+   * Sets the isRepeating flag. Recurses over structs and unions so that the
+   * flags are set correctly.
+   * @param isRepeating
+   */
+  public void setRepeating(boolean isRepeating) {
+    this.isRepeating = isRepeating;
+  }
 
-    abstract public void flatten(boolean selectedInUse, int[] sel, int size);
+  abstract public void flatten(boolean selectedInUse, int[] sel, int size);
 
     // Simplify vector by brute-force flattening noNulls if isRepeating
     // This can be used to reduce combinatorial explosion of code paths in VectorExpressions

http://git-wip-us.apache.org/repos/asf/hive/blob/7f65e36d/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/StructColumnVector.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/StructColumnVector.java
b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/StructColumnVector.java
index f7c8b05..cf07bca 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/StructColumnVector.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/StructColumnVector.java
@@ -121,4 +121,12 @@ public class StructColumnVector extends ColumnVector {
       fields[i].unFlatten();
     }
   }
+
+  @Override
+  public void setRepeating(boolean isRepeating) {
+    super.setRepeating(isRepeating);
+    for(int i=0; i < fields.length; ++i) {
+      fields[i].setRepeating(isRepeating);
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/7f65e36d/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/UnionColumnVector.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/UnionColumnVector.java
b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/UnionColumnVector.java
index 2b3b013..298d588 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/UnionColumnVector.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/UnionColumnVector.java
@@ -131,4 +131,12 @@ public class UnionColumnVector extends ColumnVector {
       fields[i].unFlatten();
     }
   }
+
+  @Override
+  public void setRepeating(boolean isRepeating) {
+    super.setRepeating(isRepeating);
+    for(int i=0; i < fields.length; ++i) {
+      fields[i].setRepeating(isRepeating);
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/7f65e36d/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java
b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java
index 7c18da6..e85491b 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java
@@ -183,4 +183,14 @@ public class VectorizedRowBatch implements Writable {
       }
     }
   }
+
+  /**
+   * Set the maximum number of rows in the batch.
+   * Data is not preserved.
+   */
+  public void ensureSize(int rows) {
+    for(int i=0; i < cols.length; ++i) {
+      cols[i].ensureSize(rows, false);
+    }
+  }
 }


Mime
View raw message