hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From prasan...@apache.org
Subject svn commit: r1661271 [10/39] - in /hive/branches/llap: ./ accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/ accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/ accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/mr/ accumulo...
Date Sat, 21 Feb 2015 02:37:03 GMT
Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFPower.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFPower.java?rev=1661271&r1=1661270&r2=1661271&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFPower.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFPower.java Sat Feb 21 02:36:54 2015
@@ -41,7 +41,6 @@ extended = "Example:\n"
 @VectorizedExpressions({FuncPowerLongToDouble.class, FuncPowerDoubleToDouble.class})
 public class GenericUDFPower extends GenericUDF {
   private final String opName;
-  private final String opDisplayName;
 
   private transient PrimitiveObjectInspector baseOI;
   private transient PrimitiveObjectInspector powerOI;
@@ -54,7 +53,6 @@ public class GenericUDFPower extends Gen
 
   public GenericUDFPower() {
     opName = getClass().getSimpleName();
-    opDisplayName = "power";
     resultOI = PrimitiveObjectInspectorFactory.writableDoubleObjectInspector;
   }
 
@@ -103,7 +101,7 @@ public class GenericUDFPower extends Gen
   @Override
   public String getDisplayString(String[] children) {
     assert (children.length == 2);
-    return opDisplayName + "(" + children[0] + ", " + children[1] + ")";
+    return getStandardDisplayString("power", children);
   }
 
   @Override

Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFPrintf.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFPrintf.java?rev=1661271&r1=1661270&r2=1661271&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFPrintf.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFPrintf.java Sat Feb 21 02:36:54 2015
@@ -141,12 +141,6 @@ public class GenericUDFPrintf extends Ge
   @Override
   public String getDisplayString(String[] children) {
     assert (children.length >= 2);
-    StringBuilder sb = new StringBuilder();
-    sb.append("printf(");
-    for (int i = 0; i < children.length - 1; i++) {
-      sb.append(children[i]).append(", ");
-    }
-    sb.append(children[children.length - 1]).append(")");
-    return sb.toString();
+    return getStandardDisplayString("printf", children);
   }
 }

Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFRound.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFRound.java?rev=1661271&r1=1661270&r2=1661271&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFRound.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFRound.java Sat Feb 21 02:36:54 2015
@@ -22,6 +22,7 @@ import org.apache.hadoop.hive.common.typ
 import org.apache.hadoop.hive.ql.exec.Description;
 import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
 import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
 import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressions;
 import org.apache.hadoop.hive.ql.exec.vector.expressions.FuncRoundWithNumDigitsDecimalToDecimal;
 import org.apache.hadoop.hive.ql.exec.vector.expressions.RoundWithNumDigitsDoubleToDouble;
@@ -83,38 +84,42 @@ public class GenericUDFRound extends Gen
           "ROUND requires one or two argument, got " + arguments.length);
     }
 
-    inputOI = (PrimitiveObjectInspector) arguments[0];
-    if (inputOI.getCategory() != Category.PRIMITIVE) {
-      throw new UDFArgumentException(
-          "ROUND input only takes primitive types, got " + inputOI.getTypeName());
+    if (arguments[0].getCategory() != Category.PRIMITIVE) {
+      throw new UDFArgumentTypeException(0,
+          "ROUND input only takes primitive types, got " + arguments[0].getTypeName());
     }
+    inputOI = (PrimitiveObjectInspector) arguments[0];
 
     if (arguments.length == 2) {
+      if (arguments[1].getCategory() != Category.PRIMITIVE) {
+        throw new UDFArgumentTypeException(1,
+            "ROUND second argument only takes primitive types, got " + arguments[1].getTypeName());
+      }
       PrimitiveObjectInspector scaleOI = (PrimitiveObjectInspector) arguments[1];
       switch (scaleOI.getPrimitiveCategory()) {
       case VOID:
         break;
       case BYTE:
         if (!(scaleOI instanceof WritableConstantByteObjectInspector)) {
-          throw new UDFArgumentException("ROUND second argument only takes constant");
+          throw new UDFArgumentTypeException(1, "ROUND second argument only takes constant");
         }
         scale = ((WritableConstantByteObjectInspector)scaleOI).getWritableConstantValue().get();
         break;
       case SHORT:
         if (!(scaleOI instanceof WritableConstantShortObjectInspector)) {
-          throw new UDFArgumentException("ROUND second argument only takes constant");
+          throw new UDFArgumentTypeException(1, "ROUND second argument only takes constant");
         }
         scale = ((WritableConstantShortObjectInspector)scaleOI).getWritableConstantValue().get();
         break;
       case INT:
         if (!(scaleOI instanceof WritableConstantIntObjectInspector)) {
-          throw new UDFArgumentException("ROUND second argument only takes constant");
+          throw new UDFArgumentTypeException(1, "ROUND second argument only takes constant");
         }
         scale = ((WritableConstantIntObjectInspector)scaleOI).getWritableConstantValue().get();
         break;
       case LONG:
         if (!(scaleOI instanceof WritableConstantLongObjectInspector)) {
-          throw new UDFArgumentException("ROUND second argument only takes constant");
+          throw new UDFArgumentTypeException(1, "ROUND second argument only takes constant");
         }
         long l = ((WritableConstantLongObjectInspector)scaleOI).getWritableConstantValue().get();
         if (l < Integer.MIN_VALUE || l > Integer.MAX_VALUE) {
@@ -123,7 +128,7 @@ public class GenericUDFRound extends Gen
         scale = (int)l;
         break;
       default:
-        throw new UDFArgumentException("ROUND second argument only takes integer constant");
+        throw new UDFArgumentTypeException(1, "ROUND second argument only takes integer constant");
       }
     }
 
@@ -151,8 +156,9 @@ public class GenericUDFRound extends Gen
       converterFromString = ObjectInspectorConverters.getConverter(inputOI, outputOI);
       break;
     default:
-      throw new UDFArgumentException("Only numeric data types are allowed for ROUND function. Got " +
-          inputType.name());
+      throw new UDFArgumentTypeException(0,
+          "Only numeric or string group data types are allowed for ROUND function. Got "
+              + inputType.name());
     }
 
     return outputOI;
@@ -240,8 +246,9 @@ public class GenericUDFRound extends Gen
        }
        return round(doubleValue, scale);
      default:
-       throw new UDFArgumentException("Only numeric data types are allowed for ROUND function. Got " +
-           inputType.name());
+      throw new UDFArgumentTypeException(0,
+          "Only numeric or string group data types are allowed for ROUND function. Got "
+              + inputType.name());
     }
   }
 
@@ -256,17 +263,7 @@ public class GenericUDFRound extends Gen
 
   @Override
   public String getDisplayString(String[] children) {
-    StringBuilder sb = new StringBuilder();
-    sb.append("round(");
-    if (children.length > 0) {
-      sb.append(children[0]);
-      for (int i = 1; i < children.length; i++) {
-        sb.append(", ");
-        sb.append(children[i]);
-      }
-    }
-    sb.append(")");
-    return sb.toString();
+    return getStandardDisplayString("round", children);
   }
 
 }

Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFSentences.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFSentences.java?rev=1661271&r1=1661270&r2=1661271&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFSentences.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFSentences.java Sat Feb 21 02:36:54 2015
@@ -128,14 +128,6 @@ public class GenericUDFSentences extends
   @Override
   public String getDisplayString(String[] children) {
     assert (children.length >= 1 && children.length <= 3);
-    String display = "sentences(" + children[0];
-    if(children.length > 1) {
-      display += ", " + children[1];
-      if(children.length > 2) {
-        display += ", " + children[2];
-      }
-    }
-    display += ")";
-    return display;
+    return getStandardDisplayString("sentences", children);
   }
 }

Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFSize.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFSize.java?rev=1661271&r1=1661270&r2=1661271&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFSize.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFSize.java Sat Feb 21 02:36:54 2015
@@ -79,6 +79,6 @@ public class GenericUDFSize extends Gene
   @Override
   public String getDisplayString(String[] children) {
     assert (children.length == 1);
-    return "size(" + children[0] + ")";
+    return getStandardDisplayString("size", children);
   }
 }

Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFSortArray.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFSortArray.java?rev=1661271&r1=1661270&r2=1661271&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFSortArray.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFSortArray.java Sat Feb 21 02:36:54 2015
@@ -116,6 +116,6 @@ public class GenericUDFSortArray extends
   @Override
   public String getDisplayString(String[] children) {
     assert (children.length == 1);
-    return "sort_array(" + children[0] + ")";
+    return getStandardDisplayString("sort_array", children);
  }
 }

Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFSplit.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFSplit.java?rev=1661271&r1=1661270&r2=1661271&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFSplit.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFSplit.java Sat Feb 21 02:36:54 2015
@@ -82,7 +82,7 @@ public class GenericUDFSplit extends Gen
   @Override
   public String getDisplayString(String[] children) {
     assert (children.length == 2);
-    return "split(" + children[0] + ", " + children[1] + ")";
+    return getStandardDisplayString("split", children);
   }
 
 }

Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFStringToMap.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFStringToMap.java?rev=1661271&r1=1661270&r2=1661271&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFStringToMap.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFStringToMap.java Sat Feb 21 02:36:54 2015
@@ -104,19 +104,7 @@ public class GenericUDFStringToMap exten
 
   @Override
   public String getDisplayString(String[] children) {
-    StringBuilder sb = new StringBuilder();
-    sb.append("str_to_map(");
     assert (children.length <= 3);
-    boolean firstChild = true;
-    for (String child : children) {
-      if (firstChild) {
-        firstChild = false;
-      } else {
-        sb.append(",");
-      }
-      sb.append(child);
-    }
-    sb.append(")");
-    return sb.toString();
+    return getStandardDisplayString("str_to_map", children, ",");
   }
 }

Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFStruct.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFStruct.java?rev=1661271&r1=1661270&r2=1661271&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFStruct.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFStruct.java Sat Feb 21 02:36:54 2015
@@ -59,15 +59,6 @@ public class GenericUDFStruct extends Ge
 
   @Override
   public String getDisplayString(String[] children) {
-    StringBuilder sb = new StringBuilder();
-    sb.append("struct(");
-    for (int i = 0; i < children.length; i++) {
-      if (i > 0) {
-        sb.append(',');
-      }
-      sb.append(children[i]);
-    }
-    sb.append(')');
-    return sb.toString();
+    return getStandardDisplayString("struct", children, ",");
   }
 }

Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTranslate.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTranslate.java?rev=1661271&r1=1661270&r2=1661271&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTranslate.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTranslate.java Sat Feb 21 02:36:54 2015
@@ -287,7 +287,7 @@ public class GenericUDFTranslate extends
   @Override
   public String getDisplayString(String[] children) {
     assert (children.length == 3);
-    return "translate(" + children[0] + ", " + children[1] + ", " + children[2] + ")";
+    return getStandardDisplayString("translate", children);
   }
 
 }

Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUpper.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUpper.java?rev=1661271&r1=1661270&r2=1661271&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUpper.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUpper.java Sat Feb 21 02:36:54 2015
@@ -107,17 +107,7 @@ public class GenericUDFUpper extends Gen
 
   @Override
   public String getDisplayString(String[] children) {
-    StringBuilder sb = new StringBuilder();
-    sb.append("upper(");
-    if (children.length > 0) {
-      sb.append(children[0]);
-      for (int i = 1; i < children.length; i++) {
-        sb.append(",");
-        sb.append(children[i]);
-      }
-    }
-    sb.append(")");
-    return sb.toString();
+    return getStandardDisplayString("upper", children);
   }
 
 }

Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFInline.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFInline.java?rev=1661271&r1=1661270&r2=1661271&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFInline.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFInline.java Sat Feb 21 02:36:54 2015
@@ -19,6 +19,7 @@
 package org.apache.hadoop.hive.ql.udf.generic;
 
 import java.util.ArrayList;
+import java.util.List;
 
 import org.apache.hadoop.hive.ql.exec.Description;
 import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
@@ -57,8 +58,11 @@ public class GenericUDTFInline extends G
 
   @Override
   public void process(Object[] os) throws HiveException {
-    for (Object row : new ArrayList<Object>(li.getList(os[0]))) {
-      forward(row);
+    List<?> list = li.getList(os[0]);
+    if (list != null && !list.isEmpty()) {
+      for (Object row : list.toArray()) {
+        forward(row);
+      }
     }
   }
 

Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/NumDistinctValueEstimator.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/NumDistinctValueEstimator.java?rev=1661271&r1=1661270&r2=1661271&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/NumDistinctValueEstimator.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/NumDistinctValueEstimator.java Sat Feb 21 02:36:54 2015
@@ -35,18 +35,18 @@ public class NumDistinctValueEstimator {
    * independent. As a consequence, the hash values will not distribute uniformly from 0 to 2^p-1
    * thus introducing errors in the estimates.
    */
-  private static final int bitVectorSize = 31;
-  private int numBitVectors;
+  private static final int BIT_VECTOR_SIZE = 31;
+  private final int numBitVectors;
 
   // Refer to Flajolet-Martin'86 for the value of phi
-  private final double phi =  0.77351;
+  private static final double PHI = 0.77351;
 
-  private int[] a;
-  private int[] b;
-  private  FastBitSet[] bitVector = new FastBitSet[numBitVectors];
+  private final int[] a;
+  private final int[] b;
+  private final FastBitSet[] bitVector;
 
-  private Random aValue;
-  private Random bValue;
+  private final Random aValue;
+  private final Random bValue;
 
   /* Create a new distinctValueEstimator
    */
@@ -54,7 +54,7 @@ public class NumDistinctValueEstimator {
     this.numBitVectors = numBitVectors;
     bitVector = new FastBitSet[numBitVectors];
     for (int i=0; i< numBitVectors; i++) {
-      bitVector[i] = new FastBitSet(bitVectorSize);
+      bitVector[i] = new FastBitSet(BIT_VECTOR_SIZE);
     }
 
     a = new int[numBitVectors];
@@ -98,23 +98,30 @@ public class NumDistinctValueEstimator {
       b[i] = randVal;
 
       if (a[i] < 0) {
-        a[i] = a[i] + (1 << bitVectorSize - 1);
+        a[i] = a[i] + (1 << BIT_VECTOR_SIZE - 1);
       }
 
       if (b[i] < 0) {
-        b[i] = b[i] + (1 << bitVectorSize - 1);
+        b[i] = b[i] + (1 << BIT_VECTOR_SIZE - 1);
       }
     }
   }
 
   public NumDistinctValueEstimator(String s, int numBitVectors) {
-    FastBitSet b[] = deserialize(s, numBitVectors);
+    this.numBitVectors = numBitVectors;
+    FastBitSet bitVectorDeser[] = deserialize(s, numBitVectors);
     bitVector = new FastBitSet[numBitVectors];
     for(int i=0; i <numBitVectors; i++) {
-       bitVector[i] = new FastBitSet(bitVectorSize);
+       bitVector[i] = new FastBitSet(BIT_VECTOR_SIZE);
        bitVector[i].clear();
-       bitVector[i].or(b[i]);
+       bitVector[i].or(bitVectorDeser[i]);
     }
+
+    a = null;
+    b = null;
+
+    aValue = null;
+    bValue = null;
   }
 
   /**
@@ -135,7 +142,7 @@ public class NumDistinctValueEstimator {
   }
 
   public int getBitVectorSize() {
-    return bitVectorSize;
+    return BIT_VECTOR_SIZE;
   }
 
   public void printNumDistinctValueEstimator() {
@@ -145,7 +152,7 @@ public class NumDistinctValueEstimator {
     LOG.debug("Number of Vectors:");
     LOG.debug(numBitVectors);
     LOG.debug("Vector Size: ");
-    LOG.debug(bitVectorSize);
+    LOG.debug(BIT_VECTOR_SIZE);
 
     for (int i=0; i < numBitVectors; i++) {
       t = t + bitVector[i].toString();
@@ -173,7 +180,7 @@ public class NumDistinctValueEstimator {
   private FastBitSet[] deserialize(String s, int numBitVectors) {
     FastBitSet[] b = new FastBitSet[numBitVectors];
     for (int j=0; j < numBitVectors; j++) {
-      b[j] = new FastBitSet(bitVectorSize);
+      b[j] = new FastBitSet(BIT_VECTOR_SIZE);
       b[j].clear();
     }
 
@@ -219,7 +226,7 @@ public class NumDistinctValueEstimator {
   }
 
   private int generateHash(long v, int hashNum) {
-    int mod = (1<<bitVectorSize) - 1;
+    int mod = (1<<BIT_VECTOR_SIZE) - 1;
     long tempHash = a[hashNum] * v  + b[hashNum];
     tempHash %= mod;
     int hash = (int) tempHash;
@@ -234,7 +241,7 @@ public class NumDistinctValueEstimator {
   }
 
   private int generateHashForPCSA(long v) {
-    int mod = 1 << (bitVectorSize - 1) - 1;
+    int mod = 1 << (BIT_VECTOR_SIZE - 1) - 1;
     long tempHash = a[0] * v + b[0];
     tempHash %= mod;
     int hash = (int) tempHash;
@@ -259,8 +266,8 @@ public class NumDistinctValueEstimator {
       int index;
 
       // Find the index of the least significant bit that is 1
-      for (index=0; index<bitVectorSize; index++) {
-        if (hash % 2 == 1) {
+      for (index=0; index<BIT_VECTOR_SIZE; index++) {
+        if (hash % 2 != 0) {
           break;
         }
         hash = hash >> 1;
@@ -277,8 +284,8 @@ public class NumDistinctValueEstimator {
     int index;
 
     // Find the index of the least significant bit that is 1
-    for (index=0; index<bitVectorSize; index++) {
-      if (rho % 2 == 1) {
+    for (index=0; index<BIT_VECTOR_SIZE; index++) {
+      if (rho % 2 != 0) {
         break;
       }
       rho = rho >> 1;
@@ -321,13 +328,13 @@ public class NumDistinctValueEstimator {
 
     for (int i=0; i < numBitVectors; i++) {
       int index = 0;
-      while (bitVector[i].get(index) && index < bitVectorSize) {
+      while (bitVector[i].get(index) && index < BIT_VECTOR_SIZE) {
         index = index + 1;
       }
       S = S + index;
     }
 
-    numDistinctValues = ((numBitVectors/phi) * Math.pow(2.0, S/numBitVectors));
+    numDistinctValues = ((numBitVectors/PHI) * Math.pow(2.0, S/numBitVectors));
     return ((long)numDistinctValues);
   }
 
@@ -345,7 +352,7 @@ public class NumDistinctValueEstimator {
     }
 
     avgLeastSigZero =
-        (double)(sumLeastSigZero/(numBitVectors * 1.0)) - (Math.log(phi)/Math.log(2.0));
+        (double)(sumLeastSigZero/(numBitVectors * 1.0)) - (Math.log(PHI)/Math.log(2.0));
     numDistinctValues = Math.pow(2.0, avgLeastSigZero);
     return ((long)(numDistinctValues));
   }

Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/MatchPath.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/MatchPath.java?rev=1661271&r1=1661270&r2=1661271&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/MatchPath.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/MatchPath.java Sat Feb 21 02:36:54 2015
@@ -28,6 +28,7 @@ import org.apache.hadoop.hive.ql.exec.Ex
 import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluatorFactory;
 import org.apache.hadoop.hive.ql.exec.PTFPartition;
 import org.apache.hadoop.hive.ql.exec.PTFPartition.PTFPartitionIterator;
+import org.apache.hadoop.hive.ql.exec.Utilities;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.parse.ASTNode;
 import org.apache.hadoop.hive.ql.parse.PTFTranslator;
@@ -195,6 +196,20 @@ public class MatchPath extends TableFunc
 
       setOutputOI(OI);
     }
+
+    @Override
+    public List<String> getReferencedColumns() throws SemanticException {
+      MatchPath matchPath = (MatchPath) evaluator;
+      List<String> columns = new ArrayList<>();
+      for (ExprNodeDesc exprNode : matchPath.resultExprInfo.resultExprNodes) {
+        Utilities.mergeUniqElems(columns, exprNode.getCols());
+      }
+      for (ExprNodeDesc exprNode : matchPath.symInfo.symbolExprsDecs) {
+        Utilities.mergeUniqElems(columns, exprNode.getCols());
+      }
+      return columns;
+    }
+    
     /*
      * validate and setup patternStr
      */
@@ -356,6 +371,7 @@ public class MatchPath extends TableFunc
 
   static class SymbolsInfo {
     int sz;
+    ArrayList<ExprNodeDesc> symbolExprsDecs;
     ArrayList<ExprNodeEvaluator> symbolExprsEvaluators;
     ArrayList<ObjectInspector> symbolExprsOIs;
     ArrayList<String> symbolExprsNames;
@@ -366,6 +382,7 @@ public class MatchPath extends TableFunc
       symbolExprsEvaluators = new ArrayList<ExprNodeEvaluator>(sz);
       symbolExprsOIs = new ArrayList<ObjectInspector>(sz);
       symbolExprsNames = new ArrayList<String>(sz);
+      symbolExprsDecs = new ArrayList<>(sz);
     }
 
     void add(String name, PTFExpressionDef arg)
@@ -373,6 +390,7 @@ public class MatchPath extends TableFunc
       symbolExprsNames.add(name);
       symbolExprsEvaluators.add(arg.getExprEvaluator());
       symbolExprsOIs.add(arg.getOI());
+      symbolExprsDecs.add(arg.getExprNode());
     }
   }
 
@@ -749,8 +767,7 @@ public class MatchPath extends TableFunc
       /*
        * create SelectListOI
        */
-      selectListInputOI = (StructObjectInspector)
-          PTFTranslator.getStandardStructOI(selectListInputRowResolver);
+      selectListInputOI = PTFTranslator.getStandardStructOI(selectListInputRowResolver);
     }
 
     private void fixResultExprString()

Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/NoopWithMap.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/NoopWithMap.java?rev=1661271&r1=1661270&r2=1661271&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/NoopWithMap.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/NoopWithMap.java Sat Feb 21 02:36:54 2015
@@ -29,11 +29,6 @@ import org.apache.hadoop.hive.serde2.obj
 
 public class NoopWithMap extends Noop
 {
-  @Override
-  public PTFPartition execute(PTFPartition iPart) throws HiveException
-  {
-    return iPart;
-  }
 
   @Override
   protected PTFPartition _transformRawInput(PTFPartition iPart) throws HiveException

Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionEvaluator.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionEvaluator.java?rev=1661271&r1=1661270&r2=1661271&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionEvaluator.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionEvaluator.java Sat Feb 21 02:36:54 2015
@@ -30,7 +30,6 @@ import org.apache.hadoop.hive.ql.metadat
 import org.apache.hadoop.hive.ql.plan.PTFDesc;
 import org.apache.hadoop.hive.ql.plan.ptf.PartitionedTableFunctionDef;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
 
 /*
@@ -60,7 +59,7 @@ import org.apache.hadoop.hive.serde2.obj
  */
 
 /**
- * Based on Hive {@link GenericUDAFEvaluator}. Break up the responsibility of the old AsbtractTableFunction
+ * Based on Hive {@link GenericUDAFEvaluator}. Break up the responsibility of the old AbstractTableFunction
  * class into a Resolver and Evaluator.
  * <p>
  * The Evaluator also holds onto the {@link TableFunctionDef}. This provides information
@@ -79,7 +78,7 @@ import org.apache.hadoop.hive.serde2.obj
  */
 public abstract class TableFunctionEvaluator {
   /*
-   * how is this different from the OutpuShape set on the TableDef.
+   * how is this different from the OutputShape set on the TableDef.
    * This is the OI of the object coming out of the PTF.
    * It is put in an output Partition whose Serde is usually LazyBinarySerde.
    * So the next PTF (or Operator) in the chain gets a LazyBinaryStruct.

Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionResolver.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionResolver.java?rev=1661271&r1=1661270&r2=1661271&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionResolver.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionResolver.java Sat Feb 21 02:36:54 2015
@@ -60,7 +60,7 @@ public abstract class TableFunctionResol
   /*
    * - called during translation.
    * - invokes createEvaluator which must be implemented by a subclass
-   * - sets up the evaluator with references to the TableDef, PartitionClass, PartitonMemsize and
+   * - sets up the evaluator with references to the TableDef, PartitionClass, PartitionMemsize and
    *   the transformsRawInput boolean.
    */
   public void initialize(HiveConf cfg, PTFDesc ptfDesc, PartitionedTableFunctionDef tDef)
@@ -193,4 +193,14 @@ public abstract class TableFunctionResol
    * a subclass must provide the {@link TableFunctionEvaluator} instance.
    */
   protected abstract TableFunctionEvaluator createEvaluator(PTFDesc ptfDesc, PartitionedTableFunctionDef tDef);
+
+  /**
+   * Provide referenced columns names to be used in partition function
+   *
+   * @return null for unknown (will get all columns from table including virtual columns) 
+   * @throws SemanticException
+   */
+  public List<String> getReferencedColumns() throws SemanticException {
+    return null;
+  }
 }

Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java?rev=1661271&r1=1661270&r2=1661271&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java Sat Feb 21 02:36:54 2015
@@ -69,8 +69,7 @@ public class WindowingTableFunction exte
   public void execute(PTFPartitionIterator<Object> pItr, PTFPartition outP) throws HiveException {
     ArrayList<List<?>> oColumns = new ArrayList<List<?>>();
     PTFPartition iPart = pItr.getPartition();
-    StructObjectInspector inputOI;
-    inputOI = (StructObjectInspector) iPart.getOutputOI();
+    StructObjectInspector inputOI = iPart.getOutputOI();
 
     WindowTableFunctionDef wTFnDef = (WindowTableFunctionDef) getTableDef();
     Order order = wTFnDef.getOrder().getExpressions().get(0).getOrder();
@@ -145,7 +144,8 @@ public class WindowingTableFunction exte
     return true;
   }
 
-  private boolean streamingPossible(Configuration cfg, WindowFunctionDef wFnDef) {
+  private boolean streamingPossible(Configuration cfg, WindowFunctionDef wFnDef)
+      throws HiveException {
     WindowFrameDef wdwFrame = wFnDef.getWindowFrame();
     WindowFunctionInfo wFnInfo = FunctionRegistry.getWindowFunctionInfo(wFnDef
         .getName());
@@ -202,7 +202,7 @@ public class WindowingTableFunction exte
    * So no Unbounded Preceding or Following.
    */
   @SuppressWarnings("resource")
-  private int[] setCanAcceptInputAsStream(Configuration cfg) {
+  private int[] setCanAcceptInputAsStream(Configuration cfg) throws HiveException {
 
     canAcceptInputAsStream = false;
 
@@ -514,7 +514,6 @@ public class WindowingTableFunction exte
       i++;
     }
 
-    i=0;
     for(i=0; i < iPart.getOutputOI().getAllStructFieldRefs().size(); i++) {
       output.add(null);
     }

Modified: hive/branches/llap/ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto?rev=1661271&r1=1661270&r2=1661271&view=diff
==============================================================================
--- hive/branches/llap/ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto (original)
+++ hive/branches/llap/ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto Sat Feb 21 02:36:54 2015
@@ -16,7 +16,9 @@
  * limitations under the License.
  */
 
-package org.apache.hadoop.hive.ql.io.orc;
+package orc.proto;
+
+option java_package = "org.apache.hadoop.hive.ql.io.orc";
 
 message IntegerStatistics  {
   optional sint64 minimum = 1;
@@ -86,6 +88,15 @@ message RowIndex {
   repeated RowIndexEntry entry = 1;
 }
 
+message BloomFilter {
+  optional uint32 numHashFunctions = 1;
+  repeated fixed64 bitset = 2;
+}
+
+message BloomFilterIndex {
+  repeated BloomFilter bloomFilter = 1;
+}
+
 message Stream {
   // if you add new index stream kinds, you need to make sure to update
   // StreamName to ensure it is added to the stripe in the right area
@@ -97,8 +108,9 @@ message Stream {
     DICTIONARY_COUNT = 4;
     SECONDARY = 5;
     ROW_INDEX = 6;
+    BLOOM_FILTER = 7;
   }
-  required Kind kind = 1;
+  optional Kind kind = 1;
   optional uint32 column = 2;
   optional uint64 length = 3;
 }
@@ -110,7 +122,7 @@ message ColumnEncoding {
     DIRECT_V2 = 2;
     DICTIONARY_V2 = 3;
   }
-  required Kind kind = 1;
+  optional Kind kind = 1;
   optional uint32 dictionarySize = 2;
 }
 
@@ -140,7 +152,7 @@ message Type {
     VARCHAR = 16;
     CHAR = 17;
   }
-  required Kind kind = 1;
+  optional Kind kind = 1;
   repeated uint32 subtypes = 2 [packed=true];
   repeated string fieldNames = 3;
   optional uint32 maximumLength = 4;
@@ -157,8 +169,8 @@ message StripeInformation {
 }
 
 message UserMetadataItem {
-  required string name = 1;
-  required bytes value = 2;
+  optional string name = 1;
+  optional bytes value = 2;
 }
 
 message StripeStatistics {

Modified: hive/branches/llap/ql/src/test/org/apache/hadoop/hive/metastore/TestMetastoreExpr.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/test/org/apache/hadoop/hive/metastore/TestMetastoreExpr.java?rev=1661271&r1=1661270&r2=1661271&view=diff
==============================================================================
--- hive/branches/llap/ql/src/test/org/apache/hadoop/hive/metastore/TestMetastoreExpr.java (original)
+++ hive/branches/llap/ql/src/test/org/apache/hadoop/hive/metastore/TestMetastoreExpr.java Sat Feb 21 02:36:54 2015
@@ -26,12 +26,9 @@ import java.util.Stack;
 import junit.framework.TestCase;
 
 import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
 import org.apache.hadoop.hive.metastore.api.Database;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
 import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
-import org.apache.hadoop.hive.metastore.api.MetaException;
 import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
 import org.apache.hadoop.hive.metastore.api.Order;
 import org.apache.hadoop.hive.metastore.api.Partition;
@@ -90,14 +87,14 @@ public class TestMetastoreExpr extends T
     }
   }
 
-  private static void silentDropDatabase(String dbName) throws MetaException, TException {
+  private static void silentDropDatabase(String dbName) throws TException {
     try {
       for (String tableName : client.getTables(dbName, "*")) {
         client.dropTable(dbName, tableName);
       }
       client.dropDatabase(dbName);
-    } catch (NoSuchObjectException e) {
-    } catch (InvalidOperationException e) {
+    } catch (NoSuchObjectException ignore) {
+    } catch (InvalidOperationException ignore) {
     }
   }
 
@@ -153,16 +150,16 @@ public class TestMetastoreExpr extends T
       client.listPartitionsByExpr(dbName, tblName,
           new byte[] { 'f', 'o', 'o' }, null, (short)-1, new ArrayList<Partition>());
       fail("Should have thrown IncompatibleMetastoreException");
-    } catch (IMetaStoreClient.IncompatibleMetastoreException ex) {
+    } catch (IMetaStoreClient.IncompatibleMetastoreException ignore) {
     }
 
     // Invalid expression => throw some exception, but not incompatible metastore.
     try {
       checkExpr(-1, dbName, tblName, e.val(31).intCol("p3").pred(">", 2).build());
       fail("Should have thrown");
-    } catch (IMetaStoreClient.IncompatibleMetastoreException ex) {
+    } catch (IMetaStoreClient.IncompatibleMetastoreException ignore) {
       fail("Should not have thrown IncompatibleMetastoreException");
-    } catch (Exception ex) {
+    } catch (Exception ignore) {
     }
   }
 
@@ -198,7 +195,7 @@ public class TestMetastoreExpr extends T
       for (int i = 0; i < args; ++i) {
         children.add(stack.pop());
       }
-      stack.push(new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo,
+      stack.push(new ExprNodeGenericFuncDesc(ti,
           FunctionRegistry.getFunctionInfo(name).getGenericUDF(), children));
       return this;
     }
@@ -249,8 +246,7 @@ public class TestMetastoreExpr extends T
   }
 
   private void addPartition(HiveMetaStoreClient client, Table table,
-      List<String> vals, String location) throws InvalidObjectException,
-        AlreadyExistsException, MetaException, TException {
+      List<String> vals, String location) throws TException {
 
     Partition part = new Partition();
     part.setDbName(table.getDbName());

Modified: hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/TestHiveKVResultCache.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/TestHiveKVResultCache.java?rev=1661271&r1=1661270&r2=1661271&view=diff
==============================================================================
--- hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/TestHiveKVResultCache.java (original)
+++ hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/TestHiveKVResultCache.java Sat Feb 21 02:36:54 2015
@@ -27,8 +27,6 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.concurrent.LinkedBlockingQueue;
 
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.io.HiveKey;
 import org.apache.hadoop.io.BytesWritable;
 import org.junit.Test;
@@ -42,8 +40,7 @@ public class TestHiveKVResultCache {
   @Test
   public void testSimple() throws Exception {
     // Create KV result cache object, add one (k,v) pair and retrieve them.
-    HiveConf conf = new HiveConf();
-    HiveKVResultCache cache = new HiveKVResultCache(conf);
+    HiveKVResultCache cache = new HiveKVResultCache();
 
     HiveKey key = new HiveKey("key".getBytes(), "key".hashCode());
     BytesWritable value = new BytesWritable("value".getBytes());
@@ -60,10 +57,9 @@ public class TestHiveKVResultCache {
 
   @Test
   public void testSpilling() throws Exception {
-    HiveConf conf = new HiveConf();
-    HiveKVResultCache cache = new HiveKVResultCache(conf);
+    HiveKVResultCache cache = new HiveKVResultCache();
 
-    final int recordCount = HiveKVResultCache.IN_MEMORY_CACHE_SIZE * 3;
+    final int recordCount = HiveKVResultCache.IN_MEMORY_NUM_ROWS * 3;
 
     // Test using the same cache where first n rows are inserted then cache is cleared.
     // Next reuse the same cache and insert another m rows and verify the cache stores correctly.
@@ -104,10 +100,18 @@ public class TestHiveKVResultCache {
   @Test
   public void testResultList() throws Exception {
     scanAndVerify(10000, 0, 0, "a", "b");
+    scanAndVerify(10000, 511, 0, "a", "b");
+    scanAndVerify(10000, 511 * 2, 0, "a", "b");
+    scanAndVerify(10000, 511, 10, "a", "b");
+    scanAndVerify(10000, 511 * 2, 10, "a", "b");
     scanAndVerify(10000, 512, 0, "a", "b");
     scanAndVerify(10000, 512 * 2, 0, "a", "b");
-    scanAndVerify(10000, 512, 10, "a", "b");
-    scanAndVerify(10000, 512 * 2, 10, "a", "b");
+    scanAndVerify(10000, 512, 3, "a", "b");
+    scanAndVerify(10000, 512 * 6, 10, "a", "b");
+    scanAndVerify(10000, 512 * 7, 5, "a", "b");
+    scanAndVerify(10000, 512 * 9, 19, "a", "b");
+    scanAndVerify(10000, 1, 0, "a", "b");
+    scanAndVerify(10000, 1, 1, "a", "b");
   }
 
   private static void scanAndVerify(
@@ -176,8 +180,8 @@ public class TestHiveKVResultCache {
     // A queue to notify separateRowGenerator to generate the next batch of rows.
     private LinkedBlockingQueue<Boolean> queue;
 
-    MyHiveFunctionResultList(Configuration conf, Iterator inputIterator) {
-      super(conf, inputIterator);
+    MyHiveFunctionResultList(Iterator inputIterator) {
+      super(inputIterator);
     }
 
     void init(long rows, int threshold, int separate, String p1, String p2) {
@@ -258,8 +262,7 @@ public class TestHiveKVResultCache {
   private static long scanResultList(long rows, int threshold, int separate,
       List<Tuple2<HiveKey, BytesWritable>> output, String prefix1, String prefix2) {
     final long iteratorCount = threshold == 0 ? 1 : rows * (100 - separate) / 100 / threshold;
-    MyHiveFunctionResultList resultList = new MyHiveFunctionResultList(
-        new HiveConf(), new Iterator() {
+    MyHiveFunctionResultList resultList = new MyHiveFunctionResultList(new Iterator() {
       // Input record iterator, not used
       private int i = 0;
       @Override

Modified: hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestFileDump.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestFileDump.java?rev=1661271&r1=1661270&r2=1661271&view=diff
==============================================================================
--- hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestFileDump.java (original)
+++ hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestFileDump.java Sat Feb 21 02:36:54 2015
@@ -21,19 +21,6 @@ package org.apache.hadoop.hive.ql.io.orc
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
 
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.common.type.HiveChar;
-import org.apache.hadoop.hive.common.type.HiveDecimal;
-import org.apache.hadoop.hive.common.type.HiveVarchar;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
-import org.apache.hive.common.util.HiveTestUtils;
-import org.junit.Before;
-import org.junit.Test;
-
 import java.io.BufferedReader;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
@@ -49,6 +36,19 @@ import java.util.List;
 import java.util.Map;
 import java.util.Random;
 
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.type.HiveChar;
+import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.common.type.HiveVarchar;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hive.common.util.HiveTestUtils;
+import org.junit.Before;
+import org.junit.Test;
+
 public class TestFileDump {
 
   Path workDir = new Path(System.getProperty("test.tmp.dir"));
@@ -303,4 +303,101 @@ public class TestFileDump {
 
     checkOutput(outputFilename, workDir + File.separator + outputFilename);
   }
+
+  @Test
+  public void testBloomFilter() throws Exception {
+    ObjectInspector inspector;
+    synchronized (TestOrcFile.class) {
+      inspector = ObjectInspectorFactory.getReflectionObjectInspector
+          (MyRecord.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
+    }
+    conf.set(HiveConf.ConfVars.HIVE_ORC_ENCODING_STRATEGY.varname, "COMPRESSION");
+    OrcFile.WriterOptions options = OrcFile.writerOptions(conf)
+        .fileSystem(fs)
+        .inspector(inspector)
+        .stripeSize(100000)
+        .compress(CompressionKind.ZLIB)
+        .bufferSize(10000)
+        .rowIndexStride(1000)
+        .bloomFilterColumns("s");
+    Writer writer = OrcFile.createWriter(testFilePath, options);
+    Random r1 = new Random(1);
+    String[] words = new String[]{"It", "was", "the", "best", "of", "times,",
+        "it", "was", "the", "worst", "of", "times,", "it", "was", "the", "age",
+        "of", "wisdom,", "it", "was", "the", "age", "of", "foolishness,", "it",
+        "was", "the", "epoch", "of", "belief,", "it", "was", "the", "epoch",
+        "of", "incredulity,", "it", "was", "the", "season", "of", "Light,",
+        "it", "was", "the", "season", "of", "Darkness,", "it", "was", "the",
+        "spring", "of", "hope,", "it", "was", "the", "winter", "of", "despair,",
+        "we", "had", "everything", "before", "us,", "we", "had", "nothing",
+        "before", "us,", "we", "were", "all", "going", "direct", "to",
+        "Heaven,", "we", "were", "all", "going", "direct", "the", "other",
+        "way"};
+    for(int i=0; i < 21000; ++i) {
+      writer.addRow(new MyRecord(r1.nextInt(), r1.nextLong(),
+          words[r1.nextInt(words.length)]));
+    }
+    writer.close();
+    PrintStream origOut = System.out;
+    String outputFilename = "orc-file-dump-bloomfilter.out";
+    FileOutputStream myOut = new FileOutputStream(workDir + File.separator + outputFilename);
+
+    // replace stdout and run command
+    System.setOut(new PrintStream(myOut));
+    FileDump.main(new String[]{testFilePath.toString(), "--rowindex=3"});
+    System.out.flush();
+    System.setOut(origOut);
+
+
+    checkOutput(outputFilename, workDir + File.separator + outputFilename);
+  }
+
+  @Test
+  public void testBloomFilter2() throws Exception {
+    ObjectInspector inspector;
+    synchronized (TestOrcFile.class) {
+      inspector = ObjectInspectorFactory.getReflectionObjectInspector
+          (MyRecord.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
+    }
+    conf.set(HiveConf.ConfVars.HIVE_ORC_ENCODING_STRATEGY.varname, "COMPRESSION");
+    OrcFile.WriterOptions options = OrcFile.writerOptions(conf)
+        .fileSystem(fs)
+        .inspector(inspector)
+        .stripeSize(100000)
+        .compress(CompressionKind.ZLIB)
+        .bufferSize(10000)
+        .rowIndexStride(1000)
+        .bloomFilterColumns("l")
+        .bloomFilterFpp(0.01);
+    Writer writer = OrcFile.createWriter(testFilePath, options);
+    Random r1 = new Random(1);
+    String[] words = new String[]{"It", "was", "the", "best", "of", "times,",
+        "it", "was", "the", "worst", "of", "times,", "it", "was", "the", "age",
+        "of", "wisdom,", "it", "was", "the", "age", "of", "foolishness,", "it",
+        "was", "the", "epoch", "of", "belief,", "it", "was", "the", "epoch",
+        "of", "incredulity,", "it", "was", "the", "season", "of", "Light,",
+        "it", "was", "the", "season", "of", "Darkness,", "it", "was", "the",
+        "spring", "of", "hope,", "it", "was", "the", "winter", "of", "despair,",
+        "we", "had", "everything", "before", "us,", "we", "had", "nothing",
+        "before", "us,", "we", "were", "all", "going", "direct", "to",
+        "Heaven,", "we", "were", "all", "going", "direct", "the", "other",
+        "way"};
+    for(int i=0; i < 21000; ++i) {
+      writer.addRow(new MyRecord(r1.nextInt(), r1.nextLong(),
+          words[r1.nextInt(words.length)]));
+    }
+    writer.close();
+    PrintStream origOut = System.out;
+    String outputFilename = "orc-file-dump-bloomfilter2.out";
+    FileOutputStream myOut = new FileOutputStream(workDir + File.separator + outputFilename);
+
+    // replace stdout and run command
+    System.setOut(new PrintStream(myOut));
+    FileDump.main(new String[]{testFilePath.toString(), "--rowindex=2"});
+    System.out.flush();
+    System.setOut(origOut);
+
+
+    checkOutput(outputFilename, workDir + File.separator + outputFilename);
+  }
 }

Modified: hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java?rev=1661271&r1=1661270&r2=1661271&view=diff
==============================================================================
--- hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java (original)
+++ hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java Sat Feb 21 02:36:54 2015
@@ -515,6 +515,10 @@ public class TestOrcFile {
       Object row = rows.next(null);
       assertEquals(tslist.get(idx++).getNanos(), ((TimestampWritable) row).getNanos());
     }
+    assertEquals(1, OrcUtils.getFlattenedColumnsCount(inspector));
+    boolean[] expected = new boolean[] {false};
+    boolean[] included = OrcUtils.includeColumns("", "ts", inspector);
+    assertEquals(true, Arrays.equals(expected, included));
   }
 
   @Test
@@ -538,6 +542,19 @@ public class TestOrcFile {
     Reader reader = OrcFile.createReader(testFilePath,
         OrcFile.readerOptions(conf).filesystem(fs));
 
+    assertEquals(3, OrcUtils.getFlattenedColumnsCount(inspector));
+    boolean[] expected = new boolean[] {false, false, true};
+    boolean[] included = OrcUtils.includeColumns("string1", "bytes1,string1", inspector);
+    assertEquals(true, Arrays.equals(expected, included));
+
+    expected = new boolean[] {false, false, false};
+    included = OrcUtils.includeColumns("", "bytes1,string1", inspector);
+    assertEquals(true, Arrays.equals(expected, included));
+
+    expected = new boolean[] {false, false, false};
+    included = OrcUtils.includeColumns(null, "bytes1,string1", inspector);
+    assertEquals(true, Arrays.equals(expected, included));
+
     // check the stats
     ColumnStatistics[] stats = reader.getStatistics();
     assertEquals(4, stats[0].getNumberOfValues());
@@ -634,6 +651,12 @@ public class TestOrcFile {
     writer.close();
     Reader reader = OrcFile.createReader(testFilePath,
         OrcFile.readerOptions(conf).filesystem(fs));
+
+    assertEquals(3, OrcUtils.getFlattenedColumnsCount(inspector));
+    boolean[] expected = new boolean[] {false, true, false};
+    boolean[] included = OrcUtils.includeColumns("int1", "int1,string1", inspector);
+    assertEquals(true, Arrays.equals(expected, included));
+
     Metadata metadata = reader.getMetadata();
     int numStripes = metadata.getStripeStatistics().size();
     assertEquals(3, numStripes);
@@ -672,7 +695,7 @@ public class TestOrcFile {
     assertEquals(5000, ((StringColumnStatistics)ss3.getColumnStatistics()[2]).getSum());
 
     RecordReaderImpl recordReader = (RecordReaderImpl) reader.rows();
-    OrcProto.RowIndex[] index = recordReader.readRowIndex(0, null);
+    OrcProto.RowIndex[] index = recordReader.readRowIndex(0, null).getRowGroupIndex();
     assertEquals(3, index.length);
     List<OrcProto.RowIndexEntry> items = index[1].getEntryList();
     assertEquals(1, items.size());
@@ -682,7 +705,7 @@ public class TestOrcFile {
     assertEquals(0, items.get(0).getPositions(2));
     assertEquals(1, 
                  items.get(0).getStatistics().getIntStatistics().getMinimum());
-    index = recordReader.readRowIndex(1, null);
+    index = recordReader.readRowIndex(1, null).getRowGroupIndex();
     assertEquals(3, index.length);
     items = index[1].getEntryList();
     assertEquals(2, 
@@ -715,6 +738,44 @@ public class TestOrcFile {
     Reader reader = OrcFile.createReader(testFilePath,
         OrcFile.readerOptions(conf).filesystem(fs));
 
+    assertEquals(24, OrcUtils.getFlattenedColumnsCount(inspector));
+    boolean[] expected = new boolean[] {false, false, false, false, false,
+        false, false, false, false, false,
+        false, false, false, false, false,
+        false, false, false, false, false,
+        false, false, false, false};
+    boolean[] included = OrcUtils.includeColumns("",
+        "boolean1,byte1,short1,int1,long1,float1,double1,bytes1,string1,middle,list,map", inspector);
+    assertEquals(true, Arrays.equals(expected, included));
+
+    expected = new boolean[] {false, true, false, false, false,
+        false, false, false, false, true,
+        true, true, true, true, true,
+        false, false, false, false, true,
+        true, true, true, true};
+    included = OrcUtils.includeColumns("boolean1,string1,middle,map",
+        "boolean1,byte1,short1,int1,long1,float1,double1,bytes1,string1,middle,list,map", inspector);
+    assertEquals(true, Arrays.equals(expected, included));
+
+    expected = new boolean[] {false, true, false, false, false,
+        false, false, false, false, true,
+        true, true, true, true, true,
+        false, false, false, false, true,
+        true, true, true, true};
+    included = OrcUtils.includeColumns("boolean1,string1,middle,map",
+        "boolean1,byte1,short1,int1,long1,float1,double1,bytes1,string1,middle,list,map", inspector);
+    assertEquals(true, Arrays.equals(expected, included));
+
+    expected = new boolean[] {false, true, true, true, true,
+        true, true, true, true, true,
+        true, true, true, true, true,
+        true, true, true, true, true,
+        true, true, true, true};
+    included = OrcUtils.includeColumns(
+        "boolean1,byte1,short1,int1,long1,float1,double1,bytes1,string1,middle,list,map",
+        "boolean1,byte1,short1,int1,long1,float1,double1,bytes1,string1,middle,list,map", inspector);
+    assertEquals(true, Arrays.equals(expected, included));
+
     Metadata metadata = reader.getMetadata();
 
     // check the stats
@@ -1183,6 +1244,20 @@ public class TestOrcFile {
     writer.close();
     Reader reader = OrcFile.createReader(testFilePath,
         OrcFile.readerOptions(conf).filesystem(fs));
+
+    assertEquals(6, OrcUtils.getFlattenedColumnsCount(inspector));
+    boolean[] expected = new boolean[] {false, false, false, false, false, false};
+    boolean[] included = OrcUtils.includeColumns("", "time,union,decimal", inspector);
+    assertEquals(true, Arrays.equals(expected, included));
+
+    expected = new boolean[] {false, true, false, false, false, true};
+    included = OrcUtils.includeColumns("time,decimal", "time,union,decimal", inspector);
+    assertEquals(true, Arrays.equals(expected, included));
+
+    expected = new boolean[] {false, false, true, true, true, false};
+    included = OrcUtils.includeColumns("union", "time,union,decimal", inspector);
+    assertEquals(true, Arrays.equals(expected, included));
+
     assertEquals(false, reader.getMetadataKeys().iterator().hasNext());
     assertEquals(5309, reader.getNumberOfRows());
     DecimalColumnStatistics stats =



Mime
View raw message