hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From br...@apache.org
Subject svn commit: r1629563 [13/33] - in /hive/branches/spark: ./ accumulo-handler/ beeline/ beeline/src/java/org/apache/hive/beeline/ bin/ bin/ext/ common/ common/src/java/org/apache/hadoop/hive/conf/ common/src/test/org/apache/hadoop/hive/common/type/ contr...
Date Mon, 06 Oct 2014 04:00:54 GMT
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java?rev=1629563&r1=1629562&r2=1629563&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java Mon Oct  6 04:00:39 2014
@@ -26,6 +26,8 @@ import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Set;
 
+import com.google.common.collect.Interner;
+import com.google.common.collect.Interners;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.Path;
@@ -278,6 +280,11 @@ public abstract class TaskCompiler {
         tsk.setRetryCmdWhenFail(true);
       }
     }
+
+    Interner<TableDesc> interner = Interners.newStrongInterner();
+    for (Task<? extends Serializable> rootTask : rootTasks) {
+      GenMapRedUtils.internTableDesc(rootTask, interner);
+    }
   }
 
 

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java?rev=1629563&r1=1629562&r2=1629563&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java Mon Oct  6 04:00:39 2014
@@ -36,7 +36,9 @@ import org.apache.hadoop.hive.conf.HiveC
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.ql.Context;
 import org.apache.hadoop.hive.ql.exec.AppMasterEventOperator;
+import org.apache.hadoop.hive.ql.exec.CommonMergeJoinOperator;
 import org.apache.hadoop.hive.ql.exec.ConditionalTask;
+import org.apache.hadoop.hive.ql.exec.DummyStoreOperator;
 import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
 import org.apache.hadoop.hive.ql.exec.FilterOperator;
 import org.apache.hadoop.hive.ql.exec.JoinOperator;
@@ -62,6 +64,7 @@ import org.apache.hadoop.hive.ql.metadat
 import org.apache.hadoop.hive.ql.optimizer.ConstantPropagate;
 import org.apache.hadoop.hive.ql.optimizer.ConvertJoinMapJoin;
 import org.apache.hadoop.hive.ql.optimizer.DynamicPartitionPruningOptimization;
+import org.apache.hadoop.hive.ql.optimizer.MergeJoinProc;
 import org.apache.hadoop.hive.ql.optimizer.ReduceSinkMapJoinProc;
 import org.apache.hadoop.hive.ql.optimizer.RemoveDynamicPruningBySize;
 import org.apache.hadoop.hive.ql.optimizer.SetReducerParallelism;
@@ -330,10 +333,17 @@ public class TezCompiler extends TaskCom
     opRules.put(new RuleRegExp("No more walking on ReduceSink-MapJoin",
         MapJoinOperator.getOperatorName() + "%"), new ReduceSinkMapJoinProc());
 
+    opRules.put(new RuleRegExp("Recoginze a Sorted Merge Join operator to setup the right edge and"
+        + " stop traversing the DummyStore-MapJoin", CommonMergeJoinOperator.getOperatorName()
+        + "%"), new MergeJoinProc());
+
     opRules.put(new RuleRegExp("Split Work + Move/Merge - FileSink",
         FileSinkOperator.getOperatorName() + "%"),
         new CompositeProcessor(new FileSinkProcessor(), genTezWork));
 
+    opRules.put(new RuleRegExp("Split work - DummyStore", DummyStoreOperator.getOperatorName()
+        + "%"), genTezWork);
+
     opRules.put(new RuleRegExp("Handle Potential Analyze Command",
         TableScanOperator.getOperatorName() + "%"),
         new ProcessAnalyzeTable(GenTezUtils.getUtils()));

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java?rev=1629563&r1=1629562&r2=1629563&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java Mon Oct  6 04:00:39 2014
@@ -54,6 +54,19 @@ public class TypeCheckCtx implements Nod
 
   private boolean allowDistinctFunctions;
 
+  private final boolean allowGBExprElimination;
+
+  private final boolean allowAllColRef;
+
+  private final boolean allowFunctionStar;
+
+  private final boolean allowWindowing;
+
+  // "[]" : LSQUARE/INDEX Expression
+  private final boolean allowIndexExpr;
+
+  private final boolean allowSubQueryExpr;
+
   /**
    * Constructor.
    *
@@ -61,10 +74,23 @@ public class TypeCheckCtx implements Nod
    *          The input row resolver of the previous operator.
    */
   public TypeCheckCtx(RowResolver inputRR) {
+    this(inputRR, false, true, true, true, true, true, true, true);
+  }
+
+  public TypeCheckCtx(RowResolver inputRR, boolean allowStatefulFunctions,
+      boolean allowDistinctFunctions, boolean allowGBExprElimination, boolean allowAllColRef,
+      boolean allowFunctionStar, boolean allowWindowing,
+      boolean allowIndexExpr, boolean allowSubQueryExpr) {
     setInputRR(inputRR);
     error = null;
-    allowStatefulFunctions = false;
-    allowDistinctFunctions = true;
+    this.allowStatefulFunctions = allowStatefulFunctions;
+    this.allowDistinctFunctions = allowDistinctFunctions;
+    this.allowGBExprElimination = allowGBExprElimination;
+    this.allowAllColRef = allowAllColRef;
+    this.allowFunctionStar = allowFunctionStar;
+    this.allowWindowing = allowWindowing;
+    this.allowIndexExpr = allowIndexExpr;
+    this.allowSubQueryExpr = allowSubQueryExpr;
   }
 
   /**
@@ -98,7 +124,8 @@ public class TypeCheckCtx implements Nod
   }
 
   /**
-   * @param allowStatefulFunctions whether to allow stateful UDF invocations
+   * @param allowStatefulFunctions
+   *          whether to allow stateful UDF invocations
    */
   public void setAllowStatefulFunctions(boolean allowStatefulFunctions) {
     this.allowStatefulFunctions = allowStatefulFunctions;
@@ -136,7 +163,31 @@ public class TypeCheckCtx implements Nod
     this.allowDistinctFunctions = allowDistinctFunctions;
   }
 
-  public boolean isAllowDistinctFunctions() {
+  public boolean getAllowDistinctFunctions() {
     return allowDistinctFunctions;
   }
+
+  public boolean getAllowGBExprElimination() {
+    return allowGBExprElimination;
+  }
+
+  public boolean getallowAllColRef() {
+    return allowAllColRef;
+  }
+
+  public boolean getallowFunctionStar() {
+    return allowFunctionStar;
+  }
+
+  public boolean getallowWindowing() {
+    return allowWindowing;
+  }
+
+  public boolean getallowIndexExpr() {
+    return allowIndexExpr;
+  }
+
+  public boolean getallowSubQueryExpr() {
+    return allowSubQueryExpr;
+  }
 }

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java?rev=1629563&r1=1629562&r2=1629563&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java Mon Oct  6 04:00:39 2014
@@ -80,12 +80,12 @@ import org.apache.hadoop.hive.serde2.typ
  * expression Node Descriptor trees. They also introduce the correct conversion
  * functions to do proper implicit conversion.
  */
-public final class TypeCheckProcFactory {
+public class TypeCheckProcFactory {
 
   protected static final Log LOG = LogFactory.getLog(TypeCheckProcFactory.class
       .getName());
 
-  private TypeCheckProcFactory() {
+  protected TypeCheckProcFactory() {
     // prevent instantiation
   }
 
@@ -118,7 +118,7 @@ public final class TypeCheckProcFactory 
     RowResolver input = ctx.getInputRR();
     ExprNodeDesc desc = null;
 
-    if ((ctx == null) || (input == null)) {
+    if ((ctx == null) || (input == null) || (!ctx.getAllowGBExprElimination())) {
       return null;
     }
 
@@ -137,8 +137,13 @@ public final class TypeCheckProcFactory 
     return desc;
   }
 
-  public static Map<ASTNode, ExprNodeDesc> genExprNode(ASTNode expr,
-      TypeCheckCtx tcCtx) throws SemanticException {
+  public static Map<ASTNode, ExprNodeDesc> genExprNode(ASTNode expr, TypeCheckCtx tcCtx)
+      throws SemanticException {
+    return genExprNode(expr, tcCtx, new TypeCheckProcFactory());
+  }
+
+  protected static Map<ASTNode, ExprNodeDesc> genExprNode(ASTNode expr,
+      TypeCheckCtx tcCtx, TypeCheckProcFactory tf) throws SemanticException {
     // Create the walker, the rules dispatcher and the context.
     // create a walker which walks the tree in a DFS manner while maintaining
     // the operator stack. The dispatcher
@@ -146,13 +151,13 @@ public final class TypeCheckProcFactory 
     Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
 
     opRules.put(new RuleRegExp("R1", HiveParser.TOK_NULL + "%"),
-        getNullExprProcessor());
+        tf.getNullExprProcessor());
     opRules.put(new RuleRegExp("R2", HiveParser.Number + "%|" +
         HiveParser.TinyintLiteral + "%|" +
         HiveParser.SmallintLiteral + "%|" +
         HiveParser.BigintLiteral + "%|" +
         HiveParser.DecimalLiteral + "%"),
-        getNumExprProcessor());
+        tf.getNumExprProcessor());
     opRules
         .put(new RuleRegExp("R3", HiveParser.Identifier + "%|"
         + HiveParser.StringLiteral + "%|" + HiveParser.TOK_CHARSETLITERAL + "%|"
@@ -162,18 +167,18 @@ public final class TypeCheckProcFactory 
         + HiveParser.KW_ARRAY + "%|" + HiveParser.KW_MAP + "%|"
         + HiveParser.KW_STRUCT + "%|" + HiveParser.KW_EXISTS + "%|"
         + HiveParser.TOK_SUBQUERY_OP_NOTIN + "%"),
-        getStrExprProcessor());
+        tf.getStrExprProcessor());
     opRules.put(new RuleRegExp("R4", HiveParser.KW_TRUE + "%|"
-        + HiveParser.KW_FALSE + "%"), getBoolExprProcessor());
-    opRules.put(new RuleRegExp("R5", HiveParser.TOK_DATELITERAL + "%"), getDateExprProcessor());
+        + HiveParser.KW_FALSE + "%"), tf.getBoolExprProcessor());
+    opRules.put(new RuleRegExp("R5", HiveParser.TOK_DATELITERAL + "%"), tf.getDateExprProcessor());
     opRules.put(new RuleRegExp("R6", HiveParser.TOK_TABLE_OR_COL + "%"),
-        getColumnExprProcessor());
+        tf.getColumnExprProcessor());
     opRules.put(new RuleRegExp("R7", HiveParser.TOK_SUBQUERY_OP + "%"),
-        getSubQueryExprProcessor());
+        tf.getSubQueryExprProcessor());
 
     // The dispatcher fires the processor corresponding to the closest matching
     // rule and passes the context along
-    Dispatcher disp = new DefaultRuleDispatcher(getDefaultExprProcessor(),
+    Dispatcher disp = new DefaultRuleDispatcher(tf.getDefaultExprProcessor(),
         opRules, tcCtx);
     GraphWalker ogw = new DefaultGraphWalker(disp);
 
@@ -229,7 +234,7 @@ public final class TypeCheckProcFactory 
    *
    * @return NullExprProcessor.
    */
-  public static NullExprProcessor getNullExprProcessor() {
+  public NullExprProcessor getNullExprProcessor() {
     return new NullExprProcessor();
   }
 
@@ -304,7 +309,7 @@ public final class TypeCheckProcFactory 
    *
    * @return NumExprProcessor.
    */
-  public static NumExprProcessor getNumExprProcessor() {
+  public NumExprProcessor getNumExprProcessor() {
     return new NumExprProcessor();
   }
 
@@ -362,7 +367,7 @@ public final class TypeCheckProcFactory 
    *
    * @return StrExprProcessor.
    */
-  public static StrExprProcessor getStrExprProcessor() {
+  public StrExprProcessor getStrExprProcessor() {
     return new StrExprProcessor();
   }
 
@@ -408,7 +413,7 @@ public final class TypeCheckProcFactory 
    *
    * @return BoolExprProcessor.
    */
-  public static BoolExprProcessor getBoolExprProcessor() {
+  public BoolExprProcessor getBoolExprProcessor() {
     return new BoolExprProcessor();
   }
 
@@ -449,7 +454,7 @@ public final class TypeCheckProcFactory 
    *
    * @return DateExprProcessor.
    */
-  public static DateExprProcessor getDateExprProcessor() {
+  public DateExprProcessor getDateExprProcessor() {
     return new DateExprProcessor();
   }
 
@@ -546,7 +551,7 @@ public final class TypeCheckProcFactory 
    *
    * @return ColumnExprProcessor.
    */
-  public static ColumnExprProcessor getColumnExprProcessor() {
+  public ColumnExprProcessor getColumnExprProcessor() {
     return new ColumnExprProcessor();
   }
 
@@ -613,7 +618,7 @@ public final class TypeCheckProcFactory 
       windowingTokens.add(HiveParser.TOK_TABSORTCOLNAMEDESC);
     }
 
-    private static boolean isRedundantConversionFunction(ASTNode expr,
+    protected static boolean isRedundantConversionFunction(ASTNode expr,
         boolean isFunction, ArrayList<ExprNodeDesc> children) {
       if (!isFunction) {
         return false;
@@ -700,7 +705,30 @@ public final class TypeCheckProcFactory 
       return getFuncExprNodeDescWithUdfData(udfName, null, children);
     }
 
-    static ExprNodeDesc getXpathOrFuncExprNodeDesc(ASTNode expr,
+    protected void validateUDF(ASTNode expr, boolean isFunction, TypeCheckCtx ctx, FunctionInfo fi,
+        List<ExprNodeDesc> children, GenericUDF genericUDF) throws SemanticException {
+      // Detect UDTF's in nested SELECT, GROUP BY, etc as they aren't
+      // supported
+      if (fi.getGenericUDTF() != null) {
+        throw new SemanticException(ErrorMsg.UDTF_INVALID_LOCATION.getMsg());
+      }
+      // UDAF in filter condition, group-by caluse, param of funtion, etc.
+      if (fi.getGenericUDAFResolver() != null) {
+        if (isFunction) {
+          throw new SemanticException(ErrorMsg.UDAF_INVALID_LOCATION.getMsg((ASTNode) expr
+              .getChild(0)));
+        } else {
+          throw new SemanticException(ErrorMsg.UDAF_INVALID_LOCATION.getMsg(expr));
+        }
+      }
+      if (!ctx.getAllowStatefulFunctions() && (genericUDF != null)) {
+        if (FunctionRegistry.isStateful(genericUDF)) {
+          throw new SemanticException(ErrorMsg.UDF_STATEFUL_INVALID_LOCATION.getMsg());
+        }
+      }
+    }
+
+    protected ExprNodeDesc getXpathOrFuncExprNodeDesc(ASTNode expr,
         boolean isFunction, ArrayList<ExprNodeDesc> children, TypeCheckCtx ctx)
         throws SemanticException, UDFArgumentException {
       // return the child directly if the conversion is redundant.
@@ -713,6 +741,7 @@ public final class TypeCheckProcFactory 
       ExprNodeDesc desc;
       if (funcText.equals(".")) {
         // "." : FIELD Expression
+
         assert (children.size() == 2);
         // Only allow constant field name for now
         assert (children.get(1) instanceof ExprNodeConstantDesc);
@@ -727,23 +756,22 @@ public final class TypeCheckProcFactory 
         // Allow accessing a field of list element structs directly from a list
         boolean isList = (object.getTypeInfo().getCategory() == ObjectInspector.Category.LIST);
         if (isList) {
-          objectTypeInfo = ((ListTypeInfo) objectTypeInfo)
-              .getListElementTypeInfo();
+          objectTypeInfo = ((ListTypeInfo) objectTypeInfo).getListElementTypeInfo();
         }
         if (objectTypeInfo.getCategory() != Category.STRUCT) {
           throw new SemanticException(ErrorMsg.INVALID_DOT.getMsg(expr));
         }
-        TypeInfo t = ((StructTypeInfo) objectTypeInfo)
-            .getStructFieldTypeInfo(fieldNameString);
+        TypeInfo t = ((StructTypeInfo) objectTypeInfo).getStructFieldTypeInfo(fieldNameString);
         if (isList) {
           t = TypeInfoFactory.getListTypeInfo(t);
         }
 
-        desc = new ExprNodeFieldDesc(t, children.get(0), fieldNameString,
-            isList);
-
+        desc = new ExprNodeFieldDesc(t, children.get(0), fieldNameString, isList);
       } else if (funcText.equals("[")) {
         // "[]" : LSQUARE/INDEX Expression
+        if (!ctx.getallowIndexExpr())
+          throw new SemanticException(ErrorMsg.INVALID_FUNCTION.getMsg(expr));
+
         assert (children.size() == 2);
 
         // Check whether this is a list or a map
@@ -759,8 +787,7 @@ public final class TypeCheckProcFactory 
 
           // Calculate TypeInfo
           TypeInfo t = ((ListTypeInfo) myt).getListElementTypeInfo();
-          desc = new ExprNodeGenericFuncDesc(t, FunctionRegistry
-              .getGenericUDFForIndex(), children);
+          desc = new ExprNodeGenericFuncDesc(t, FunctionRegistry.getGenericUDFForIndex(), children);
         } else if (myt.getCategory() == Category.MAP) {
           if (!FunctionRegistry.implicitConvertible(children.get(1).getTypeInfo(),
               ((MapTypeInfo) myt).getMapKeyTypeInfo())) {
@@ -769,11 +796,9 @@ public final class TypeCheckProcFactory 
           }
           // Calculate TypeInfo
           TypeInfo t = ((MapTypeInfo) myt).getMapValueTypeInfo();
-          desc = new ExprNodeGenericFuncDesc(t, FunctionRegistry
-              .getGenericUDFForIndex(), children);
+          desc = new ExprNodeGenericFuncDesc(t, FunctionRegistry.getGenericUDFForIndex(), children);
         } else {
-          throw new SemanticException(ErrorMsg.NON_COLLECTION_TYPE.getMsg(expr,
-              myt.getTypeName()));
+          throw new SemanticException(ErrorMsg.NON_COLLECTION_TYPE.getMsg(expr, myt.getTypeName()));
         }
       } else {
         // other operators or functions
@@ -825,26 +850,7 @@ public final class TypeCheckProcFactory 
           }
         }
 
-        // Detect UDTF's in nested SELECT, GROUP BY, etc as they aren't
-        // supported
-        if (fi.getGenericUDTF() != null) {
-          throw new SemanticException(ErrorMsg.UDTF_INVALID_LOCATION.getMsg());
-        }
-        // UDAF in filter condition, group-by caluse, param of funtion, etc.
-        if (fi.getGenericUDAFResolver() != null) {
-          if (isFunction) {
-            throw new SemanticException(ErrorMsg.UDAF_INVALID_LOCATION.
-                getMsg((ASTNode) expr.getChild(0)));
-          } else {
-            throw new SemanticException(ErrorMsg.UDAF_INVALID_LOCATION.getMsg(expr));
-          }
-        }
-        if (!ctx.getAllowStatefulFunctions() && (genericUDF != null)) {
-          if (FunctionRegistry.isStateful(genericUDF)) {
-            throw new SemanticException(
-              ErrorMsg.UDF_STATEFUL_INVALID_LOCATION.getMsg());
-          }
-        }
+        validateUDF(expr, isFunction, ctx, fi, children, genericUDF);
 
         // Try to infer the type of the constant only if there are two
         // nodes, one of them is column and the other is numeric const
@@ -955,6 +961,24 @@ public final class TypeCheckProcFactory 
       return false;
     }
 
+    protected ExprNodeColumnDesc processQualifiedColRef(TypeCheckCtx ctx, ASTNode expr,
+        Object... nodeOutputs) throws SemanticException {
+      RowResolver input = ctx.getInputRR();
+      String tableAlias = BaseSemanticAnalyzer.unescapeIdentifier(expr.getChild(0).getChild(0)
+          .getText());
+      // NOTE: tableAlias must be a valid non-ambiguous table alias,
+      // because we've checked that in TOK_TABLE_OR_COL's process method.
+      ColumnInfo colInfo = input.get(tableAlias, ((ExprNodeConstantDesc) nodeOutputs[1]).getValue()
+          .toString());
+
+      if (colInfo == null) {
+        ctx.setError(ErrorMsg.INVALID_COLUMN.getMsg(expr.getChild(1)), expr);
+        return null;
+      }
+      return new ExprNodeColumnDesc(colInfo.getType(), colInfo.getInternalName(),
+          colInfo.getTabAlias(), colInfo.getIsVirtualCol());
+    }
+
     @Override
     public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
         Object... nodeOutputs) throws SemanticException {
@@ -1004,7 +1028,11 @@ public final class TypeCheckProcFactory 
        * The difference is that there is translation for Window related tokens, so we just
        * return null;
        */
-      if ( windowingTokens.contains(expr.getType())) {
+      if (windowingTokens.contains(expr.getType())) {
+        if (!ctx.getallowWindowing())
+          throw new SemanticException(SemanticAnalyzer.generateErrorMessage(expr,
+              ErrorMsg.INVALID_FUNCTION.getMsg("Windowing is not supported in the context")));
+
         return null;
       }
 
@@ -1013,6 +1041,11 @@ public final class TypeCheckProcFactory 
       }
 
       if (expr.getType() == HiveParser.TOK_ALLCOLREF) {
+        if (!ctx.getallowAllColRef())
+          throw new SemanticException(SemanticAnalyzer.generateErrorMessage(expr,
+              ErrorMsg.INVALID_COLUMN
+                  .getMsg("All column reference is not supported in the context")));
+
         RowResolver input = ctx.getInputRR();
         ExprNodeColumnListDesc columnList = new ExprNodeColumnListDesc();
         assert expr.getChildCount() <= 1;
@@ -1050,22 +1083,7 @@ public final class TypeCheckProcFactory 
       if (expr.getType() == HiveParser.DOT
           && expr.getChild(0).getType() == HiveParser.TOK_TABLE_OR_COL
           && nodeOutputs[0] == null) {
-
-        RowResolver input = ctx.getInputRR();
-        String tableAlias = BaseSemanticAnalyzer.unescapeIdentifier(expr
-            .getChild(0).getChild(0).getText());
-        // NOTE: tableAlias must be a valid non-ambiguous table alias,
-        // because we've checked that in TOK_TABLE_OR_COL's process method.
-        ColumnInfo colInfo = input.get(tableAlias,
-            ((ExprNodeConstantDesc) nodeOutputs[1]).getValue().toString());
-
-        if (colInfo == null) {
-          ctx.setError(ErrorMsg.INVALID_COLUMN.getMsg(expr.getChild(1)), expr);
-          return null;
-        }
-        return new ExprNodeColumnDesc(colInfo.getType(), colInfo
-            .getInternalName(), colInfo.getTabAlias(), colInfo
-            .getIsVirtualCol());
+        return processQualifiedColRef(ctx, expr, nodeOutputs);
       }
 
       // Return nulls for conversion operators
@@ -1080,7 +1098,7 @@ public final class TypeCheckProcFactory 
           expr.getType() == HiveParser.TOK_FUNCTIONSTAR ||
           expr.getType() == HiveParser.TOK_FUNCTIONDI);
 
-      if (!ctx.isAllowDistinctFunctions() && expr.getType() == HiveParser.TOK_FUNCTIONDI) {
+      if (!ctx.getAllowDistinctFunctions() && expr.getType() == HiveParser.TOK_FUNCTIONDI) {
         throw new SemanticException(
             SemanticAnalyzer.generateErrorMessage(expr, ErrorMsg.DISTINCT_NOT_SUPPORTED.getMsg()));
       }
@@ -1099,6 +1117,11 @@ public final class TypeCheckProcFactory 
       }
 
       if (expr.getType() == HiveParser.TOK_FUNCTIONSTAR) {
+        if (!ctx.getallowFunctionStar())
+        throw new SemanticException(SemanticAnalyzer.generateErrorMessage(expr,
+            ErrorMsg.INVALID_COLUMN
+                .getMsg(".* reference is not supported in the context")));
+
         RowResolver input = ctx.getInputRR();
         for (ColumnInfo colInfo : input.getColumnInfos()) {
           if (!colInfo.getIsVirtualCol()) {
@@ -1111,8 +1134,7 @@ public final class TypeCheckProcFactory 
       // If any of the children contains null, then return a null
       // this is a hack for now to handle the group by case
       if (children.contains(null)) {
-        RowResolver input = ctx.getInputRR();
-        List<String> possibleColumnNames = input.getReferenceableColumnAliases(null, -1);
+        List<String> possibleColumnNames = getReferenceableColumnAliases(ctx);
         String reason = String.format("(possible column names are: %s)",
             StringUtils.join(possibleColumnNames, ", "));
         ctx.setError(ErrorMsg.INVALID_COLUMN.getMsg(expr.getChild(0), reason),
@@ -1135,6 +1157,9 @@ public final class TypeCheckProcFactory 
       }
     }
 
+    protected List<String> getReferenceableColumnAliases(TypeCheckCtx ctx) {
+      return ctx.getInputRR().getReferenceableColumnAliases(null, -1);
+    }
   }
 
   /**
@@ -1142,7 +1167,7 @@ public final class TypeCheckProcFactory 
    *
    * @return DefaultExprProcessor.
    */
-  public static DefaultExprProcessor getDefaultExprProcessor() {
+  public DefaultExprProcessor getDefaultExprProcessor() {
     return new DefaultExprProcessor();
   }
 
@@ -1160,13 +1185,18 @@ public final class TypeCheckProcFactory 
         return null;
       }
 
+      ASTNode expr = (ASTNode) nd;
+      ASTNode sqNode = (ASTNode) expr.getParent().getChild(1);
+
+      if (!ctx.getallowSubQueryExpr())
+        throw new SemanticException(SemanticAnalyzer.generateErrorMessage(sqNode,
+            ErrorMsg.UNSUPPORTED_SUBQUERY_EXPRESSION.getMsg()));
+
       ExprNodeDesc desc = TypeCheckProcFactory.processGByExpr(nd, procCtx);
       if (desc != null) {
         return desc;
       }
 
-      ASTNode expr = (ASTNode) nd;
-      ASTNode sqNode = (ASTNode) expr.getParent().getChild(1);
       /*
        * Restriction.1.h :: SubQueries only supported in the SQL Where Clause.
        */
@@ -1182,7 +1212,7 @@ public final class TypeCheckProcFactory 
    *
    * @return DateExprProcessor.
    */
-  public static SubQueryExprProcessor getSubQueryExprProcessor() {
+  public SubQueryExprProcessor getSubQueryExprProcessor() {
     return new SubQueryExprProcessor();
   }
 }

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java?rev=1629563&r1=1629562&r2=1629563&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java Mon Oct  6 04:00:39 2014
@@ -17,6 +17,13 @@
  */
 package org.apache.hadoop.hive.ql.parse;
 
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.ql.Context;
@@ -27,19 +34,12 @@ import org.apache.hadoop.hive.ql.hooks.W
 import org.apache.hadoop.hive.ql.io.AcidUtils;
 import org.apache.hadoop.hive.ql.lib.Node;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.InvalidTableException;
 import org.apache.hadoop.hive.ql.metadata.Table;
 import org.apache.hadoop.hive.ql.metadata.VirtualColumn;
 import org.apache.hadoop.hive.ql.session.SessionState;
 
 
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-
 /**
  * A subclass of the {@link org.apache.hadoop.hive.ql.parse.SemanticAnalyzer} that just handles
  * update and delete statements.  It works by rewriting the updates and deletes into insert
@@ -128,11 +128,16 @@ public class UpdateDeleteSemanticAnalyze
     Table mTable;
     try {
       mTable = db.getTable(tableName[0], tableName[1]);
+    } catch (InvalidTableException e) {
+      LOG.error("Failed to find table " + getDotName(tableName) + " got exception "
+          + e.getMessage());
+      throw new SemanticException(ErrorMsg.INVALID_TABLE.getMsg(getDotName(tableName)), e);
     } catch (HiveException e) {
-      LOG.error("Failed to find table " + getDotName(tableName) + " got exception " +
-          e.getMessage());
-      throw new SemanticException(ErrorMsg.INVALID_TABLE, getDotName(tableName));
+      LOG.error("Failed to find table " + getDotName(tableName) + " got exception "
+          + e.getMessage());
+      throw new SemanticException(e.getMessage(), e);
     }
+
     List<FieldSchema> partCols = mTable.getPartCols();
 
     rewrittenQueryStr.append("insert into table ");
@@ -343,8 +348,10 @@ public class UpdateDeleteSemanticAnalyze
 
       // Add the setRCols to the input list
       for (String colName : setRCols) {
-        columnAccessInfo.add(Table.getCompleteName(mTable.getDbName(), mTable.getTableName()),
+        if(columnAccessInfo != null) {//assuming this means we are not doing Auth
+          columnAccessInfo.add(Table.getCompleteName(mTable.getDbName(), mTable.getTableName()),
             colName);
+        }
       }
     }
 
@@ -386,7 +393,7 @@ public class UpdateDeleteSemanticAnalyze
       setRCols.add(colName.getText());
     } else if (node.getChildren() != null) {
       for (Node n : node.getChildren()) {
-        addSetRCols(node, setRCols);
+        addSetRCols((ASTNode)n, setRCols);
       }
     }
   }

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java?rev=1629563&r1=1629562&r2=1629563&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java Mon Oct  6 04:00:39 2014
@@ -110,10 +110,12 @@ public class AlterTableDesc extends DDLD
    * @param newComment
    * @param newType
    */
-  public AlterTableDesc(String tblName, String oldColName, String newColName,
+  public AlterTableDesc(String tblName, HashMap<String, String> partSpec,
+      String oldColName, String newColName,
       String newType, String newComment, boolean first, String afterCol) {
     super();
     oldName = tblName;
+    this.partSpec = partSpec;
     this.oldColName = oldColName;
     this.newColName = newColName;
     newColType = newType;
@@ -142,11 +144,12 @@ public class AlterTableDesc extends DDLD
    * @param newCols
    *          new columns to be added
    */
-  public AlterTableDesc(String name, List<FieldSchema> newCols,
+  public AlterTableDesc(String name, HashMap<String, String> partSpec, List<FieldSchema> newCols,
       AlterTableTypes alterType) {
     op = alterType;
     oldName = name;
     this.newCols = new ArrayList<FieldSchema>(newCols);
+    this.partSpec = partSpec;
   }
 
   /**

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java?rev=1629563&r1=1629562&r2=1629563&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java Mon Oct  6 04:00:39 2014
@@ -41,6 +41,7 @@ public abstract class BaseWork extends A
   // Their function is mainly as root ops to give the mapjoin the correct
   // schema info.
   List<HashTableDummyOperator> dummyOps;
+  int tag;
 
   public BaseWork() {}
 
@@ -100,7 +101,7 @@ public abstract class BaseWork extends A
 
     // add all children
     opStack.addAll(opSet);
-    
+
     while(!opStack.empty()) {
       Operator<?> op = opStack.pop();
       returnSet.add(op);
@@ -139,4 +140,12 @@ public abstract class BaseWork extends A
   }
 
   public abstract void configureJobConf(JobConf job);
+
+  public void setTag(int tag) {
+    this.tag = tag;
+  }
+
+  public int getTag() {
+    return tag;
+  }
 }

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/DropTableDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/DropTableDesc.java?rev=1629563&r1=1629562&r2=1629563&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/DropTableDesc.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/DropTableDesc.java Mon Oct  6 04:00:39 2014
@@ -52,6 +52,7 @@ public class DropTableDesc extends DDLDe
   ArrayList<PartSpec> partSpecs;
   boolean expectView;
   boolean ifExists;
+  boolean ifPurge;
   boolean ignoreProtection;
 
   public DropTableDesc() {
@@ -59,12 +60,14 @@ public class DropTableDesc extends DDLDe
 
   /**
    * @param tableName
+   * @param ifPurge
    */
-  public DropTableDesc(String tableName, boolean expectView, boolean ifExists) {
+  public DropTableDesc(String tableName, boolean expectView, boolean ifExists, boolean ifPurge) {
     this.tableName = tableName;
     this.partSpecs = null;
     this.expectView = expectView;
     this.ifExists = ifExists;
+    this.ifPurge = ifPurge;
     this.ignoreProtection = false;
   }
 
@@ -149,4 +152,19 @@ public class DropTableDesc extends DDLDe
   public void setIfExists(boolean ifExists) {
     this.ifExists = ifExists;
   }
+
+  /**
+   *  @return whether Purge was specified
+   */
+  public boolean getIfPurge() {
+      return ifPurge;
+  }
+
+  /**
+   * @param ifPurge
+   *          set whether Purge was specified
+   */
+  public void setIfPurge(boolean ifPurge) {
+      this.ifPurge = ifPurge;
+  }
 }

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java?rev=1629563&r1=1629562&r2=1629563&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java Mon Oct  6 04:00:39 2014
@@ -25,6 +25,7 @@ import org.apache.hadoop.hive.serde.serd
 import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.BaseCharTypeInfo;
 import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
@@ -93,7 +94,7 @@ public class ExprNodeConstantDesc extend
       return "null";
     }
 
-    if (typeInfo.getTypeName().equals(serdeConstants.STRING_TYPE_NAME)) {
+    if (typeInfo.getTypeName().equals(serdeConstants.STRING_TYPE_NAME) || typeInfo instanceof BaseCharTypeInfo) {
       return "'" + value.toString() + "'";
     } else if (typeInfo.getTypeName().equals(serdeConstants.BINARY_TYPE_NAME)) {
       byte[] bytes = (byte[]) value;

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java?rev=1629563&r1=1629562&r2=1629563&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java Mon Oct  6 04:00:39 2014
@@ -372,5 +372,42 @@ public class ExprNodeDescUtils {
     } catch (Exception e) {
       return null;
     }
-  }
+	}
+
+	public static void getExprNodeColumnDesc(List<ExprNodeDesc> exprDescList,
+			Map<Integer, ExprNodeDesc> hashCodeTocolumnDescMap) {
+		for (ExprNodeDesc exprNodeDesc : exprDescList) {
+			getExprNodeColumnDesc(exprNodeDesc, hashCodeTocolumnDescMap);
+		}
+	}
+
+	/**
+	 * Get Map of ExprNodeColumnDesc HashCode to ExprNodeColumnDesc.
+	 * 
+	 * @param exprDesc
+	 * @param hashCodeTocolumnDescMap
+	 *            Assumption: If two ExprNodeColumnDesc have same hash code then
+	 *            they are logically referring to same projection
+	 */
+	public static void getExprNodeColumnDesc(ExprNodeDesc exprDesc,
+			Map<Integer, ExprNodeDesc> hashCodeTocolumnDescMap) {
+		if (exprDesc instanceof ExprNodeColumnDesc) {
+			hashCodeTocolumnDescMap.put(
+					((ExprNodeColumnDesc) exprDesc).hashCode(),
+					((ExprNodeColumnDesc) exprDesc));
+		} else if (exprDesc instanceof ExprNodeColumnListDesc) {
+			for (ExprNodeDesc child : ((ExprNodeColumnListDesc) exprDesc)
+					.getChildren()) {
+				getExprNodeColumnDesc(child, hashCodeTocolumnDescMap);
+			}
+		} else if (exprDesc instanceof ExprNodeGenericFuncDesc) {
+			for (ExprNodeDesc child : ((ExprNodeGenericFuncDesc) exprDesc)
+					.getChildren()) {
+				getExprNodeColumnDesc(child, hashCodeTocolumnDescMap);
+			}
+		} else if (exprDesc instanceof ExprNodeFieldDesc) {
+			getExprNodeColumnDesc(((ExprNodeFieldDesc) exprDesc).getDesc(),
+					hashCodeTocolumnDescMap);
+		}
+	}
 }

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java?rev=1629563&r1=1629562&r2=1629563&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java Mon Oct  6 04:00:39 2014
@@ -69,6 +69,7 @@ public class MapJoinDesc extends JoinDes
 
   // Hash table memory usage allowed; used in case of non-staged mapjoin.
   private float hashtableMemoryUsage;
+  protected boolean genJoinKeys = true;
 
   public MapJoinDesc() {
     bigTableBucketNumMapping = new LinkedHashMap<String, Integer>();
@@ -122,6 +123,7 @@ public class MapJoinDesc extends JoinDes
     }
   }
 
+  @Explain(displayName = "input vertices")
   public Map<Integer, String> getParentToInput() {
     return parentToInput;
   }
@@ -331,4 +333,16 @@ public class MapJoinDesc extends JoinDes
   public boolean getCustomBucketMapJoin() {
     return this.customBucketMapJoin;
   }
+
+  public boolean isMapSideJoin() {
+    return true;
+  }
+
+  public void setGenJoinKeys(boolean genJoinKeys) {
+    this.genJoinKeys = genJoinKeys;
+  }
+
+  public boolean getGenJoinKeys() {
+    return genJoinKeys;
+  }
 }

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java?rev=1629563&r1=1629562&r2=1629563&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java Mon Oct  6 04:00:39 2014
@@ -30,6 +30,7 @@ import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
 
+import com.google.common.collect.Interner;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.Path;
@@ -96,6 +97,7 @@ public class MapWork extends BaseWork {
   private Long minSplitSize;
   private Long minSplitSizePerNode;
   private Long minSplitSizePerRack;
+  private final int tag = 0;
 
   //use sampled partitioning
   private int samplingType;
@@ -126,6 +128,8 @@ public class MapWork extends BaseWork {
   private Map<String, List<ExprNodeDesc>> eventSourcePartKeyExprMap =
       new LinkedHashMap<String, List<ExprNodeDesc>>();
 
+  private boolean doSplitsGrouping = true;
+
   public MapWork() {}
 
   public MapWork(String name) {
@@ -195,6 +199,22 @@ public class MapWork extends BaseWork {
     }
   }
 
+  public void internTable(Interner<TableDesc> interner) {
+    if (aliasToPartnInfo != null) {
+      for (PartitionDesc part : aliasToPartnInfo.values()) {
+        if (part == null) {
+          continue;
+        }
+        part.intern(interner);
+      }
+    }
+    if (pathToPartitionInfo != null) {
+      for (PartitionDesc part : pathToPartitionInfo.values()) {
+        part.intern(interner);
+      }
+    }
+  }
+
   /**
    * @return the aliasToPartnInfo
    */
@@ -567,4 +587,12 @@ public class MapWork extends BaseWork {
   public void setEventSourcePartKeyExprMap(Map<String, List<ExprNodeDesc>> map) {
     this.eventSourcePartKeyExprMap = map;
   }
+
+  public void setDoSplitsGrouping(boolean doSplitsGrouping) {
+    this.doSplitsGrouping = doSplitsGrouping;
+  }
+
+  public boolean getDoSplitsGrouping() {
+    return this.doSplitsGrouping;
+  }
 }

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/OpTraits.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/OpTraits.java?rev=1629563&r1=1629562&r2=1629563&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/OpTraits.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/OpTraits.java Mon Oct  6 04:00:39 2014
@@ -20,17 +20,16 @@ package org.apache.hadoop.hive.ql.plan;
 
 import java.util.List;
 
-import org.apache.hadoop.hive.ql.metadata.Table;
-import org.apache.hadoop.hive.ql.parse.PrunedPartitionList;
-
 public class OpTraits {
-  
+
   List<List<String>> bucketColNames;
+  List<List<String>> sortColNames;
   int numBuckets;
-  
-  public OpTraits(List<List<String>> bucketColNames, int numBuckets) {
+
+  public OpTraits(List<List<String>> bucketColNames, int numBuckets, List<List<String>> sortColNames) {
     this.bucketColNames = bucketColNames;
     this.numBuckets = numBuckets;
+    this.sortColNames = sortColNames;
   }
 
   public List<List<String>> getBucketColNames() {
@@ -42,10 +41,18 @@ public class OpTraits {
   }
 
   public void setBucketColNames(List<List<String>> bucketColNames) {
-    this.bucketColNames = bucketColNames;    
+    this.bucketColNames = bucketColNames;
   }
 
   public void setNumBuckets(int numBuckets) {
     this.numBuckets = numBuckets;
   }
+
+  public void setSortColNames(List<List<String>> sortColNames) {
+    this.sortColNames = sortColNames;
+  }
+
+  public List<List<String>> getSortCols() {
+    return sortColNames;
+  }
 }

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java?rev=1629563&r1=1629562&r2=1629563&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java Mon Oct  6 04:00:39 2014
@@ -48,12 +48,10 @@ import org.apache.hadoop.util.Reflection
 public class PartitionDesc implements Serializable, Cloneable {
 
   static {
-    TABLE_INTERNER = Interners.newWeakInterner();
     STRING_INTERNER = Interners.newWeakInterner();
     CLASS_INTERNER = Interners.newWeakInterner();
   }
 
-  private static final Interner<TableDesc> TABLE_INTERNER;
   private static final Interner<String> STRING_INTERNER;
   private static final Interner<Class<?>> CLASS_INTERNER;
 
@@ -73,12 +71,12 @@ public class PartitionDesc implements Se
   }
 
   public PartitionDesc(final TableDesc table, final LinkedHashMap<String, String> partSpec) {
-    setTableDesc(table);
+    this.tableDesc = table;
     this.partSpec = partSpec;
   }
 
   public PartitionDesc(final Partition part) throws HiveException {
-    setTableDesc(Utilities.getTableDesc(part.getTable()));
+    this.tableDesc = Utilities.getTableDesc(part.getTable());
     setProperties(part.getMetadataFromPartitionSchema());
     partSpec = part.getSpec();
     setInputFileFormatClass(part.getInputFormatClass());
@@ -86,7 +84,7 @@ public class PartitionDesc implements Se
   }
 
   public PartitionDesc(final Partition part,final TableDesc tblDesc) throws HiveException {
-    setTableDesc(tblDesc);
+    this.tableDesc = tblDesc;
     setProperties(part.getSchemaFromTableSchema(tblDesc.getProperties())); // each partition maintains a large properties
     partSpec = part.getSpec();
     setOutputFileFormatClass(part.getInputFormatClass());
@@ -99,7 +97,7 @@ public class PartitionDesc implements Se
   }
 
   public void setTableDesc(TableDesc tableDesc) {
-    this.tableDesc = TABLE_INTERNER.intern(tableDesc);
+    this.tableDesc = tableDesc;
   }
 
   @Explain(displayName = "partition values")
@@ -266,4 +264,8 @@ public class PartitionDesc implements Se
       baseFileName = path;
     }
   }
+
+  public void intern(Interner<TableDesc> interner) {
+    this.tableDesc = interner.intern(tableDesc);
+  }
 }

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java?rev=1629563&r1=1629562&r2=1629563&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java Mon Oct  6 04:00:39 2014
@@ -19,12 +19,13 @@
 package org.apache.hadoop.hive.ql.plan;
 
 import java.io.Serializable;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.LinkedList;
 import java.util.LinkedHashMap;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -46,6 +47,22 @@ import org.apache.hadoop.mapred.JobConf;
 @Explain(displayName = "Tez")
 public class TezWork extends AbstractOperatorDesc {
 
+  public enum VertexType {
+    AUTO_INITIALIZED_EDGES, // no custom vertex or edge
+    INITIALIZED_EDGES, // custom vertex and custom edge but single MR Input
+    MULTI_INPUT_INITIALIZED_EDGES, // custom vertex, custom edge and multi MR Input
+    MULTI_INPUT_UNINITIALIZED_EDGES // custom vertex, no custom edge, multi MR Input
+    ;
+
+    public static boolean isCustomInputType(VertexType vertex) {
+      if ((vertex == null) || (vertex == AUTO_INITIALIZED_EDGES)) {
+        return false;
+      } else {
+        return true;
+      }
+    }
+  }
+
   private static transient final Log LOG = LogFactory.getLog(TezWork.class);
 
   private static int counter;
@@ -56,6 +73,7 @@ public class TezWork extends AbstractOpe
   private final Map<BaseWork, List<BaseWork>> invertedWorkGraph = new HashMap<BaseWork, List<BaseWork>>();
   private final Map<Pair<BaseWork, BaseWork>, TezEdgeProperty> edgeProperties =
       new HashMap<Pair<BaseWork, BaseWork>, TezEdgeProperty>();
+  private final Map<BaseWork, VertexType> workVertexTypeMap = new HashMap<BaseWork, VertexType>();
 
   public TezWork(String name) {
     this.name = name + ":" + (++counter);
@@ -305,15 +323,23 @@ public class TezWork extends AbstractOpe
       work.configureJobConf(jobConf);
     }
     String[] newTmpJars = jobConf.getStrings(MR_JAR_PROPERTY);
-    if (oldTmpJars != null && (oldTmpJars.length != 0)) {
-      if (newTmpJars != null && (newTmpJars.length != 0)) {
-        String[] combinedTmpJars = new String[newTmpJars.length + oldTmpJars.length];
-        System.arraycopy(oldTmpJars, 0, combinedTmpJars, 0, oldTmpJars.length);
-        System.arraycopy(newTmpJars, 0, combinedTmpJars, oldTmpJars.length, newTmpJars.length);
-        jobConf.setStrings(MR_JAR_PROPERTY, combinedTmpJars);
+    if (oldTmpJars != null || newTmpJars != null) {
+      String[] finalTmpJars;
+      if (oldTmpJars == null || oldTmpJars.length == 0) {
+        // Avoid a copy when oldTmpJars is null or empty
+        finalTmpJars = newTmpJars;
+      } else if (newTmpJars == null || newTmpJars.length == 0) {
+        // Avoid a copy when newTmpJars is null or empty
+        finalTmpJars = oldTmpJars;
       } else {
-        jobConf.setStrings(MR_JAR_PROPERTY, oldTmpJars);
+        // Both are non-empty, only copy now
+        finalTmpJars = new String[oldTmpJars.length + newTmpJars.length];
+        System.arraycopy(oldTmpJars, 0, finalTmpJars, 0, oldTmpJars.length);
+        System.arraycopy(newTmpJars, 0, finalTmpJars, oldTmpJars.length, newTmpJars.length);
       }
+
+      jobConf.setStrings(MR_JAR_PROPERTY, finalTmpJars);
+      return finalTmpJars;
     }
     return newTmpJars;
    }
@@ -332,4 +358,40 @@ public class TezWork extends AbstractOpe
     ImmutablePair workPair = new ImmutablePair(a, b);
     edgeProperties.put(workPair, edgeProp);
   }
+
+  public void setVertexType(BaseWork w, VertexType incomingVertexType) {
+    VertexType vertexType = workVertexTypeMap.get(w);
+    if (vertexType == null) {
+      vertexType = VertexType.AUTO_INITIALIZED_EDGES;
+    }
+    switch (vertexType) {
+    case INITIALIZED_EDGES:
+      if (incomingVertexType == VertexType.MULTI_INPUT_UNINITIALIZED_EDGES) {
+        vertexType = VertexType.MULTI_INPUT_INITIALIZED_EDGES;
+      }
+      break;
+
+    case MULTI_INPUT_INITIALIZED_EDGES:
+      // nothing to do
+      break;
+
+    case MULTI_INPUT_UNINITIALIZED_EDGES:
+      if (incomingVertexType == VertexType.INITIALIZED_EDGES) {
+        vertexType = VertexType.MULTI_INPUT_INITIALIZED_EDGES;
+      }
+      break;
+
+    case AUTO_INITIALIZED_EDGES:
+      vertexType = incomingVertexType;
+      break;
+
+    default:
+      break;
+    }
+    workVertexTypeMap.put(w, vertexType);
+  }
+
+  public VertexType getVertexType(BaseWork w) {
+    return workVertexTypeMap.get(w);
+  }
 }

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/AuthorizationPreEventListener.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/AuthorizationPreEventListener.java?rev=1629563&r1=1629562&r2=1629563&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/AuthorizationPreEventListener.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/AuthorizationPreEventListener.java Mon Oct  6 04:00:39 2014
@@ -23,7 +23,9 @@ import java.util.List;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.classification.InterfaceAudience.Private;
 import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.metastore.MetaStorePreEventListener;
 import org.apache.hadoop.hive.metastore.MetaStoreUtils;
 import org.apache.hadoop.hive.metastore.TableType;
@@ -40,6 +42,8 @@ import org.apache.hadoop.hive.metastore.
 import org.apache.hadoop.hive.metastore.events.PreDropPartitionEvent;
 import org.apache.hadoop.hive.metastore.events.PreDropTableEvent;
 import org.apache.hadoop.hive.metastore.events.PreEventContext;
+import org.apache.hadoop.hive.metastore.events.PreReadDatabaseEvent;
+import org.apache.hadoop.hive.metastore.events.PreReadTableEvent;
 import org.apache.hadoop.hive.ql.metadata.AuthorizationException;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.metadata.HiveUtils;
@@ -54,6 +58,7 @@ import org.apache.hadoop.hive.ql.securit
  * metastore PreEventContexts, such as the adding/dropping and altering
  * of databases, tables and partitions.
  */
+@Private
 public class AuthorizationPreEventListener extends MetaStorePreEventListener {
 
   public static final Log LOG = LogFactory.getLog(
@@ -136,6 +141,12 @@ public class AuthorizationPreEventListen
     case ALTER_TABLE:
       authorizeAlterTable((PreAlterTableEvent)context);
       break;
+    case READ_TABLE:
+      authorizeReadTable((PreReadTableEvent)context);
+      break;
+    case READ_DATABASE:
+      authorizeReadDatabase((PreReadDatabaseEvent)context);
+      break;
     case ADD_PARTITION:
       authorizeAddPartition((PreAddPartitionEvent)context);
       break;
@@ -162,6 +173,44 @@ public class AuthorizationPreEventListen
 
   }
 
+  private void authorizeReadTable(PreReadTableEvent context) throws InvalidOperationException,
+      MetaException {
+    if (!isReadAuthzEnabled()) {
+      return;
+    }
+    try {
+      org.apache.hadoop.hive.ql.metadata.Table wrappedTable = new TableWrapper(context.getTable());
+      for (HiveMetastoreAuthorizationProvider authorizer : tAuthorizers.get()) {
+        authorizer.authorize(wrappedTable, new Privilege[] { Privilege.SELECT }, null);
+      }
+    } catch (AuthorizationException e) {
+      throw invalidOperationException(e);
+    } catch (HiveException e) {
+      throw metaException(e);
+    }
+  }
+
+  private void authorizeReadDatabase(PreReadDatabaseEvent context)
+      throws InvalidOperationException, MetaException {
+    if (!isReadAuthzEnabled()) {
+      return;
+    }
+    try {
+      for (HiveMetastoreAuthorizationProvider authorizer : tAuthorizers.get()) {
+        authorizer.authorize(new Database(context.getDatabase()),
+            new Privilege[] { Privilege.SELECT }, null);
+      }
+    } catch (AuthorizationException e) {
+      throw invalidOperationException(e);
+    } catch (HiveException e) {
+      throw metaException(e);
+    }
+  }
+
+  private boolean isReadAuthzEnabled() {
+    return tConfig.get().getBoolean(ConfVars.HIVE_METASTORE_AUTHORIZATION_AUTH_READS.varname, true);
+  }
+
   private void authorizeAuthorizationAPICall() throws InvalidOperationException, MetaException {
     for (HiveMetastoreAuthorizationProvider authorizer : tAuthorizers.get()) {
       try {
@@ -358,7 +407,7 @@ public class AuthorizationPreEventListen
     public PartitionWrapper(org.apache.hadoop.hive.metastore.api.Partition mapiPart,
         PreEventContext context) throws HiveException, NoSuchObjectException, MetaException {
       org.apache.hadoop.hive.metastore.api.Partition wrapperApiPart = mapiPart.deepCopy();
-      org.apache.hadoop.hive.metastore.api.Table t = context.getHandler().get_table(
+      org.apache.hadoop.hive.metastore.api.Table t = context.getHandler().get_table_core(
           mapiPart.getDbName(), mapiPart.getTableName());
       if (wrapperApiPart.getSd() == null){
         // In the cases of create partition, by the time this event fires, the partition

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/HiveAuthorizationProviderBase.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/HiveAuthorizationProviderBase.java?rev=1629563&r1=1629562&r2=1629563&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/HiveAuthorizationProviderBase.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/HiveAuthorizationProviderBase.java Mon Oct  6 04:00:39 2014
@@ -85,7 +85,7 @@ public abstract class HiveAuthorizationP
         return hiveClient.getDatabase(dbName);
       } else {
         try {
-          return handler.get_database(dbName);
+          return handler.get_database_core(dbName);
         } catch (NoSuchObjectException e) {
           throw new HiveException(e);
         } catch (MetaException e) {

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAccessControlException.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAccessControlException.java?rev=1629563&r1=1629562&r2=1629563&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAccessControlException.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAccessControlException.java Mon Oct  6 04:00:39 2014
@@ -27,7 +27,7 @@ import org.apache.hadoop.hive.ql.metadat
  * an error while performing authorization, and not a authorization being
  * denied.
  */
-@LimitedPrivate(value = { "" })
+@LimitedPrivate(value = { "Apache Argus (incubating)" })
 @Evolving
 public class HiveAccessControlException extends HiveException{
 

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAccessController.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAccessController.java?rev=1629563&r1=1629562&r2=1629563&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAccessController.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAccessController.java Mon Oct  6 04:00:39 2014
@@ -19,8 +19,7 @@ package org.apache.hadoop.hive.ql.securi
 
 import java.util.List;
 
-import org.apache.hadoop.hive.common.classification.InterfaceAudience.LimitedPrivate;
-import org.apache.hadoop.hive.common.classification.InterfaceStability.Evolving;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.hive.conf.HiveConf;
 
 /**
@@ -28,8 +27,7 @@ import org.apache.hadoop.hive.conf.HiveC
  * create/drop roles, and commands to read the state of authorization rules.
  * Methods here have corresponding methods in HiveAuthorizer, check method documentation there.
  */
-@LimitedPrivate(value = { "" })
-@Evolving
+@Private
 public interface HiveAccessController {
 
   void grantPrivileges(List<HivePrincipal> hivePrincipals, List<HivePrivilege> hivePrivileges,

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthorizationValidator.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthorizationValidator.java?rev=1629563&r1=1629562&r2=1629563&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthorizationValidator.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthorizationValidator.java Mon Oct  6 04:00:39 2014
@@ -19,15 +19,13 @@ package org.apache.hadoop.hive.ql.securi
 
 import java.util.List;
 
-import org.apache.hadoop.hive.common.classification.InterfaceAudience.LimitedPrivate;
-import org.apache.hadoop.hive.common.classification.InterfaceStability.Evolving;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
 
 /**
  * Interface used to check if user has privileges to perform certain action.
  * Methods here have corresponding methods in HiveAuthorizer, check method documentation there.
  */
-@LimitedPrivate(value = { "" })
-@Evolving
+@Private
 public interface HiveAuthorizationValidator {
 
   /**

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthorizer.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthorizer.java?rev=1629563&r1=1629562&r2=1629563&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthorizer.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthorizer.java Mon Oct  6 04:00:39 2014
@@ -34,7 +34,7 @@ import org.apache.hadoop.hive.ql.securit
  *  statements and does not make assumptions about the privileges needed for a hive operation.
  * This is referred to as V2 authorizer in other parts of the code.
  */
-@LimitedPrivate(value = { "" })
+@LimitedPrivate(value = { "Apache Argus (incubating)" })
 @Evolving
 public interface HiveAuthorizer {
 

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthorizerFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthorizerFactory.java?rev=1629563&r1=1629562&r2=1629563&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthorizerFactory.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthorizerFactory.java Mon Oct  6 04:00:39 2014
@@ -27,7 +27,7 @@ import org.apache.hadoop.hive.ql.securit
  * create  {@link HiveAuthorizer} instance used for hive authorization.
  *
  */
-@LimitedPrivate(value = { "" })
+@LimitedPrivate(value = { "Apache Argus (incubating)" })
 @Evolving
 public interface HiveAuthorizerFactory {
   /**

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthzContext.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthzContext.java?rev=1629563&r1=1629562&r2=1629563&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthzContext.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthzContext.java Mon Oct  6 04:00:39 2014
@@ -25,7 +25,7 @@ import org.apache.hadoop.hive.common.cla
  * auditing and/or authorization.
  * It is an immutable class. Builder inner class is used instantiate it.
  */
-@LimitedPrivate(value = { "" })
+@LimitedPrivate(value = { "Apache Argus (incubating)" })
 @Evolving
 public final class HiveAuthzContext {
 

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthzPluginException.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthzPluginException.java?rev=1629563&r1=1629562&r2=1629563&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthzPluginException.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthzPluginException.java Mon Oct  6 04:00:39 2014
@@ -27,7 +27,7 @@ import org.apache.hadoop.hive.ql.metadat
  * an error while performing authorization, and not a authorization being
  * denied.
  */
-@LimitedPrivate(value = { "" })
+@LimitedPrivate(value = { "Apache Argus (incubating)" })
 @Evolving
 public class HiveAuthzPluginException extends HiveException{
 

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthzSessionContext.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthzSessionContext.java?rev=1629563&r1=1629562&r2=1629563&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthzSessionContext.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveAuthzSessionContext.java Mon Oct  6 04:00:39 2014
@@ -24,7 +24,7 @@ import org.apache.hadoop.hive.common.cla
  * Provides session context information.
  * It is an immutable class. Builder inner class is used instantiate it.
  */
-@LimitedPrivate(value = { "" })
+@LimitedPrivate(value = { "Apache Argus (incubating)" })
 @Evolving
 public final class HiveAuthzSessionContext {
 

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveMetastoreClientFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveMetastoreClientFactory.java?rev=1629563&r1=1629562&r2=1629563&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveMetastoreClientFactory.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveMetastoreClientFactory.java Mon Oct  6 04:00:39 2014
@@ -28,7 +28,7 @@ import org.apache.hadoop.hive.metastore.
  * But Hive class is not a public interface, so this factory helps in hiding Hive
  * class from the authorization interface users.
  */
-@LimitedPrivate(value = { "" })
+@LimitedPrivate(value = { "Apache Argus (incubating)" })
 @Evolving
 public interface HiveMetastoreClientFactory {
   IMetaStoreClient getHiveMetastoreClient() throws HiveAuthzPluginException;

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java?rev=1629563&r1=1629562&r2=1629563&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java Mon Oct  6 04:00:39 2014
@@ -23,7 +23,7 @@ import org.apache.hadoop.hive.common.cla
 /**
  * List of hive operations types.
  */
-@LimitedPrivate(value = { "" })
+@LimitedPrivate(value = { "Apache Argus (incubating)" })
 @Evolving
 public enum HiveOperationType {
   EXPLAIN,

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrincipal.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrincipal.java?rev=1629563&r1=1629562&r2=1629563&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrincipal.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrincipal.java Mon Oct  6 04:00:39 2014
@@ -23,7 +23,7 @@ import org.apache.hadoop.hive.common.cla
 /**
  * Represents the user or role in grant/revoke statements
  */
-@LimitedPrivate(value = { "" })
+@LimitedPrivate(value = { "Apache Argus (incubating)" })
 @Evolving
 public class HivePrincipal implements Comparable<HivePrincipal> {
 

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilege.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilege.java?rev=1629563&r1=1629562&r2=1629563&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilege.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilege.java Mon Oct  6 04:00:39 2014
@@ -27,7 +27,7 @@ import org.apache.hadoop.hive.ql.securit
 /**
  * Represents the hive privilege being granted/revoked
  */
-@LimitedPrivate(value = { "" })
+@LimitedPrivate(value = { "Apache Argus (incubating)" })
 @Evolving
 public class HivePrivilege implements Comparable<HivePrivilege> {
   @Override
@@ -97,6 +97,7 @@ public class HivePrivilege implements Co
     return supportedScope != null && supportedScope.contains(scope.name());
   }
 
+  @Override
   public int compareTo(HivePrivilege privilege) {
     int compare = columns != null ?
         (privilege.columns != null ? compare(columns, privilege.columns) : 1) :

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilegeInfo.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilegeInfo.java?rev=1629563&r1=1629562&r2=1629563&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilegeInfo.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilegeInfo.java Mon Oct  6 04:00:39 2014
@@ -23,7 +23,7 @@ import org.apache.hadoop.hive.common.cla
 /**
  * Represents a privilege granted for an object to a principal
  */
-@LimitedPrivate(value = { "" })
+@LimitedPrivate(value = { "Apache Argus (incubating)" })
 @Evolving
 public class HivePrivilegeInfo{
   private final HivePrincipal principal;

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilegeObject.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilegeObject.java?rev=1629563&r1=1629562&r2=1629563&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilegeObject.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilegeObject.java Mon Oct  6 04:00:39 2014
@@ -22,14 +22,19 @@ import java.util.Collection;
 import java.util.Iterator;
 import java.util.List;
 
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
 import org.apache.hadoop.hive.common.classification.InterfaceAudience.LimitedPrivate;
-import org.apache.hadoop.hive.common.classification.InterfaceStability.Unstable;
 
 /**
- * Represents the object on which privilege is being granted/revoked
+ * Represents the object on which privilege is being granted/revoked, and objects
+ * being used in queries.
+ *
+ * Check the get* function documentation for information on what value it returns based on
+ * the {@link HivePrivilegeObjectType}.
+ *
  */
-@LimitedPrivate(value = { "" })
-@Unstable
+@LimitedPrivate(value = { "Apache Argus (incubating)" })
+@Evolving
 public class HivePrivilegeObject implements Comparable<HivePrivilegeObject> {
 
   @Override
@@ -77,9 +82,20 @@ public class HivePrivilegeObject impleme
     return o1.size() > o2.size() ? 1 : (o1.size() < o2.size() ? -1 : 0);
   }
 
+  /**
+   * Note that GLOBAL, PARTITION, COLUMN fields are populated only for Hive's old default
+   * authorization mode.
+   * When the authorization manager is an instance of HiveAuthorizerFactory, these types are not
+   * used.
+   */
   public enum HivePrivilegeObjectType {
     GLOBAL, DATABASE, TABLE_OR_VIEW, PARTITION, COLUMN, LOCAL_URI, DFS_URI, COMMAND_PARAMS, FUNCTION
-  } ;
+  };
+
+  /**
+   * When {@link HiveOperationType} is QUERY, this action type is set so that it is possible
+   * to determine if the action type on this object is an INSERT or INSERT_OVERWRITE
+   */
   public enum HivePrivObjectActionType {
     OTHER, INSERT, INSERT_OVERWRITE, UPDATE, DELETE
   };
@@ -139,6 +155,9 @@ public class HivePrivilegeObject impleme
     return type;
   }
 
+  /**
+   * @return the db name if type is DATABASE, TABLE, or FUNCTION
+   */
   public String getDbname() {
     return dbname;
   }
@@ -150,6 +169,10 @@ public class HivePrivilegeObject impleme
     return objectName;
   }
 
+  /**
+   * See javadoc of {@link HivePrivObjectActionType}
+   * @return action type
+   */
   public HivePrivObjectActionType getActionType() {
     return actionType;
   }
@@ -158,12 +181,15 @@ public class HivePrivilegeObject impleme
     return commandParams;
   }
 
+  /**
+   * @return  partiton key information. Used only for old default authorization mode.
+   */
   public List<String> getPartKeys() {
     return partKeys;
   }
 
   /**
-   * Applicable columns in this object
+   * Applicable columns in this object, when the type is {@link HivePrivilegeObjectType.TABLE}
    * In case of DML read operations, this is the set of columns being used.
    * Column information is not set for DDL operations and for tables being written into
    * @return list of applicable columns

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveRoleGrant.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveRoleGrant.java?rev=1629563&r1=1629562&r2=1629563&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveRoleGrant.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveRoleGrant.java Mon Oct  6 04:00:39 2014
@@ -26,7 +26,7 @@ import com.google.common.collect.Compari
 /**
  * Represents a grant of a role to a principal
  */
-@LimitedPrivate(value = { "" })
+@LimitedPrivate(value = { "Apache Argus (incubating)" })
 @Evolving
 public class HiveRoleGrant implements Comparable<HiveRoleGrant> {
 

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java?rev=1629563&r1=1629562&r2=1629563&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java Mon Oct  6 04:00:39 2014
@@ -1249,7 +1249,7 @@ public class SessionState {
 
     try {
       if (tezSessionState != null) {
-        TezSessionPoolManager.getInstance().close(tezSessionState);
+        TezSessionPoolManager.getInstance().close(tezSessionState, false);
       }
     } catch (Exception e) {
       LOG.info("Error closing tez session", e);

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java?rev=1629563&r1=1629562&r2=1629563&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java Mon Oct  6 04:00:39 2014
@@ -18,8 +18,15 @@
 
 package org.apache.hadoop.hive.ql.stats;
 
-import com.google.common.base.Joiner;
-import com.google.common.collect.Lists;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.FileSystem;
@@ -80,19 +87,14 @@ import org.apache.hadoop.hive.serde2.obj
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableTimestampObjectInspector;
 import org.apache.hadoop.io.BytesWritable;
 
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import com.google.common.base.Joiner;
+import com.google.common.collect.Lists;
 
 public class StatsUtils {
 
   private static final Log LOG = LogFactory.getLog(StatsUtils.class.getName());
 
+
   /**
    * Collect table, partition and column level statistics
    * @param conf
@@ -109,15 +111,34 @@ public class StatsUtils {
   public static Statistics collectStatistics(HiveConf conf, PrunedPartitionList partList,
       Table table, TableScanOperator tableScanOperator) throws HiveException {
 
-    Statistics stats = new Statistics();
-
     // column level statistics are required only for the columns that are needed
     List<ColumnInfo> schema = tableScanOperator.getSchema().getSignature();
     List<String> neededColumns = tableScanOperator.getNeededColumns();
+    List<String> referencedColumns = tableScanOperator.getReferencedColumns();
+
+    return collectStatistics(conf, partList, table, schema, neededColumns, referencedColumns);
+  }
+
+  private static Statistics collectStatistics(HiveConf conf, PrunedPartitionList partList,
+      Table table, List<ColumnInfo> schema, List<String> neededColumns,
+      List<String> referencedColumns) throws HiveException {
+
     boolean fetchColStats =
         HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_STATS_FETCH_COLUMN_STATS);
     boolean fetchPartStats =
         HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_STATS_FETCH_PARTITION_STATS);
+
+    return collectStatistics(conf, partList, table, schema, neededColumns, referencedColumns,
+        fetchColStats, fetchPartStats);
+  }
+
+  public static Statistics collectStatistics(HiveConf conf, PrunedPartitionList partList,
+      Table table, List<ColumnInfo> schema, List<String> neededColumns,
+      List<String> referencedColumns, boolean fetchColStats, boolean fetchPartStats)
+      throws HiveException {
+
+    Statistics stats = new Statistics();
+
     float deserFactor =
         HiveConf.getFloatVar(conf, HiveConf.ConfVars.HIVE_STATS_DESERIALIZATION_FACTOR);
 
@@ -207,7 +228,6 @@ public class StatsUtils {
           stats.getBasicStatsState().equals(State.COMPLETE)) {
         stats.setBasicStatsState(State.PARTIAL);
       }
-      boolean haveFullStats = fetchColStats;
       if (fetchColStats) {
         List<String> partNames = new ArrayList<String>(partList.getNotDeniedPartns().size());
         for (Partition part : partList.getNotDeniedPartns()) {
@@ -215,37 +235,84 @@ public class StatsUtils {
         }
         Map<String, String> colToTabAlias = new HashMap<String, String>();
         neededColumns = processNeededColumns(schema, neededColumns, colToTabAlias);
-        AggrStats aggrStats = Hive.get().getAggrColStatsFor(table.getDbName(), table.getTableName(), neededColumns, partNames);
+        AggrStats aggrStats = Hive.get().getAggrColStatsFor(table.getDbName(), table.getTableName(),
+            neededColumns, partNames);
         if (null == aggrStats) {
-          haveFullStats = false;
+          // There are some partitions with no state (or we didn't fetch any state).
+          // Update the stats with empty list to reflect that in the
+          // state/initialize structures.
+          List<ColStatistics> emptyStats = Lists.newArrayList();
+
+          // add partition column stats
+          addParitionColumnStats(neededColumns, referencedColumns, schema, table, partList,
+              emptyStats);
+
+          stats.addToColumnStats(emptyStats);
+          stats.updateColumnStatsState(deriveStatType(emptyStats, referencedColumns));
         } else {
           List<ColumnStatisticsObj> colStats = aggrStats.getColStats();
           if (colStats.size() != neededColumns.size()) {
-            LOG.debug("Column stats requested for : " + neededColumns.size() + " columns. Able to retrieve"
-                + " for " + colStats.size() + " columns");
+            LOG.debug("Column stats requested for : " + neededColumns.size() + " columns. Able to" +
+                " retrieve for " + colStats.size() + " columns");
           }
-          List<ColStatistics> columnStats = convertColStats(colStats, table.getTableName(), colToTabAlias);
+          List<ColStatistics> columnStats = convertColStats(colStats, table.getTableName(),
+              colToTabAlias);
+
+          addParitionColumnStats(neededColumns, referencedColumns, schema, table, partList,
+              columnStats);
+
           stats.addToColumnStats(columnStats);
-          State colState = deriveStatType(columnStats, neededColumns);
+          State colState = deriveStatType(columnStats, referencedColumns);
           if (aggrStats.getPartsFound() != partNames.size() && colState != State.NONE) {
-            LOG.debug("Column stats requested for : " + partNames.size() +" partitions. "
-              + "Able to retrieve for " + aggrStats.getPartsFound() + " partitions");
+            LOG.debug("Column stats requested for : " + partNames.size() + " partitions. "
+                + "Able to retrieve for " + aggrStats.getPartsFound() + " partitions");
             colState = State.PARTIAL;
           }
           stats.setColumnStatsState(colState);
         }
       }
-      // There are some partitions with no state (or we didn't fetch any state).
-      // Update the stats with empty list to reflect that in the state/initialize structures.
-      if (!haveFullStats) {
-        List<ColStatistics> emptyStats = Lists.<ColStatistics>newArrayList();
-        stats.addToColumnStats(emptyStats);
-        stats.updateColumnStatsState(deriveStatType(emptyStats, neededColumns));
-      }
     }
     return stats;
   }
 
+  private static void addParitionColumnStats(List<String> neededColumns,
+      List<String> referencedColumns, List<ColumnInfo> schema, Table table,
+      PrunedPartitionList partList, List<ColStatistics> colStats)
+      throws HiveException {
+
+    // extra columns is difference between referenced columns vs needed
+    // columns. The difference could be partition columns.
+    List<String> extraCols = Lists.newArrayList(referencedColumns);
+    if (referencedColumns.size() > neededColumns.size()) {
+      extraCols.removeAll(neededColumns);
+      for (String col : extraCols) {
+        for (ColumnInfo ci : schema) {
+          // conditions for being partition column
+          if (col.equals(ci.getInternalName()) && ci.getIsVirtualCol() &&
+              !ci.isHiddenVirtualCol()) {
+            // currently metastore does not store column stats for
+            // partition column, so we calculate the NDV from pruned
+            // partition list
+            ColStatistics partCS = new ColStatistics(table.getTableName(),
+                ci.getInternalName(), ci.getType().getTypeName());
+            long numPartitions = getNDVPartitionColumn(partList.getPartitions(),
+                ci.getInternalName());
+            partCS.setCountDistint(numPartitions);
+            colStats.add(partCS);
+          }
+        }
+      }
+    }
+  }
+
+  public static int getNDVPartitionColumn(Set<Partition> partitions, String partColName) {
+    Set<String> distinctVals = new HashSet<String>(partitions.size());
+    for (Partition partition : partitions) {
+      distinctVals.add(partition.getSpec().get(partColName));
+    }
+    return distinctVals.size();
+  }
+
   private static void setUnknownRcDsToAverage(
       List<Long> rowCounts, List<Long> dataSizes, int avgRowSize) {
     if (LOG.isDebugEnabled()) {
@@ -751,7 +818,8 @@ public class StatsUtils {
         || colType.equalsIgnoreCase(serdeConstants.FLOAT_TYPE_NAME)) {
       return JavaDataModel.get().primitive1();
     } else if (colType.equalsIgnoreCase(serdeConstants.DOUBLE_TYPE_NAME)
-        || colType.equalsIgnoreCase(serdeConstants.BIGINT_TYPE_NAME)) {
+        || colType.equalsIgnoreCase(serdeConstants.BIGINT_TYPE_NAME)
+        || colType.equalsIgnoreCase("long")) {
       return JavaDataModel.get().primitive2();
     } else if (colType.equalsIgnoreCase(serdeConstants.TIMESTAMP_TYPE_NAME)) {
       return JavaDataModel.get().lengthOfTimestamp();
@@ -780,7 +848,8 @@ public class StatsUtils {
       return JavaDataModel.get().lengthForIntArrayOfSize(length);
     } else if (colType.equalsIgnoreCase(serdeConstants.DOUBLE_TYPE_NAME)) {
       return JavaDataModel.get().lengthForDoubleArrayOfSize(length);
-    } else if (colType.equalsIgnoreCase(serdeConstants.BIGINT_TYPE_NAME)) {
+    } else if (colType.equalsIgnoreCase(serdeConstants.BIGINT_TYPE_NAME)
+        || colType.equalsIgnoreCase("long")) {
       return JavaDataModel.get().lengthForLongArrayOfSize(length);
     } else if (colType.equalsIgnoreCase(serdeConstants.BINARY_TYPE_NAME)) {
       return JavaDataModel.get().lengthForByteArrayOfSize(length);
@@ -876,7 +945,7 @@ public class StatsUtils {
       Statistics parentStats, Map<String, ExprNodeDesc> colExprMap, RowSchema rowSchema) {
 
     List<ColStatistics> cs = Lists.newArrayList();
-    if (colExprMap != null) {
+    if (colExprMap != null  && rowSchema != null) {
       for (ColumnInfo ci : rowSchema.getSignature()) {
         String outColName = ci.getInternalName();
         outColName = StatsUtils.stripPrefixFromColumnName(outColName);
@@ -1042,10 +1111,8 @@ public class StatsUtils {
 
   /**
    * Get basic stats of table
-   * @param dbName
-   *          - database name
-   * @param tabName
-   *          - table name
+   * @param table
+   *          - table
    * @param statType
    *          - type of stats
    * @return value of stats

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java?rev=1629563&r1=1629562&r2=1629563&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java Mon Oct  6 04:00:39 2014
@@ -76,7 +76,7 @@ public class Initiator extends Compactor
         // don't doom the entire thread.
         try {
           ShowCompactResponse currentCompactions = txnHandler.showCompact(new ShowCompactRequest());
-          ValidTxnList txns = TxnHandler.createValidTxnList(txnHandler.getOpenTxns());
+          ValidTxnList txns = TxnHandler.createValidTxnList(txnHandler.getOpenTxns(), 0);
           Set<CompactionInfo> potentials = txnHandler.findPotentialCompactions(abortedThreshold);
           LOG.debug("Found " + potentials.size() + " potential compactions, " +
               "checking to see if we should compact any of them");

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java?rev=1629563&r1=1629562&r2=1629563&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java Mon Oct  6 04:00:39 2014
@@ -120,7 +120,7 @@ public class Worker extends CompactorThr
 
         final boolean isMajor = ci.isMajorCompaction();
         final ValidTxnList txns =
-            TxnHandler.createValidTxnList(txnHandler.getOpenTxns());
+            TxnHandler.createValidTxnList(txnHandler.getOpenTxns(), 0);
         final StringBuffer jobName = new StringBuffer(name);
         jobName.append("-compactor-");
         jobName.append(ci.getFullPartitionName());

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog.java?rev=1629563&r1=1629562&r2=1629563&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog.java Mon Oct  6 04:00:39 2014
@@ -52,29 +52,6 @@ public class UDFLog extends UDFMath {
   }
 
   /**
-   * Get the logarithm of the given decimal with the given base.
-   */
-  public DoubleWritable evaluate(DoubleWritable base, HiveDecimalWritable writable) {
-    if (base == null || writable == null) {
-      return null;
-    }
-    double d = writable.getHiveDecimal().bigDecimalValue().doubleValue();
-    return log(base.get(), d);
-  }
-
-  /**
-   * Get the logarithm of input with the given decimal as the base.
-   */
-  public DoubleWritable evaluate(HiveDecimalWritable base, DoubleWritable d) {
-    if (base == null || d == null) {
-      return null;
-    }
-
-    double b = base.getHiveDecimal().bigDecimalValue().doubleValue();
-    return log(b, d.get());
-  }
-
-  /**
    * Get the logarithm of the given decimal input with the given decimal base.
    */
   public DoubleWritable evaluate(HiveDecimalWritable baseWritable, HiveDecimalWritable writable) {



Mime
View raw message