hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From br...@apache.org
Subject svn commit: r1640263 [7/12] - in /hive/branches/spark: ./ accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/ accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/mr/ accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/predicate/ accu...
Date Tue, 18 Nov 2014 00:48:46 GMT
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java Tue Nov 18 00:48:40 2014
@@ -57,7 +57,9 @@ import org.apache.hadoop.hive.ql.udf.gen
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr;
 import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters;
 import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
 import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
 
@@ -420,9 +422,10 @@ public class PartitionPruner implements 
 
     String defaultPartitionName = conf.getVar(HiveConf.ConfVars.DEFAULTPARTITIONNAME);
     List<String> partCols = extractPartColNames(tab);
+    List<PrimitiveTypeInfo> partColTypeInfos = extractPartColTypes(tab);
 
     boolean hasUnknownPartitions = prunePartitionNames(
-        partCols, prunerExpr, defaultPartitionName, partNames);
+        partCols, partColTypeInfos, prunerExpr, defaultPartitionName, partNames);
     perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.PRUNE_LISTING);
 
     perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.PARTITION_RETRIEVING);
@@ -442,19 +445,30 @@ public class PartitionPruner implements 
     return partCols;
   }
 
+  private static List<PrimitiveTypeInfo> extractPartColTypes(Table tab) {
+    List<FieldSchema> pCols = tab.getPartCols();
+    List<PrimitiveTypeInfo> partColTypeInfos = new ArrayList<PrimitiveTypeInfo>(pCols.size());
+    for (FieldSchema pCol : pCols) {
+      partColTypeInfos.add(TypeInfoFactory.getPrimitiveTypeInfo(pCol.getType()));
+    }
+    return partColTypeInfos;
+  }
+
   /**
    * Prunes partition names to see if they match the prune expression.
-   * @param columnNames name of partition columns
+   * @param partColumnNames name of partition columns
+   * @param partColumnTypeInfos types of partition columns
    * @param prunerExpr The expression to match.
    * @param defaultPartitionName name of default partition
    * @param partNames Partition names to filter. The list is modified in place.
    * @return Whether the list has any partitions for which the expression may or may not match.
    */
-  public static boolean prunePartitionNames(List<String> columnNames, ExprNodeGenericFuncDesc prunerExpr,
+  public static boolean prunePartitionNames(List<String> partColumnNames,
+      List<PrimitiveTypeInfo> partColumnTypeInfos, ExprNodeGenericFuncDesc prunerExpr,
       String defaultPartitionName, List<String> partNames) throws HiveException, MetaException {
     // Prepare the expression to filter on the columns.
     ObjectPair<PrimitiveObjectInspector, ExprNodeEvaluator> handle =
-        PartExprEvalUtils.prepareExpr(prunerExpr, columnNames);
+        PartExprEvalUtils.prepareExpr(prunerExpr, partColumnNames, partColumnTypeInfos);
 
     // Filter the name list. Removing elements one by one can be slow on e.g. ArrayList,
     // so let's create a new list and copy it if we don't have a linked list
@@ -462,8 +476,8 @@ public class PartitionPruner implements 
     List<String> partNamesSeq = inPlace ? partNames : new LinkedList<String>(partNames);
 
     // Array for the values to pass to evaluator.
-    ArrayList<String> values = new ArrayList<String>(columnNames.size());
-    for (int i = 0; i < columnNames.size(); ++i) {
+    ArrayList<String> values = new ArrayList<String>(partColumnNames.size());
+    for (int i = 0; i < partColumnNames.size(); ++i) {
       values.add(null);
     }
 
@@ -473,8 +487,17 @@ public class PartitionPruner implements 
       String partName = partIter.next();
       Warehouse.makeValsFromName(partName, values);
 
+      ArrayList<Object> convertedValues = new ArrayList<Object>(values.size());
+      for(int i=0; i<values.size(); i++) {
+        Object o = ObjectInspectorConverters.getConverter(
+            PrimitiveObjectInspectorFactory.javaStringObjectInspector,
+            PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(partColumnTypeInfos.get(i)))
+            .convert(values.get(i));
+        convertedValues.add(o);
+      }
+
       // Evaluate the expression tree.
-      Boolean isNeeded = (Boolean)PartExprEvalUtils.evaluateExprOnPart(handle, values);
+      Boolean isNeeded = (Boolean)PartExprEvalUtils.evaluateExprOnPart(handle, convertedValues);
       boolean isUnknown = (isNeeded == null);
       if (!isUnknown && !isNeeded) {
         partIter.remove();

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java Tue Nov 18 00:48:40 2014
@@ -22,6 +22,8 @@ import static org.apache.hadoop.hive.ql.
 import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_DATABASEPROPERTIES;
 
 import java.io.Serializable;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.ArrayList;
@@ -229,7 +231,7 @@ public class DDLSemanticAnalyzer extends
     reservedPartitionValues.add(HiveConf.getVar(conf, ConfVars.METASTORE_INT_ORIGINAL));
     reservedPartitionValues.add(HiveConf.getVar(conf, ConfVars.METASTORE_INT_ARCHIVED));
     reservedPartitionValues.add(HiveConf.getVar(conf, ConfVars.METASTORE_INT_EXTRACTED));
-    hiveAuthorizationTaskFactory = new HiveAuthorizationTaskFactoryImpl(conf, db);
+    hiveAuthorizationTaskFactory = createAuthorizationTaskFactory(conf, db);
   }
 
   @Override
@@ -3419,4 +3421,30 @@ public class DDLSemanticAnalyzer extends
       throw new SemanticException(e);
     }
   }
+
+  private HiveAuthorizationTaskFactory createAuthorizationTaskFactory(HiveConf conf, Hive db) {
+    Class<? extends HiveAuthorizationTaskFactory> authProviderClass = conf.
+        getClass(HiveConf.ConfVars.HIVE_AUTHORIZATION_TASK_FACTORY.varname,
+            HiveAuthorizationTaskFactoryImpl.class,
+            HiveAuthorizationTaskFactory.class);
+    String msg = "Unable to create instance of " + authProviderClass.getName() + ": ";
+    try {
+      Constructor<? extends HiveAuthorizationTaskFactory> constructor =
+          authProviderClass.getConstructor(HiveConf.class, Hive.class);
+      return constructor.newInstance(conf, db);
+    } catch (NoSuchMethodException e) {
+      throw new IllegalStateException(msg + e.getMessage(), e);
+    } catch (SecurityException e) {
+      throw new IllegalStateException(msg + e.getMessage(), e);
+    } catch (InstantiationException e) {
+      throw new IllegalStateException(msg + e.getMessage(), e);
+    } catch (IllegalAccessException e) {
+      throw new IllegalStateException(msg + e.getMessage(), e);
+    } catch (IllegalArgumentException e) {
+      throw new IllegalStateException(msg + e.getMessage(), e);
+    } catch (InvocationTargetException e) {
+      throw new IllegalStateException(msg + e.getMessage(), e);
+    }
+  }
+
 }

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g Tue Nov 18 00:48:40 2014
@@ -283,6 +283,8 @@ KW_ROLE: 'ROLE';
 KW_ROLES: 'ROLES';
 KW_INNER: 'INNER';
 KW_EXCHANGE: 'EXCHANGE';
+KW_URI: 'URI';
+KW_SERVER : 'SERVER';
 KW_ADMIN: 'ADMIN';
 KW_OWNER: 'OWNER';
 KW_PRINCIPALS: 'PRINCIPALS';

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g Tue Nov 18 00:48:40 2014
@@ -109,6 +109,7 @@ TOK_DATE;
 TOK_DATELITERAL;
 TOK_DATETIME;
 TOK_TIMESTAMP;
+TOK_TIMESTAMPLITERAL;
 TOK_STRING;
 TOK_CHAR;
 TOK_VARCHAR;
@@ -340,6 +341,8 @@ TOK_VIRTUAL_TABLE;
 TOK_VIRTUAL_TABREF;
 TOK_ANONYMOUS;
 TOK_COL_NAME;
+TOK_URI_TYPE;
+TOK_SERVER_TYPE;
 }
 
 
@@ -1485,11 +1488,15 @@ privilegeObject
 privObject
     : (KW_DATABASE|KW_SCHEMA) identifier -> ^(TOK_DB_TYPE identifier)
     | KW_TABLE? tableName partitionSpec? -> ^(TOK_TABLE_TYPE tableName partitionSpec?)
+    | KW_URI (path=StringLiteral) ->  ^(TOK_URI_TYPE $path)
+    | KW_SERVER identifier -> ^(TOK_SERVER_TYPE identifier)
     ;
 
 privObjectCols
     : (KW_DATABASE|KW_SCHEMA) identifier -> ^(TOK_DB_TYPE identifier)
     | KW_TABLE? tableName (LPAREN cols=columnNameList RPAREN)? partitionSpec? -> ^(TOK_TABLE_TYPE tableName $cols? partitionSpec?)
+    | KW_URI (path=StringLiteral) ->  ^(TOK_URI_TYPE $path)
+    | KW_SERVER identifier -> ^(TOK_SERVER_TYPE identifier)
     ;
 
 privilegeList

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g Tue Nov 18 00:48:40 2014
@@ -218,6 +218,7 @@ constant
     :
     Number
     | dateLiteral
+    | timestampLiteral
     | StringLiteral
     | stringLiteralSequence
     | BigintLiteral
@@ -250,6 +251,14 @@ dateLiteral
     }
     ;
 
+timestampLiteral
+    :
+    KW_TIMESTAMP StringLiteral ->
+    {
+      adaptor.create(TOK_TIMESTAMPLITERAL, $StringLiteral.text)
+    }
+    ;
+
 expression
 @init { gParent.pushMsg("expression specification", state); }
 @after { gParent.popMsg(state); }
@@ -260,7 +269,6 @@ expression
 atomExpression
     :
     KW_NULL -> TOK_NULL
-    | dateLiteral
     | constant
     | castExpression
     | caseExpression
@@ -545,5 +553,5 @@ principalIdentifier
 
 nonReserved
     :
-    KW_TRUE | KW_FALSE | KW_LIKE | KW_EXISTS | KW_ASC | KW_DESC | KW_ORDER | KW_GROUP | KW_BY | KW_AS | KW_INSERT | KW_OVERWRITE | KW_OUTER | KW_LEFT | KW_RIGHT | KW_FULL | KW_PARTITION | KW_PARTITIONS | KW_TABLE | KW_TABLES | KW_COLUMNS | KW_INDEX | KW_INDEXES | KW_REBUILD | KW_FUNCTIONS | KW_SHOW | KW_MSCK | KW_REPAIR | KW_DIRECTORY | KW_LOCAL | KW_USING | KW_CLUSTER | KW_DISTRIBUTE | KW_SORT | KW_UNION | KW_LOAD | KW_EXPORT | KW_IMPORT | KW_DATA | KW_INPATH | KW_IS | KW_NULL | KW_CREATE | KW_EXTERNAL | KW_ALTER | KW_CHANGE | KW_FIRST | KW_AFTER | KW_DESCRIBE | KW_DROP | KW_RENAME | KW_IGNORE | KW_PROTECTION | KW_TO | KW_COMMENT | KW_BOOLEAN | KW_TINYINT | KW_SMALLINT | KW_INT | KW_BIGINT | KW_FLOAT | KW_DOUBLE | KW_DATE | KW_DATETIME | KW_TIMESTAMP | KW_DECIMAL | KW_STRING | KW_ARRAY | KW_STRUCT | KW_UNIONTYPE | KW_PARTITIONED | KW_CLUSTERED | KW_SORTED | KW_INTO | KW_BUCKETS | KW_ROW | KW_ROWS | KW_FORMAT | KW_DELIMITED | KW_FIELDS | KW_TERMINATED | KW_ESCAPED | KW_COLLECTION | 
 KW_ITEMS | KW_KEYS | KW_KEY_TYPE | KW_LINES | KW_STORED | KW_FILEFORMAT | KW_INPUTFORMAT | KW_OUTPUTFORMAT | KW_INPUTDRIVER | KW_OUTPUTDRIVER | KW_OFFLINE | KW_ENABLE | KW_DISABLE | KW_READONLY | KW_NO_DROP | KW_LOCATION | KW_BUCKET | KW_OUT | KW_OF | KW_PERCENT | KW_ADD | KW_REPLACE | KW_RLIKE | KW_REGEXP | KW_TEMPORARY | KW_EXPLAIN | KW_FORMATTED | KW_PRETTY | KW_DEPENDENCY | KW_LOGICAL | KW_SERDE | KW_WITH | KW_DEFERRED | KW_SERDEPROPERTIES | KW_DBPROPERTIES | KW_LIMIT | KW_SET | KW_UNSET | KW_TBLPROPERTIES | KW_IDXPROPERTIES | KW_VALUE_TYPE | KW_ELEM_TYPE | KW_MAPJOIN | KW_STREAMTABLE | KW_HOLD_DDLTIME | KW_CLUSTERSTATUS | KW_UTC | KW_UTCTIMESTAMP | KW_LONG | KW_DELETE | KW_PLUS | KW_MINUS | KW_FETCH | KW_INTERSECT | KW_VIEW | KW_IN | KW_DATABASES | KW_MATERIALIZED | KW_SCHEMA | KW_SCHEMAS | KW_GRANT | KW_REVOKE | KW_SSL | KW_UNDO | KW_LOCK | KW_LOCKS | KW_UNLOCK | KW_SHARED | KW_EXCLUSIVE | KW_PROCEDURE | KW_UNSIGNED | KW_WHILE | KW_READ | KW_READS | KW_PURGE | KW_RANGE | KW_AN
 ALYZE | KW_BEFORE | KW_BETWEEN | KW_BOTH | KW_BINARY | KW_CONTINUE | KW_CURSOR | KW_TRIGGER | KW_RECORDREADER | KW_RECORDWRITER | KW_SEMI | KW_LATERAL | KW_TOUCH | KW_ARCHIVE | KW_UNARCHIVE | KW_COMPUTE | KW_STATISTICS | KW_USE | KW_OPTION | KW_CONCATENATE | KW_SHOW_DATABASE | KW_UPDATE | KW_RESTRICT | KW_CASCADE | KW_SKEWED | KW_ROLLUP | KW_CUBE | KW_DIRECTORIES | KW_FOR | KW_GROUPING | KW_SETS | KW_TRUNCATE | KW_NOSCAN | KW_USER | KW_ROLE | KW_ROLES | KW_INNER | KW_DEFINED | KW_ADMIN | KW_JAR | KW_FILE | KW_OWNER | KW_PRINCIPALS | KW_ALL | KW_DEFAULT | KW_NONE | KW_COMPACT | KW_COMPACTIONS | KW_TRANSACTIONS | KW_REWRITE | KW_AUTHORIZATION | KW_VALUES
+    KW_TRUE | KW_FALSE | KW_LIKE | KW_EXISTS | KW_ASC | KW_DESC | KW_ORDER | KW_GROUP | KW_BY | KW_AS | KW_INSERT | KW_OVERWRITE | KW_OUTER | KW_LEFT | KW_RIGHT | KW_FULL | KW_PARTITION | KW_PARTITIONS | KW_TABLE | KW_TABLES | KW_COLUMNS | KW_INDEX | KW_INDEXES | KW_REBUILD | KW_FUNCTIONS | KW_SHOW | KW_MSCK | KW_REPAIR | KW_DIRECTORY | KW_LOCAL | KW_USING | KW_CLUSTER | KW_DISTRIBUTE | KW_SORT | KW_UNION | KW_LOAD | KW_EXPORT | KW_IMPORT | KW_DATA | KW_INPATH | KW_IS | KW_NULL | KW_CREATE | KW_EXTERNAL | KW_ALTER | KW_CHANGE | KW_FIRST | KW_AFTER | KW_DESCRIBE | KW_DROP | KW_RENAME | KW_IGNORE | KW_PROTECTION | KW_TO | KW_COMMENT | KW_BOOLEAN | KW_TINYINT | KW_SMALLINT | KW_INT | KW_BIGINT | KW_FLOAT | KW_DOUBLE | KW_DATE | KW_DATETIME | KW_TIMESTAMP | KW_DECIMAL | KW_STRING | KW_ARRAY | KW_STRUCT | KW_UNIONTYPE | KW_PARTITIONED | KW_CLUSTERED | KW_SORTED | KW_INTO | KW_BUCKETS | KW_ROW | KW_ROWS | KW_FORMAT | KW_DELIMITED | KW_FIELDS | KW_TERMINATED | KW_ESCAPED | KW_COLLECTION | 
 KW_ITEMS | KW_KEYS | KW_KEY_TYPE | KW_LINES | KW_STORED | KW_FILEFORMAT | KW_INPUTFORMAT | KW_OUTPUTFORMAT | KW_INPUTDRIVER | KW_OUTPUTDRIVER | KW_OFFLINE | KW_ENABLE | KW_DISABLE | KW_READONLY | KW_NO_DROP | KW_LOCATION | KW_BUCKET | KW_OUT | KW_OF | KW_PERCENT | KW_ADD | KW_REPLACE | KW_RLIKE | KW_REGEXP | KW_TEMPORARY | KW_EXPLAIN | KW_FORMATTED | KW_PRETTY | KW_DEPENDENCY | KW_LOGICAL | KW_SERDE | KW_WITH | KW_DEFERRED | KW_SERDEPROPERTIES | KW_DBPROPERTIES | KW_LIMIT | KW_SET | KW_UNSET | KW_TBLPROPERTIES | KW_IDXPROPERTIES | KW_VALUE_TYPE | KW_ELEM_TYPE | KW_MAPJOIN | KW_STREAMTABLE | KW_HOLD_DDLTIME | KW_CLUSTERSTATUS | KW_UTC | KW_UTCTIMESTAMP | KW_LONG | KW_DELETE | KW_PLUS | KW_MINUS | KW_FETCH | KW_INTERSECT | KW_VIEW | KW_IN | KW_DATABASES | KW_MATERIALIZED | KW_SCHEMA | KW_SCHEMAS | KW_GRANT | KW_REVOKE | KW_SSL | KW_UNDO | KW_LOCK | KW_LOCKS | KW_UNLOCK | KW_SHARED | KW_EXCLUSIVE | KW_PROCEDURE | KW_UNSIGNED | KW_WHILE | KW_READ | KW_READS | KW_PURGE | KW_RANGE | KW_AN
 ALYZE | KW_BEFORE | KW_BETWEEN | KW_BOTH | KW_BINARY | KW_CONTINUE | KW_CURSOR | KW_TRIGGER | KW_RECORDREADER | KW_RECORDWRITER | KW_SEMI | KW_LATERAL | KW_TOUCH | KW_ARCHIVE | KW_UNARCHIVE | KW_COMPUTE | KW_STATISTICS | KW_USE | KW_OPTION | KW_CONCATENATE | KW_SHOW_DATABASE | KW_UPDATE | KW_RESTRICT | KW_CASCADE | KW_SKEWED | KW_ROLLUP | KW_CUBE | KW_DIRECTORIES | KW_FOR | KW_GROUPING | KW_SETS | KW_TRUNCATE | KW_NOSCAN | KW_USER | KW_ROLE | KW_ROLES | KW_INNER | KW_DEFINED | KW_ADMIN | KW_JAR | KW_FILE | KW_OWNER | KW_PRINCIPALS | KW_ALL | KW_DEFAULT | KW_NONE | KW_COMPACT | KW_COMPACTIONS | KW_TRANSACTIONS | KW_REWRITE | KW_AUTHORIZATION | KW_VALUES | KW_URI | KW_SERVER
     ;

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Tue Nov 18 00:48:40 2014
@@ -51,6 +51,7 @@ import org.apache.hadoop.hive.metastore.
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.metastore.api.MetaException;
 import org.apache.hadoop.hive.metastore.api.Order;
+import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
 import org.apache.hadoop.hive.ql.ErrorMsg;
 import org.apache.hadoop.hive.ql.QueryProperties;
 import org.apache.hadoop.hive.ql.exec.AbstractMapJoinOperator;
@@ -318,9 +319,6 @@ public class SemanticAnalyzer extends Ba
 
   private static final String VALUES_TMP_TABLE_NAME_PREFIX = "Values__Tmp__Table__";
 
-  @VisibleForTesting
-  static final String ACID_TABLE_PROPERTY = "transactional";
-
   private HashMap<TableScanOperator, ExprNodeDesc> opToPartPruner;
   private HashMap<TableScanOperator, PrunedPartitionList> opToPartList;
   private HashMap<String, Operator<? extends OperatorDesc>> topOps;
@@ -2764,11 +2762,14 @@ public class SemanticAnalyzer extends Ba
   // TODO: make aliases unique, otherwise needless rewriting takes place
   private Integer genColListRegex(String colRegex, String tabAlias, ASTNode sel,
     ArrayList<ExprNodeDesc> col_list, HashSet<ColumnInfo> excludeCols, RowResolver input,
-    Integer pos, RowResolver output, List<String> aliases, boolean ensureUniqueCols)
-      throws SemanticException {
+    RowResolver colSrcRR, Integer pos, RowResolver output, List<String> aliases,
+    boolean ensureUniqueCols) throws SemanticException {
 
+    if (colSrcRR == null) {
+      colSrcRR = input;
+    }
     // The table alias should exist
-    if (tabAlias != null && !input.hasTableAlias(tabAlias)) {
+    if (tabAlias != null && !colSrcRR.hasTableAlias(tabAlias)) {
       throw new SemanticException(ErrorMsg.INVALID_TABLE_ALIAS.getMsg(sel));
     }
 
@@ -2797,7 +2798,7 @@ public class SemanticAnalyzer extends Ba
     // For expr "*", aliases should be iterated in the order they are specified
     // in the query.
     for (String alias : aliases) {
-      HashMap<String, ColumnInfo> fMap = input.getFieldMap(alias);
+      HashMap<String, ColumnInfo> fMap = colSrcRR.getFieldMap(alias);
       if (fMap == null) {
         continue;
       }
@@ -2808,8 +2809,11 @@ public class SemanticAnalyzer extends Ba
         if (excludeCols != null && excludeCols.contains(colInfo)) {
           continue; // This was added during plan generation.
         }
+        // First, look up the column from the source against which * is to be resolved.
+        // We'd later translated this into the column from proper input, if it's valid.
+        // TODO: excludeCols may be possible to remove using the same technique.
         String name = colInfo.getInternalName();
-        String[] tmp = input.reverseLookup(name);
+        String[] tmp = colSrcRR.reverseLookup(name);
 
         // Skip the colinfos which are not for this particular alias
         if (tabAlias != null && !tmp[0].equalsIgnoreCase(tabAlias)) {
@@ -2825,6 +2829,27 @@ public class SemanticAnalyzer extends Ba
           continue;
         }
 
+        // If input (GBY) is different than the source of columns, find the same column in input.
+        // TODO: This is fraught with peril.
+        if (input != colSrcRR) {
+          colInfo = input.get(tabAlias, tmp[1]);
+          if (colInfo == null) {
+            LOG.error("Cannot find colInfo for " + tabAlias + "." + tmp[1]
+                + ", derived from [" + colSrcRR + "], in [" + input + "]");
+            throw new SemanticException(ErrorMsg.NON_KEY_EXPR_IN_GROUPBY, tmp[1]);
+          }
+          String oldCol = null;
+          if (LOG.isDebugEnabled()) {
+            oldCol = name + " => " + (tmp == null ? "null" : (tmp[0] + "." + tmp[1]));
+          }
+          name = colInfo.getInternalName();
+          tmp = input.reverseLookup(name);
+          if (LOG.isDebugEnabled()) {
+            String newCol = name + " => " + (tmp == null ? "null" : (tmp[0] + "." + tmp[1]));
+            LOG.debug("Translated [" + oldCol + "] to [" + newCol + "]");
+          }
+        }
+
         ColumnInfo oColInfo = inputColsProcessed.get(colInfo);
         if (oColInfo == null) {
           ExprNodeColumnDesc expr = new ExprNodeColumnDesc(colInfo.getType(),
@@ -3423,11 +3448,10 @@ public class SemanticAnalyzer extends Ba
   }
 
 
-  private Operator<?> genSelectPlan(String dest, QB qb, Operator<?> input)
-      throws SemanticException {
+  private Operator<?> genSelectPlan(String dest, QB qb, Operator<?> input,
+      Operator<?> inputForSelectStar) throws SemanticException {
     ASTNode selExprList = qb.getParseInfo().getSelForClause(dest);
-
-    Operator<?> op = genSelectPlan(selExprList, qb, input, false);
+    Operator<?> op = genSelectPlan(selExprList, qb, input, inputForSelectStar, false);
 
     if (LOG.isDebugEnabled()) {
       LOG.debug("Created Select Plan for clause: " + dest);
@@ -3437,8 +3461,8 @@ public class SemanticAnalyzer extends Ba
   }
 
   @SuppressWarnings("nls")
-  private Operator<?> genSelectPlan(ASTNode selExprList, QB qb,
-      Operator<?> input, boolean outerLV) throws SemanticException {
+  private Operator<?> genSelectPlan(ASTNode selExprList, QB qb, Operator<?> input,
+      Operator<?> inputForSelectStar, boolean outerLV) throws SemanticException {
 
     if (LOG.isDebugEnabled()) {
       LOG.debug("tree: " + selExprList.toStringTree());
@@ -3449,6 +3473,10 @@ public class SemanticAnalyzer extends Ba
     ASTNode trfm = null;
     Integer pos = Integer.valueOf(0);
     RowResolver inputRR = opParseCtx.get(input).getRowResolver();
+    RowResolver starRR = null;
+    if (inputForSelectStar != null && inputForSelectStar != input) {
+      starRR = opParseCtx.get(inputForSelectStar).getRowResolver();
+    }
     // SELECT * or SELECT TRANSFORM(*)
     boolean selectStar = false;
     int posn = 0;
@@ -3494,7 +3522,7 @@ public class SemanticAnalyzer extends Ba
       }
       if (isUDTF && (selectStar = udtfExprType == HiveParser.TOK_FUNCTIONSTAR)) {
         genColListRegex(".*", null, (ASTNode) udtfExpr.getChild(0),
-            col_list, null, inputRR, pos, out_rwsch, qb.getAliases(), false);
+            col_list, null, inputRR, starRR, pos, out_rwsch, qb.getAliases(), false);
       }
     }
 
@@ -3547,7 +3575,7 @@ public class SemanticAnalyzer extends Ba
     }
 
     if (LOG.isDebugEnabled()) {
-      LOG.debug("genSelectPlan: input = " + inputRR.toString());
+      LOG.debug("genSelectPlan: input = " + inputRR + " starRr = " + starRR);
     }
 
     // For UDTF's, skip the function name to get the expressions
@@ -3616,7 +3644,7 @@ public class SemanticAnalyzer extends Ba
       if (expr.getType() == HiveParser.TOK_ALLCOLREF) {
         pos = genColListRegex(".*", expr.getChildCount() == 0 ? null
             : getUnescapedName((ASTNode) expr.getChild(0)).toLowerCase(),
-            expr, col_list, null, inputRR, pos, out_rwsch, qb.getAliases(), false);
+            expr, col_list, null, inputRR, starRR, pos, out_rwsch, qb.getAliases(), false);
         selectStar = true;
       } else if (expr.getType() == HiveParser.TOK_TABLE_OR_COL && !hasAsClause
           && !inputRR.getIsExprResolver()
@@ -3625,7 +3653,7 @@ public class SemanticAnalyzer extends Ba
         // This can only happen without AS clause
         // We don't allow this for ExprResolver - the Group By case
         pos = genColListRegex(unescapeIdentifier(expr.getChild(0).getText()),
-            null, expr, col_list, null, inputRR, pos, out_rwsch, qb.getAliases(), false);
+            null, expr, col_list, null, inputRR, starRR, pos, out_rwsch, qb.getAliases(), false);
       } else if (expr.getType() == HiveParser.DOT
           && expr.getChild(0).getType() == HiveParser.TOK_TABLE_OR_COL
           && inputRR.hasTableAlias(unescapeIdentifier(expr.getChild(0)
@@ -3637,7 +3665,7 @@ public class SemanticAnalyzer extends Ba
         // We don't allow this for ExprResolver - the Group By case
         pos = genColListRegex(unescapeIdentifier(expr.getChild(1).getText()),
             unescapeIdentifier(expr.getChild(0).getChild(0).getText().toLowerCase()),
-             expr, col_list, null, inputRR, pos, out_rwsch, qb.getAliases(), false);
+             expr, col_list, null, inputRR, starRR, pos, out_rwsch, qb.getAliases(), false);
       } else {
         // Case when this is an expression
         TypeCheckCtx tcCtx = new TypeCheckCtx(inputRR);
@@ -5165,7 +5193,8 @@ public class SemanticAnalyzer extends Ba
       Operator groupByOperatorInfo = genGroupByPlanGroupByOperator(parseInfo,
           dest, curr, reduceSinkOperatorInfo, GroupByDesc.Mode.COMPLETE, null);
 
-      curr = genPostGroupByBodyPlan(groupByOperatorInfo, dest, qb, aliasToOpInfo);
+      // TODO: should we pass curr instead of null?
+      curr = genPostGroupByBodyPlan(groupByOperatorInfo, dest, qb, aliasToOpInfo, null);
     }
 
     return curr;
@@ -8782,7 +8811,7 @@ public class SemanticAnalyzer extends Ba
       for (String dest : ks) {
         curr = input;
         curr = genGroupByPlan2MRMultiGroupBy(dest, qb, curr);
-        curr = genSelectPlan(dest, qb, curr);
+        curr = genSelectPlan(dest, qb, curr, null); // TODO: we may need to pass "input" here instead of null
         Integer limit = qbp.getDestLimit(dest);
         if (limit != null) {
           curr = genLimitMapRedPlan(dest, qb, curr, limit.intValue(), true);
@@ -8839,6 +8868,8 @@ public class SemanticAnalyzer extends Ba
                 ASTNode whereExpr = qb.getParseInfo().getWhrForClause(dest);
                 curr = genFilterPlan((ASTNode) whereExpr.getChild(0), qb, curr, aliasToOpInfo, false);
               }
+              // Preserve operator before the GBY - we'll use it to resolve '*'
+              Operator<?> gbySource = curr;
 
               if (qbp.getAggregationExprsForClause(dest).size() != 0
                   || getGroupByForClause(qbp, dest).size() > 0) {
@@ -8863,8 +8894,12 @@ public class SemanticAnalyzer extends Ba
                   curr = genGroupByPlan1MR(dest, qb, curr);
                 }
               }
+              if (LOG.isDebugEnabled()) {
+                LOG.debug("RR before GB " + opParseCtx.get(gbySource).getRowResolver()
+                    + " after GB " + opParseCtx.get(curr).getRowResolver());
+              }
 
-              curr = genPostGroupByBodyPlan(curr, dest, qb, aliasToOpInfo);
+              curr = genPostGroupByBodyPlan(curr, dest, qb, aliasToOpInfo, gbySource);
             }
           } else {
             curr = genGroupByPlan1ReduceMultiGBY(commonGroupByDestGroup, qb, input, aliasToOpInfo);
@@ -8891,7 +8926,7 @@ public class SemanticAnalyzer extends Ba
   }
 
   private Operator genPostGroupByBodyPlan(Operator curr, String dest, QB qb,
-      Map<String, Operator> aliasToOpInfo)
+      Map<String, Operator> aliasToOpInfo, Operator gbySource)
       throws SemanticException {
 
     QBParseInfo qbp = qb.getParseInfo();
@@ -8909,7 +8944,7 @@ public class SemanticAnalyzer extends Ba
       curr = genWindowingPlan(qb.getWindowingSpec(dest), curr);
     }
 
-    curr = genSelectPlan(dest, qb, curr);
+    curr = genSelectPlan(dest, qb, curr, gbySource);
     Integer limit = qbp.getDestLimit(dest);
 
     // Expressions are not supported currently without a alias.
@@ -9866,7 +9901,7 @@ public class SemanticAnalyzer extends Ba
     // Get the UDTF Path
     QB blankQb = new QB(null, null, false);
     Operator udtfPath = genSelectPlan((ASTNode) lateralViewTree
-        .getChild(0), blankQb, lvForward,
+        .getChild(0), blankQb, lvForward, null,
         lateralViewTree.getType() == HiveParser.TOK_LATERAL_VIEW_OUTER);
     // add udtf aliases to QB
     for (String udtfAlias : blankQb.getAliases()) {
@@ -12415,7 +12450,9 @@ public class SemanticAnalyzer extends Ba
   private boolean isAcidTable(Table tab) {
     if (tab == null) return false;
     if (!SessionState.get().getTxnMgr().supportsAcid()) return false;
-    return tab.getProperty(ACID_TABLE_PROPERTY) != null;
+    String tableIsTransactional =
+        tab.getProperty(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL);
+    return tableIsTransactional != null && tableIsTransactional.equalsIgnoreCase("true");
   }
 
   private boolean isAcidOutputFormat(Class<? extends HiveOutputFormat> of) {
@@ -14203,8 +14240,8 @@ public class SemanticAnalyzer extends Ba
         if (expr.getType() == HiveParser.TOK_ALLCOLREF) {
           pos = genColListRegex(".*",
               expr.getChildCount() == 0 ? null : getUnescapedName((ASTNode) expr.getChild(0))
-                  .toLowerCase(), expr, col_list, excludedColumns, inputRR, pos, out_rwsch,
-                  tabAliasesForAllProjs, true);
+                  .toLowerCase(), expr, col_list, excludedColumns, inputRR, null, pos,
+                  out_rwsch, tabAliasesForAllProjs, true);
           selectStar = true;
         } else if (expr.getType() == HiveParser.TOK_TABLE_OR_COL && !hasAsClause
             && !inputRR.getIsExprResolver()
@@ -14213,7 +14250,8 @@ public class SemanticAnalyzer extends Ba
           // This can only happen without AS clause
           // We don't allow this for ExprResolver - the Group By case
           pos = genColListRegex(unescapeIdentifier(expr.getChild(0).getText()), null, expr,
-              col_list, excludedColumns, inputRR, pos, out_rwsch, tabAliasesForAllProjs, true);
+              col_list, excludedColumns, inputRR, null, pos, out_rwsch, tabAliasesForAllProjs,
+              true);
         } else if (expr.getType() == HiveParser.DOT
             && expr.getChild(0).getType() == HiveParser.TOK_TABLE_OR_COL
             && inputRR.hasTableAlias(unescapeIdentifier(expr.getChild(0).getChild(0).getText()
@@ -14224,7 +14262,8 @@ public class SemanticAnalyzer extends Ba
           // We don't allow this for ExprResolver - the Group By case
           pos = genColListRegex(unescapeIdentifier(expr.getChild(1).getText()),
               unescapeIdentifier(expr.getChild(0).getChild(0).getText().toLowerCase()), expr,
-              col_list, excludedColumns, inputRR, pos, out_rwsch, tabAliasesForAllProjs, true);
+              col_list, excludedColumns, inputRR, null, pos, out_rwsch, tabAliasesForAllProjs,
+              true);
         } else if (expr.toStringTree().contains("TOK_FUNCTIONDI") && !(srcRel instanceof HiveAggregateRel)) {
           // Likely a malformed query eg, select hash(distinct c1) from t1;
           throw new OptiqSemanticException("Distinct without an aggreggation.");

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java Tue Nov 18 00:48:40 2014
@@ -150,6 +150,7 @@ public class TezCompiler extends TaskCom
           LOG.info("Found cycle in operator plan...");
           cycleFree = false;
           removeEventOperator(component);
+          break;
         }
       }
       LOG.info("Cycle free: " + cycleFree);
@@ -227,7 +228,7 @@ public class TezCompiler extends TaskCom
     for (Operator<?> child : children) {
       if (!indexes.containsKey(child)) {
         connect(child, index, nodes, indexes, lowLinks, components);
-        lowLinks.put(child, Math.min(lowLinks.get(o), lowLinks.get(child)));
+        lowLinks.put(o, Math.min(lowLinks.get(o), lowLinks.get(child)));
       } else if (nodes.contains(child)) {
         lowLinks.put(o, Math.min(lowLinks.get(o), indexes.get(child)));
       }

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java Tue Nov 18 00:48:40 2014
@@ -19,6 +19,7 @@
 package org.apache.hadoop.hive.ql.parse;
 
 import java.sql.Date;
+import java.sql.Timestamp;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -73,6 +74,8 @@ import org.apache.hadoop.hive.serde2.typ
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
 import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo;
 
+import com.google.common.collect.Lists;
+
 
 /**
  * The Factory for creating typecheck processors. The typecheck processors are
@@ -170,7 +173,8 @@ public class TypeCheckProcFactory {
         tf.getStrExprProcessor());
     opRules.put(new RuleRegExp("R4", HiveParser.KW_TRUE + "%|"
         + HiveParser.KW_FALSE + "%"), tf.getBoolExprProcessor());
-    opRules.put(new RuleRegExp("R5", HiveParser.TOK_DATELITERAL + "%"), tf.getDateExprProcessor());
+    opRules.put(new RuleRegExp("R5", HiveParser.TOK_DATELITERAL + "%|"
+        + HiveParser.TOK_TIMESTAMPLITERAL + "%"), tf.getDateTimeExprProcessor());
     opRules.put(new RuleRegExp("R6", HiveParser.TOK_TABLE_OR_COL + "%"),
         tf.getColumnExprProcessor());
     opRules.put(new RuleRegExp("R7", HiveParser.TOK_SUBQUERY_OP + "%"),
@@ -182,9 +186,8 @@ public class TypeCheckProcFactory {
         opRules, tcCtx);
     GraphWalker ogw = new DefaultGraphWalker(disp);
 
-    // Create a list of topop nodes
-    ArrayList<Node> topNodes = new ArrayList<Node>();
-    topNodes.add(expr);
+    // Create a list of top nodes
+    ArrayList<Node> topNodes = Lists.<Node>newArrayList(expr);
     HashMap<Node, Object> nodeOutputs = new LinkedHashMap<Node, Object>();
     ogw.startWalking(topNodes, nodeOutputs);
 
@@ -420,7 +423,7 @@ public class TypeCheckProcFactory {
   /**
    * Processor for date constants.
    */
-  public static class DateExprProcessor implements NodeProcessor {
+  public static class DateTimeExprProcessor implements NodeProcessor {
 
     @Override
     public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
@@ -437,14 +440,24 @@ public class TypeCheckProcFactory {
       }
 
       ASTNode expr = (ASTNode) nd;
+      String timeString = BaseSemanticAnalyzer.stripQuotes(expr.getText());
 
       // Get the string value and convert to a Date value.
       try {
-        String dateString = BaseSemanticAnalyzer.stripQuotes(expr.getText());
-        Date date = Date.valueOf(dateString);
-        return new ExprNodeConstantDesc(TypeInfoFactory.dateTypeInfo, date);
-      } catch (IllegalArgumentException err) {
-        throw new SemanticException("Unable to convert date literal string to date value.", err);
+        // todo replace below with joda-time, which supports timezone
+        if (expr.getType() == HiveParser.TOK_DATELITERAL) {
+          PrimitiveTypeInfo typeInfo = TypeInfoFactory.dateTypeInfo;
+          return new ExprNodeConstantDesc(typeInfo,
+              Date.valueOf(timeString));
+        }
+        if (expr.getType() == HiveParser.TOK_TIMESTAMPLITERAL) {
+          return new ExprNodeConstantDesc(TypeInfoFactory.timestampTypeInfo,
+              Timestamp.valueOf(timeString));
+        }
+        throw new IllegalArgumentException("Invalid time literal type " + expr.getType());
+      } catch (Exception err) {
+        throw new SemanticException(
+            "Unable to convert time literal '" + timeString + "' to time value.", err);
       }
     }
   }
@@ -454,8 +467,8 @@ public class TypeCheckProcFactory {
    *
    * @return DateExprProcessor.
    */
-  public DateExprProcessor getDateExprProcessor() {
-    return new DateExprProcessor();
+  public DateTimeExprProcessor getDateTimeExprProcessor() {
+    return new DateTimeExprProcessor();
   }
 
   /**

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/authorization/HiveAuthorizationTaskFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/authorization/HiveAuthorizationTaskFactory.java?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/authorization/HiveAuthorizationTaskFactory.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/authorization/HiveAuthorizationTaskFactory.java Tue Nov 18 00:48:40 2014
@@ -21,6 +21,8 @@ import java.io.Serializable;
 import java.util.HashSet;
 
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.classification.InterfaceAudience.LimitedPrivate;
+import org.apache.hadoop.hive.common.classification.InterfaceStability.Evolving;
 import org.apache.hadoop.hive.ql.exec.Task;
 import org.apache.hadoop.hive.ql.hooks.ReadEntity;
 import org.apache.hadoop.hive.ql.hooks.WriteEntity;
@@ -32,6 +34,8 @@ import org.apache.hadoop.hive.ql.parse.S
  * tasks. Every method in this class may return null, indicating no task
  * needs to be executed or can throw a SemanticException.
  */
+@LimitedPrivate(value = { "Apache Hive, Apache Sentry (incubating)" })
+@Evolving
 public interface HiveAuthorizationTaskFactory {
   public Task<? extends Serializable> createCreateRoleTask(ASTNode node, HashSet<ReadEntity> inputs,
       HashSet<WriteEntity> outputs) throws SemanticException;

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/authorization/HiveAuthorizationTaskFactoryImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/authorization/HiveAuthorizationTaskFactoryImpl.java?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/authorization/HiveAuthorizationTaskFactoryImpl.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/authorization/HiveAuthorizationTaskFactoryImpl.java Tue Nov 18 00:48:40 2014
@@ -238,7 +238,7 @@ public class HiveAuthorizationTaskFactor
     return subject;
   }
 
-  private PrivilegeObjectDesc parsePrivObject(ASTNode ast) throws SemanticException {
+  protected PrivilegeObjectDesc parsePrivObject(ASTNode ast) throws SemanticException {
     PrivilegeObjectDesc subject = new PrivilegeObjectDesc();
     ASTNode child = (ASTNode) ast.getChild(0);
     ASTNode gchild = (ASTNode)child.getChild(0);
@@ -246,6 +246,8 @@ public class HiveAuthorizationTaskFactor
       subject.setTable(true);
       String[] qualified = BaseSemanticAnalyzer.getQualifiedTableName(gchild);
       subject.setObject(BaseSemanticAnalyzer.getDotName(qualified));
+    } else if (child.getType() == HiveParser.TOK_URI_TYPE || child.getType() == HiveParser.TOK_SERVER_TYPE) {
+      throw new SemanticException("Hive authorization does not support the URI or SERVER objects");
     } else {
       subject.setTable(false);
       subject.setObject(BaseSemanticAnalyzer.unescapeIdentifier(gchild.getText()));

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/MergeJoinWork.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/MergeJoinWork.java?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/MergeJoinWork.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/MergeJoinWork.java Tue Nov 18 00:48:40 2014
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.apache.hadoop.hive.ql.plan;
 
 import java.util.ArrayList;

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsAggregator.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsAggregator.java?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsAggregator.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsAggregator.java Tue Nov 18 00:48:40 2014
@@ -134,6 +134,7 @@ public class JDBCStatsAggregator impleme
       }
     };
 
+    fileID = JDBCStatsUtils.truncateRowId(fileID);
     String keyPrefix = Utilities.escapeSqlLike(fileID) + "%";
     for (int failures = 0;; failures++) {
       try {
@@ -147,7 +148,7 @@ public class JDBCStatsAggregator impleme
         if (result.next()) {
           retval = result.getLong(1);
         } else {
-          LOG.warn("Warning. Nothing published. Nothing aggregated.");
+          LOG.warn("Nothing published. Nothing aggregated.");
           return null;
         }
         return Long.toString(retval);
@@ -217,6 +218,7 @@ public class JDBCStatsAggregator impleme
     };
     try {
 
+      rowID = JDBCStatsUtils.truncateRowId(rowID);
       String keyPrefix = Utilities.escapeSqlLike(rowID) + "%";
 
       PreparedStatement delStmt = Utilities.prepareWithRetry(conn,

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java Tue Nov 18 00:48:40 2014
@@ -139,7 +139,11 @@ public class JDBCStatsPublisher implemen
           + " stats: " + JDBCStatsUtils.getSupportedStatistics());
       return false;
     }
-    LOG.info("Stats publishing for key " + fileID);
+    String rowId = JDBCStatsUtils.truncateRowId(fileID);
+    if (LOG.isInfoEnabled()) {
+      String truncateSuffix = (rowId != fileID) ? " (from " + fileID + ")" : ""; // object equality
+      LOG.info("Stats publishing for key " + rowId + truncateSuffix);
+    }
 
     Utilities.SQLCommand<Void> execUpdate = new Utilities.SQLCommand<Void>() {
       @Override
@@ -153,7 +157,7 @@ public class JDBCStatsPublisher implemen
 
     for (int failures = 0;; failures++) {
       try {
-        insStmt.setString(1, fileID);
+        insStmt.setString(1, rowId);
         for (int i = 0; i < JDBCStatsUtils.getSupportedStatistics().size(); i++) {
           insStmt.setString(i + 2, stats.get(supportedStatistics.get(i)));
         }
@@ -172,10 +176,10 @@ public class JDBCStatsPublisher implemen
             for (i = 0; i < JDBCStatsUtils.getSupportedStatistics().size(); i++) {
               updStmt.setString(i + 1, stats.get(supportedStatistics.get(i)));
             }
-            updStmt.setString(supportedStatistics.size() + 1, fileID);
+            updStmt.setString(supportedStatistics.size() + 1, rowId);
             updStmt.setString(supportedStatistics.size() + 2,
                 stats.get(JDBCStatsUtils.getBasicStat()));
-            updStmt.setString(supportedStatistics.size() + 3, fileID);
+            updStmt.setString(supportedStatistics.size() + 3, rowId);
             Utilities.executeWithRetry(execUpdate, updStmt, waitWindow, maxRetries);
             return true;
           } catch (SQLRecoverableException ue) {

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsSetupConstants.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsSetupConstants.java?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsSetupConstants.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsSetupConstants.java Tue Nov 18 00:48:40 2014
@@ -34,4 +34,7 @@ public final class JDBCStatsSetupConstan
 
   public static final String PART_STAT_RAW_DATA_SIZE_COLUMN_NAME = "RAW_DATA_SIZE";
 
+  // 255 is an old value that we will keep for now; it can be increased to 4000; limits are
+  // MySQL - 65535, SQL Server - 8000, Oracle - 4000, Derby - 32762, Postgres - large.
+  public static final int ID_COLUMN_VARCHAR_SIZE = 255;
 }

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsUtils.java?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsUtils.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsUtils.java Tue Nov 18 00:48:40 2014
@@ -24,6 +24,7 @@ import java.util.List;
 import java.util.Map;
 
 import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.hadoop.util.hash.MurmurHash;
 
 public class JDBCStatsUtils {
 
@@ -124,9 +125,10 @@ public class JDBCStatsUtils {
    * Prepares CREATE TABLE query
    */
   public static String getCreate(String comment) {
-    String create = "CREATE TABLE /* " + comment + " */ " + JDBCStatsUtils.getStatTableName() +
-          " (" + getTimestampColumnName() + " TIMESTAMP DEFAULT CURRENT_TIMESTAMP, " +
-          JDBCStatsUtils.getIdColumnName() + " VARCHAR(255) PRIMARY KEY ";
+    String create = "CREATE TABLE /* " + comment + " */ " + JDBCStatsUtils.getStatTableName()
+        + " (" + getTimestampColumnName() + " TIMESTAMP DEFAULT CURRENT_TIMESTAMP, "
+        + JDBCStatsUtils.getIdColumnName() + " VARCHAR("
+        + JDBCStatsSetupConstants.ID_COLUMN_VARCHAR_SIZE + ") PRIMARY KEY ";
     for (int i = 0; i < supportedStats.size(); i++) {
       create += ", " + getStatColumnName(supportedStats.get(i)) + " BIGINT ";
     }
@@ -191,4 +193,13 @@ public class JDBCStatsUtils {
     return delete;
   }
 
+  /**
+   * Make sure the row ID fits into the row ID column in the table.
+   * @param rowId Row ID.
+   * @return Resulting row ID truncated to correct length, if necessary.
+   */
+  public static String truncateRowId(String rowId) {
+    return (rowId.length() <= JDBCStatsSetupConstants.ID_COLUMN_VARCHAR_SIZE)
+        ? rowId : Integer.toHexString(MurmurHash.getInstance().hash(rowId.getBytes()));
+  }
 }

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java Tue Nov 18 00:48:40 2014
@@ -69,7 +69,7 @@ public class Cleaner extends CompactorTh
       // This is solely for testing.  It checks if the test has set the looped value to false,
       // and if so remembers that and then sets it to true at the end.  We have to check here
       // first to make sure we go through a complete iteration of the loop before resetting it.
-      boolean setLooped = !looped.boolVal;
+      boolean setLooped = !looped.get();
       // Make sure nothing escapes this run method and kills the metastore at large,
       // so wrap it in a big catch Throwable statement.
       try {
@@ -137,16 +137,16 @@ public class Cleaner extends CompactorTh
 
         // Now, go back to bed until it's time to do this again
         long elapsedTime = System.currentTimeMillis() - startedAt;
-        if (elapsedTime >= cleanerCheckInterval || stop.boolVal)  continue;
+        if (elapsedTime >= cleanerCheckInterval || stop.get())  continue;
         else Thread.sleep(cleanerCheckInterval - elapsedTime);
       } catch (Throwable t) {
         LOG.error("Caught an exception in the main loop of compactor cleaner, " +
             StringUtils.stringifyException(t));
       }
       if (setLooped) {
-        looped.boolVal = true;
+        looped.set(true);
       }
-    } while (!stop.boolVal);
+    } while (!stop.get());
   }
 
   private Set<Long> findRelatedLocks(CompactionInfo ci, ShowLocksResponse locksResponse) {

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java Tue Nov 18 00:48:40 2014
@@ -40,6 +40,7 @@ import java.io.IOException;
 import java.security.PrivilegedExceptionAction;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
  * Superclass for all threads in the compactor.
@@ -52,8 +53,8 @@ abstract class CompactorThread extends T
   protected CompactionTxnHandler txnHandler;
   protected RawStore rs;
   protected int threadId;
-  protected BooleanPointer stop;
-  protected BooleanPointer looped;
+  protected AtomicBoolean stop;
+  protected AtomicBoolean looped;
 
   @Override
   public void setHiveConf(HiveConf conf) {
@@ -67,7 +68,7 @@ abstract class CompactorThread extends T
   }
 
   @Override
-  public void init(BooleanPointer stop, BooleanPointer looped) throws MetaException {
+  public void init(AtomicBoolean stop, AtomicBoolean looped) throws MetaException {
     this.stop = stop;
     this.looped = looped;
     setPriority(MIN_PRIORITY);

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java Tue Nov 18 00:48:40 2014
@@ -33,6 +33,7 @@ import org.apache.hadoop.hive.metastore.
 import org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement;
 import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
 import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
 import org.apache.hadoop.hive.metastore.txn.CompactionInfo;
 import org.apache.hadoop.hive.metastore.txn.TxnHandler;
 import org.apache.hadoop.hive.ql.io.AcidUtils;
@@ -44,6 +45,7 @@ import java.security.PrivilegedException
 import java.util.List;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
  * A class to initiate compactions.  This will run in a separate thread.
@@ -52,8 +54,6 @@ public class Initiator extends Compactor
   static final private String CLASS_NAME = Initiator.class.getName();
   static final private Log LOG = LogFactory.getLog(CLASS_NAME);
 
-  static final private String NO_COMPACTION = "NO_AUTO_COMPACTION";
-
   private long checkInterval;
 
   @Override
@@ -85,9 +85,8 @@ public class Initiator extends Compactor
             try {
               Table t = resolveTable(ci);
               // check if no compaction set for this table
-              if (t.getParameters().get(NO_COMPACTION) != null) {
-                LOG.info("Table " + tableName(t) + " marked " +  NO_COMPACTION +
-                    " so we will not compact it.");
+              if (noAutoCompactSet(t)) {
+                LOG.info("Table " + tableName(t) + " marked true so we will not compact it.");
                 continue;
               }
 
@@ -126,10 +125,10 @@ public class Initiator extends Compactor
         }
 
         long elapsedTime = System.currentTimeMillis() - startedAt;
-        if (elapsedTime >= checkInterval || stop.boolVal)  continue;
+        if (elapsedTime >= checkInterval || stop.get())  continue;
         else Thread.sleep(checkInterval - elapsedTime);
 
-      } while (!stop.boolVal);
+      } while (!stop.get());
     } catch (Throwable t) {
       LOG.error("Caught an exception in the main loop of compactor initiator, exiting " +
           StringUtils.stringifyException(t));
@@ -137,7 +136,7 @@ public class Initiator extends Compactor
   }
 
   @Override
-  public void init(BooleanPointer stop, BooleanPointer looped) throws MetaException {
+  public void init(AtomicBoolean stop, AtomicBoolean looped) throws MetaException {
     super.init(stop, looped);
     checkInterval =
         conf.getTimeVar(HiveConf.ConfVars.HIVE_COMPACTOR_CHECK_INTERVAL, TimeUnit.MILLISECONDS) ;
@@ -278,4 +277,16 @@ public class Initiator extends Compactor
     rqst.setRunas(runAs);
     txnHandler.compact(rqst);
   }
+
+  // Because TABLE_NO_AUTO_COMPACT was originally assumed to be NO_AUTO_COMPACT and then was moved
+  // to no_auto_compact, we need to check it in both cases.
+  private boolean noAutoCompactSet(Table t) {
+    String noAutoCompact =
+        t.getParameters().get(hive_metastoreConstants.TABLE_NO_AUTO_COMPACT);
+    if (noAutoCompact == null) {
+      noAutoCompact =
+          t.getParameters().get(hive_metastoreConstants.TABLE_NO_AUTO_COMPACT.toUpperCase());
+    }
+    return noAutoCompact != null && noAutoCompact.equalsIgnoreCase("true");
+  }
 }

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java Tue Nov 18 00:48:40 2014
@@ -42,6 +42,7 @@ import java.security.PrivilegedException
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
  * A class to do compactions.  This will run in a separate thread.  It will spin on the
@@ -77,7 +78,7 @@ public class Worker extends CompactorThr
       do {
         CompactionInfo ci = txnHandler.findNextToCompact(name);
 
-        if (ci == null && !stop.boolVal) {
+        if (ci == null && !stop.get()) {
           try {
             Thread.sleep(SLEEP_TIME);
             continue;
@@ -160,7 +161,7 @@ public class Worker extends CompactorThr
               ".  Marking clean to avoid repeated failures, " + StringUtils.stringifyException(e));
           txnHandler.markCleaned(ci);
         }
-      } while (!stop.boolVal);
+      } while (!stop.get());
     } catch (Throwable t) {
       LOG.error("Caught an exception in the main loop of compactor worker " + name +
           ", exiting " + StringUtils.stringifyException(t));
@@ -168,7 +169,7 @@ public class Worker extends CompactorThr
   }
 
   @Override
-  public void init(BooleanPointer stop, BooleanPointer looped) throws MetaException {
+  public void init(AtomicBoolean stop, AtomicBoolean looped) throws MetaException {
     super.init(stop, looped);
 
     StringBuilder name = new StringBuilder(hostname());

Modified: hive/branches/spark/ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto (original)
+++ hive/branches/spark/ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto Tue Nov 18 00:48:40 2014
@@ -191,8 +191,15 @@ message PostScript {
   optional uint64 footerLength = 1;
   optional CompressionKind compression = 2;
   optional uint64 compressionBlockSize = 3;
+  // the version of the file format
+  //   [0, 11] = Hive 0.11
+  //   [0, 12] = Hive 0.12
   repeated uint32 version = 4 [packed = true];
   optional uint64 metadataLength = 5;
+  // Version of the writer:
+  //   0 (or missing) = original
+  //   1 = HIVE-8732 fixed
+  optional uint32 writerVersion = 6;
   // Leave this last in the record
   optional string magic = 8000;
 }

Modified: hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/TestFileSinkOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/TestFileSinkOperator.java?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/TestFileSinkOperator.java (original)
+++ hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/TestFileSinkOperator.java Tue Nov 18 00:48:40 2014
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.apache.hadoop.hive.ql.exec;
 
 import org.apache.commons.logging.Log;

Modified: hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestDecimalUtil.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestDecimalUtil.java?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestDecimalUtil.java (original)
+++ hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestDecimalUtil.java Tue Nov 18 00:48:40 2014
@@ -38,8 +38,9 @@ public class TestDecimalUtil {
     DecimalUtil.floor(0, d1, dcv);
     Assert.assertEquals(0, expected1.compareTo(dcv.vector[0].getHiveDecimal()));
 
+    // As of HIVE-8745, these decimal values should be trimmed of trailing zeros.
     HiveDecimal d2 = HiveDecimal.create("23.00000");
-    Assert.assertEquals(5, d2.scale());
+    Assert.assertEquals(0, d2.scale());
     HiveDecimal expected2 = HiveDecimal.create("23");
     DecimalUtil.floor(0, d2, dcv);
     Assert.assertEquals(0, expected2.compareTo(dcv.vector[0].getHiveDecimal()));
@@ -50,19 +51,19 @@ public class TestDecimalUtil {
     Assert.assertEquals(0, expected3.compareTo(dcv.vector[0].getHiveDecimal()));
 
     HiveDecimal d4 = HiveDecimal.create("-17.00000");
-    Assert.assertEquals(5, d4.scale());
+    Assert.assertEquals(0, d4.scale());
     HiveDecimal expected4 = HiveDecimal.create("-17");
     DecimalUtil.floor(0, d4, dcv);
     Assert.assertEquals(0, expected4.compareTo(dcv.vector[0].getHiveDecimal()));
 
     HiveDecimal d5 = HiveDecimal.create("-0.30000");
-    Assert.assertEquals(5, d5.scale());
+    Assert.assertEquals(1, d5.scale());
     HiveDecimal expected5 = HiveDecimal.create("-1");
     DecimalUtil.floor(0, d5, dcv);
     Assert.assertEquals(0, expected5.compareTo(dcv.vector[0].getHiveDecimal()));
 
     HiveDecimal d6 = HiveDecimal.create("0.30000");
-    Assert.assertEquals(5, d6.scale());
+    Assert.assertEquals(1, d6.scale());
     HiveDecimal expected6 = HiveDecimal.create("0");
     DecimalUtil.floor(0, d6, dcv);
     Assert.assertEquals(0, expected6.compareTo(dcv.vector[0].getHiveDecimal()));
@@ -76,8 +77,9 @@ public class TestDecimalUtil {
     DecimalUtil.ceiling(0, d1, dcv);
     Assert.assertEquals(0, expected1.compareTo(dcv.vector[0].getHiveDecimal()));
 
+    // As of HIVE-8745, these decimal values should be trimmed of trailing zeros.
     HiveDecimal d2 = HiveDecimal.create("23.00000");
-    Assert.assertEquals(5, d2.scale());
+    Assert.assertEquals(0, d2.scale());
     HiveDecimal expected2 = HiveDecimal.create("23");
     DecimalUtil.ceiling(0, d2, dcv);
     Assert.assertEquals(0, expected2.compareTo(dcv.vector[0].getHiveDecimal()));
@@ -88,19 +90,19 @@ public class TestDecimalUtil {
     Assert.assertEquals(0, expected3.compareTo(dcv.vector[0].getHiveDecimal()));
 
     HiveDecimal d4 = HiveDecimal.create("-17.00000");
-    Assert.assertEquals(5, d4.scale());
+    Assert.assertEquals(0, d4.scale());
     HiveDecimal expected4 = HiveDecimal.create("-17");
     DecimalUtil.ceiling(0, d4, dcv);
     Assert.assertEquals(0, expected4.compareTo(dcv.vector[0].getHiveDecimal()));
 
     HiveDecimal d5 = HiveDecimal.create("-0.30000");
-    Assert.assertEquals(5, d5.scale());
+    Assert.assertEquals(1, d5.scale());
     HiveDecimal expected5 = HiveDecimal.create("0");
     DecimalUtil.ceiling(0, d5, dcv);
     Assert.assertEquals(0, expected5.compareTo(dcv.vector[0].getHiveDecimal()));
 
     HiveDecimal d6 = HiveDecimal.create("0.30000");
-    Assert.assertEquals(5, d6.scale());
+    Assert.assertEquals(1, d6.scale());
     HiveDecimal expected6 = HiveDecimal.create("1");
     DecimalUtil.ceiling(0, d6, dcv);
     Assert.assertEquals(0, expected6.compareTo(dcv.vector[0].getHiveDecimal()));
@@ -127,8 +129,9 @@ public class TestDecimalUtil {
     DecimalUtil.round(0, d1, dcv);
     Assert.assertEquals(0, expected1.compareTo(dcv.vector[0].getHiveDecimal()));
 
+    // As of HIVE-8745, these decimal values should be trimmed of trailing zeros.
     HiveDecimal d2 = HiveDecimal.create("23.00000");
-    Assert.assertEquals(5, d2.scale());
+    Assert.assertEquals(0, d2.scale());
     HiveDecimal expected2 = HiveDecimal.create("23");
     DecimalUtil.round(0, d2, dcv);
     Assert.assertEquals(0, expected2.compareTo(dcv.vector[0].getHiveDecimal()));
@@ -139,7 +142,7 @@ public class TestDecimalUtil {
     Assert.assertEquals(0, expected3.compareTo(dcv.vector[0].getHiveDecimal()));
 
     HiveDecimal d4 = HiveDecimal.create("-17.00000");
-    Assert.assertEquals(5, d4.scale());
+    Assert.assertEquals(0, d4.scale());
     HiveDecimal expected4 = HiveDecimal.create("-17");
     DecimalUtil.round(0, d4, dcv);
     Assert.assertEquals(0, expected4.compareTo(dcv.vector[0].getHiveDecimal()));
@@ -163,8 +166,9 @@ public class TestDecimalUtil {
     DecimalUtil.round(0, d1, dcv);
     Assert.assertEquals(0, expected1.compareTo(dcv.vector[0].getHiveDecimal()));
 
+    // As of HIVE-8745, these decimal values should be trimmed of trailing zeros.
     HiveDecimal d2 = HiveDecimal.create("23.56700");
-    Assert.assertEquals(5, d2.scale());
+    Assert.assertEquals(3, d2.scale());
     HiveDecimal expected2 = HiveDecimal.create("23.567");
     DecimalUtil.round(0, d2, dcv);
     Assert.assertEquals(0, expected2.compareTo(dcv.vector[0].getHiveDecimal()));
@@ -175,7 +179,7 @@ public class TestDecimalUtil {
     Assert.assertEquals(0, expected3.compareTo(dcv.vector[0].getHiveDecimal()));
 
     HiveDecimal d4 = HiveDecimal.create("-17.23400");
-    Assert.assertEquals(5, d4.scale());
+    Assert.assertEquals(3, d4.scale());
     HiveDecimal expected4 = HiveDecimal.create("-17.234");
     DecimalUtil.round(0, d4, dcv);
     Assert.assertEquals(0, expected4.compareTo(dcv.vector[0].getHiveDecimal()));
@@ -204,8 +208,9 @@ public class TestDecimalUtil {
     DecimalUtil.negate(0, d2, dcv);
     Assert.assertEquals(0, expected2.compareTo(dcv.vector[0].getHiveDecimal()));
 
+    // As of HIVE-8745, these decimal values should be trimmed of trailing zeros.
     HiveDecimal d3 = HiveDecimal.create("0.00000");
-    Assert.assertEquals(5, d3.scale());
+    Assert.assertEquals(0, d3.scale());
     HiveDecimal expected3 = HiveDecimal.create("0");
     DecimalUtil.negate(0, d3, dcv);
     Assert.assertEquals(0, expected3.compareTo(dcv.vector[0].getHiveDecimal()));
@@ -223,7 +228,7 @@ public class TestDecimalUtil {
     Assert.assertEquals(-1, lcv.vector[0]);
 
     HiveDecimal d3 = HiveDecimal.create("0.00000");
-    Assert.assertEquals(5, d3.scale());
+    Assert.assertEquals(0, d3.scale());
     d3.setScale(5);
     DecimalUtil.sign(0, d3, lcv);
     Assert.assertEquals(0, lcv.vector[0]);

Modified: hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTypeCasts.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTypeCasts.java?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTypeCasts.java (original)
+++ hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTypeCasts.java Tue Nov 18 00:48:40 2014
@@ -323,18 +323,19 @@ public class TestVectorTypeCasts {
     expr.evaluate(b);
     BytesColumnVector r = (BytesColumnVector) b.cols[1];
 
-    byte[] v = toBytes("1.10");
+    // As of HIVE-8745, these decimal values should be trimmed of trailing zeros.
+    byte[] v = toBytes("1.1");
     assertTrue(((Integer) v.length).toString() + " " + r.length[0], v.length == r.length[0]);
     Assert.assertEquals(0,
         StringExpr.compare(v, 0, v.length,
             r.vector[0], r.start[0], r.length[0]));
 
-    v = toBytes("-2.20");
+    v = toBytes("-2.2");
     Assert.assertEquals(0,
         StringExpr.compare(v, 0, v.length,
             r.vector[1], r.start[1], r.length[1]));
 
-    v = toBytes("9999999999999999.00");
+    v = toBytes("9999999999999999");
     Assert.assertEquals(0,
         StringExpr.compare(v, 0, v.length,
             r.vector[2], r.start[2], r.length[2]));

Modified: hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java (original)
+++ hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java Tue Nov 18 00:48:40 2014
@@ -21,27 +21,6 @@ import static org.junit.Assert.assertArr
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.sql.Date;
-import java.sql.Timestamp;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-import java.util.TimeZone;
-import java.util.TreeSet;
-
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.BlockLocation;
 import org.apache.hadoop.fs.FSDataInputStream;
@@ -66,9 +45,9 @@ import org.apache.hadoop.hive.ql.io.Comb
 import org.apache.hadoop.hive.ql.io.HiveInputFormat;
 import org.apache.hadoop.hive.ql.io.HiveOutputFormat;
 import org.apache.hadoop.hive.ql.io.InputFormatChecker;
-import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentFactory;
 import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf;
 import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
+import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentFactory;
 import org.apache.hadoop.hive.ql.plan.MapWork;
 import org.apache.hadoop.hive.ql.plan.PartitionDesc;
 import org.apache.hadoop.hive.ql.plan.TableDesc;
@@ -104,6 +83,27 @@ import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TestName;
 
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.sql.Date;
+import java.sql.Timestamp;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.TimeZone;
+import java.util.TreeSet;
+
 public class TestInputOutputFormat {
 
   Path workDir = new Path(System.getProperty("test.tmp.dir","target/tmp"));
@@ -1032,6 +1032,24 @@ public class TestInputOutputFormat {
     reader.close();
   }
 
+  static class SimpleRow implements Writable {
+    Text z;
+
+    public SimpleRow(Text t) {
+      this.z = t;
+    }
+
+    @Override
+    public void write(DataOutput dataOutput) throws IOException {
+      throw new UnsupportedOperationException("unsupported");
+    }
+
+    @Override
+    public void readFields(DataInput dataInput) throws IOException {
+      throw new UnsupportedOperationException("unsupported");
+    }
+  }
+
   static class NestedRow implements Writable {
     int z;
     MyRow r;
@@ -1620,14 +1638,14 @@ public class TestInputOutputFormat {
     assertEquals("mock:/combinationAcid/p=0/base_0000010/bucket_00000",
         split.getPath().toString());
     assertEquals(0, split.getStart());
-    assertEquals(580, split.getLength());
+    assertEquals(582, split.getLength());
     split = (HiveInputFormat.HiveInputSplit) splits[1];
     assertEquals("org.apache.hadoop.hive.ql.io.orc.OrcInputFormat",
         split.inputFormatClassName());
     assertEquals("mock:/combinationAcid/p=0/base_0000010/bucket_00001",
         split.getPath().toString());
     assertEquals(0, split.getStart());
-    assertEquals(601, split.getLength());
+    assertEquals(603, split.getLength());
     CombineHiveInputFormat.CombineHiveInputSplit combineSplit =
         (CombineHiveInputFormat.CombineHiveInputSplit) splits[2];
     assertEquals(BUCKETS, combineSplit.getNumPaths());
@@ -1635,7 +1653,7 @@ public class TestInputOutputFormat {
       assertEquals("mock:/combinationAcid/p=1/00000" + bucket + "_0",
           combineSplit.getPath(bucket).toString());
       assertEquals(0, combineSplit.getOffset(bucket));
-      assertEquals(225, combineSplit.getLength(bucket));
+      assertEquals(227, combineSplit.getLength(bucket));
     }
     String[] hosts = combineSplit.getLocations();
     assertEquals(2, hosts.length);
@@ -1685,4 +1703,89 @@ public class TestInputOutputFormat {
     assertEquals("cost", leaves.get(0).getColumnName());
     assertEquals(PredicateLeaf.Operator.IS_NULL, leaves.get(0).getOperator());
   }
+
+  @Test
+  @SuppressWarnings("unchecked,deprecation")
+  public void testSplitElimination() throws Exception {
+    Properties properties = new Properties();
+    StructObjectInspector inspector;
+    synchronized (TestOrcFile.class) {
+      inspector = (StructObjectInspector)
+          ObjectInspectorFactory.getReflectionObjectInspector(NestedRow.class,
+              ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
+    }
+    SerDe serde = new OrcSerde();
+    OutputFormat<?, ?> outFormat = new OrcOutputFormat();
+    conf.setInt("mapred.max.split.size", 50);
+    RecordWriter writer =
+        outFormat.getRecordWriter(fs, conf, testFilePath.toString(),
+            Reporter.NULL);
+    writer.write(NullWritable.get(),
+        serde.serialize(new NestedRow(1,2,3), inspector));
+    writer.write(NullWritable.get(),
+        serde.serialize(new NestedRow(4,5,6), inspector));
+    writer.write(NullWritable.get(),
+        serde.serialize(new NestedRow(7,8,9), inspector));
+    writer.close(Reporter.NULL);
+    serde = new OrcSerde();
+    SearchArgument sarg =
+        SearchArgumentFactory.newBuilder()
+            .startAnd()
+            .lessThan("z", new Integer(0))
+            .end()
+            .build();
+    conf.set("sarg.pushdown", sarg.toKryo());
+    conf.set("hive.io.file.readcolumn.names", "z,r");
+    properties.setProperty("columns", "z,r");
+    properties.setProperty("columns.types", "int:struct<x:int,y:int>");
+    SerDeUtils.initializeSerDe(serde, conf, properties, null);
+    inspector = (StructObjectInspector) serde.getObjectInspector();
+    InputFormat<?,?> in = new OrcInputFormat();
+    FileInputFormat.setInputPaths(conf, testFilePath.toString());
+    InputSplit[] splits = in.getSplits(conf, 1);
+    assertEquals(0, splits.length);
+  }
+
+  @Test
+  @SuppressWarnings("unchecked,deprecation")
+  public void testSplitEliminationNullStats() throws Exception {
+    Properties properties = new Properties();
+    StructObjectInspector inspector;
+    synchronized (TestOrcFile.class) {
+      inspector = (StructObjectInspector)
+          ObjectInspectorFactory.getReflectionObjectInspector(SimpleRow.class,
+              ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
+    }
+    SerDe serde = new OrcSerde();
+    OutputFormat<?, ?> outFormat = new OrcOutputFormat();
+    conf.setInt("mapred.max.split.size", 50);
+    RecordWriter writer =
+        outFormat.getRecordWriter(fs, conf, testFilePath.toString(),
+            Reporter.NULL);
+    writer.write(NullWritable.get(),
+        serde.serialize(new SimpleRow(null), inspector));
+    writer.write(NullWritable.get(),
+        serde.serialize(new SimpleRow(null), inspector));
+    writer.write(NullWritable.get(),
+        serde.serialize(new SimpleRow(null), inspector));
+    writer.close(Reporter.NULL);
+    serde = new OrcSerde();
+    SearchArgument sarg =
+        SearchArgumentFactory.newBuilder()
+            .startAnd()
+            .lessThan("z", new String("foo"))
+            .end()
+            .build();
+    conf.set("sarg.pushdown", sarg.toKryo());
+    conf.set("hive.io.file.readcolumn.names", "z");
+    properties.setProperty("columns", "z");
+    properties.setProperty("columns.types", "string");
+    SerDeUtils.initializeSerDe(serde, conf, properties, null);
+    inspector = (StructObjectInspector) serde.getObjectInspector();
+    InputFormat<?,?> in = new OrcInputFormat();
+    FileInputFormat.setInputPaths(conf, testFilePath.toString());
+    InputSplit[] splits = in.getSplits(conf, 1);
+    assertEquals(0, splits.length);
+  }
+
 }

Modified: hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/primitive/TestParquetByteInspector.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/primitive/TestParquetByteInspector.java?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/primitive/TestParquetByteInspector.java (original)
+++ hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/primitive/TestParquetByteInspector.java Tue Nov 18 00:48:40 2014
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.apache.hadoop.hive.ql.io.parquet.serde.primitive;
 
 import static org.junit.Assert.assertEquals;

Modified: hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/primitive/TestParquetShortInspector.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/primitive/TestParquetShortInspector.java?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/primitive/TestParquetShortInspector.java (original)
+++ hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/primitive/TestParquetShortInspector.java Tue Nov 18 00:48:40 2014
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.apache.hadoop.hive.ql.io.parquet.serde.primitive;
 
 import static org.junit.Assert.assertEquals;

Modified: hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java (original)
+++ hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java Tue Nov 18 00:48:40 2014
@@ -2830,7 +2830,7 @@ public class TestSearchArgumentImpl {
             .build();
     assertEquals("leaf-0 = (LESS_THAN x 1970-01-11)\n" +
         "leaf-1 = (LESS_THAN_EQUALS y hi)\n" +
-        "leaf-2 = (EQUALS z 1.0)\n" +
+        "leaf-2 = (EQUALS z 1)\n" +
         "expr = (and leaf-0 leaf-1 leaf-2)", sarg.toString());
 
     sarg = SearchArgumentFactory.newBuilder()

Modified: hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java (original)
+++ hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java Tue Nov 18 00:48:40 2014
@@ -32,6 +32,7 @@ import org.apache.hadoop.fs.FSDataInputS
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
 import org.apache.hadoop.hive.ql.Context;
 import org.apache.hadoop.hive.ql.QueryPlan;
 import org.apache.hadoop.hive.ql.exec.ExplainTask;
@@ -267,7 +268,7 @@ public class TestUpdateDeleteSemanticAna
     // I have to create the tables here (rather than in setup()) because I need the Hive
     // connection, which is conviently created by the semantic analyzer.
     Map<String, String> params = new HashMap<String, String>(1);
-    params.put(SemanticAnalyzer.ACID_TABLE_PROPERTY, "true");
+    params.put(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL, "true");
     db.createTable("T", Arrays.asList("a", "b"), null, OrcInputFormat.class,
         OrcOutputFormat.class, 2, Arrays.asList("a"), params);
     db.createTable("U", Arrays.asList("a", "b"), Arrays.asList("ds"), OrcInputFormat.class,

Modified: hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/parse/authorization/TestHiveAuthorizationTaskFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/parse/authorization/TestHiveAuthorizationTaskFactory.java?rev=1640263&r1=1640262&r2=1640263&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/parse/authorization/TestHiveAuthorizationTaskFactory.java (original)
+++ hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/parse/authorization/TestHiveAuthorizationTaskFactory.java Tue Nov 18 00:48:40 2014
@@ -22,18 +22,23 @@ import java.util.HashMap;
 import junit.framework.Assert;
 
 import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.metastore.api.PrincipalType;
 import org.apache.hadoop.hive.ql.Context;
 import org.apache.hadoop.hive.ql.metadata.Hive;
 import org.apache.hadoop.hive.ql.metadata.Partition;
 import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.parse.ASTNode;
 import org.apache.hadoop.hive.ql.parse.DDLSemanticAnalyzer;
+import org.apache.hadoop.hive.ql.parse.HiveParser;
 import org.apache.hadoop.hive.ql.parse.ParseDriver;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
 import org.apache.hadoop.hive.ql.plan.DDLWork;
 import org.apache.hadoop.hive.ql.plan.GrantDesc;
 import org.apache.hadoop.hive.ql.plan.GrantRevokeRoleDDL;
 import org.apache.hadoop.hive.ql.plan.PrincipalDesc;
 import org.apache.hadoop.hive.ql.plan.PrivilegeDesc;
+import org.apache.hadoop.hive.ql.plan.PrivilegeObjectDesc;
 import org.apache.hadoop.hive.ql.plan.RevokeDesc;
 import org.apache.hadoop.hive.ql.plan.RoleDDLDesc;
 import org.apache.hadoop.hive.ql.plan.RoleDDLDesc.RoleOperation;
@@ -47,6 +52,33 @@ import org.mockito.Mockito;
 
 public class TestHiveAuthorizationTaskFactory {
 
+  public static class DummyHiveAuthorizationTaskFactoryImpl extends HiveAuthorizationTaskFactoryImpl {
+
+    static String uriPath = "";
+    static String serverName = "";
+
+    public DummyHiveAuthorizationTaskFactoryImpl(HiveConf conf, Hive db) {
+      super(conf, db);
+    }
+
+    @Override
+    protected PrivilegeObjectDesc parsePrivObject(ASTNode ast) throws SemanticException {
+      ASTNode child = (ASTNode) ast.getChild(0);
+      ASTNode gchild = (ASTNode)child.getChild(0);
+      if (child.getType() == HiveParser.TOK_URI_TYPE) {
+        uriPath = gchild.getText().replaceAll("'", "").replaceAll("\"", "");
+      } else if (child.getType() == HiveParser.TOK_SERVER_TYPE) {
+        serverName = gchild.getText();
+      }
+      return super.parsePrivObject(ast);
+    }
+
+    public static void reset() {
+      uriPath = "";
+      serverName = "";
+    }
+  }
+
   private static final String SELECT = "SELECT";
   private static final String DB = "default";
   private static final String TABLE = "table1";
@@ -67,6 +99,8 @@ public class TestHiveAuthorizationTaskFa
   @Before
   public void setup() throws Exception {
     conf = new HiveConf();
+    conf.setVar(ConfVars.HIVE_AUTHORIZATION_TASK_FACTORY,
+        TestHiveAuthorizationTaskFactory.DummyHiveAuthorizationTaskFactoryImpl.class.getName());
     db = Mockito.mock(Hive.class);
     table = new Table(DB, TABLE);
     partition = new Partition(table);
@@ -81,6 +115,7 @@ public class TestHiveAuthorizationTaskFa
     HadoopDefaultAuthenticator auth = new HadoopDefaultAuthenticator();
     auth.setConf(conf);
     currentUser = auth.getUserName();
+    DummyHiveAuthorizationTaskFactoryImpl.reset();
 
   }
   /**
@@ -414,6 +449,34 @@ public class TestHiveAuthorizationTaskFa
     Assert.assertTrue("Expected table", grantDesc.getHiveObj().getTable());
   }
 
+  /**
+   * GRANT ALL ON URI
+   */
+  @Test
+  public void testGrantUri() throws Exception {
+    String uriPath = "/tmp";
+    try {
+      analyze("GRANT ALL ON URI '" + uriPath  + "' TO USER user2");
+      Assert.fail("Grant on URI should fail");
+    } catch (SemanticException e) {
+      Assert.assertEquals(uriPath, DummyHiveAuthorizationTaskFactoryImpl.uriPath);
+    }
+  }
+
+  /**
+   * GRANT ALL ON SERVER
+   */
+  @Test
+  public void testGrantServer() throws Exception {
+    String serverName = "foo";
+    try {
+      analyze("GRANT ALL ON SERVER " + serverName + " TO USER user2");
+      Assert.fail("Grant on Server should fail");
+    } catch (SemanticException e) {
+      Assert.assertEquals(serverName, DummyHiveAuthorizationTaskFactoryImpl.serverName);
+    }
+  }
+
   private DDLWork analyze(String command) throws Exception {
     return AuthorizationTestUtil.analyze(command, conf, db);
   }



Mime
View raw message