hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gunt...@apache.org
Subject svn commit: r1617652 [2/7] - in /hive/branches/cbo: ./ ant/ ant/src/org/apache/hadoop/hive/ant/ beeline/ beeline/src/java/org/apache/hive/beeline/ beeline/src/main/resources/ cli/src/java/org/apache/hadoop/hive/cli/ common/src/java/org/apache/hadoop/hi...
Date Wed, 13 Aug 2014 02:28:58 GMT
Modified: hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java?rev=1617652&r1=1617651&r2=1617652&view=diff
==============================================================================
--- hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java (original)
+++ hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java Wed Aug 13 02:28:54 2014
@@ -29,6 +29,8 @@ import org.apache.commons.logging.LogFac
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.FileUtils;
+import org.apache.hadoop.hive.common.ObjectPair;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
 import org.apache.hadoop.hive.metastore.api.Database;
@@ -80,11 +82,11 @@ public class HiveAlterHandler implements
     FileSystem destFs = null;
 
     boolean success = false;
-    String oldTblLoc = null;
-    String newTblLoc = null;
     boolean moveData = false;
     boolean rename = false;
     Table oldt = null;
+    List<ObjectPair<Partition, String>> altps = new ArrayList<ObjectPair<Partition, String>>();
+
     try {
       msdb.openTransaction();
       name = name.toLowerCase();
@@ -131,32 +133,32 @@ public class HiveAlterHandler implements
 
       // if this alter is a rename, the table is not a virtual view, the user
       // didn't change the default location (or new location is empty), and
-      // table is not an external table, that means useris asking metastore to
+      // table is not an external table, that means user is asking metastore to
       // move data to the new location corresponding to the new name
       if (rename
           && !oldt.getTableType().equals(TableType.VIRTUAL_VIEW.toString())
           && (oldt.getSd().getLocation().compareTo(newt.getSd().getLocation()) == 0
             || StringUtils.isEmpty(newt.getSd().getLocation()))
           && !MetaStoreUtils.isExternalTable(oldt)) {
+
+        srcPath = new Path(oldt.getSd().getLocation());
+        srcFs = wh.getFs(srcPath);
+
         // that means user is asking metastore to move data to new location
         // corresponding to the new name
         // get new location
-        newTblLoc = wh.getTablePath(msdb.getDatabase(newt.getDbName()),
-            newt.getTableName()).toString();
-        Path newTblPath = constructRenamedPath(new Path(newTblLoc),
-            new Path(newt.getSd().getLocation()));
-        newTblLoc = newTblPath.toString();
-        newt.getSd().setLocation(newTblLoc);
-        oldTblLoc = oldt.getSd().getLocation();
+        Path databasePath = constructRenamedPath(
+            wh.getDefaultDatabasePath(newt.getDbName()), srcPath);
+        destPath = new Path(databasePath, newt.getTableName());
+        destFs = wh.getFs(destPath);
+
+        newt.getSd().setLocation(destPath.toString());
         moveData = true;
+
         // check that destination does not exist otherwise we will be
         // overwriting data
-        srcPath = new Path(oldTblLoc);
-        srcFs = wh.getFs(srcPath);
-        destPath = new Path(newTblLoc);
-        destFs = wh.getFs(destPath);
         // check that src and dest are on the same file system
-        if (! equalsFileSystem(srcFs, destFs)) {
+        if (!FileUtils.equalsFileSystem(srcFs, destFs)) {
           throw new InvalidOperationException("table new location " + destPath
               + " is on a different file system than the old location "
               + srcPath + ". This operation is not supported");
@@ -176,22 +178,18 @@ public class HiveAlterHandler implements
               + destPath + " for table " + newt.getDbName() + "."
               + newt.getTableName());
         }
+        String oldTblLocPath = srcPath.toUri().getPath();
+        String newTblLocPath = destPath.toUri().getPath();
+
         // also the location field in partition
         List<Partition> parts = msdb.getPartitions(dbname, name, -1);
         for (Partition part : parts) {
           String oldPartLoc = part.getSd().getLocation();
-          Path oldPartLocPath = new Path(oldPartLoc);
-          String oldTblLocPath = new Path(oldTblLoc).toUri().getPath();
-          String newTblLocPath = new Path(newTblLoc).toUri().getPath();
           if (oldPartLoc.contains(oldTblLocPath)) {
-            Path newPartLocPath = null;
-            URI oldUri = oldPartLocPath.toUri();
-            String newPath = oldUri.getPath().replace(oldTblLocPath,
-                                                      newTblLocPath);
-
-            newPartLocPath = new Path(oldUri.getScheme(),
-                                      oldUri.getAuthority(),
-                                      newPath);
+            URI oldUri = new Path(oldPartLoc).toUri();
+            String newPath = oldUri.getPath().replace(oldTblLocPath, newTblLocPath);
+            Path newPartLocPath = new Path(oldUri.getScheme(), oldUri.getAuthority(), newPath);
+            altps.add(ObjectPair.create(part, part.getSd().getLocation()));
             part.getSd().setLocation(newPartLocPath.toString());
             msdb.alterPartition(dbname, name, part.getValues(), part);
           }
@@ -234,9 +232,23 @@ public class HiveAlterHandler implements
           try {
             msdb.openTransaction();
             msdb.alterTable(dbname, newt.getTableName(), oldt);
+            for (ObjectPair<Partition, String> pair : altps) {
+              Partition part = pair.getFirst();
+              part.getSd().setLocation(pair.getSecond());
+              msdb.alterPartition(dbname, name, part.getValues(), part);
+            }
             revertMetaDataTransaction = msdb.commitTransaction();
           } catch (Exception e1) {
-            LOG.error("Reverting metadata opeation failed During HDFS operation failed", e1);
+            // we should log this for manual rollback by administrator
+            LOG.error("Reverting metadata by HDFS operation failure failed During HDFS operation failed", e1);
+            LOG.error("Table " + Warehouse.getQualifiedName(newt) +
+                " should be renamed to " + Warehouse.getQualifiedName(oldt));
+            LOG.error("Table " + Warehouse.getQualifiedName(newt) +
+                " should have path " + srcPath);
+            for (ObjectPair<Partition, String> pair : altps) {
+              LOG.error("Partition " + Warehouse.getQualifiedName(pair.getFirst()) +
+                  " should have path " + pair.getSecond());
+            }
             if (!revertMetaDataTransaction) {
               msdb.rollbackTransaction();
             }
@@ -251,21 +263,6 @@ public class HiveAlterHandler implements
     }
   }
 
-  /**
-   * @param fs1
-   * @param fs2
-   * @return return true if both file system arguments point to same file system
-   */
-  private boolean equalsFileSystem(FileSystem fs1, FileSystem fs2) {
-    //When file system cache is disabled, you get different FileSystem objects
-    // for same file system, so '==' can't be used in such cases
-    //FileSystem api doesn't have a .equals() function implemented, so using
-    //the uri for comparison. FileSystem already uses uri+Configuration for
-    //equality in its CACHE .
-    //Once equality has been added in HDFS-4321, we should make use of it
-    return fs1.getUri().equals(fs2.getUri());
-  }
-
   public Partition alterPartition(final RawStore msdb, Warehouse wh, final String dbname,
       final String name, final List<String> part_vals, final Partition new_part)
       throws InvalidOperationException, InvalidObjectException, AlreadyExistsException,

Modified: hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java?rev=1617652&r1=1617651&r2=1617652&view=diff
==============================================================================
--- hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (original)
+++ hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java Wed Aug 13 02:28:54 2014
@@ -424,7 +424,7 @@ public class HiveMetaStore extends Thrif
 
       String partitionValidationRegex =
           hiveConf.getVar(HiveConf.ConfVars.METASTORE_PARTITION_NAME_WHITELIST_PATTERN);
-      if (partitionValidationRegex != null && partitionValidationRegex != "") {
+      if (partitionValidationRegex != null && !partitionValidationRegex.isEmpty()) {
         partitionValidationPattern = Pattern.compile(partitionValidationRegex);
       } else {
         partitionValidationPattern = null;
@@ -3269,6 +3269,9 @@ public class HiveMetaStore extends Thrif
 
       boolean success = false, indexTableCreated = false;
 
+      String[] qualified =
+          MetaStoreUtils.getQualifiedName(index.getDbName(), index.getIndexTableName());
+
       try {
         ms.openTransaction();
         Index old_index = null;
@@ -3291,7 +3294,7 @@ public class HiveMetaStore extends Thrif
         Table indexTbl = indexTable;
         if (indexTbl != null) {
           try {
-            indexTbl = ms.getTable(index.getDbName(), index.getIndexTableName());
+            indexTbl = ms.getTable(qualified[0], qualified[1]);
           } catch (Exception e) {
           }
           if (indexTbl != null) {
@@ -3312,7 +3315,7 @@ public class HiveMetaStore extends Thrif
         if (!success) {
           if (indexTableCreated) {
             try {
-              this.drop_table(index.getDbName(), index.getIndexTableName(), false);
+              drop_table(qualified[0], qualified[1], false);
             } catch (Exception e) {
             }
           }
@@ -3366,8 +3369,8 @@ public class HiveMetaStore extends Thrif
 
         String idxTblName = index.getIndexTableName();
         if (idxTblName != null) {
-          Table tbl = null;
-          tbl = this.get_table(dbName, idxTblName);
+          String[] qualified = MetaStoreUtils.getQualifiedName(index.getDbName(), idxTblName);
+          Table tbl = get_table(qualified[0], qualified[1]);
           if (tbl.getSd() == null) {
             throw new MetaException("Table metadata is corrupted");
           }

Modified: hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java?rev=1617652&r1=1617651&r2=1617652&view=diff
==============================================================================
--- hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java (original)
+++ hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java Wed Aug 13 02:28:54 2014
@@ -1538,4 +1538,12 @@ public class MetaStoreUtils {
       return part.getValues().size();
     }
   }
+
+  public static String[] getQualifiedName(String defaultDbName, String tableName) {
+    String[] names = tableName.split("\\.");
+    if (names.length == 1) {
+      return new String[] { defaultDbName, tableName};
+    }
+    return new String[] {names[0], names[1]};
+  }
 }

Modified: hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java?rev=1617652&r1=1617651&r2=1617652&view=diff
==============================================================================
--- hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (original)
+++ hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java Wed Aug 13 02:28:54 2014
@@ -2847,7 +2847,8 @@ public class ObjectStore implements RawS
           "Original table does not exist for the given index.");
     }
 
-    MTable indexTable = getMTable(index.getDbName(), index.getIndexTableName());
+    String[] qualified = MetaStoreUtils.getQualifiedName(index.getDbName(), index.getIndexTableName());
+    MTable indexTable = getMTable(qualified[0], qualified[1]);
     if (indexTable == null) {
       throw new InvalidObjectException(
           "Underlying index table does not exist for the given index.");

Modified: hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java?rev=1617652&r1=1617651&r2=1617652&view=diff
==============================================================================
--- hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java (original)
+++ hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java Wed Aug 13 02:28:54 2014
@@ -51,6 +51,7 @@ import org.apache.hadoop.hive.conf.HiveC
 import org.apache.hadoop.hive.metastore.api.Database;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.Partition;
 import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
 import org.apache.hadoop.hive.metastore.api.Table;
 import org.apache.hadoop.hive.shims.ShimLoader;
@@ -186,6 +187,14 @@ public class Warehouse {
     return getDnsPath(new Path(getDatabasePath(db), tableName.toLowerCase()));
   }
 
+  public static String getQualifiedName(Table table) {
+    return table.getDbName() + "." + table.getTableName();
+  }
+
+  public static String getQualifiedName(Partition partition) {
+    return partition.getDbName() + "." + partition.getTableName() + partition.getValues();
+  }
+
   public boolean mkdirs(Path f, boolean inheritPermCandidate) throws MetaException {
     boolean inheritPerms = HiveConf.getBoolVar(conf,
       HiveConf.ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS) && inheritPermCandidate;

Modified: hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java?rev=1617652&r1=1617651&r2=1617652&view=diff
==============================================================================
--- hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java (original)
+++ hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java Wed Aug 13 02:28:54 2014
@@ -18,6 +18,7 @@
 package org.apache.hadoop.hive.metastore.txn;
 
 import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.shims.ShimLoader;
 
 import java.sql.Connection;
 import java.sql.Driver;
@@ -201,7 +202,8 @@ public class TxnDbUtil {
     Properties prop = new Properties();
     String driverUrl = HiveConf.getVar(conf, HiveConf.ConfVars.METASTORECONNECTURLKEY);
     String user = HiveConf.getVar(conf, HiveConf.ConfVars.METASTORE_CONNECTION_USER_NAME);
-    String passwd = HiveConf.getVar(conf, HiveConf.ConfVars.METASTOREPWD);
+    String passwd = ShimLoader.getHadoopShims().getPassword(conf,
+        HiveConf.ConfVars.METASTOREPWD.varname);
     prop.put("user", user);
     prop.put("password", passwd);
     return driver.connect(driverUrl, prop);

Modified: hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java?rev=1617652&r1=1617651&r2=1617652&view=diff
==============================================================================
--- hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java (original)
+++ hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java Wed Aug 13 02:28:54 2014
@@ -32,9 +32,12 @@ import org.apache.hadoop.hive.common.Val
 import org.apache.hadoop.hive.common.ValidTxnListImpl;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.api.*;
+import org.apache.hadoop.hive.shims.ShimLoader;
 import org.apache.hadoop.util.StringUtils;
 
 import javax.sql.DataSource;
+
+import java.io.IOException;
 import java.sql.*;
 import java.util.*;
 
@@ -1602,7 +1605,13 @@ public class TxnHandler {
 
     String driverUrl = HiveConf.getVar(conf, HiveConf.ConfVars.METASTORECONNECTURLKEY);
     String user = HiveConf.getVar(conf, HiveConf.ConfVars.METASTORE_CONNECTION_USER_NAME);
-    String passwd = HiveConf.getVar(conf, HiveConf.ConfVars.METASTOREPWD);
+    String passwd;
+    try {
+      passwd = ShimLoader.getHadoopShims().getPassword(conf,
+          HiveConf.ConfVars.METASTOREPWD.varname);
+    } catch (IOException err) {
+      throw new SQLException("Error getting metastore password", err);
+    }
     String connectionPooler = HiveConf.getVar(conf,
         HiveConf.ConfVars.METASTORE_CONNECTION_POOLING_TYPE).toLowerCase();
 

Modified: hive/branches/cbo/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/cbo/pom.xml?rev=1617652&r1=1617651&r2=1617652&view=diff
==============================================================================
--- hive/branches/cbo/pom.xml (original)
+++ hive/branches/cbo/pom.xml Wed Aug 13 02:28:54 2014
@@ -146,6 +146,7 @@
     <stax.version>1.0.1</stax.version>
     <slf4j.version>1.7.5</slf4j.version>
     <ST4.version>4.0.4</ST4.version>
+    <super-csv.version>2.2.0</super-csv.version>
     <tez.version>0.4.0-incubating</tez.version>
     <tempus-fugit.version>1.1</tempus-fugit.version>
     <snappy.version>0.2</snappy.version>

Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/Driver.java?rev=1617652&r1=1617651&r2=1617652&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/Driver.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/Driver.java Wed Aug 13 02:28:54 2014
@@ -507,7 +507,7 @@ public class Driver implements CommandPr
       // get mapping of tables to columns used
       ColumnAccessInfo colAccessInfo = sem.getColumnAccessInfo();
       // colAccessInfo is set only in case of SemanticAnalyzer
-      Map<String, Set<String>> tab2Cols = colAccessInfo != null ? colAccessInfo
+      Map<String, List<String>> tab2Cols = colAccessInfo != null ? colAccessInfo
           .getTableToColumnAccessMap() : null;
       doAuthorizationV2(ss, op, inputs, outputs, command, tab2Cols);
      return;
@@ -608,7 +608,7 @@ public class Driver implements CommandPr
           Partition partition = read.getPartition();
           tbl = partition.getTable();
           // use partition level authorization
-          if (tableUsePartLevelAuth.get(tbl.getTableName()) == Boolean.TRUE) {
+          if (Boolean.TRUE.equals(tableUsePartLevelAuth.get(tbl.getTableName()))) {
             List<String> cols = part2Cols.get(partition);
             if (cols != null && cols.size() > 0) {
               authorizer.authorize(partition.getTable(),
@@ -626,7 +626,7 @@ public class Driver implements CommandPr
         // check, and the table authorization may already happened because of other
         // partitions
         if (tbl != null && !tableAuthChecked.contains(tbl.getTableName()) &&
-            !(tableUsePartLevelAuth.get(tbl.getTableName()) == Boolean.TRUE)) {
+            !(Boolean.TRUE.equals(tableUsePartLevelAuth.get(tbl.getTableName())))) {
           List<String> cols = tab2Cols.get(tbl);
           if (cols != null && cols.size() > 0) {
             authorizer.authorize(tbl, null, cols,
@@ -671,7 +671,7 @@ public class Driver implements CommandPr
           //or non-existent tho such sources may still be referenced by the TableScanOperator
           //if it's null then the partition probably doesn't exist so let's use table permission
           if (tbl.isPartitioned() &&
-              tableUsePartLevelAuth.get(tbl.getTableName()) == Boolean.TRUE) {
+              Boolean.TRUE.equals(tableUsePartLevelAuth.get(tbl.getTableName()))) {
             String alias_id = topOpMap.getKey();
 
             PrunedPartitionList partsList = PartitionPruner.prune(tableScanOp,
@@ -700,7 +700,7 @@ public class Driver implements CommandPr
   }
 
   private static void doAuthorizationV2(SessionState ss, HiveOperation op, HashSet<ReadEntity> inputs,
-      HashSet<WriteEntity> outputs, String command, Map<String, Set<String>> tab2cols) throws HiveException {
+      HashSet<WriteEntity> outputs, String command, Map<String, List<String>> tab2cols) throws HiveException {
 
     HiveAuthzContext.Builder authzContextBuilder = new HiveAuthzContext.Builder();
 
@@ -711,36 +711,14 @@ public class Driver implements CommandPr
     authzContextBuilder.setCommandString(command);
 
     HiveOperationType hiveOpType = getHiveOperationType(op);
-    List<HivePrivilegeObject> inputsHObjs = getHivePrivObjects(inputs);
-    updateInputColumnInfo(inputsHObjs, tab2cols);
+    List<HivePrivilegeObject> inputsHObjs = getHivePrivObjects(inputs, tab2cols);
+    List<HivePrivilegeObject> outputHObjs = getHivePrivObjects(outputs, null);
 
-    List<HivePrivilegeObject> outputHObjs = getHivePrivObjects(outputs);
     ss.getAuthorizerV2().checkPrivileges(hiveOpType, inputsHObjs, outputHObjs, authzContextBuilder.build());
-    return;
   }
 
-  /**
-   * Add column information for input table objects
-   * @param inputsHObjs input HivePrivilegeObject
-   * @param map table to used input columns mapping
-   */
-  private static void updateInputColumnInfo(List<HivePrivilegeObject> inputsHObjs,
-      Map<String, Set<String>> tableName2Cols) {
-    if(tableName2Cols == null) {
-      return;
-    }
-    for(HivePrivilegeObject inputObj : inputsHObjs){
-      if(inputObj.getType() != HivePrivilegeObjectType.TABLE_OR_VIEW){
-        // input columns are relevant only for tables or views
-        continue;
-      }
-      Set<String> cols = tableName2Cols.get(Table.getCompleteName(inputObj.getDbname(),
-          inputObj.getObjectName()));
-      inputObj.setColumns(cols);
-    }
-  }
-
-  private static List<HivePrivilegeObject> getHivePrivObjects(HashSet<? extends Entity> privObjects) {
+  private static List<HivePrivilegeObject> getHivePrivObjects(
+      HashSet<? extends Entity> privObjects, Map<String, List<String>> tableName2Cols) {
     List<HivePrivilegeObject> hivePrivobjs = new ArrayList<HivePrivilegeObject>();
     if(privObjects == null){
       return hivePrivobjs;
@@ -764,13 +742,17 @@ public class Driver implements CommandPr
       //support for authorization on partitions needs to be added
       String dbname = null;
       String objName = null;
+      List<String> partKeys = null;
+      List<String> columns = null;
       switch(privObject.getType()){
       case DATABASE:
-        dbname = privObject.getDatabase() == null ? null : privObject.getDatabase().getName();
+        dbname = privObject.getDatabase().getName();
         break;
       case TABLE:
-        dbname = privObject.getTable() == null ? null : privObject.getTable().getDbName();
-        objName = privObject.getTable() == null ? null : privObject.getTable().getTableName();
+        dbname = privObject.getTable().getDbName();
+        objName = privObject.getTable().getTableName();
+        columns = tableName2Cols == null ? null :
+            tableName2Cols.get(Table.getCompleteName(dbname, objName));
         break;
       case DFS_DIR:
       case LOCAL_DIR:
@@ -788,7 +770,7 @@ public class Driver implements CommandPr
       }
       HivePrivObjectActionType actionType = AuthorizationUtils.getActionType(privObject);
       HivePrivilegeObject hPrivObject = new HivePrivilegeObject(privObjType, dbname, objName,
-          actionType);
+          partKeys, columns, actionType, null);
       hivePrivobjs.add(hPrivObject);
     }
     return hivePrivobjs;

Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java?rev=1617652&r1=1617651&r2=1617652&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java Wed Aug 13 02:28:54 2014
@@ -35,7 +35,6 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -600,13 +599,9 @@ public class DDLTask extends Task<DDLWor
 
     HiveAuthorizer authorizer = getSessionAuthorizer();
     try {
-      Set<String> colSet = showGrantDesc.getColumns() != null ? new HashSet<String>(
-          showGrantDesc.getColumns()) : null;
       List<HivePrivilegeInfo> privInfos = authorizer.showPrivileges(
           AuthorizationUtils.getHivePrincipal(showGrantDesc.getPrincipalDesc()),
-          AuthorizationUtils.getHivePrivilegeObject(showGrantDesc.getHiveObj(),
-              colSet
-              ));
+          AuthorizationUtils.getHivePrivilegeObject(showGrantDesc.getHiveObj()));
       boolean testMode = conf.getBoolVar(HiveConf.ConfVars.HIVE_IN_TEST);
       writeToFile(writeGrantInfo(privInfos, testMode), showGrantDesc.getResFile());
     } catch (IOException e) {
@@ -625,7 +620,7 @@ public class DDLTask extends Task<DDLWor
     //Convert to object types used by the authorization plugin interface
     List<HivePrincipal> hivePrincipals = AuthorizationUtils.getHivePrincipals(principals);
     List<HivePrivilege> hivePrivileges = AuthorizationUtils.getHivePrivileges(privileges);
-    HivePrivilegeObject hivePrivObject = AuthorizationUtils.getHivePrivilegeObject(privSubjectDesc, null);
+    HivePrivilegeObject hivePrivObject = AuthorizationUtils.getHivePrivilegeObject(privSubjectDesc);
 
     HivePrincipal grantorPrincipal = new HivePrincipal(
         grantor, AuthorizationUtils.getHivePrincipalType(grantorType));
@@ -754,8 +749,7 @@ public class DDLTask extends Task<DDLWor
   }
 
   private int dropIndex(Hive db, DropIndexDesc dropIdx) throws HiveException {
-    db.dropIndex(SessionState.get().getCurrentDatabase(), dropIdx.getTableName(),
-        dropIdx.getIndexName(), true);
+    db.dropIndex(dropIdx.getTableName(), dropIdx.getIndexName(), true);
     return 0;
   }
 
@@ -765,11 +759,7 @@ public class DDLTask extends Task<DDLWor
       validateSerDe(crtIndex.getSerde());
     }
 
-    String indexTableName =
-      crtIndex.getIndexTableName() != null ? crtIndex.getIndexTableName() :
-        MetaStoreUtils.getIndexTableName(SessionState.get().getCurrentDatabase(),
-             crtIndex.getTableName(), crtIndex.getIndexName());
-
+    String indexTableName = crtIndex.getIndexTableName();
     if (!Utilities.isDefaultNameNode(conf)) {
       // If location is specified - ensure that it is a full qualified name
       makeLocationQualified(crtIndex, indexTableName);
@@ -792,10 +782,9 @@ public class DDLTask extends Task<DDLWor
   }
 
   private int alterIndex(Hive db, AlterIndexDesc alterIndex) throws HiveException {
-    String dbName = alterIndex.getDbName();
     String baseTableName = alterIndex.getBaseTableName();
     String indexName = alterIndex.getIndexName();
-    Index idx = db.getIndex(dbName, baseTableName, indexName);
+    Index idx = db.getIndex(baseTableName, indexName);
 
     switch(alterIndex.getOp()) {
     case ADDPROPS:
@@ -806,8 +795,7 @@ public class DDLTask extends Task<DDLWor
         Map<String, String> props = new HashMap<String, String>();
         Map<Map<String, String>, Long> basePartTs = new HashMap<Map<String, String>, Long>();
 
-        Table baseTbl = db.getTable(SessionState.get().getCurrentDatabase(),
-            baseTableName);
+        Table baseTbl = db.getTable(baseTableName);
 
         if (baseTbl.isPartitioned()) {
           List<Partition> baseParts;
@@ -854,7 +842,7 @@ public class DDLTask extends Task<DDLWor
     }
 
     try {
-      db.alterIndex(dbName, baseTableName, indexName, idx);
+      db.alterIndex(baseTableName, indexName, idx);
     } catch (InvalidOperationException e) {
       console.printError("Invalid alter operation: " + e.getMessage());
       LOG.info("alter index: " + stringifyException(e));
@@ -896,7 +884,7 @@ public class DDLTask extends Task<DDLWor
    */
   private int renamePartition(Hive db, RenamePartitionDesc renamePartitionDesc) throws HiveException {
 
-    Table tbl = db.getTable(renamePartitionDesc.getDbName(), renamePartitionDesc.getTableName());
+    Table tbl = db.getTable(renamePartitionDesc.getTableName());
 
     Partition oldPart = db.getPartition(tbl, renamePartitionDesc.getOldPartSpec(), false);
     Partition part = db.getPartition(tbl, renamePartitionDesc.getOldPartSpec(), false);
@@ -923,7 +911,7 @@ public class DDLTask extends Task<DDLWor
   private int alterTableAlterPart(Hive db, AlterTableAlterPartDesc alterPartitionDesc)
       throws HiveException {
 
-    Table tbl = db.getTable(alterPartitionDesc.getDbName(), alterPartitionDesc.getTableName());
+    Table tbl = db.getTable(alterPartitionDesc.getTableName(), true);
     String tabName = alterPartitionDesc.getTableName();
 
     // This is checked by DDLSemanticAnalyzer
@@ -1015,14 +1003,11 @@ public class DDLTask extends Task<DDLWor
   private int touch(Hive db, AlterTableSimpleDesc touchDesc)
       throws HiveException {
 
-    String dbName = touchDesc.getDbName();
-    String tblName = touchDesc.getTableName();
-
-    Table tbl = db.getTable(dbName, tblName);
+    Table tbl = db.getTable(touchDesc.getTableName());
 
     if (touchDesc.getPartSpec() == null) {
       try {
-        db.alterTable(tblName, tbl);
+        db.alterTable(touchDesc.getTableName(), tbl);
       } catch (InvalidOperationException e) {
         throw new HiveException("Uable to update table");
       }
@@ -1034,7 +1019,7 @@ public class DDLTask extends Task<DDLWor
         throw new HiveException("Specified partition does not exist");
       }
       try {
-        db.alterPartition(tblName, part);
+        db.alterPartition(touchDesc.getTableName(), part);
       } catch (InvalidOperationException e) {
         throw new HiveException(e);
       }
@@ -1173,10 +1158,8 @@ public class DDLTask extends Task<DDLWor
   private int archive(Hive db, AlterTableSimpleDesc simpleDesc,
       DriverContext driverContext)
           throws HiveException {
-    String dbName = simpleDesc.getDbName();
-    String tblName = simpleDesc.getTableName();
 
-    Table tbl = db.getTable(dbName, tblName);
+    Table tbl = db.getTable(simpleDesc.getTableName());
 
     if (tbl.getTableType() != TableType.MANAGED_TABLE) {
       throw new HiveException("ARCHIVE can only be performed on managed tables");
@@ -1378,7 +1361,7 @@ public class DDLTask extends Task<DDLWor
             authority.toString(),
             harPartitionDir.getPath()); // make in Path to ensure no slash at the end
         setArchived(p, harPath, partSpecInfo.values.size());
-        db.alterPartition(tblName, p);
+        db.alterPartition(simpleDesc.getTableName(), p);
       }
     } catch (Exception e) {
       throw new HiveException("Unable to change the partition info for HAR", e);
@@ -1399,10 +1382,8 @@ public class DDLTask extends Task<DDLWor
 
   private int unarchive(Hive db, AlterTableSimpleDesc simpleDesc)
       throws HiveException {
-    String dbName = simpleDesc.getDbName();
-    String tblName = simpleDesc.getTableName();
 
-    Table tbl = db.getTable(dbName, tblName);
+    Table tbl = db.getTable(simpleDesc.getTableName());
 
     // Means user specified a table, not a partition
     if (simpleDesc.getPartSpec() == null) {
@@ -1587,7 +1568,7 @@ public class DDLTask extends Task<DDLWor
     for(Partition p: partitions) {
       setUnArchived(p);
       try {
-        db.alterPartition(tblName, p);
+        db.alterPartition(simpleDesc.getTableName(), p);
       } catch (InvalidOperationException e) {
         throw new HiveException(e);
       }
@@ -1636,10 +1617,7 @@ public class DDLTask extends Task<DDLWor
 
   private int compact(Hive db, AlterTableSimpleDesc desc) throws HiveException {
 
-    String dbName = desc.getDbName();
-    String tblName = desc.getTableName();
-
-    Table tbl = db.getTable(dbName, tblName);
+    Table tbl = db.getTable(desc.getTableName());
 
     String partName = null;
     if (desc.getPartSpec() == null) {
@@ -1852,7 +1830,7 @@ public class DDLTask extends Task<DDLWor
     final String ROW_FORMAT = "row_format";
     final String TBL_LOCATION = "tbl_location";
     final String TBL_PROPERTIES = "tbl_properties";
-    boolean isHbaseTable = false;
+    boolean needsLocation = true;
     StringBuilder createTab_str = new StringBuilder();
 
     String tableName = showCreateTbl.getTableName();
@@ -1864,9 +1842,7 @@ public class DDLTask extends Task<DDLWor
       FileSystem fs = resFile.getFileSystem(conf);
       outStream = fs.create(resFile);
 
-      if (tbl.getStorageHandler() != null) {
-        isHbaseTable = tbl.getStorageHandler().toString().equals("org.apache.hadoop.hive.hbase.HBaseStorageHandler");
-      }
+      needsLocation = doesTableNeedLocation(tbl);
 
       if (tbl.isView()) {
         String createTab_stmt = "CREATE VIEW `" + tableName + "` AS " + tbl.getViewExpandedText();
@@ -1883,7 +1859,7 @@ public class DDLTask extends Task<DDLWor
       createTab_str.append("<" + LIST_PARTITIONS + ">\n");
       createTab_str.append("<" + SORT_BUCKET + ">\n");
       createTab_str.append("<" + ROW_FORMAT + ">\n");
-      if (!isHbaseTable) {
+      if (needsLocation) {
         createTab_str.append("LOCATION\n");
         createTab_str.append("<" + TBL_LOCATION + ">\n");
       }
@@ -2065,7 +2041,7 @@ public class DDLTask extends Task<DDLWor
       createTab_stmt.add(SORT_BUCKET, tbl_sort_bucket);
       createTab_stmt.add(ROW_FORMAT, tbl_row_format);
       // Table location should not be printed with hbase backed tables
-      if (!isHbaseTable) {
+      if (needsLocation) {
         createTab_stmt.add(TBL_LOCATION, tbl_location);
       }
       createTab_stmt.add(TBL_PROPERTIES, tbl_properties);
@@ -2233,15 +2209,7 @@ public class DDLTask extends Task<DDLWor
   public int showColumns(Hive db, ShowColumnsDesc showCols)
       throws HiveException {
 
-    String dbName = showCols.getDbName();
-    String tableName = showCols.getTableName();
-    Table table = null;
-    if (dbName == null) {
-      table = db.getTable(tableName);
-    }
-    else {
-      table = db.getTable(dbName, tableName);
-    }
+    Table table = db.getTable(showCols.getTableName());
 
     // write the results in the file
     DataOutputStream outStream = null;
@@ -3280,7 +3248,8 @@ public class DDLTask extends Task<DDLWor
     Table oldTbl = tbl.copy();
 
     if (alterTbl.getOp() == AlterTableDesc.AlterTableTypes.RENAME) {
-      tbl.setTableName(alterTbl.getNewName());
+      tbl.setDbName(Utilities.getDatabaseName(alterTbl.getNewName()));
+      tbl.setTableName(Utilities.getTableName(alterTbl.getNewName()));
     } else if (alterTbl.getOp() == AlterTableDesc.AlterTableTypes.ADDCOLS) {
       List<FieldSchema> newCols = alterTbl.getNewCols();
       List<FieldSchema> oldCols = tbl.getCols();
@@ -3933,7 +3902,7 @@ public class DDLTask extends Task<DDLWor
       tbl.getTTable().getSd().setOutputFormat(tbl.getOutputFormatClass().getName());
     }
 
-    if (!Utilities.isDefaultNameNode(conf) && tbl.getTTable().getSd().isSetLocation()) {
+    if (!Utilities.isDefaultNameNode(conf) && doesTableNeedLocation(tbl)) {
       // If location is specified - ensure that it is a full qualified name
       makeLocationQualified(tbl.getDbName(), tbl.getTTable().getSd(), tbl.getTableName());
     }
@@ -4309,10 +4278,12 @@ public class DDLTask extends Task<DDLWor
     if (crtIndex.getLocation() == null) {
       // Location is not set, leave it as-is if index doesn't belong to default DB
       // Currently all indexes are created in current DB only
-      if (db.getDatabaseCurrent().getName().equalsIgnoreCase(MetaStoreUtils.DEFAULT_DATABASE_NAME)) {
+      if (Utilities.getDatabaseName(name).equalsIgnoreCase(MetaStoreUtils.DEFAULT_DATABASE_NAME)) {
         // Default database name path is always ignored, use METASTOREWAREHOUSE and object name
         // instead
-        path = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.METASTOREWAREHOUSE), name.toLowerCase());
+        String warehouse = HiveConf.getVar(conf, ConfVars.METASTOREWAREHOUSE);
+        String tableName = Utilities.getTableName(name);
+        path = new Path(warehouse, tableName.toLowerCase());
       }
     }
     else {
@@ -4341,4 +4312,15 @@ public class DDLTask extends Task<DDLWor
               database.getName().toLowerCase() + ".db")));
     }
   }
+
+  private static boolean doesTableNeedLocation(Table tbl) {
+    // If we are ok with breaking compatibility of existing 3rd party StorageHandlers,
+    // this method could be moved to the HiveStorageHandler interface.
+    boolean retval = true;
+    if (tbl.getStorageHandler() != null) {
+      retval = !tbl.getStorageHandler().toString().equals(
+          "org.apache.hadoop.hive.hbase.HBaseStorageHandler");
+    }
+    return retval;
+  }
 }

Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java?rev=1617652&r1=1617651&r2=1617652&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java Wed Aug 13 02:28:54 2014
@@ -181,7 +181,7 @@ public class HashTableSinkOperator exten
         if (pos == posBigTableAlias) {
           continue;
         }
-        mapJoinTables[pos] = new HashMapWrapper(hconf);
+        mapJoinTables[pos] = new HashMapWrapper(hconf, -1);
         TableDesc valueTableDesc = conf.getValueTblFilteredDescs().get(pos);
         SerDe valueSerDe = (SerDe) ReflectionUtils.newInstance(valueTableDesc.getDeserializerClass(), null);
         SerDeUtils.initializeSerDe(valueSerDe, null, valueTableDesc.getProperties(), null);

Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1617652&r1=1617651&r2=1617652&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Wed Aug 13 02:28:54 2014
@@ -96,6 +96,7 @@ import org.apache.hadoop.fs.permission.F
 import org.apache.hadoop.hive.common.HiveInterruptCallback;
 import org.apache.hadoop.hive.common.HiveInterruptUtils;
 import org.apache.hadoop.hive.common.HiveStatsUtils;
+import org.apache.hadoop.hive.common.JavaUtils;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.metastore.Warehouse;
@@ -1381,9 +1382,8 @@ public final class Utilities {
   public static RCFile.Writer createRCFileWriter(JobConf jc, FileSystem fs, Path file,
       boolean isCompressed, Progressable progressable) throws IOException {
     CompressionCodec codec = null;
-    Class<?> codecClass = null;
     if (isCompressed) {
-      codecClass = FileOutputFormat.getOutputCompressorClass(jc, DefaultCodec.class);
+      Class<?> codecClass = FileOutputFormat.getOutputCompressorClass(jc, DefaultCodec.class);
       codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, jc);
     }
     return new RCFile.Writer(fs, jc, file, progressable, codec);
@@ -1972,6 +1972,7 @@ public final class Utilities {
         newPath.remove(oneurl);
       }
     }
+    JavaUtils.closeClassLoader(loader);
 
     loader = new URLClassLoader(newPath.toArray(new URL[0]));
     curThread.setContextClassLoader(loader);
@@ -2046,19 +2047,53 @@ public final class Utilities {
    * @return String array with two elements, first is db name, second is table name
    * @throws HiveException
    */
-  public static String[] getDbTableName(String dbtable) throws HiveException{
-    if(dbtable == null){
+  public static String[] getDbTableName(String dbtable) throws SemanticException {
+    return getDbTableName(SessionState.get().getCurrentDatabase(), dbtable);
+  }
+
+  public static String[] getDbTableName(String defaultDb, String dbtable) throws SemanticException {
+    if (dbtable == null) {
       return new String[2];
     }
     String[] names =  dbtable.split("\\.");
     switch (names.length) {
-    case 2:
-      return names;
-    case 1:
-      return new String [] {SessionState.get().getCurrentDatabase(), dbtable};
-    default:
-      throw new HiveException(ErrorMsg.INVALID_TABLE_NAME, dbtable);
+      case 2:
+        return names;
+      case 1:
+        return new String [] {defaultDb, dbtable};
+      default:
+        throw new SemanticException(ErrorMsg.INVALID_TABLE_NAME, dbtable);
+    }
+  }
+
+  /**
+   * Accepts qualified name which is in the form of dbname.tablename and returns dbname from it
+   *
+   * @param dbTableName
+   * @return dbname
+   * @throws SemanticException input string is not qualified name
+   */
+  public static String getDatabaseName(String dbTableName) throws SemanticException {
+    String[] split = dbTableName.split("\\.");
+    if (split.length != 2) {
+      throw new SemanticException(ErrorMsg.INVALID_TABLE_NAME, dbTableName);
+    }
+    return split[0];
+  }
+
+  /**
+   * Accepts qualified name which is in the form of dbname.tablename and returns tablename from it
+   *
+   * @param dbTableName
+   * @return tablename
+   * @throws SemanticException input string is not qualified name
+   */
+  public static String getTableName(String dbTableName) throws SemanticException {
+    String[] split = dbTableName.split("\\.");
+    if (split.length != 2) {
+      throw new SemanticException(ErrorMsg.INVALID_TABLE_NAME, dbTableName);
     }
+    return split[1];
   }
 
   public static void validateColumnNames(List<String> colNames, List<String> checkCols)

Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/BytesBytesMultiHashMap.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/BytesBytesMultiHashMap.java?rev=1617652&r1=1617651&r2=1617652&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/BytesBytesMultiHashMap.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/BytesBytesMultiHashMap.java Wed Aug 13 02:28:54 2014
@@ -145,8 +145,7 @@ public final class BytesBytesMultiHashMa
   private long[] refs;
   private int startingHashBitCount, hashBitCount;
 
-  private int metricPutConflict = 0, metricSameBitsDiffKey = 0,
-      metricSameBitsSameKey = 0, metricDiffBits = 0;
+  private int metricPutConflict = 0, metricExpands = 0, metricExpandsUs = 0;
 
   /** We have 39 bits to store list pointer from the first record; this is size limit */
   final static long MAX_WB_SIZE = ((long)1) << 38;
@@ -430,16 +429,13 @@ public final class BytesBytesMultiHashMa
    */
   private boolean isSameKey(byte[] key, int length, long ref, int hashCode) {
     if (!compareHashBits(ref, hashCode)) {
-      ++metricDiffBits;
       return false;  // Hash bits don't match.
     }
     writeBuffers.setReadPoint(getFirstRecordLengthsOffset(ref));
     int valueLength = (int)writeBuffers.readVLong(), keyLength = (int)writeBuffers.readVLong();
     long keyOffset = Ref.getOffset(ref) - (valueLength + keyLength);
     // See the comment in the other isSameKey
-    boolean result = writeBuffers.isEqual(key, length, keyOffset, keyLength);
-    if (result) { ++metricSameBitsSameKey; } else { ++metricSameBitsDiffKey; }
-    return result;
+    return writeBuffers.isEqual(key, length, keyOffset, keyLength);
   }
 
   private boolean compareHashBits(long ref, int hashCode) {
@@ -461,6 +457,7 @@ public final class BytesBytesMultiHashMa
   }
 
   private void expandAndRehash() {
+    long expandTime = System.nanoTime();
     final long[] oldRefs = refs;
     long capacity = refs.length << 1;
     validateCapacity(capacity);
@@ -492,6 +489,9 @@ public final class BytesBytesMultiHashMa
     this.largestNumberOfSteps = maxSteps;
     this.hashBitCount = newHashBitCount;
     this.resizeThreshold = (int)(capacity * loadFactor);
+    metricExpandsUs += (System.nanoTime() - expandTime);
+    ++metricExpands;
+
   }
 
   /**
@@ -703,11 +703,9 @@ public final class BytesBytesMultiHashMa
   }
 
   public void debugDumpMetrics() {
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Map metrics: keys " + this.keysAssigned + ", write conflict " + metricPutConflict
-          + ", write max dist " + largestNumberOfSteps + ", read neq " + metricDiffBits
-          + ", read eq-eq " + metricSameBitsSameKey + ", read eq-neq " + metricSameBitsDiffKey);
-    }
+    LOG.info("Map metrics: keys allocated " + this.refs.length +", keys assigned " + keysAssigned
+        + ", write conflict " + metricPutConflict  + ", write max dist " + largestNumberOfSteps
+        + ", expanded " + metricExpands + " times in " + metricExpandsUs + "us");
   }
 
   private void debugDumpKeyProbe(long keyOffset, int keyLength, int hashCode, int finalSlot) {

Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java?rev=1617652&r1=1617651&r2=1617652&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java Wed Aug 13 02:28:54 2014
@@ -71,25 +71,39 @@ public class HashMapWrapper extends Abst
   }
 
   public HashMapWrapper() {
-    this(HiveConf.ConfVars.HIVEHASHTABLETHRESHOLD.defaultIntVal,
-        HiveConf.ConfVars.HIVEHASHTABLELOADFACTOR.defaultFloatVal, false, false);
+    this(HiveConf.ConfVars.HIVEHASHTABLEKEYCOUNTADJUSTMENT.defaultFloatVal,
+        HiveConf.ConfVars.HIVEHASHTABLETHRESHOLD.defaultIntVal,
+        HiveConf.ConfVars.HIVEHASHTABLELOADFACTOR.defaultFloatVal, false, false, -1);
   }
 
-  public HashMapWrapper(Configuration hconf) {
-    this(HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVEHASHTABLETHRESHOLD),
+  public HashMapWrapper(Configuration hconf, long keyCount) {
+    this(HiveConf.getFloatVar(hconf, HiveConf.ConfVars.HIVEHASHTABLEKEYCOUNTADJUSTMENT),
+        HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVEHASHTABLETHRESHOLD),
         HiveConf.getFloatVar(hconf, HiveConf.ConfVars.HIVEHASHTABLELOADFACTOR),
         HiveConf.getBoolVar(hconf, HiveConf.ConfVars.HIVEMAPJOINLAZYHASHTABLE),
-        HiveConf.getBoolVar(hconf, HiveConf.ConfVars.HIVEMAPJOINUSEOPTIMIZEDKEYS));
+        HiveConf.getBoolVar(hconf, HiveConf.ConfVars.HIVEMAPJOINUSEOPTIMIZEDKEYS), keyCount);
   }
 
-  private HashMapWrapper(
-      int threshold, float loadFactor, boolean useLazyRows, boolean useOptimizedKeys) {
+  private HashMapWrapper(float keyCountAdj, int threshold, float loadFactor,
+      boolean useLazyRows, boolean useOptimizedKeys, long keyCount) {
     super(createConstructorMetaData(threshold, loadFactor));
+    threshold = calculateTableSize(keyCountAdj, threshold, loadFactor, keyCount);
     mHash = new HashMap<MapJoinKey, MapJoinRowContainer>(threshold, loadFactor);
     this.useLazyRows = useLazyRows;
     this.useOptimizedKeys = useOptimizedKeys;
   }
 
+  public static int calculateTableSize(
+      float keyCountAdj, int threshold, float loadFactor, long keyCount) {
+    if (keyCount >= 0 && keyCountAdj != 0) {
+      // We have statistics for the table. Size appropriately.
+      threshold = (int)Math.ceil(keyCount / (keyCountAdj * loadFactor));
+    }
+    LOG.info("Key count from statistics is " + keyCount + "; setting map size to " + threshold);
+    return threshold;
+  }
+
+
   @Override
   public MapJoinRowContainer get(MapJoinKey key) {
     return mHash.get(key);

Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java?rev=1617652&r1=1617651&r2=1617652&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java Wed Aug 13 02:28:54 2014
@@ -59,15 +59,17 @@ public class MapJoinBytesTableContainer 
 
   private List<Object> EMPTY_LIST = new ArrayList<Object>(0);
 
-  public MapJoinBytesTableContainer(Configuration hconf, MapJoinObjectSerDeContext valCtx)
-      throws SerDeException {
-    this(HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVEHASHTABLETHRESHOLD),
+  public MapJoinBytesTableContainer(Configuration hconf,
+      MapJoinObjectSerDeContext valCtx, long keyCount) throws SerDeException {
+    this(HiveConf.getFloatVar(hconf, HiveConf.ConfVars.HIVEHASHTABLEKEYCOUNTADJUSTMENT),
+        HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVEHASHTABLETHRESHOLD),
         HiveConf.getFloatVar(hconf, HiveConf.ConfVars.HIVEHASHTABLELOADFACTOR),
-        HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVEHASHTABLEWBSIZE), valCtx);
+        HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVEHASHTABLEWBSIZE), valCtx, keyCount);
   }
 
-  private MapJoinBytesTableContainer(int threshold, float loadFactor, int wbSize,
-      MapJoinObjectSerDeContext valCtx) throws SerDeException {
+  private MapJoinBytesTableContainer(float keyCountAdj, int threshold, float loadFactor,
+      int wbSize, MapJoinObjectSerDeContext valCtx, long keyCount) throws SerDeException {
+    threshold = HashMapWrapper.calculateTableSize(keyCountAdj, threshold, loadFactor, keyCount);
     hashMap = new BytesBytesMultiHashMap(threshold, loadFactor, wbSize);
   }
 

Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java?rev=1617652&r1=1617651&r2=1617652&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java Wed Aug 13 02:28:54 2014
@@ -88,6 +88,7 @@ public class HashTableLoader implements 
 
     TezContext tezContext = (TezContext) MapredContext.get();
     Map<Integer, String> parentToInput = desc.getParentToInput();
+    Map<Integer, Long> parentKeyCounts = desc.getParentKeyCounts();
 
     boolean useOptimizedTables = HiveConf.getBoolVar(
         hconf, HiveConf.ConfVars.HIVEMAPJOINUSEOPTIMIZEDTABLE);
@@ -117,8 +118,11 @@ public class HashTableLoader implements 
           }
         }
         isFirstKey = false;
+        Long keyCountObj = parentKeyCounts.get(pos);
+        long keyCount = (keyCountObj == null) ? -1 : keyCountObj.longValue();
         MapJoinTableContainer tableContainer = useOptimizedTables
-            ? new MapJoinBytesTableContainer(hconf, valCtx) : new HashMapWrapper(hconf);
+            ? new MapJoinBytesTableContainer(hconf, valCtx, keyCount)
+            : new HashMapWrapper(hconf, keyCount);
 
         while (kvReader.next()) {
           lastKey = tableContainer.putRow(keyCtx, (Writable)kvReader.getCurrentKey(),

Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java?rev=1617652&r1=1617651&r2=1617652&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java Wed Aug 13 02:28:54 2014
@@ -80,6 +80,7 @@ import org.apache.hadoop.hive.ql.plan.Ex
 import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
 import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
 import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeNullDesc;
 import org.apache.hadoop.hive.ql.udf.SettableUDF;
 import org.apache.hadoop.hive.ql.udf.UDFConv;
 import org.apache.hadoop.hive.ql.udf.UDFHex;
@@ -323,10 +324,12 @@ public class VectorizationContext {
         ve = getGenericUdfVectorExpression(expr.getGenericUDF(),
             childExpressions, mode, exprDesc.getTypeInfo());
       }
+    } else if (exprDesc instanceof ExprNodeNullDesc) {
+    	ve = getConstantVectorExpression(null, exprDesc.getTypeInfo(), mode);
     } else if (exprDesc instanceof ExprNodeConstantDesc) {
       ve = getConstantVectorExpression(((ExprNodeConstantDesc) exprDesc).getValue(), exprDesc.getTypeInfo(),
           mode);
-    }
+    } 
     if (ve == null) {
       throw new HiveException("Could not vectorize expression: "+exprDesc.getName());
     }
@@ -410,8 +413,8 @@ public class VectorizationContext {
         }
       }
     } else {
-      for (ExprNodeDesc child : children) {
-        ExprNodeDesc castExpression = getImplicitCastExpression(genericUDF, child, commonType);
+      for (ExprNodeDesc child : children) {    	
+    	ExprNodeDesc castExpression = getImplicitCastExpression(genericUDF, child, commonType);
         if (castExpression != null) {
           atleastOneCastNeeded = true;
           childrenWithCasts.add(castExpression);
@@ -652,69 +655,70 @@ public class VectorizationContext {
   }
 
   /**
-   * Handles only the special case of unary operators on a constant.
+   * Handles only the special cases of cast/+ve/-ve operator on a constant.
    * @param exprDesc
-   * @return The same expression if no folding done, else return the constant
+   * @return The same expression if no evaluation done, else return the constant
    *         expression.
    * @throws HiveException
    */
-  ExprNodeDesc foldConstantsForUnaryExpression(ExprNodeDesc exprDesc) throws HiveException {
-    if (!(exprDesc instanceof ExprNodeGenericFuncDesc)) {
-      return exprDesc;
-    }
-    
-    if (exprDesc.getChildren() == null || (exprDesc.getChildren().size() != 1) ) {
-      return exprDesc;
-    }
-
-    ExprNodeConstantDesc foldedChild = null;
-    if (!( exprDesc.getChildren().get(0) instanceof ExprNodeConstantDesc)) {
-
-      // try recursive folding
-      ExprNodeDesc expr = foldConstantsForUnaryExpression(exprDesc.getChildren().get(0));
-      if (expr instanceof ExprNodeConstantDesc) {
-        foldedChild = (ExprNodeConstantDesc) expr;
-      }
-    } else {
-      foldedChild = (ExprNodeConstantDesc) exprDesc.getChildren().get(0);
-    }
-
-    if (foldedChild == null) {
-      return exprDesc;
-    }
-
-    ObjectInspector childoi = foldedChild.getWritableObjectInspector();
-    GenericUDF gudf = ((ExprNodeGenericFuncDesc) exprDesc).getGenericUDF();
-
-    if (gudf instanceof GenericUDFOPNegative || gudf instanceof GenericUDFOPPositive
-        || castExpressionUdfs.contains(gudf.getClass())
-        || ((gudf instanceof GenericUDFBridge)
-            && castExpressionUdfs.contains(((GenericUDFBridge) gudf).getUdfClass()))) {
-      ExprNodeEvaluator<?> evaluator = ExprNodeEvaluatorFactory.get(exprDesc);
-      ObjectInspector output = evaluator.initialize(childoi);
-      Object constant = evaluator.evaluate(null);
-      Object java = ObjectInspectorUtils.copyToStandardJavaObject(constant, output);  
-      return new ExprNodeConstantDesc(exprDesc.getTypeInfo(), java);
-     }
-
-    return exprDesc;
+  ExprNodeDesc evaluateCastOnConstants(ExprNodeDesc exprDesc) throws HiveException {
+	  if (!(exprDesc instanceof ExprNodeGenericFuncDesc)) {
+		  return exprDesc;
+	  }
+      
+	  if (exprDesc.getChildren() == null || (exprDesc.getChildren().size() != 1) ) {
+		  return exprDesc;
+	  }
+  
+	  ExprNodeConstantDesc foldedChild = null;
+	  if (!( exprDesc.getChildren().get(0) instanceof ExprNodeConstantDesc)) {
+		  
+		  // try recursive folding
+		  ExprNodeDesc expr = evaluateCastOnConstants(exprDesc.getChildren().get(0));
+		  if (expr instanceof ExprNodeConstantDesc) {
+			  foldedChild = (ExprNodeConstantDesc) expr;
+		  }
+	  } else {
+		  foldedChild = (ExprNodeConstantDesc) exprDesc.getChildren().get(0);
+	  }
+  
+	  if (foldedChild == null) {
+		  return exprDesc;
+	  }
+  
+	  ObjectInspector childoi = foldedChild.getWritableObjectInspector();
+	  GenericUDF gudf = ((ExprNodeGenericFuncDesc) exprDesc).getGenericUDF();
+      
+	  // Only evaluate +ve/-ve or cast on constant or recursive casting.
+	  if (gudf instanceof GenericUDFOPNegative || gudf instanceof GenericUDFOPPositive ||
+			  castExpressionUdfs.contains(gudf.getClass())
+			  || ((gudf instanceof GenericUDFBridge)
+					  && castExpressionUdfs.contains(((GenericUDFBridge) gudf).getUdfClass()))) {
+		  ExprNodeEvaluator<?> evaluator = ExprNodeEvaluatorFactory.get(exprDesc);
+		  ObjectInspector output = evaluator.initialize(childoi);
+		  Object constant = evaluator.evaluate(null);
+		  Object java = ObjectInspectorUtils.copyToStandardJavaObject(constant, output);  
+		  return new ExprNodeConstantDesc(exprDesc.getTypeInfo(), java);
+	  }
+  
+	  return exprDesc;
   }
-
-  /* Fold simple unary expressions in all members of the input list and return new list
+  
+  /* For cast on constant operator in all members of the input list and return new list
    * containing results.
    */
-  private List<ExprNodeDesc> foldConstantsForUnaryExprs(List<ExprNodeDesc> childExpr)
-      throws HiveException {
-    List<ExprNodeDesc> constantFoldedChildren = new ArrayList<ExprNodeDesc>();
-    if (childExpr != null) {
-      for (ExprNodeDesc expr : childExpr) {
-        expr = this.foldConstantsForUnaryExpression(expr);
-        constantFoldedChildren.add(expr);
-      }
-    }
-    return constantFoldedChildren;
+  private List<ExprNodeDesc> evaluateCastOnConstants(List<ExprNodeDesc> childExpr)
+		  throws HiveException {
+	  List<ExprNodeDesc> evaluatedChildren = new ArrayList<ExprNodeDesc>();
+	  if (childExpr != null) {
+        for (ExprNodeDesc expr : childExpr) {
+        	expr = this.evaluateCastOnConstants(expr);
+        	evaluatedChildren.add(expr);
+        }
+	  }
+	  return evaluatedChildren;
   }
-
+      
   private VectorExpression getConstantVectorExpression(Object constantValue, TypeInfo typeInfo,
       Mode mode) throws HiveException {
     String type =  typeInfo.getTypeName();
@@ -903,8 +907,9 @@ public class VectorizationContext {
   private VectorExpression getGenericUdfVectorExpression(GenericUDF udf,
       List<ExprNodeDesc> childExpr, Mode mode, TypeInfo returnType) throws HiveException {
 
-    List<ExprNodeDesc> constantFoldedChildren = foldConstantsForUnaryExprs(childExpr);
-    childExpr = constantFoldedChildren;
+	List<ExprNodeDesc> castedChildren = evaluateCastOnConstants(childExpr);
+	childExpr = castedChildren;	  
+	  
     //First handle special cases
     if (udf instanceof GenericUDFBetween) {
       return getBetweenFilterExpression(childExpr, mode, returnType);
@@ -928,15 +933,15 @@ public class VectorizationContext {
       }
     } else if (udf instanceof GenericUDFToDecimal) {
       return getCastToDecimal(childExpr, returnType);
-    }
-
+    } 
+    
     // Now do a general lookup
     Class<?> udfClass = udf.getClass();
     if (udf instanceof GenericUDFBridge) {
       udfClass = ((GenericUDFBridge) udf).getUdfClass();
     }
 
-    VectorExpression ve = getVectorExpressionForUdf(udfClass, constantFoldedChildren, mode, returnType);
+    VectorExpression ve = getVectorExpressionForUdf(udfClass, castedChildren, mode, returnType);
 
     if (ve == null) {
       throw new HiveException("Udf: "+udf.getClass().getSimpleName()+", is not supported");
@@ -998,7 +1003,7 @@ public class VectorizationContext {
       }
     }
   }
-
+      
   /**
    * Create a filter or boolean-valued expression for column IN ( <list-of-constants> )
    */
@@ -1006,13 +1011,11 @@ public class VectorizationContext {
       throws HiveException {
     ExprNodeDesc colExpr = childExpr.get(0);
 
-    TypeInfo colTypeInfo = colExpr.getTypeInfo();
     String colType = colExpr.getTypeString();
 
     // prepare arguments for createVectorExpression
-    List<ExprNodeDesc> childrenForInList =
-        foldConstantsForUnaryExprs(childExpr.subList(1, childExpr.size()));
-
+    List<ExprNodeDesc> childrenForInList =  evaluateCastOnConstants(childExpr.subList(1, childExpr.size()));	
+    
     /* This method assumes that the IN list has no NULL entries. That is enforced elsewhere,
      * in the Vectorizer class. If NULL is passed in as a list entry, behavior is not defined.
      * If in the future, NULL values are allowed in the IN list, be sure to handle 3-valued
@@ -1107,16 +1110,116 @@ public class VectorizationContext {
       return getCastToString(childExpr, returnType);
     }
     return null;
-  }
-
+  } 
+  
+  private Decimal128 castConstantToDecimal(Object scalar, TypeInfo type) throws HiveException {
+	  PrimitiveTypeInfo ptinfo = (PrimitiveTypeInfo) type;
+	  String typename = type.getTypeName();
+	  Decimal128 d = new Decimal128();
+	  int scale = HiveDecimalUtils.getScaleForType(ptinfo);
+	  switch (ptinfo.getPrimitiveCategory()) {
+	  case FLOAT:
+		  float floatVal = ((Float) scalar).floatValue();
+		  d.update(floatVal, (short) scale);
+		  break;
+	  case DOUBLE:
+		  double doubleVal = ((Double) scalar).doubleValue();
+		  d.update(doubleVal, (short) scale);
+		  break;
+	  case BYTE:
+		  byte byteVal = ((Byte) scalar).byteValue();
+		  d.update(byteVal, (short) scale);
+		  break;
+	  case SHORT:
+		  short shortVal = ((Short) scalar).shortValue();
+		  d.update(shortVal, (short) scale);
+		  break;
+	  case INT:
+		  int intVal = ((Integer) scalar).intValue();
+		  d.update(intVal, (short) scale);
+		  break;
+	  case LONG:
+		  long longVal = ((Long) scalar).longValue();
+		  d.update(longVal, (short) scale);
+		  break;
+	  case DECIMAL:
+		  HiveDecimal decimalVal = (HiveDecimal) scalar;
+		  d.update(decimalVal.unscaledValue(), (short) scale);
+		  break;
+	  default:
+		  throw new HiveException("Unsupported type "+typename+" for cast to Decimal128");
+	  }
+	  return d;
+  }
+
+  private String castConstantToString(Object scalar, TypeInfo type) throws HiveException {
+	  PrimitiveTypeInfo ptinfo = (PrimitiveTypeInfo) type;
+	  String typename = type.getTypeName();
+	  switch (ptinfo.getPrimitiveCategory()) {
+	  case FLOAT:
+	  case DOUBLE:
+	  case BYTE:
+	  case SHORT:
+	  case INT:
+	  case LONG:
+		  return ((Number) scalar).toString();
+	  case DECIMAL:
+		  HiveDecimal decimalVal = (HiveDecimal) scalar;
+		  return decimalVal.toString();
+	  default:
+		  throw new HiveException("Unsupported type "+typename+" for cast to String");
+	  }
+  }
+
+  private Double castConstantToDouble(Object scalar, TypeInfo type) throws HiveException {
+	  PrimitiveTypeInfo ptinfo = (PrimitiveTypeInfo) type;
+	  String typename = type.getTypeName();
+	  switch (ptinfo.getPrimitiveCategory()) {
+	  case FLOAT:
+	  case DOUBLE:
+	  case BYTE:
+	  case SHORT:
+	  case INT:
+	  case LONG:
+		  return ((Number) scalar).doubleValue();
+	  case DECIMAL:
+		  HiveDecimal decimalVal = (HiveDecimal) scalar;
+		  return decimalVal.doubleValue();
+	  default:
+		  throw new HiveException("Unsupported type "+typename+" for cast to Double");
+	  }
+  }  
+
+  private Long castConstantToLong(Object scalar, TypeInfo type) throws HiveException {
+	  PrimitiveTypeInfo ptinfo = (PrimitiveTypeInfo) type;
+	  String typename = type.getTypeName();
+	  switch (ptinfo.getPrimitiveCategory()) {
+	  case FLOAT:
+	  case DOUBLE:
+	  case BYTE:
+	  case SHORT:
+	  case INT:
+	  case LONG:
+		  return ((Number) scalar).longValue();
+	  case DECIMAL:
+		  HiveDecimal decimalVal = (HiveDecimal) scalar;
+		  return decimalVal.longValue();
+	  default:
+		  throw new HiveException("Unsupported type "+typename+" for cast to Long");
+	  }
+  }    
+  
   private VectorExpression getCastToDecimal(List<ExprNodeDesc> childExpr, TypeInfo returnType)
       throws HiveException {
     ExprNodeDesc child = childExpr.get(0);
     String inputType = childExpr.get(0).getTypeString();
     if (child instanceof ExprNodeConstantDesc) {
-      // Don't do constant folding here.  Wait until the optimizer is changed to do it.
-      // Family of related JIRAs: HIVE-7421, HIVE-7422, and HIVE-7424.
-      return null;
+     // Return a constant vector expression
+      Object constantValue = ((ExprNodeConstantDesc) child).getValue();
+      Decimal128 decimalValue = castConstantToDecimal(constantValue, child.getTypeInfo());
+      return getConstantVectorExpression(decimalValue, returnType, Mode.PROJECTION);    	
+    } else if (child instanceof ExprNodeNullDesc) {
+    	return getConstantVectorExpression(null, returnType, Mode.PROJECTION); 
     }
     if (isIntFamily(inputType)) {
       return createVectorExpression(CastLongToDecimal.class, childExpr, Mode.PROJECTION, returnType);
@@ -1131,16 +1234,19 @@ public class VectorizationContext {
       return createVectorExpression(CastTimestampToDecimal.class, childExpr, Mode.PROJECTION, returnType);
     }
     throw new HiveException("Unhandled cast input type: " + inputType);
-  }
-
+  }  
+  
   private VectorExpression getCastToString(List<ExprNodeDesc> childExpr, TypeInfo returnType)
       throws HiveException {
     ExprNodeDesc child = childExpr.get(0);
     String inputType = childExpr.get(0).getTypeString();
     if (child instanceof ExprNodeConstantDesc) {
-      // Don't do constant folding here.  Wait until the optimizer is changed to do it.
-      // Family of related JIRAs: HIVE-7421, HIVE-7422, and HIVE-7424.
-      return null;
+        // Return a constant vector expression
+        Object constantValue = ((ExprNodeConstantDesc) child).getValue();
+        String strValue = castConstantToString(constantValue, child.getTypeInfo());
+        return getConstantVectorExpression(strValue, returnType, Mode.PROJECTION); 
+    } else if (child instanceof ExprNodeNullDesc) {
+    	return getConstantVectorExpression(null, returnType, Mode.PROJECTION); 
     }
     if (inputType.equals("boolean")) {
       // Boolean must come before the integer family. It's a special case.
@@ -1164,9 +1270,12 @@ public class VectorizationContext {
     ExprNodeDesc child = childExpr.get(0);
     String inputType = childExpr.get(0).getTypeString();
     if (child instanceof ExprNodeConstantDesc) {
-      // Don't do constant folding here.  Wait until the optimizer is changed to do it.
-      // Family of related JIRAs: HIVE-7421, HIVE-7422, and HIVE-7424.
-      return null;
+        // Return a constant vector expression
+        Object constantValue = ((ExprNodeConstantDesc) child).getValue();
+        Double doubleValue = castConstantToDouble(constantValue, child.getTypeInfo());
+        return getConstantVectorExpression(doubleValue, returnType, Mode.PROJECTION);     	
+    } else if (child instanceof ExprNodeNullDesc) {
+    	return getConstantVectorExpression(null, returnType, Mode.PROJECTION); 
     }
     if (isIntFamily(inputType)) {
       return createVectorExpression(CastLongToDouble.class, childExpr, Mode.PROJECTION, returnType);
@@ -1191,6 +1300,8 @@ public class VectorizationContext {
       // Don't do constant folding here.  Wait until the optimizer is changed to do it.
       // Family of related JIRAs: HIVE-7421, HIVE-7422, and HIVE-7424.
       return null;
+    } else if (child instanceof ExprNodeNullDesc) {
+    	return getConstantVectorExpression(null, TypeInfoFactory.booleanTypeInfo, Mode.PROJECTION); 
     }
     // Long and double are handled using descriptors, string needs to be specially handled.
     if (inputType.equals("string")) {
@@ -1215,9 +1326,12 @@ public class VectorizationContext {
     ExprNodeDesc child = childExpr.get(0);
     String inputType = childExpr.get(0).getTypeString();
     if (child instanceof ExprNodeConstantDesc) {
-      // Don't do constant folding here.  Wait until the optimizer is changed to do it.
-      // Family of related JIRAs: HIVE-7421, HIVE-7422, and HIVE-7424.
-      return null;
+        // Return a constant vector expression
+        Object constantValue = ((ExprNodeConstantDesc) child).getValue();
+        Long longValue = castConstantToLong(constantValue, child.getTypeInfo());
+        return getConstantVectorExpression(longValue, TypeInfoFactory.longTypeInfo, Mode.PROJECTION);    	
+    } else if (child instanceof ExprNodeNullDesc) {
+    	return getConstantVectorExpression(null, TypeInfoFactory.longTypeInfo, Mode.PROJECTION); 
     }
     // Float family, timestamp are handled via descriptor based lookup, int family needs
     // special handling.
@@ -1281,7 +1395,7 @@ public class VectorizationContext {
     String colType = commonType.getTypeName();
 
     // prepare arguments for createVectorExpression
-    List<ExprNodeDesc> childrenAfterNot = foldConstantsForUnaryExprs(castChildren);
+    List<ExprNodeDesc> childrenAfterNot = evaluateCastOnConstants(castChildren);
 
     // determine class
     Class<?> cl = null;

Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java?rev=1617652&r1=1617651&r2=1617652&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java Wed Aug 13 02:28:54 2014
@@ -37,6 +37,7 @@ import org.apache.hadoop.hive.serde2.obj
 import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
 import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
 import org.apache.hadoop.io.BooleanWritable;
+import org.apache.hadoop.io.BytesWritable;
 import org.apache.hadoop.io.DataOutputBuffer;
 import org.apache.hadoop.io.FloatWritable;
 import org.apache.hadoop.io.IntWritable;
@@ -123,6 +124,7 @@ public class VectorizedBatchUtil {
         case DOUBLE:
           cvList.add(new DoubleColumnVector(VectorizedRowBatch.DEFAULT_SIZE));
           break;
+        case BINARY:
         case STRING:
           cvList.add(new BytesColumnVector(VectorizedRowBatch.DEFAULT_SIZE));
           break;
@@ -237,7 +239,7 @@ public class VectorizedBatchUtil {
       // float/double. String types have no default value for null.
       switch (poi.getPrimitiveCategory()) {
       case BOOLEAN: {
-        LongColumnVector lcv = (LongColumnVector) batch.cols[off+i];
+        LongColumnVector lcv = (LongColumnVector) batch.cols[off + i];
         if (writableCol != null) {
           lcv.vector[rowIndex] = ((BooleanWritable) writableCol).get() ? 1 : 0;
           lcv.isNull[rowIndex] = false;
@@ -248,7 +250,7 @@ public class VectorizedBatchUtil {
       }
         break;
       case BYTE: {
-        LongColumnVector lcv = (LongColumnVector) batch.cols[off+i];
+        LongColumnVector lcv = (LongColumnVector) batch.cols[off + i];
         if (writableCol != null) {
           lcv.vector[rowIndex] = ((ByteWritable) writableCol).get();
           lcv.isNull[rowIndex] = false;
@@ -259,7 +261,7 @@ public class VectorizedBatchUtil {
       }
         break;
       case SHORT: {
-        LongColumnVector lcv = (LongColumnVector) batch.cols[off+i];
+        LongColumnVector lcv = (LongColumnVector) batch.cols[off + i];
         if (writableCol != null) {
           lcv.vector[rowIndex] = ((ShortWritable) writableCol).get();
           lcv.isNull[rowIndex] = false;
@@ -270,7 +272,7 @@ public class VectorizedBatchUtil {
       }
         break;
       case INT: {
-        LongColumnVector lcv = (LongColumnVector) batch.cols[off+i];
+        LongColumnVector lcv = (LongColumnVector) batch.cols[off + i];
         if (writableCol != null) {
           lcv.vector[rowIndex] = ((IntWritable) writableCol).get();
           lcv.isNull[rowIndex] = false;
@@ -281,7 +283,7 @@ public class VectorizedBatchUtil {
       }
         break;
       case LONG: {
-        LongColumnVector lcv = (LongColumnVector) batch.cols[off+i];
+        LongColumnVector lcv = (LongColumnVector) batch.cols[off + i];
         if (writableCol != null) {
           lcv.vector[rowIndex] = ((LongWritable) writableCol).get();
           lcv.isNull[rowIndex] = false;
@@ -292,7 +294,7 @@ public class VectorizedBatchUtil {
       }
         break;
       case DATE: {
-        LongColumnVector lcv = (LongColumnVector) batch.cols[off+i];
+        LongColumnVector lcv = (LongColumnVector) batch.cols[off + i];
         if (writableCol != null) {
           lcv.vector[rowIndex] = ((DateWritable) writableCol).getDays();
           lcv.isNull[rowIndex] = false;
@@ -303,7 +305,7 @@ public class VectorizedBatchUtil {
       }
         break;
       case FLOAT: {
-        DoubleColumnVector dcv = (DoubleColumnVector) batch.cols[off+i];
+        DoubleColumnVector dcv = (DoubleColumnVector) batch.cols[off + i];
         if (writableCol != null) {
           dcv.vector[rowIndex] = ((FloatWritable) writableCol).get();
           dcv.isNull[rowIndex] = false;
@@ -314,7 +316,7 @@ public class VectorizedBatchUtil {
       }
         break;
       case DOUBLE: {
-        DoubleColumnVector dcv = (DoubleColumnVector) batch.cols[off+i];
+        DoubleColumnVector dcv = (DoubleColumnVector) batch.cols[off + i];
         if (writableCol != null) {
           dcv.vector[rowIndex] = ((DoubleWritable) writableCol).get();
           dcv.isNull[rowIndex] = false;
@@ -325,7 +327,7 @@ public class VectorizedBatchUtil {
       }
         break;
       case TIMESTAMP: {
-        LongColumnVector lcv = (LongColumnVector) batch.cols[off+i];
+        LongColumnVector lcv = (LongColumnVector) batch.cols[off + i];
         if (writableCol != null) {
           Timestamp t = ((TimestampWritable) writableCol).getTimestamp();
           lcv.vector[rowIndex] = TimestampUtils.getTimeNanoSec(t);
@@ -336,8 +338,27 @@ public class VectorizedBatchUtil {
         }
       }
         break;
+      case BINARY: {
+        BytesColumnVector bcv = (BytesColumnVector) batch.cols[off + i];
+        if (writableCol != null) {
+            bcv.isNull[rowIndex] = false;
+            BytesWritable bw = (BytesWritable) writableCol;
+            byte[] bytes = bw.getBytes();
+            int start = buffer.getLength();
+            int length = bytes.length;
+            try {
+              buffer.write(bytes, 0, length);
+            } catch (IOException ioe) {
+              throw new IllegalStateException("bad write", ioe);
+            }
+            bcv.setRef(rowIndex, buffer.getData(), start, length);
+        } else {
+          setNullColIsNullValue(bcv, rowIndex);
+        }
+      }
+        break;
       case STRING: {
-        BytesColumnVector bcv = (BytesColumnVector) batch.cols[off+i];
+        BytesColumnVector bcv = (BytesColumnVector) batch.cols[off + i];
         if (writableCol != null) {
           bcv.isNull[rowIndex] = false;
           Text colText = (Text) writableCol;
@@ -355,7 +376,7 @@ public class VectorizedBatchUtil {
       }
         break;
       case DECIMAL:
-        DecimalColumnVector dcv = (DecimalColumnVector) batch.cols[off+i];
+        DecimalColumnVector dcv = (DecimalColumnVector) batch.cols[off + i];
         if (writableCol != null) {
           dcv.isNull[rowIndex] = false;
           HiveDecimalWritable wobj = (HiveDecimalWritable) writableCol;

Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedColumnarSerDe.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedColumnarSerDe.java?rev=1617652&r1=1617651&r2=1617652&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedColumnarSerDe.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedColumnarSerDe.java Wed Aug 13 02:28:54 2014
@@ -152,13 +152,20 @@ public class VectorizedColumnarSerDe ext
                 ByteBuffer b = Text.encode(String.valueOf(dcv.vector[rowIndex]));
                 serializeVectorStream.write(b.array(), 0, b.limit());
                 break;
-              case STRING:
+              case BINARY: {
+                BytesColumnVector bcv = (BytesColumnVector) batch.cols[k];
+                byte[] bytes = bcv.vector[rowIndex];
+                serializeVectorStream.write(bytes, 0, bytes.length);
+              }
+                break;
+              case STRING: {
                 BytesColumnVector bcv = (BytesColumnVector) batch.cols[k];
                 LazyUtils.writeEscaped(serializeVectorStream, bcv.vector[rowIndex],
                     bcv.start[rowIndex],
                     bcv.length[rowIndex],
                     serdeParams.isEscaped(), serdeParams.getEscapeChar(), serdeParams
                         .getNeedsEscape());
+              }
                 break;
               case TIMESTAMP:
                 LongColumnVector tcv = (LongColumnVector) batch.cols[k];

Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java?rev=1617652&r1=1617651&r2=1617652&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java Wed Aug 13 02:28:54 2014
@@ -278,7 +278,7 @@ public class VectorizedRowBatchCtx {
         case PRIMITIVE: {
           PrimitiveObjectInspector poi = (PrimitiveObjectInspector) foi;
           // Vectorization currently only supports the following data types:
-          // BOOLEAN, BYTE, SHORT, INT, LONG, FLOAT, DOUBLE, STRING, TIMESTAMP,
+          // BOOLEAN, BYTE, SHORT, INT, LONG, FLOAT, DOUBLE, BINARY, STRING, TIMESTAMP,
           // DATE and DECIMAL
           switch (poi.getPrimitiveCategory()) {
           case BOOLEAN:
@@ -294,6 +294,7 @@ public class VectorizedRowBatchCtx {
           case DOUBLE:
             result.cols[j] = new DoubleColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
             break;
+          case BINARY:
           case STRING:
             result.cols[j] = new BytesColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
             break;
@@ -404,7 +405,7 @@ public class VectorizedRowBatchCtx {
             lcv.isNull[0] = true;
             lcv.isRepeating = true;
           } else { 
-            lcv.fill((Boolean)value == true ? 1 : 0);
+            lcv.fill((Boolean) value == true ? 1 : 0);
             lcv.isNull[0] = false;
           }
         }
@@ -417,7 +418,7 @@ public class VectorizedRowBatchCtx {
             lcv.isNull[0] = true;
             lcv.isRepeating = true;
           } else { 
-            lcv.fill((Byte)value);
+            lcv.fill((Byte) value);
             lcv.isNull[0] = false;
           }
         }
@@ -430,7 +431,7 @@ public class VectorizedRowBatchCtx {
             lcv.isNull[0] = true;
             lcv.isRepeating = true;
           } else { 
-            lcv.fill((Short)value);
+            lcv.fill((Short) value);
             lcv.isNull[0] = false;
           }
         }
@@ -443,7 +444,7 @@ public class VectorizedRowBatchCtx {
             lcv.isNull[0] = true;
             lcv.isRepeating = true;
           } else { 
-            lcv.fill((Integer)value);
+            lcv.fill((Integer) value);
             lcv.isNull[0] = false;
           }          
         }
@@ -456,7 +457,7 @@ public class VectorizedRowBatchCtx {
             lcv.isNull[0] = true;
             lcv.isRepeating = true;
           } else { 
-            lcv.fill((Long)value);
+            lcv.fill((Long) value);
             lcv.isNull[0] = false;
           }          
         }
@@ -469,7 +470,7 @@ public class VectorizedRowBatchCtx {
             lcv.isNull[0] = true;
             lcv.isRepeating = true;
           } else { 
-            lcv.fill(((Date)value).getTime());
+            lcv.fill(((Date) value).getTime());
             lcv.isNull[0] = false;
           }          
         }
@@ -521,17 +522,31 @@ public class VectorizedRowBatchCtx {
             dv.isNull[0] = true;
             dv.isRepeating = true;
           } else {
-            HiveDecimal hd = (HiveDecimal)(value);
-            dv.vector[0] = new Decimal128(hd.toString(), (short)hd.scale());
+            HiveDecimal hd = (HiveDecimal) value;
+            dv.vector[0] = new Decimal128(hd.toString(), (short) hd.scale());
             dv.isRepeating = true;
             dv.isNull[0] = false;      
           }
         }
         break;
-          
+
+        case BINARY: {
+            BytesColumnVector bcv = (BytesColumnVector) batch.cols[colIndex];
+            byte[] bytes = (byte[]) value;
+            if (bytes == null) {
+              bcv.noNulls = false;
+              bcv.isNull[0] = true;
+              bcv.isRepeating = true;
+            } else {
+              bcv.fill(bytes);
+              bcv.isNull[0] = false;
+            }
+          }
+          break;
+
         case STRING: {
           BytesColumnVector bcv = (BytesColumnVector) batch.cols[colIndex];
-          String sVal = (String)value;
+          String sVal = (String) value;
           if (sVal == null) {
             bcv.noNulls = false;
             bcv.isNull[0] = true;

Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ConstantVectorExpression.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ConstantVectorExpression.java?rev=1617652&r1=1617651&r2=1617652&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ConstantVectorExpression.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ConstantVectorExpression.java Wed Aug 13 02:28:54 2014
@@ -187,13 +187,14 @@ public class ConstantVectorExpression ex
 
   public void setTypeString(String typeString) {
     this.outputType = typeString;
-    if ("string".equalsIgnoreCase(typeString)) {
+    if (VectorizationContext.isStringFamily(typeString)) {
       this.type = Type.BYTES;
-    } else if ("double".equalsIgnoreCase(typeString)) {
+    } else if (VectorizationContext.isFloatFamily(typeString)) {
       this.type = Type.DOUBLE;
-    } else if (VectorizationContext.decimalTypePattern.matcher(typeString).matches()){
+    } else if (VectorizationContext.isDecimalFamily(typeString)){
       this.type = Type.DECIMAL;
     } else {
+      // everything else that does not belong to string, double, decimal is treated as long.	
       this.type = Type.LONG;
     }
   }

Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java?rev=1617652&r1=1617651&r2=1617652&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java Wed Aug 13 02:28:54 2014
@@ -50,25 +50,12 @@ import org.apache.hadoop.hive.serde2.obj
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableHiveVarcharObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableIntObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableLongObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableHiveVarcharObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableTimestampObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableShortObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableStringObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableTimestampObjectInspector;
-import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.VoidObjectInspector;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
-import org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo;
-import org.apache.hadoop.io.BooleanWritable;
-import org.apache.hadoop.io.FloatWritable;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
 
 /**
  * VectorExpressionWritableFactory helper class for generating VectorExpressionWritable objects.
@@ -364,7 +351,6 @@ public final class VectorExpressionWrite
      */
     public static VectorExpressionWriter genVectorExpressionWritable(ExprNodeDesc nodeDesc)
       throws HiveException {
-      String nodeType = nodeDesc.getTypeString();
       ObjectInspector objectInspector = nodeDesc.getWritableObjectInspector();
       if (null == objectInspector) {
         objectInspector = TypeInfoUtils
@@ -408,6 +394,9 @@ public final class VectorExpressionWrite
           case LONG:
             return genVectorExpressionWritableLong(
                 (SettableLongObjectInspector) fieldObjInspector);
+          case VOID:
+              return genVectorExpressionWritableVoid(
+                  (VoidObjectInspector) fieldObjInspector);        	  
           case BINARY:
             return genVectorExpressionWritableBinary(
                 (SettableBinaryObjectInspector) fieldObjInspector);
@@ -722,6 +711,39 @@ public final class VectorExpressionWrite
     }.init(fieldObjInspector);
   }
 
+  private static VectorExpressionWriter genVectorExpressionWritableVoid(
+	      VoidObjectInspector fieldObjInspector) throws HiveException {
+	    return new VectorExpressionWriterLong() {
+	      private Object obj;
+	      
+	      public VectorExpressionWriter init(VoidObjectInspector objInspector) 
+	          throws HiveException {
+	        super.init(objInspector);
+	        this.obj = initValue(null);
+	        return this;
+	      }
+	      
+	      @Override
+	      public Object writeValue(long value) throws HiveException {
+	        return this.obj;
+	      }
+	      
+	      @Override
+	      public Object setValue(Object field, long value) throws HiveException {
+	        if (null == field) {
+	          field = initValue(null);
+	        }
+	        return field;
+	      }
+
+	      @Override
+	      public Object initValue(Object ignored) {
+	        return ((VoidObjectInspector) this.objectInspector).copyObject(null);
+	      }
+	    }.init(fieldObjInspector);
+	  }
+  
+  
   private static VectorExpressionWriter genVectorExpressionWritableInt(
       SettableIntObjectInspector fieldObjInspector) throws HiveException {
     return new VectorExpressionWriterLong() {



Mime
View raw message