hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ser...@apache.org
Subject [38/52] [abbrv] hive git commit: HIVE-11621 Fix TestMiniTezCliDriver test failures when HBase Metastore is used (Daniel Dai via gates)
Date Tue, 22 Sep 2015 21:32:19 GMT
HIVE-11621 Fix TestMiniTezCliDriver test failures when HBase Metastore is used (Daniel Dai via gates)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f014f0da
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f014f0da
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f014f0da

Branch: refs/heads/llap
Commit: f014f0da457d1e959cb7da2824f6cf2c5ee0c971
Parents: 4d66206
Author: Alan Gates <gates@hortonworks.com>
Authored: Fri Aug 28 10:38:49 2015 -0700
Committer: Alan Gates <gates@hortonworks.com>
Committed: Fri Aug 28 10:38:49 2015 -0700

----------------------------------------------------------------------
 .../apache/hadoop/hive/ant/QTestGenTask.java    |  11 ++
 data/conf/tez/hive-site.xml                     |   9 ++
 itests/qtest/pom.xml                            |   3 +-
 .../org/apache/hadoop/hive/ql/QTestUtil.java    |  80 +++++-------
 .../hadoop/hive/metastore/HiveAlterHandler.java |  22 ++--
 .../hive/metastore/hbase/HBaseReadWrite.java    |  17 ++-
 .../hadoop/hive/metastore/hbase/HBaseStore.java | 127 +++++++++++++++----
 .../hadoop/hive/metastore/hbase/HBaseUtils.java |  60 ++++++---
 .../hadoop/hive/metastore/hbase/StatsCache.java |   2 +-
 .../stats/ColumnStatsAggregatorFactory.java     |   9 +-
 .../apache/hadoop/hive/ql/stats/StatsUtils.java |   3 +
 .../dynpart_sort_opt_vectorization.q            |   2 +
 .../clientpositive/dynpart_sort_optimization.q  |   2 +
 .../tez/dynpart_sort_opt_vectorization.q.out    |  12 +-
 .../tez/dynpart_sort_optimization.q.out         |  12 +-
 ql/src/test/templates/TestCliDriver.vm          |   3 +-
 16 files changed, 248 insertions(+), 126 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/f014f0da/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java
----------------------------------------------------------------------
diff --git a/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java b/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java
index 7fffe13..8b1c4fe 100644
--- a/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java
+++ b/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java
@@ -149,6 +149,8 @@ public class QTestGenTask extends Task {
 
   private String cleanupScript;
 
+  private String useHBaseMetastore;
+
   public void setHadoopVersion(String ver) {
     this.hadoopVersion = ver;
   }
@@ -221,6 +223,14 @@ public class QTestGenTask extends Task {
     this.cleanupScript = cleanupScript;
   }
 
+  public String getUseHBaseMetastore() {
+    return useHBaseMetastore;
+  }
+
+  public void setUseHBaseMetastore(String useHBaseMetastore) {
+    this.useHBaseMetastore = useHBaseMetastore;
+  }
+
   public void setHiveRootDirectory(File hiveRootDirectory) {
     try {
       this.hiveRootDirectory = hiveRootDirectory.getCanonicalPath();
@@ -530,6 +540,7 @@ public class QTestGenTask extends Task {
       ctx.put("hadoopVersion", hadoopVersion);
       ctx.put("initScript", initScript);
       ctx.put("cleanupScript", cleanupScript);
+      ctx.put("useHBaseMetastore", useHBaseMetastore);
 
       File outFile = new File(outDir, className + ".java");
       FileWriter writer = new FileWriter(outFile);

http://git-wip-us.apache.org/repos/asf/hive/blob/f014f0da/data/conf/tez/hive-site.xml
----------------------------------------------------------------------
diff --git a/data/conf/tez/hive-site.xml b/data/conf/tez/hive-site.xml
index e0238aa..bcda3ea 100644
--- a/data/conf/tez/hive-site.xml
+++ b/data/conf/tez/hive-site.xml
@@ -253,5 +253,14 @@
   </description>
 </property>
 
+<property>
+  <name>hive.metastore.fastpath</name>
+  <value>true</value>
+</property>
+
+<property>
+  <name>hive.metastore.rawstore.impl</name>
+  <value>org.apache.hadoop.hive.metastore.hbase.HBaseStore</value>
+</property>
 
 </configuration>

http://git-wip-us.apache.org/repos/asf/hive/blob/f014f0da/itests/qtest/pom.xml
----------------------------------------------------------------------
diff --git a/itests/qtest/pom.xml b/itests/qtest/pom.xml
index c2cb2f6..664068b 100644
--- a/itests/qtest/pom.xml
+++ b/itests/qtest/pom.xml
@@ -509,7 +509,8 @@
                               logDirectory="${project.build.directory}/qfile-results/clientpositive/"
                               hadoopVersion="${active.hadoop.version}"
                               initScript="${initScript}"
-                              cleanupScript="q_test_cleanup.sql"/>
+                              cleanupScript="q_test_cleanup.sql"
+                              useHBaseMetastore="true"/>
 
                     <qtestgen hiveRootDirectory="${basedir}/${hive.path.to.root}/"
                               outputDirectory="${project.build.directory}/generated-test-sources/java/org/apache/hadoop/hive/cli/"

http://git-wip-us.apache.org/repos/asf/hive/blob/f014f0da/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index 358fdbd..9f112ad 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -84,7 +84,6 @@ import org.apache.hadoop.hive.metastore.MetaStoreUtils;
 import org.apache.hadoop.hive.metastore.Warehouse;
 import org.apache.hadoop.hive.metastore.api.Index;
 import org.apache.hadoop.hive.metastore.hbase.HBaseReadWrite;
-import org.apache.hadoop.hive.metastore.hbase.TephraHBaseConnection;
 import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
 import org.apache.hadoop.hive.ql.exec.Task;
 import org.apache.hadoop.hive.ql.exec.Utilities;
@@ -109,6 +108,8 @@ import org.apache.hadoop.hive.shims.HadoopShims;
 import org.apache.hadoop.hive.shims.ShimLoader;
 import org.apache.hadoop.util.Shell;
 import org.apache.hive.common.util.StreamPrinter;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
 import org.apache.tools.ant.BuildException;
 import org.apache.zookeeper.WatchedEvent;
 import org.apache.zookeeper.Watcher;
@@ -166,12 +167,12 @@ public class QTestUtil {
 
   private final String initScript;
   private final String cleanupScript;
+  private boolean useHBaseMetastore = false;
 
   public interface SuiteAddTestFunctor {
     public void addTestToSuite(TestSuite suite, Object setup, String tName);
   }
   private HBaseTestingUtility utility;
-  private boolean snapshotTaken = false;
 
   static {
     for (String srcTable : System.getProperty("test.src.tables", "").trim().split(",")) {
@@ -348,61 +349,46 @@ public class QTestUtil {
     return "jceks://file" + new Path(keyDir, "test.jks").toUri();
   }
 
-  private void rebuildHBase() throws Exception {
-    HBaseAdmin admin = utility.getHBaseAdmin();
-    if (!snapshotTaken) {
-      for (String tableName : HBaseReadWrite.tableNames) {
-        List<byte[]> families = HBaseReadWrite.columnFamilies.get(tableName);
-        HTableDescriptor desc = new HTableDescriptor(
-            TableName.valueOf(tableName));
-        for (byte[] family : families) {
-          HColumnDescriptor columnDesc = new HColumnDescriptor(family);
-          desc.addFamily(columnDesc);
-        }
-        try {
-          admin.disableTable(tableName);
-          admin.deleteTable(tableName);
-        } catch (IOException e) {
-          System.out.println(e.getMessage());
-        }
-        admin.createTable(desc);
-      }
-    } else {
-      for (String tableName : HBaseReadWrite.tableNames) {
-        admin.disableTable(tableName);
-        admin.restoreSnapshot("snapshot_" + tableName);
-        admin.enableTable(tableName);
-      }
-      try {
-        db.createDatabase(new org.apache.hadoop.hive.metastore.api.Database(
-            DEFAULT_DATABASE_NAME, DEFAULT_DATABASE_COMMENT, new Warehouse(conf)
-                .getDefaultDatabasePath(DEFAULT_DATABASE_NAME).toString(), null));
-      } catch (Exception e) {
-        // Ignore if default database already exist
-      }
-      SessionState.get().setCurrentDatabase(DEFAULT_DATABASE_NAME);
-    }
-    admin.close();
-  }
-
   private void startMiniHBaseCluster() throws Exception {
     utility = new HBaseTestingUtility();
     utility.startMiniCluster();
     conf = new HiveConf(utility.getConfiguration(), Driver.class);
-    rebuildHBase();
+    conf = new HiveConf(utility.getConfiguration(), Driver.class);
+    HBaseAdmin admin = utility.getHBaseAdmin();
+    for (String tableName : HBaseReadWrite.tableNames) {
+      List<byte[]> families = HBaseReadWrite.columnFamilies.get(tableName);
+      HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
+      for (byte[] family : families) {
+        HColumnDescriptor columnDesc = new HColumnDescriptor(family);
+        desc.addFamily(columnDesc);
+      }
+      admin.createTable(desc);
+    }
+    admin.close();
     HBaseReadWrite.getInstance(conf);
   }
 
   public QTestUtil(String outDir, String logDir, MiniClusterType clusterType,
       String confDir, String hadoopVer, String initScript, String cleanupScript)
     throws Exception {
+    this(outDir, logDir, clusterType, confDir, hadoopVer, initScript, cleanupScript, false);
+  }
+  public QTestUtil(String outDir, String logDir, MiniClusterType clusterType,
+      String confDir, String hadoopVer, String initScript, String cleanupScript, boolean useHBaseMetastore)
+    throws Exception {
     this.outDir = outDir;
     this.logDir = logDir;
+    this.useHBaseMetastore = useHBaseMetastore;
+
+    Logger hadoopLog = Logger.getLogger("org.apache.hadoop");
+    hadoopLog.setLevel(Level.INFO);
     if (confDir != null && !confDir.isEmpty()) {
       HiveConf.setHiveSiteLocation(new URL("file://"+ new File(confDir).toURI().getPath() + "/hive-site.xml"));
       System.out.println("Setting hive-site: "+HiveConf.getHiveSiteLocation());
     }
-    startMiniHBaseCluster();
+    if (useHBaseMetastore) {
+      startMiniHBaseCluster();
+    }
     conf = new HiveConf(Driver.class);
     this.hadoopVer = getHadoopMainVersion(hadoopVer);
     qMap = new TreeMap<String, String>();
@@ -491,7 +477,9 @@ public class QTestUtil {
         sparkSession = null;
       }
     }
-    utility.shutdownMiniCluster();
+    if (useHBaseMetastore) {
+      utility.shutdownMiniCluster();
+    }
     if (mr != null) {
       mr.shutdown();
       mr = null;
@@ -779,8 +767,6 @@ public class QTestUtil {
       return;
     }
 
-    rebuildHBase();
-
     clearTablesCreatedDuringTests();
     clearKeysCreatedInTests();
 
@@ -880,12 +866,6 @@ public class QTestUtil {
     cliDriver.processLine(initCommands);
 
     conf.setBoolean("hive.test.init.phase", false);
-
-    HBaseAdmin admin = utility.getHBaseAdmin();
-    for (String tableName : HBaseReadWrite.tableNames) {
-      admin.snapshot("snapshot_" + tableName, tableName);
-    }
-    snapshotTaken = true;
   }
 
   public void init() throws Exception {

http://git-wip-us.apache.org/repos/asf/hive/blob/f014f0da/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
index f402f73..d9382ff 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
@@ -668,17 +668,19 @@ public class HiveAlterHandler implements AlterHandler {
           }
 
           List<ColumnStatisticsObj> statsObjs = cs.getStatsObj();
-          for (ColumnStatisticsObj statsObj : statsObjs) {
-            boolean found = false;
-            for (FieldSchema newCol : newCols) {
-              if (statsObj.getColName().equalsIgnoreCase(newCol.getName())
-                  && statsObj.getColType().equals(newCol.getType())) {
-                found = true;
-                break;
+          if (statsObjs != null) {
+            for (ColumnStatisticsObj statsObj : statsObjs) {
+              boolean found = false;
+              for (FieldSchema newCol : newCols) {
+                if (statsObj.getColName().equalsIgnoreCase(newCol.getName())
+                    && statsObj.getColType().equals(newCol.getType())) {
+                  found = true;
+                  break;
+                }
+              }
+              if (!found) {
+                msdb.deleteTableColumnStatistics(dbName, tableName, statsObj.getColName());
               }
-            }
-            if (!found) {
-              msdb.deleteTableColumnStatistics(dbName, tableName, statsObj.getColName());
             }
           }
         }

http://git-wip-us.apache.org/repos/asf/hive/blob/f014f0da/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
index f1336dc..8a1448c 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
@@ -680,7 +680,13 @@ public class HBaseReadWrite {
         firstStar = i;
         break;
       } else {
-        keyElements.add(partVals.get(i));
+        // empty string equals to null partition,
+        // means star
+        if (partVals.get(i).equals("")) {
+          break;
+        } else {
+          keyElements.add(partVals.get(i));
+        }
       }
     }
 
@@ -693,7 +699,7 @@ public class HBaseReadWrite {
     }
     keyPrefix = HBaseUtils.buildPartitionKey(dbName, tableName,
         HBaseUtils.getPartitionKeyTypes(table.getPartitionKeys().subList(0, keyElements.size()-2)),
-          keyElements.subList(0, keyElements.size()-2));
+          keyElements.subList(2, keyElements.size()));
 
     // Now, build a filter out of the remaining keys
     List<PartitionKeyComparator.Range> ranges = new ArrayList<PartitionKeyComparator.Range>();
@@ -809,7 +815,7 @@ public class HBaseReadWrite {
     for (int i = 0; i < numToFetch && iter.hasNext(); i++) {
       Result result = iter.next();
       HBaseUtils.StorageDescriptorParts sdParts = HBaseUtils.deserializePartition(dbName, tableName,
-          tablePartitions, result.getRow(), result.getValue(CATALOG_CF, CATALOG_COL));
+          tablePartitions, result.getRow(), result.getValue(CATALOG_CF, CATALOG_COL), staticConf);
       StorageDescriptor sd = getStorageDescriptor(sdParts.sdHash);
       HBaseUtils.assembleStorageDescriptor(sd, sdParts);
       parts.add(sdParts.containingPartition);
@@ -1604,9 +1610,10 @@ public class HBaseReadWrite {
             // recontruct the key.  We have to pull the dbName and tableName out of the key to
             // find the partition values.
             byte[] key = results[i].getRow();
-            String[] reconstructedKey = HBaseUtils.parseKey(key);
+            List<String> reconstructedKey = HBaseUtils.parseKey(key, HBaseUtils.getPartitionNames(getTable(dbName, tblName).getPartitionKeys()),
+                HBaseUtils.getPartitionKeyTypes(getTable(dbName, tblName).getPartitionKeys()));
             List<String> reconstructedPartVals =
-                Arrays.asList(reconstructedKey).subList(2, reconstructedKey.length);
+                reconstructedKey.subList(2, reconstructedKey.size());
             String partName = valToPartMap.get(reconstructedPartVals);
             assert partName != null;
             csd.setIsTblLevel(false);

http://git-wip-us.apache.org/repos/asf/hive/blob/f014f0da/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
index f30fcab..568a347 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
@@ -20,15 +20,18 @@ package org.apache.hadoop.hive.metastore.hbase;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.cache.CacheLoader;
+
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.common.FileUtils;
+import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.HiveMetaStore;
 import org.apache.hadoop.hive.metastore.PartFilterExprUtil;
 import org.apache.hadoop.hive.metastore.PartitionExpressionProxy;
 import org.apache.hadoop.hive.metastore.RawStore;
+import org.apache.hadoop.hive.metastore.Warehouse;
 import org.apache.hadoop.hive.metastore.api.AggrStats;
 import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
 import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId;
@@ -64,6 +67,9 @@ import org.apache.hadoop.hive.metastore.hbase.HBaseFilterPlanUtil.PlanResult;
 import org.apache.hadoop.hive.metastore.hbase.HBaseFilterPlanUtil.ScanPlan;
 import org.apache.hadoop.hive.metastore.parser.ExpressionTree;
 import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hive.common.util.HiveStringUtils;
 import org.apache.thrift.TException;
 
 import java.io.IOException;
@@ -71,6 +77,7 @@ import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
@@ -150,7 +157,7 @@ public class HBaseStore implements RawStore {
     boolean commit = false;
     openTransaction();
     try {
-      Database db = getHBase().getDb(name);
+      Database db = getHBase().getDb(HiveStringUtils.normalizeIdentifier(name));
       if (db == null) {
         throw new NoSuchObjectException("Unable to find db " + name);
       }
@@ -169,7 +176,7 @@ public class HBaseStore implements RawStore {
     boolean commit = false;
     openTransaction();
     try {
-      getHBase().deleteDb(dbname);
+      getHBase().deleteDb(HiveStringUtils.normalizeIdentifier(dbname));
       commit = true;
       return true;
     } catch (IOException e) {
@@ -259,7 +266,8 @@ public class HBaseStore implements RawStore {
     boolean commit = false;
     openTransaction();
     try {
-      getHBase().deleteTable(dbName, tableName);
+      getHBase().deleteTable(HiveStringUtils.normalizeIdentifier(dbName),
+          HiveStringUtils.normalizeIdentifier(tableName));
       commit = true;
       return true;
     } catch (IOException e) {
@@ -275,7 +283,8 @@ public class HBaseStore implements RawStore {
     boolean commit = false;
     openTransaction();
     try {
-      Table table = getHBase().getTable(dbName, tableName);
+      Table table = getHBase().getTable(HiveStringUtils.normalizeIdentifier(dbName),
+          HiveStringUtils.normalizeIdentifier(tableName));
       if (table == null) {
         LOG.debug("Unable to find table " + tableNameForErrorMsg(dbName, tableName));
       }
@@ -334,7 +343,8 @@ public class HBaseStore implements RawStore {
     boolean commit = false;
     openTransaction();
     try {
-      Partition part = getHBase().getPartition(dbName, tableName, part_vals);
+      Partition part = getHBase().getPartition(HiveStringUtils.normalizeIdentifier(dbName),
+          HiveStringUtils.normalizeIdentifier(tableName), part_vals);
       if (part == null) {
         throw new NoSuchObjectException("Unable to find partition " +
             partNameForErrorMsg(dbName, tableName, part_vals));
@@ -355,7 +365,8 @@ public class HBaseStore implements RawStore {
     boolean commit = false;
     openTransaction();
     try {
-      boolean exists = getHBase().getPartition(dbName, tableName, part_vals) != null;
+      boolean exists = getHBase().getPartition(HiveStringUtils.normalizeIdentifier(dbName),
+          HiveStringUtils.normalizeIdentifier(tableName), part_vals) != null;
       commit = true;
       return exists;
     } catch (IOException e) {
@@ -372,9 +383,11 @@ public class HBaseStore implements RawStore {
     boolean commit = false;
     openTransaction();
     try {
-      getHBase().deletePartition(dbName, tableName, part_vals);
+      getHBase().deletePartition(HiveStringUtils.normalizeIdentifier(dbName),
+          HiveStringUtils.normalizeIdentifier(tableName), part_vals);
       // Drop any cached stats that reference this partitions
-      getHBase().getStatsCache().invalidate(dbName, tableName,
+      getHBase().getStatsCache().invalidate(HiveStringUtils.normalizeIdentifier(dbName),
+          HiveStringUtils.normalizeIdentifier(tableName),
           buildExternalPartName(dbName, tableName, part_vals));
       commit = true;
       return true;
@@ -393,7 +406,8 @@ public class HBaseStore implements RawStore {
     boolean commit = false;
     openTransaction();
     try {
-      List<Partition> parts = getHBase().scanPartitionsInTable(dbName, tableName, max);
+      List<Partition> parts = getHBase().scanPartitionsInTable(HiveStringUtils.normalizeIdentifier(dbName),
+          HiveStringUtils.normalizeIdentifier(tableName), max);
       commit = true;
       return parts;
     } catch (IOException e) {
@@ -410,7 +424,8 @@ public class HBaseStore implements RawStore {
     boolean commit = false;
     openTransaction();
     try {
-      getHBase().replaceTable(getHBase().getTable(dbname, name), newTable);
+      getHBase().replaceTable(getHBase().getTable(HiveStringUtils.normalizeIdentifier(dbname),
+          HiveStringUtils.normalizeIdentifier(name)), newTable);
       if (newTable.getPartitionKeys() != null && newTable.getPartitionKeys().size() > 0
           && !name.equals(newTable.getTableName())) {
         // They renamed the table, so we need to change each partition as well, since it changes
@@ -443,7 +458,8 @@ public class HBaseStore implements RawStore {
     boolean commit = false;
     openTransaction();
     try {
-      List<Table> tables = getHBase().scanTables(dbName, likeToRegex(pattern));
+      List<Table> tables = getHBase().scanTables(HiveStringUtils.normalizeIdentifier(dbName),
+          likeToRegex(pattern));
       List<String> tableNames = new ArrayList<String>(tables.size());
       for (Table table : tables) tableNames.add(table.getTableName());
       commit = true;
@@ -462,7 +478,12 @@ public class HBaseStore implements RawStore {
     boolean commit = false;
     openTransaction();
     try {
-      List<Table> tables = getHBase().getTables(dbname, tableNames);
+      List<String> normalizedTableNames = new ArrayList<String>(tableNames.size());
+      for (String tableName : tableNames) {
+        normalizedTableNames.add(HiveStringUtils.normalizeIdentifier(tableName));
+      }
+      List<Table> tables = getHBase().getTables(HiveStringUtils.normalizeIdentifier(dbname),
+          normalizedTableNames);
       commit = true;
       return tables;
     } catch (IOException e) {
@@ -491,10 +512,12 @@ public class HBaseStore implements RawStore {
     boolean commit = false;
     openTransaction();
     try {
-      List<Partition> parts = getHBase().scanPartitionsInTable(db_name, tbl_name, max_parts);
+      List<Partition> parts = getHBase().scanPartitionsInTable(HiveStringUtils.normalizeIdentifier(db_name),
+          HiveStringUtils.normalizeIdentifier(tbl_name), max_parts);
       if (parts == null) return null;
       List<String> names = new ArrayList<String>(parts.size());
-      Table table = getHBase().getTable(db_name, tbl_name);
+      Table table = getHBase().getTable(HiveStringUtils.normalizeIdentifier(db_name),
+          HiveStringUtils.normalizeIdentifier(tbl_name));
       for (Partition p : parts) {
         names.add(buildExternalPartName(table, p));
       }
@@ -521,10 +544,12 @@ public class HBaseStore implements RawStore {
     boolean commit = false;
     openTransaction();
     try {
-      Partition oldPart = getHBase().getPartition(db_name, tbl_name, part_vals);
+      Partition oldPart = getHBase().getPartition(HiveStringUtils.normalizeIdentifier(db_name),
+          HiveStringUtils.normalizeIdentifier(tbl_name), part_vals);
       getHBase().replacePartition(oldPart, new_part);
       // Drop any cached stats that reference this partitions
-      getHBase().getStatsCache().invalidate(db_name, tbl_name,
+      getHBase().getStatsCache().invalidate(HiveStringUtils.normalizeIdentifier(db_name),
+          HiveStringUtils.normalizeIdentifier(tbl_name),
           buildExternalPartName(db_name, tbl_name, part_vals));
       commit = true;
     } catch (IOException e) {
@@ -542,11 +567,14 @@ public class HBaseStore implements RawStore {
     boolean commit = false;
     openTransaction();
     try {
-      List<Partition> oldParts = getHBase().getPartitions(db_name, tbl_name,
-          HBaseUtils.getPartitionKeyTypes(getTable(db_name, tbl_name).getPartitionKeys()), part_vals_list);
+      List<Partition> oldParts = getHBase().getPartitions(HiveStringUtils.normalizeIdentifier(db_name),
+          HiveStringUtils.normalizeIdentifier(tbl_name),
+          HBaseUtils.getPartitionKeyTypes(getTable(HiveStringUtils.normalizeIdentifier(db_name),
+          HiveStringUtils.normalizeIdentifier(tbl_name)).getPartitionKeys()), part_vals_list);
       getHBase().replacePartitions(oldParts, new_parts);
       for (List<String> part_vals : part_vals_list) {
-        getHBase().getStatsCache().invalidate(db_name, tbl_name,
+        getHBase().getStatsCache().invalidate(HiveStringUtils.normalizeIdentifier(db_name),
+            HiveStringUtils.normalizeIdentifier(tbl_name),
             buildExternalPartName(db_name, tbl_name, part_vals));
       }
       commit = true;
@@ -604,7 +632,8 @@ public class HBaseStore implements RawStore {
     boolean commit = false;
     openTransaction();
     try {
-      getPartitionsByExprInternal(dbName, tblName, exprTree, maxParts, result);
+      getPartitionsByExprInternal(HiveStringUtils.normalizeIdentifier(dbName),
+          HiveStringUtils.normalizeIdentifier(tblName), exprTree, maxParts, result);
       return result;
     } finally {
       commitOrRoleBack(commit);
@@ -616,22 +645,62 @@ public class HBaseStore implements RawStore {
                                      String defaultPartitionName, short maxParts,
                                      List<Partition> result) throws TException {
     final ExpressionTree exprTree = PartFilterExprUtil.makeExpressionTree(expressionProxy, expr);
-    // TODO: investigate if there should be any role for defaultPartitionName in this
-    // implementation. direct sql code path in ObjectStore does not use it.
-
+    dbName = HiveStringUtils.normalizeIdentifier(dbName);
+    tblName = HiveStringUtils.normalizeIdentifier(tblName);
+    Table table = getTable(dbName, tblName);
     boolean commit = false;
     openTransaction();
     try {
-      return getPartitionsByExprInternal(dbName, tblName, exprTree, maxParts, result);
+      if (exprTree == null) {
+        List<String> partNames = new LinkedList<String>();
+        boolean hasUnknownPartitions = getPartitionNamesPrunedByExprNoTxn(
+            table, expr, defaultPartitionName, maxParts, partNames);
+        result.addAll(getPartitionsByNames(dbName, tblName, partNames));
+        return hasUnknownPartitions;
+      } else {
+        return getPartitionsByExprInternal(dbName, tblName, exprTree, maxParts, result);
+      }
     } finally {
       commitOrRoleBack(commit);
     }
   }
 
+  /**
+   * Gets the partition names from a table, pruned using an expression.
+   * @param table Table.
+   * @param expr Expression.
+   * @param defaultPartName Default partition name from job config, if any.
+   * @param maxParts Maximum number of partition names to return.
+   * @param result The resulting names.
+   * @return Whether the result contains any unknown partitions.
+   * @throws NoSuchObjectException
+   */
+  private boolean getPartitionNamesPrunedByExprNoTxn(Table table, byte[] expr,
+      String defaultPartName, short maxParts, List<String> result) throws MetaException, NoSuchObjectException {
+    List<Partition> parts = getPartitions(
+        table.getDbName(), table.getTableName(), maxParts);
+    for (Partition part : parts) {
+      result.add(Warehouse.makePartName(table.getPartitionKeys(), part.getValues()));
+    }
+    List<String> columnNames = new ArrayList<String>();
+    List<PrimitiveTypeInfo> typeInfos = new ArrayList<PrimitiveTypeInfo>();
+    for (FieldSchema fs : table.getPartitionKeys()) {
+      columnNames.add(fs.getName());
+      typeInfos.add(TypeInfoFactory.getPrimitiveTypeInfo(fs.getType()));
+    }
+    if (defaultPartName == null || defaultPartName.isEmpty()) {
+      defaultPartName = HiveConf.getVar(getConf(), HiveConf.ConfVars.DEFAULTPARTITIONNAME);
+    }
+    return expressionProxy.filterPartitionsByExpr(
+        columnNames, typeInfos, expr, defaultPartName, result);
+  }
+
   private boolean getPartitionsByExprInternal(String dbName, String tblName,
       ExpressionTree exprTree, short maxParts, List<Partition> result) throws MetaException,
       NoSuchObjectException {
 
+    dbName = HiveStringUtils.normalizeIdentifier(dbName);
+    tblName = HiveStringUtils.normalizeIdentifier(tblName);
     Table table = getTable(dbName, tblName);
     if (table == null) {
       throw new NoSuchObjectException("Unable to find table " + dbName + "." + tblName);
@@ -1453,7 +1522,8 @@ public class HBaseStore implements RawStore {
         listPartitionsPsWithAuth(db_name, tbl_name, part_vals, max_parts, null, null);
     List<String> partNames = new ArrayList<String>(parts.size());
     for (Partition part : parts) {
-      partNames.add(buildExternalPartName(db_name, tbl_name, part.getValues()));
+      partNames.add(buildExternalPartName(HiveStringUtils.normalizeIdentifier(db_name),
+          HiveStringUtils.normalizeIdentifier(tbl_name), part.getValues()));
     }
     return partNames;
   }
@@ -1468,7 +1538,8 @@ public class HBaseStore implements RawStore {
     boolean commit = false;
     openTransaction();
     try {
-      List<Partition> parts = getHBase().scanPartitions(db_name, tbl_name, part_vals, max_parts);
+      List<Partition> parts = getHBase().scanPartitions(HiveStringUtils.normalizeIdentifier(db_name),
+          HiveStringUtils.normalizeIdentifier(tbl_name), part_vals, max_parts);
       commit = true;
       return parts;
     } catch (IOException e) {
@@ -1596,7 +1667,7 @@ public class HBaseStore implements RawStore {
               getHBase().getStatsCache().get(dbName, tblName, partNames, colName);
           if (oneCol.getColStatsSize() > 0) {
             assert oneCol.getColStatsSize() == 1;
-            aggrStats.setPartsFound(aggrStats.getPartsFound() + oneCol.getPartsFound());
+            aggrStats.setPartsFound(oneCol.getPartsFound());
             aggrStats.addToColStats(oneCol.getColStats().get(0));
           }
         } catch (CacheLoader.InvalidCacheLoadException e) {
@@ -2204,7 +2275,7 @@ public class HBaseStore implements RawStore {
     List<String> vals = new ArrayList<String>();
     String[] kvp = name.split("/");
     for (String kv : kvp) {
-      vals.add(kv.substring(kv.indexOf('=') + 1));
+      vals.add(FileUtils.unescapePathName(kv.substring(kv.indexOf('=') + 1)));
     }
     return vals;
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/f014f0da/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseUtils.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseUtils.java
index 841afd4..cc90a76 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseUtils.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseUtils.java
@@ -26,6 +26,7 @@ import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.api.AggrStats;
 import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData;
 import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
@@ -67,6 +68,7 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
 import org.apache.hadoop.io.BytesWritable;
 import org.apache.hive.common.util.BloomFilter;
+import org.apache.hive.common.util.HiveStringUtils;
 
 import java.io.IOException;
 import java.nio.charset.Charset;
@@ -115,9 +117,22 @@ class HBaseUtils {
     return protoKey.getBytes(ENCODING);
   }
 
-  static String[] parseKey(byte[] serialized) {
-    String munged = new String(serialized, ENCODING);
-    return munged.split(KEY_SEPARATOR_STR);
+  static List<String> parseKey(byte[] serialized, List<String> partNames, List<String> partTypes) {
+    BinarySortableSerDe serDe = new BinarySortableSerDe();
+    Properties props = new Properties();
+    props.setProperty(serdeConstants.LIST_COLUMNS, "dbName,tableName," + StringUtils.join(partNames, ","));
+    props.setProperty(serdeConstants.LIST_COLUMN_TYPES, "string,string," + StringUtils.join(partTypes, ","));
+    List<String> partVals = null;
+    try {
+      serDe.initialize(new Configuration(), props);
+      List deserializedkeys = ((List)serDe.deserialize(new BytesWritable(serialized)));
+      partVals = new ArrayList<String>();
+      for (Object deserializedkey : deserializedkeys) {
+        partVals.add(deserializedkey.toString());
+      }
+    } catch (SerDeException e) {
+    }
+    return partVals;
   }
 
   private static HbaseMetastoreProto.Parameters buildParameters(Map<String, String> params) {
@@ -240,9 +255,16 @@ class HBaseUtils {
 
   private static PrincipalPrivilegeSet buildPrincipalPrivilegeSet(
       HbaseMetastoreProto.PrincipalPrivilegeSet proto) throws InvalidProtocolBufferException {
-    PrincipalPrivilegeSet pps = new PrincipalPrivilegeSet();
-    pps.setUserPrivileges(convertPrincipalPrivilegeSetEntries(proto.getUsersList()));
-    pps.setRolePrivileges(convertPrincipalPrivilegeSetEntries(proto.getRolesList()));
+    PrincipalPrivilegeSet pps = null;
+    if (!proto.getUsersList().isEmpty() || !proto.getRolesList().isEmpty()) {
+      pps = new PrincipalPrivilegeSet();
+      if (!proto.getUsersList().isEmpty()) {
+        pps.setUserPrivileges(convertPrincipalPrivilegeSetEntries(proto.getUsersList()));
+      }
+      if (!proto.getRolesList().isEmpty()) {
+        pps.setRolePrivileges(convertPrincipalPrivilegeSetEntries(proto.getRolesList()));
+      }
+    }
     return pps;
   }
   /**
@@ -339,7 +361,7 @@ class HBaseUtils {
    */
   static byte[][] serializeDatabase(Database db) {
     byte[][] result = new byte[2][];
-    result[0] = buildKey(db.getName());
+    result[0] = buildKey(HiveStringUtils.normalizeIdentifier(db.getName()));
     HbaseMetastoreProto.Database.Builder builder = HbaseMetastoreProto.Database.newBuilder();
 
     if (db.getDescription() != null) builder.setDescription(db.getDescription());
@@ -696,8 +718,10 @@ class HBaseUtils {
     sd.setNumBuckets(proto.getNumBuckets());
     if (proto.hasSerdeInfo()) {
       SerDeInfo serde = new SerDeInfo();
-      serde.setName(proto.getSerdeInfo().getName());
-      serde.setSerializationLib(proto.getSerdeInfo().getSerializationLib());
+      serde.setName(proto.getSerdeInfo().hasName()?
+          proto.getSerdeInfo().getName():null);
+      serde.setSerializationLib(proto.getSerdeInfo().hasSerializationLib()?
+          proto.getSerdeInfo().getSerializationLib():null);
       serde.setParameters(buildParameters(proto.getSerdeInfo().getParameters()));
       sd.setSerdeInfo(serde);
     }
@@ -848,8 +872,8 @@ class HBaseUtils {
    * @return A struct that contains the partition plus parts of the storage descriptor
    */
   static StorageDescriptorParts deserializePartition(String dbName, String tableName, List<FieldSchema> partitions,
-      byte[] key, byte[] serialized) throws InvalidProtocolBufferException {
-    List keys = deserializePartitionKey(partitions, key);
+      byte[] key, byte[] serialized, Configuration conf) throws InvalidProtocolBufferException {
+    List keys = deserializePartitionKey(partitions, key, conf);
     return deserializePartition(dbName, tableName, keys, serialized);
   }
 
@@ -886,7 +910,8 @@ class HBaseUtils {
     return k.split(KEY_SEPARATOR_STR);
   }
 
-  private static List<String> deserializePartitionKey(List<FieldSchema> partitions, byte[] key) {
+  private static List<String> deserializePartitionKey(List<FieldSchema> partitions, byte[] key,
+      Configuration conf) {
     StringBuffer names = new StringBuffer();
     names.append("dbName,tableName,");
     StringBuffer types = new StringBuffer();
@@ -908,7 +933,8 @@ class HBaseUtils {
       List deserializedkeys = ((List)serDe.deserialize(new BytesWritable(key))).subList(2, partitions.size()+2);
       List<String> partitionKeys = new ArrayList<String>();
       for (Object deserializedKey : deserializedkeys) {
-        partitionKeys.add(deserializedKey.toString());
+        partitionKeys.add(deserializedKey!=null?deserializedKey.toString():
+          HiveConf.getVar(conf, HiveConf.ConfVars.DEFAULTPARTITIONNAME));
       }
       return partitionKeys;
     } catch (SerDeException e) {
@@ -924,7 +950,8 @@ class HBaseUtils {
    */
   static byte[][] serializeTable(Table table, byte[] sdHash) {
     byte[][] result = new byte[2][];
-    result[0] = buildKey(table.getDbName(), table.getTableName());
+    result[0] = buildKey(HiveStringUtils.normalizeIdentifier(table.getDbName()),
+        HiveStringUtils.normalizeIdentifier(table.getTableName()));
     HbaseMetastoreProto.Table.Builder builder = HbaseMetastoreProto.Table.newBuilder();
     if (table.getOwner() != null) builder.setOwner(table.getOwner());
     builder
@@ -952,7 +979,10 @@ class HBaseUtils {
     if (table.getPrivileges() != null) {
       builder.setPrivileges(buildPrincipalPrivilegeSet(table.getPrivileges()));
     }
-    builder.setIsTemporary(table.isTemporary());
+    // Set only if table is temporary
+    if (table.isTemporary()) {
+      builder.setIsTemporary(table.isTemporary());
+    }
     result[1] = builder.build().toByteArray();
     return result;
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/f014f0da/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/StatsCache.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/StatsCache.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/StatsCache.java
index 0d3ed40..42efe94 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/StatsCache.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/StatsCache.java
@@ -99,7 +99,7 @@ class StatsCache {
                   for (ColumnStatisticsObj cso : cs.getStatsObj()) {
                     if (statsObj == null) {
                       statsObj = ColumnStatsAggregatorFactory.newColumnStaticsObj(key.colName,
-                          cso.getStatsData().getSetField());
+                          cso.getColType(), cso.getStatsData().getSetField());
                     }
                     if (aggregator == null) {
                       aggregator = ColumnStatsAggregatorFactory.getColumnStatsAggregator(

http://git-wip-us.apache.org/repos/asf/hive/blob/f014f0da/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/ColumnStatsAggregatorFactory.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/ColumnStatsAggregatorFactory.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/ColumnStatsAggregatorFactory.java
index ebecfe3..a8dbc1f 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/ColumnStatsAggregatorFactory.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/stats/ColumnStatsAggregatorFactory.java
@@ -53,39 +53,34 @@ public class ColumnStatsAggregatorFactory {
     }
   }
 
-  public static ColumnStatisticsObj newColumnStaticsObj(String colName, _Fields type) {
+  public static ColumnStatisticsObj newColumnStaticsObj(String colName, String colType, _Fields type) {
     ColumnStatisticsObj cso = new ColumnStatisticsObj();
     ColumnStatisticsData csd = new ColumnStatisticsData();
     cso.setColName(colName);
+    cso.setColType(colType);
     switch (type) {
     case BOOLEAN_STATS:
       csd.setBooleanStats(new BooleanColumnStatsData());
-      cso.setColType("boolean");
       break;
 
     case LONG_STATS:
       csd.setLongStats(new LongColumnStatsData());
-      cso.setColType("long");
       break;
 
     case DOUBLE_STATS:
       csd.setDoubleStats(new DoubleColumnStatsData());
-      cso.setColType("double");
       break;
 
     case STRING_STATS:
       csd.setStringStats(new StringColumnStatsData());
-      cso.setColType("string");
       break;
 
     case BINARY_STATS:
       csd.setBinaryStats(new BinaryColumnStatsData());
-      cso.setColType("binary");
       break;
 
     case DECIMAL_STATS:
       csd.setDecimalStats(new DecimalColumnStatsData());
-      cso.setColType("decimal");
       break;
 
     default:

http://git-wip-us.apache.org/repos/asf/hive/blob/f014f0da/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
index 55aea0e..351cb2b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
@@ -737,6 +737,9 @@ public class StatsUtils {
   }
 
   private static List<ColStatistics> convertColStats(List<ColumnStatisticsObj> colStats, String tabName) {
+    if (colStats==null) {
+      return new ArrayList<ColStatistics>();
+    }
     List<ColStatistics> stats = new ArrayList<ColStatistics>(colStats.size());
     for (ColumnStatisticsObj statObj : colStats) {
       ColStatistics cs = getColStatistics(statObj, tabName, statObj.getColName());

http://git-wip-us.apache.org/repos/asf/hive/blob/f014f0da/ql/src/test/queries/clientpositive/dynpart_sort_opt_vectorization.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/dynpart_sort_opt_vectorization.q b/ql/src/test/queries/clientpositive/dynpart_sort_opt_vectorization.q
index 8001081..7e94f23 100644
--- a/ql/src/test/queries/clientpositive/dynpart_sort_opt_vectorization.q
+++ b/ql/src/test/queries/clientpositive/dynpart_sort_opt_vectorization.q
@@ -123,6 +123,7 @@ insert overwrite table over1k_part2_orc partition(ds="foo",t) select si,i,b,f,t
 desc formatted over1k_part2_orc partition(ds="foo",t=27);
 desc formatted over1k_part2_orc partition(ds="foo",t="__HIVE_DEFAULT_PARTITION__");
 
+-- SORT_BEFORE_DIFF
 select * from over1k_part2_orc;
 select count(*) from over1k_part2_orc;
 
@@ -132,6 +133,7 @@ insert overwrite table over1k_part2_orc partition(ds="foo",t) select si,i,b,f,t
 desc formatted over1k_part2_orc partition(ds="foo",t=27);
 desc formatted over1k_part2_orc partition(ds="foo",t="__HIVE_DEFAULT_PARTITION__");
 
+-- SORT_BEFORE_DIFF
 select * from over1k_part2_orc;
 select count(*) from over1k_part2_orc;
 

http://git-wip-us.apache.org/repos/asf/hive/blob/f014f0da/ql/src/test/queries/clientpositive/dynpart_sort_optimization.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/dynpart_sort_optimization.q b/ql/src/test/queries/clientpositive/dynpart_sort_optimization.q
index f842efe..ea670e9 100644
--- a/ql/src/test/queries/clientpositive/dynpart_sort_optimization.q
+++ b/ql/src/test/queries/clientpositive/dynpart_sort_optimization.q
@@ -117,6 +117,7 @@ insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from
 desc formatted over1k_part2 partition(ds="foo",t=27);
 desc formatted over1k_part2 partition(ds="foo",t="__HIVE_DEFAULT_PARTITION__");
 
+-- SORT_BEFORE_DIFF
 select * from over1k_part2;
 select count(*) from over1k_part2;
 
@@ -126,6 +127,7 @@ insert overwrite table over1k_part2 partition(ds="foo",t) select si,i,b,f,t from
 desc formatted over1k_part2 partition(ds="foo",t=27);
 desc formatted over1k_part2 partition(ds="foo",t="__HIVE_DEFAULT_PARTITION__");
 
+-- SORT_BEFORE_DIFF
 select * from over1k_part2;
 select count(*) from over1k_part2;
 

http://git-wip-us.apache.org/repos/asf/hive/blob/f014f0da/ql/src/test/results/clientpositive/tez/dynpart_sort_opt_vectorization.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/dynpart_sort_opt_vectorization.q.out b/ql/src/test/results/clientpositive/tez/dynpart_sort_opt_vectorization.q.out
index 4451046..f0fc221 100644
--- a/ql/src/test/results/clientpositive/tez/dynpart_sort_opt_vectorization.q.out
+++ b/ql/src/test/results/clientpositive/tez/dynpart_sort_opt_vectorization.q.out
@@ -1835,13 +1835,15 @@ Bucket Columns:     	[]
 Sort Columns:       	[]                  	 
 Storage Desc Params:	 	 
 	serialization.format	1                   
-PREHOOK: query: select * from over1k_part2_orc
+PREHOOK: query: -- SORT_BEFORE_DIFF
+select * from over1k_part2_orc
 PREHOOK: type: QUERY
 PREHOOK: Input: default@over1k_part2_orc
 PREHOOK: Input: default@over1k_part2_orc@ds=foo/t=27
 PREHOOK: Input: default@over1k_part2_orc@ds=foo/t=__HIVE_DEFAULT_PARTITION__
 #### A masked pattern was here ####
-POSTHOOK: query: select * from over1k_part2_orc
+POSTHOOK: query: -- SORT_BEFORE_DIFF
+select * from over1k_part2_orc
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@over1k_part2_orc
 POSTHOOK: Input: default@over1k_part2_orc@ds=foo/t=27
@@ -1980,13 +1982,15 @@ Bucket Columns:     	[]
 Sort Columns:       	[]                  	 
 Storage Desc Params:	 	 
 	serialization.format	1                   
-PREHOOK: query: select * from over1k_part2_orc
+PREHOOK: query: -- SORT_BEFORE_DIFF
+select * from over1k_part2_orc
 PREHOOK: type: QUERY
 PREHOOK: Input: default@over1k_part2_orc
 PREHOOK: Input: default@over1k_part2_orc@ds=foo/t=27
 PREHOOK: Input: default@over1k_part2_orc@ds=foo/t=__HIVE_DEFAULT_PARTITION__
 #### A masked pattern was here ####
-POSTHOOK: query: select * from over1k_part2_orc
+POSTHOOK: query: -- SORT_BEFORE_DIFF
+select * from over1k_part2_orc
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@over1k_part2_orc
 POSTHOOK: Input: default@over1k_part2_orc@ds=foo/t=27

http://git-wip-us.apache.org/repos/asf/hive/blob/f014f0da/ql/src/test/results/clientpositive/tez/dynpart_sort_optimization.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/dynpart_sort_optimization.q.out b/ql/src/test/results/clientpositive/tez/dynpart_sort_optimization.q.out
index cb001b9..8d4c1b7 100644
--- a/ql/src/test/results/clientpositive/tez/dynpart_sort_optimization.q.out
+++ b/ql/src/test/results/clientpositive/tez/dynpart_sort_optimization.q.out
@@ -1735,13 +1735,15 @@ Bucket Columns:     	[]
 Sort Columns:       	[]                  	 
 Storage Desc Params:	 	 
 	serialization.format	1                   
-PREHOOK: query: select * from over1k_part2
+PREHOOK: query: -- SORT_BEFORE_DIFF
+select * from over1k_part2
 PREHOOK: type: QUERY
 PREHOOK: Input: default@over1k_part2
 PREHOOK: Input: default@over1k_part2@ds=foo/t=27
 PREHOOK: Input: default@over1k_part2@ds=foo/t=__HIVE_DEFAULT_PARTITION__
 #### A masked pattern was here ####
-POSTHOOK: query: select * from over1k_part2
+POSTHOOK: query: -- SORT_BEFORE_DIFF
+select * from over1k_part2
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@over1k_part2
 POSTHOOK: Input: default@over1k_part2@ds=foo/t=27
@@ -1880,13 +1882,15 @@ Bucket Columns:     	[]
 Sort Columns:       	[]                  	 
 Storage Desc Params:	 	 
 	serialization.format	1                   
-PREHOOK: query: select * from over1k_part2
+PREHOOK: query: -- SORT_BEFORE_DIFF
+select * from over1k_part2
 PREHOOK: type: QUERY
 PREHOOK: Input: default@over1k_part2
 PREHOOK: Input: default@over1k_part2@ds=foo/t=27
 PREHOOK: Input: default@over1k_part2@ds=foo/t=__HIVE_DEFAULT_PARTITION__
 #### A masked pattern was here ####
-POSTHOOK: query: select * from over1k_part2
+POSTHOOK: query: -- SORT_BEFORE_DIFF
+select * from over1k_part2
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@over1k_part2
 POSTHOOK: Input: default@over1k_part2@ds=foo/t=27

http://git-wip-us.apache.org/repos/asf/hive/blob/f014f0da/ql/src/test/templates/TestCliDriver.vm
----------------------------------------------------------------------
diff --git a/ql/src/test/templates/TestCliDriver.vm b/ql/src/test/templates/TestCliDriver.vm
index ae449c5..01745da 100644
--- a/ql/src/test/templates/TestCliDriver.vm
+++ b/ql/src/test/templates/TestCliDriver.vm
@@ -45,13 +45,14 @@ public class $className extends TestCase {
     String hiveConfDir = "$hiveConfDir";
     String initScript = "$initScript";
     String cleanupScript = "$cleanupScript";
+    boolean useHBaseMetastore = Boolean.valueOf("$useHBaseMetastore");
     try {
       String hadoopVer = "$hadoopVersion";
       if (!hiveConfDir.isEmpty()) {
         hiveConfDir = HIVE_ROOT + hiveConfDir;
       }
       qt = new QTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR,
-      hiveConfDir, hadoopVer, initScript, cleanupScript);
+      hiveConfDir, hadoopVer, initScript, cleanupScript, useHBaseMetastore);
 
       // do a one time initialization
       qt.cleanUp();


Mime
View raw message