hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gunt...@apache.org
Subject svn commit: r1562653 [9/10] - in /hive/trunk: itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/ metastore/if/ metastore/src/gen/thrift/gen-cpp/ metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ metastore/src/gen...
Date Thu, 30 Jan 2014 00:24:29 GMT
Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java?rev=1562653&r1=1562652&r2=1562653&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java Thu Jan 30 00:24:28 2014
@@ -134,6 +134,8 @@ import org.datanucleus.store.rdbms.excep
 
 import org.antlr.runtime.Token;
 
+import com.google.common.collect.Lists;
+
 
 /**
  * This class is the interface between the application logic and the database
@@ -1468,35 +1470,23 @@ public class ObjectStore implements RawS
   }
 
   public List<Partition> getPartitions(
-      String dbName, String tableName, int maxParts) throws MetaException {
+      String dbName, String tableName, int maxParts) throws MetaException, NoSuchObjectException {
     return getPartitionsInternal(dbName, tableName, maxParts, true, true);
   }
 
-  protected List<Partition> getPartitionsInternal(String dbName, String tblName,
-      int maxParts, boolean allowSql, boolean allowJdo) throws MetaException {
-    dbName = dbName.toLowerCase();
-    tblName = tblName.toLowerCase();
-    GetPartsHelper ctx = new GetPartsHelper(dbName, tblName, allowSql, allowJdo);
-    try {
-      ctx.start(false);
-      if (ctx.canUseDirectSql()) {
-        try {
-          Integer max = (maxParts < 0) ? null : maxParts;
-          ctx.setResult(directSql.getPartitions(dbName, tblName, max));
-        } catch (Exception ex) {
-          ctx.handleDirectSqlError(ex);
-        }
-      }
-
-      if (!ctx.canUseDirectSql()) {
-        ctx.setResult(convertToParts(listMPartitions(dbName, tblName, maxParts)));
+  protected List<Partition> getPartitionsInternal(
+      String dbName, String tblName, final int maxParts, boolean allowSql, boolean allowJdo)
+          throws MetaException, NoSuchObjectException {
+    return new GetListHelper<Partition>(dbName, tblName, allowSql, allowJdo) {
+      protected List<Partition> getSqlResult(GetHelper<List<Partition>> ctx) throws MetaException {
+        Integer max = (maxParts < 0) ? null : maxParts;
+        return directSql.getPartitions(dbName, tblName, max);
+      }
+      protected List<Partition> getJdoResult(
+          GetHelper<List<Partition>> ctx) throws MetaException, NoSuchObjectException {
+        return convertToParts(listMPartitions(dbName, tblName, maxParts));
       }
-      return ctx.commit();
-    } catch (NoSuchObjectException ex) {
-      throw new MetaException(ex.getMessage());
-    } finally {
-      ctx.close();
-    }
+    }.run(false);
   }
 
   @Override
@@ -1789,28 +1779,17 @@ public class ObjectStore implements RawS
   }
 
   protected List<Partition> getPartitionsByNamesInternal(String dbName, String tblName,
-      List<String> partNames, boolean allowSql, boolean allowJdo)
+      final List<String> partNames, boolean allowSql, boolean allowJdo)
           throws MetaException, NoSuchObjectException {
-    dbName = dbName.toLowerCase();
-    tblName = tblName.toLowerCase();
-    GetPartsHelper ctx = new GetPartsHelper(dbName, tblName, allowSql, allowJdo);
-    try {
-      ctx.start(false);
-      if (ctx.canUseDirectSql()) {
-        try {
-          ctx.setResult(directSql.getPartitionsViaSqlFilter(dbName, tblName, partNames, null));
-        } catch (Exception ex) {
-          ctx.handleDirectSqlError(ex);
-        }
-      }
-
-      if (!ctx.canUseDirectSql()) {
-        ctx.setResult(getPartitionsViaOrmFilter(dbName, tblName, partNames));
+    return new GetListHelper<Partition>(dbName, tblName, allowSql, allowJdo) {
+      protected List<Partition> getSqlResult(GetHelper<List<Partition>> ctx) throws MetaException {
+        return directSql.getPartitionsViaSqlFilter(dbName, tblName, partNames, null);
+      }
+      protected List<Partition> getJdoResult(
+          GetHelper<List<Partition>> ctx) throws MetaException, NoSuchObjectException {
+        return getPartitionsViaOrmFilter(dbName, tblName, partNames);
       }
-      return ctx.commit();
-    } finally {
-      ctx.close();
-    }
+    }.run(false);
   }
 
   @Override
@@ -1820,8 +1799,8 @@ public class ObjectStore implements RawS
         dbName, tblName, expr, defaultPartitionName, maxParts, result, true, true);
   }
 
-  protected boolean getPartitionsByExprInternal(String dbName, String tblName,
-      byte[] expr, String defaultPartitionName, short maxParts, List<Partition> result,
+  protected boolean getPartitionsByExprInternal(String dbName, String tblName, final byte[] expr,
+      final String defaultPartitionName, final  short maxParts, List<Partition> result,
       boolean allowSql, boolean allowJdo) throws TException {
     assert result != null;
 
@@ -1840,48 +1819,43 @@ public class ObjectStore implements RawS
     //       If forcing everyone to use thick client is out of the question, maybe we could
     //       parse the filter into standard hive expressions and not all this separate tree
     //       Filter.g stuff. That way this method and ...ByFilter would just be merged.
-    ExpressionTree exprTree = makeExpressionTree(filter);
+    final ExpressionTree exprTree = makeExpressionTree(filter);
 
-    dbName = dbName.toLowerCase();
-    tblName = tblName.toLowerCase();
-    GetPartsHelper ctx = new GetPartsHelper(dbName, tblName, allowSql, allowJdo);
-    boolean hasUnknownPartitions = false;
-    try {
-      ctx.start(true);
-      if (ctx.canUseDirectSql()) {
-        try {
-          // If we have some sort of expression tree, try SQL filter pushdown.
-          boolean haveResult = (exprTree != null) && ctx.setResult(
-              directSql.getPartitionsViaSqlFilter(ctx.getTable(), exprTree, null));
-          if (!haveResult) {
-            // We couldn't do SQL filter pushdown. Get names via normal means.
-            List<String> partNames = new LinkedList<String>();
-            hasUnknownPartitions = getPartitionNamesPrunedByExprNoTxn(
-                ctx.getTable(), expr, defaultPartitionName, maxParts, partNames);
-            ctx.setResult(directSql.getPartitionsViaSqlFilter(dbName, tblName, partNames, null));
-          }
-        } catch (Exception ex) {
-          ctx.handleDirectSqlError(ex);
+    final AtomicBoolean hasUnknownPartitions = new AtomicBoolean(false);
+    result.addAll(new GetListHelper<Partition>(dbName, tblName, allowSql, allowJdo) {
+      protected List<Partition> getSqlResult(GetHelper<List<Partition>> ctx) throws MetaException {
+        // If we have some sort of expression tree, try SQL filter pushdown.
+        List<Partition> result = null;
+        if (exprTree != null) {
+          result = directSql.getPartitionsViaSqlFilter(ctx.getTable(), exprTree, null);
+        }
+        if (result == null) {
+          // We couldn't do SQL filter pushdown. Get names via normal means.
+          List<String> partNames = new LinkedList<String>();
+          hasUnknownPartitions.set(getPartitionNamesPrunedByExprNoTxn(
+              ctx.getTable(), expr, defaultPartitionName, maxParts, partNames));
+          result = directSql.getPartitionsViaSqlFilter(dbName, tblName, partNames, null);
         }
+        return result;
       }
-
-      if (!ctx.canUseDirectSql()) {
+      protected List<Partition> getJdoResult(
+          GetHelper<List<Partition>> ctx) throws MetaException, NoSuchObjectException {
         // If we have some sort of expression tree, try JDOQL filter pushdown.
-        boolean haveResult = (exprTree != null) && ctx.setResult(
-            getPartitionsViaOrmFilter(ctx.getTable(), exprTree, maxParts, false));
-        if (!haveResult) {
+        List<Partition> result = null;
+        if (exprTree != null) {
+          result = getPartitionsViaOrmFilter(ctx.getTable(), exprTree, maxParts, false);
+        }
+        if (result == null) {
           // We couldn't do JDOQL filter pushdown. Get names via normal means.
           List<String> partNames = new ArrayList<String>();
-          hasUnknownPartitions = getPartitionNamesPrunedByExprNoTxn(
-              ctx.getTable(), expr, defaultPartitionName, maxParts, partNames);
-          ctx.setResult(getPartitionsViaOrmFilter(dbName, tblName, partNames));
+          hasUnknownPartitions.set(getPartitionNamesPrunedByExprNoTxn(
+              ctx.getTable(), expr, defaultPartitionName, maxParts, partNames));
+          result = getPartitionsViaOrmFilter(dbName, tblName, partNames);
         }
+        return result;
       }
-      result.addAll(ctx.commit());
-    } finally {
-      ctx.close();
-    }
-    return hasUnknownPartitions;
+    }.run(true));
+    return hasUnknownPartitions.get();
   }
 
   private class LikeChecker extends ExpressionTree.TreeVisitor {
@@ -2046,23 +2020,22 @@ public class ObjectStore implements RawS
     return getPartitionsByFilterInternal(dbName, tblName, filter, maxParts, true, true);
   }
 
-  /** Helper class for getting partitions w/transaction, direct SQL, perf logging, etc. */
-  private class GetPartsHelper {
-    private final boolean isInTxn, doTrace, allowSql, allowJdo;
+  /** Helper class for getting stuff w/transaction, direct SQL, perf logging, etc. */
+  private abstract class GetHelper<T> {
+    private final boolean isInTxn, doTrace, allowJdo;
     private boolean doUseDirectSql;
     private long start;
     private Table table;
-    private String dbName = null, tblName = null;
-    boolean success = false;
-    private List<Partition> results = null;
+    protected final String dbName, tblName;
+    private boolean success = false;
+    protected T results = null;
 
-    public GetPartsHelper(String dbName, String tblName, boolean allowSql, boolean allowJdo)
+    public GetHelper(String dbName, String tblName, boolean allowSql, boolean allowJdo)
         throws MetaException {
       assert allowSql || allowJdo;
-      this.allowSql = allowSql;
       this.allowJdo = allowJdo;
-      this.dbName = dbName;
-      this.tblName = tblName;
+      this.dbName = dbName.toLowerCase();
+      this.tblName = tblName.toLowerCase();
       this.doTrace = LOG.isDebugEnabled();
       this.isInTxn = isActiveTransaction();
 
@@ -2077,7 +2050,38 @@ public class ObjectStore implements RawS
       this.doUseDirectSql = allowSql && isConfigEnabled && directSql.isCompatibleDatastore();
     }
 
-    public void start(boolean initTable) throws MetaException, NoSuchObjectException {
+    protected abstract String describeResult();
+    protected abstract T getSqlResult(GetHelper<T> ctx) throws MetaException;
+    protected abstract T getJdoResult(
+        GetHelper<T> ctx) throws MetaException, NoSuchObjectException;
+
+    public T run(boolean initTable) throws MetaException, NoSuchObjectException {
+      try {
+        start(initTable);
+        if (doUseDirectSql) {
+          try {
+            setResult(getSqlResult(this));
+          } catch (Exception ex) {
+            handleDirectSqlError(ex);
+          }
+        }
+        if (!doUseDirectSql) {
+          setResult(getJdoResult(this));
+        }
+        return commit();
+      } catch (NoSuchObjectException ex) {
+        throw ex;
+      } catch (MetaException ex) {
+        throw ex;
+      } catch (Exception ex) {
+        LOG.error("", ex);
+        throw new MetaException(ex.getMessage());
+      } finally {
+        close();
+      }
+    }
+
+    private void start(boolean initTable) throws MetaException, NoSuchObjectException {
       start = doTrace ? System.nanoTime() : 0;
       openTransaction();
       if (initTable) {
@@ -2085,12 +2089,12 @@ public class ObjectStore implements RawS
       }
     }
 
-    public boolean setResult(List<Partition> results) {
+    private boolean setResult(T results) {
       this.results = results;
       return this.results != null;
     }
 
-    public void handleDirectSqlError(Exception ex) throws MetaException, NoSuchObjectException {
+    private void handleDirectSqlError(Exception ex) throws MetaException, NoSuchObjectException {
       LOG.error("Direct SQL failed" + (allowJdo ? ", falling back to ORM" : ""), ex);
       if (!allowJdo) {
         if (ex instanceof MetaException) {
@@ -2115,61 +2119,72 @@ public class ObjectStore implements RawS
       this.doUseDirectSql = false;
     }
 
-    public List<Partition> commit() {
+    private T commit() {
       success = commitTransaction();
       if (doTrace) {
-        LOG.debug(results.size() + " partition retrieved using " + (doUseDirectSql ? "SQL" : "ORM")
+        LOG.debug(describeResult() + " retrieved using " + (doUseDirectSql ? "SQL" : "ORM")
             + " in " + ((System.nanoTime() - start) / 1000000.0) + "ms");
       }
       return results;
     }
 
-    public void close() {
+    private void close() {
       if (!success) {
         rollbackTransaction();
       }
     }
 
-    public boolean canUseDirectSql() {
-      return doUseDirectSql;
-    }
-
     public Table getTable() {
       return table;
     }
   }
 
+  private abstract class GetListHelper<T> extends GetHelper<List<T>> {
+    public GetListHelper(
+        String dbName, String tblName, boolean allowSql, boolean allowJdo) throws MetaException {
+      super(dbName, tblName, allowSql, allowJdo);
+    }
+
+    @Override
+    protected String describeResult() {
+      return results.size() + " entries";
+    }
+  }
+
+  private abstract class GetStatHelper extends GetHelper<ColumnStatistics> {
+    public GetStatHelper(
+        String dbName, String tblName, boolean allowSql, boolean allowJdo) throws MetaException {
+      super(dbName, tblName, allowSql, allowJdo);
+    }
+
+    @Override
+    protected String describeResult() {
+      return "statistics for " + (results == null ? 0 : results.getStatsObjSize()) + " columns";
+    }
+  }
+
   protected List<Partition> getPartitionsByFilterInternal(String dbName, String tblName,
-      String filter, short maxParts, boolean allowSql, boolean allowJdo)
+      String filter, final short maxParts, boolean allowSql, boolean allowJdo)
       throws MetaException, NoSuchObjectException {
-    ExpressionTree tree = (filter != null && !filter.isEmpty())
+    final ExpressionTree tree = (filter != null && !filter.isEmpty())
         ? getFilterParser(filter).tree : ExpressionTree.EMPTY_TREE;
 
-    dbName = dbName.toLowerCase();
-    tblName = tblName.toLowerCase();
-    GetPartsHelper ctx = new GetPartsHelper(dbName, tblName, allowSql, allowJdo);
-    try {
-      ctx.start(true);
-      if (ctx.canUseDirectSql()) {
-        try {
-          if (!ctx.setResult(directSql.getPartitionsViaSqlFilter(
-              ctx.getTable(), tree, (maxParts < 0) ? null : (int)maxParts))) {
-            // Cannot push down SQL filter. The message has been logged internally.
-            // This is not an error so don't roll back, just go to JDO.
-            ctx.disableDirectSql();
-          }
-        } catch (Exception ex) {
-          ctx.handleDirectSqlError(ex);
-        }
+    return new GetListHelper<Partition>(dbName, tblName, allowSql, allowJdo) {
+      protected List<Partition> getSqlResult(GetHelper<List<Partition>> ctx) throws MetaException {
+        List<Partition> parts = directSql.getPartitionsViaSqlFilter(
+            ctx.getTable(), tree, (maxParts < 0) ? null : (int)maxParts);
+        if (parts == null) {
+          // Cannot push down SQL filter. The message has been logged internally.
+          // This is not an error so don't roll back, just go to JDO.
+          ctx.disableDirectSql();
+        }
+        return parts;
+      }
+      protected List<Partition> getJdoResult(
+          GetHelper<List<Partition>> ctx) throws MetaException, NoSuchObjectException {
+        return getPartitionsViaOrmFilter(ctx.getTable(), tree, maxParts, true);
       }
-
-      if (!ctx.canUseDirectSql()) {
-        ctx.setResult(getPartitionsViaOrmFilter(ctx.getTable(), tree, maxParts, true));
-      }
-      return ctx.commit();
-    } finally {
-      ctx.close();
-    }
+    }.run(true);
   }
 
   /**
@@ -2178,14 +2193,19 @@ public class ObjectStore implements RawS
    * @param tblName Table name.
    * @return Table object.
    */
-  private Table ensureGetTable(
+  private MTable ensureGetMTable(
       String dbName, String tblName) throws NoSuchObjectException, MetaException {
     MTable mtable = getMTable(dbName, tblName);
     if (mtable == null) {
       throw new NoSuchObjectException("Specified database/table does not exist : "
           + dbName + "." + tblName);
     }
-    return convertToTable(mtable);
+    return mtable;
+  }
+
+  private Table ensureGetTable(
+      String dbName, String tblName) throws NoSuchObjectException, MetaException {
+    return convertToTable(ensureGetMTable(dbName, tblName));
   }
 
   private FilterParser getFilterParser(String filter) throws MetaException {
@@ -4937,296 +4957,30 @@ public class ObjectStore implements RawS
     }
   }
 
-  // Methods to persist, maintain and retrieve Column Statistics
-  private MTableColumnStatistics convertToMTableColumnStatistics(ColumnStatisticsDesc statsDesc,
-      ColumnStatisticsObj statsObj) throws NoSuchObjectException,
-      MetaException, InvalidObjectException
-  {
-     if (statsObj == null || statsDesc == null) {
-       throw new InvalidObjectException("Invalid column stats object");
-     }
-
-     String dbName = statsDesc.getDbName();
-     String tableName = statsDesc.getTableName();
-     MTable table = getMTable(dbName, tableName);
-
-     if (table == null) {
-       throw new NoSuchObjectException("Table " + tableName +
-       " for which stats is gathered doesn't exist.");
-     }
-
-     MTableColumnStatistics mColStats = new MTableColumnStatistics();
-     mColStats.setTable(table);
-     mColStats.setDbName(statsDesc.getDbName());
-     mColStats.setTableName(statsDesc.getTableName());
-     mColStats.setLastAnalyzed(statsDesc.getLastAnalyzed());
-     mColStats.setColName(statsObj.getColName());
-     mColStats.setColType(statsObj.getColType());
-
-     if (statsObj.getStatsData().isSetBooleanStats()) {
-       BooleanColumnStatsData boolStats = statsObj.getStatsData().getBooleanStats();
-       mColStats.setBooleanStats(boolStats.getNumTrues(), boolStats.getNumFalses(),
-           boolStats.getNumNulls());
-     } else if (statsObj.getStatsData().isSetLongStats()) {
-       LongColumnStatsData longStats = statsObj.getStatsData().getLongStats();
-       mColStats.setLongStats(longStats.getNumNulls(), longStats.getNumDVs(),
-           longStats.getLowValue(), longStats.getHighValue());
-     } else if (statsObj.getStatsData().isSetDoubleStats()) {
-       DoubleColumnStatsData doubleStats = statsObj.getStatsData().getDoubleStats();
-       mColStats.setDoubleStats(doubleStats.getNumNulls(), doubleStats.getNumDVs(),
-           doubleStats.getLowValue(), doubleStats.getHighValue());
-     } else if (statsObj.getStatsData().isSetStringStats()) {
-       StringColumnStatsData stringStats = statsObj.getStatsData().getStringStats();
-       mColStats.setStringStats(stringStats.getNumNulls(), stringStats.getNumDVs(),
-         stringStats.getMaxColLen(), stringStats.getAvgColLen());
-     } else if (statsObj.getStatsData().isSetBinaryStats()) {
-       BinaryColumnStatsData binaryStats = statsObj.getStatsData().getBinaryStats();
-       mColStats.setBinaryStats(binaryStats.getNumNulls(), binaryStats.getMaxColLen(),
-         binaryStats.getAvgColLen());
-     }
-     return mColStats;
-  }
-
-  private ColumnStatisticsObj getTableColumnStatisticsObj(MTableColumnStatistics mStatsObj) {
-    ColumnStatisticsObj statsObj = new ColumnStatisticsObj();
-    statsObj.setColType(mStatsObj.getColType());
-    statsObj.setColName(mStatsObj.getColName());
-    String colType = mStatsObj.getColType().toLowerCase();
-    ColumnStatisticsData colStatsData = new ColumnStatisticsData();
-
-    if (colType.equals("boolean")) {
-      BooleanColumnStatsData boolStats = new BooleanColumnStatsData();
-      boolStats.setNumFalses(mStatsObj.getNumFalses());
-      boolStats.setNumTrues(mStatsObj.getNumTrues());
-      boolStats.setNumNulls(mStatsObj.getNumNulls());
-      colStatsData.setBooleanStats(boolStats);
-    } else if (colType.equals("string") ||
-        colType.startsWith("varchar") || colType.startsWith("char")) {
-      StringColumnStatsData stringStats = new StringColumnStatsData();
-      stringStats.setNumNulls(mStatsObj.getNumNulls());
-      stringStats.setAvgColLen(mStatsObj.getAvgColLen());
-      stringStats.setMaxColLen(mStatsObj.getMaxColLen());
-      stringStats.setNumDVs(mStatsObj.getNumDVs());
-      colStatsData.setStringStats(stringStats);
-    } else if (colType.equals("binary")) {
-      BinaryColumnStatsData binaryStats = new BinaryColumnStatsData();
-      binaryStats.setNumNulls(mStatsObj.getNumNulls());
-      binaryStats.setAvgColLen(mStatsObj.getAvgColLen());
-      binaryStats.setMaxColLen(mStatsObj.getMaxColLen());
-      colStatsData.setBinaryStats(binaryStats);
-    } else if (colType.equals("bigint") || colType.equals("int") ||
-        colType.equals("smallint") || colType.equals("tinyint") ||
-        colType.equals("timestamp")) {
-      LongColumnStatsData longStats = new LongColumnStatsData();
-      longStats.setNumNulls(mStatsObj.getNumNulls());
-      longStats.setHighValue(mStatsObj.getLongHighValue());
-      longStats.setLowValue(mStatsObj.getLongLowValue());
-      longStats.setNumDVs(mStatsObj.getNumDVs());
-      colStatsData.setLongStats(longStats);
-   } else if (colType.equals("double") || colType.equals("float")) {
-     DoubleColumnStatsData doubleStats = new DoubleColumnStatsData();
-     doubleStats.setNumNulls(mStatsObj.getNumNulls());
-     doubleStats.setHighValue(mStatsObj.getDoubleHighValue());
-     doubleStats.setLowValue(mStatsObj.getDoubleLowValue());
-     doubleStats.setNumDVs(mStatsObj.getNumDVs());
-     colStatsData.setDoubleStats(doubleStats);
-   }
-   statsObj.setStatsData(colStatsData);
-   return statsObj;
-  }
-
-  private ColumnStatisticsDesc getTableColumnStatisticsDesc(MTableColumnStatistics mStatsObj) {
-    ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc();
-    statsDesc.setIsTblLevel(true);
-    statsDesc.setDbName(mStatsObj.getDbName());
-    statsDesc.setTableName(mStatsObj.getTableName());
-    statsDesc.setLastAnalyzed(mStatsObj.getLastAnalyzed());
-    return statsDesc;
-  }
-
-  private ColumnStatistics convertToTableColumnStatistics(MTableColumnStatistics mStatsObj)
-    throws MetaException
-  {
-    if (mStatsObj == null) {
-      return null;
-    }
-
-    ColumnStatisticsDesc statsDesc = getTableColumnStatisticsDesc(mStatsObj);
-    ColumnStatisticsObj statsObj = getTableColumnStatisticsObj(mStatsObj);
-    List<ColumnStatisticsObj> statsObjs = new ArrayList<ColumnStatisticsObj>();
-    statsObjs.add(statsObj);
-
-    ColumnStatistics colStats = new ColumnStatistics();
-    colStats.setStatsDesc(statsDesc);
-    colStats.setStatsObj(statsObjs);
-    return colStats;
-  }
-
-  private MPartitionColumnStatistics convertToMPartitionColumnStatistics(ColumnStatisticsDesc
-    statsDesc, ColumnStatisticsObj statsObj, List<String> partVal)
-    throws MetaException, NoSuchObjectException
-  {
-    if (statsDesc == null || statsObj == null || partVal == null) {
-      return null;
-    }
-
-    MPartition partition  = getMPartition(statsDesc.getDbName(), statsDesc.getTableName(), partVal);
-
-    if (partition == null) {
-      throw new NoSuchObjectException("Partition for which stats is gathered doesn't exist.");
-    }
+  private void writeMTableColumnStatistics(Table table, MTableColumnStatistics mStatsObj)
+    throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException {
+    String dbName = mStatsObj.getDbName();
+    String tableName = mStatsObj.getTableName();
+    String colName = mStatsObj.getColName();
 
-    MPartitionColumnStatistics mColStats = new MPartitionColumnStatistics();
-    mColStats.setPartition(partition);
-    mColStats.setDbName(statsDesc.getDbName());
-    mColStats.setTableName(statsDesc.getTableName());
-    mColStats.setPartitionName(statsDesc.getPartName());
-    mColStats.setLastAnalyzed(statsDesc.getLastAnalyzed());
-    mColStats.setColName(statsObj.getColName());
-    mColStats.setColType(statsObj.getColType());
-
-    if (statsObj.getStatsData().isSetBooleanStats()) {
-      BooleanColumnStatsData boolStats = statsObj.getStatsData().getBooleanStats();
-      mColStats.setBooleanStats(boolStats.getNumTrues(), boolStats.getNumFalses(),
-          boolStats.getNumNulls());
-    } else if (statsObj.getStatsData().isSetLongStats()) {
-      LongColumnStatsData longStats = statsObj.getStatsData().getLongStats();
-      mColStats.setLongStats(longStats.getNumNulls(), longStats.getNumDVs(),
-          longStats.getLowValue(), longStats.getHighValue());
-    } else if (statsObj.getStatsData().isSetDoubleStats()) {
-      DoubleColumnStatsData doubleStats = statsObj.getStatsData().getDoubleStats();
-      mColStats.setDoubleStats(doubleStats.getNumNulls(), doubleStats.getNumDVs(),
-          doubleStats.getLowValue(), doubleStats.getHighValue());
-    } else if (statsObj.getStatsData().isSetStringStats()) {
-      StringColumnStatsData stringStats = statsObj.getStatsData().getStringStats();
-      mColStats.setStringStats(stringStats.getNumNulls(), stringStats.getNumDVs(),
-        stringStats.getMaxColLen(), stringStats.getAvgColLen());
-    } else if (statsObj.getStatsData().isSetBinaryStats()) {
-      BinaryColumnStatsData binaryStats = statsObj.getStatsData().getBinaryStats();
-      mColStats.setBinaryStats(binaryStats.getNumNulls(), binaryStats.getMaxColLen(),
-        binaryStats.getAvgColLen());
+    LOG.info("Updating table level column statistics for db=" + dbName + " tableName=" + tableName
+      + " colName=" + colName);
+    validateTableCols(table, Lists.newArrayList(colName));
+
+    List<MTableColumnStatistics> oldStats =
+        getMTableColumnStatistics(table, Lists.newArrayList(colName));
+
+    if (!oldStats.isEmpty()) {
+      assert oldStats.size() == 1;
+      StatObjectConverter.setFieldsIntoOldStats(mStatsObj, oldStats.get(0));
+    } else {
+      pm.makePersistent(mStatsObj);
     }
-    return mColStats;
   }
 
-  private void writeMTableColumnStatistics(MTableColumnStatistics mStatsObj)
-    throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException
-  {
-     String dbName = mStatsObj.getDbName();
-     String tableName = mStatsObj.getTableName();
-     String colName = mStatsObj.getColName();
-
-     LOG.info("Updating table level column statistics for db=" + dbName + " tableName=" + tableName
-       + " colName=" + colName);
-
-     MTable mTable = getMTable(mStatsObj.getDbName(), mStatsObj.getTableName());
-     boolean foundCol = false;
-
-     if (mTable == null) {
-        throw new
-          NoSuchObjectException("Table " + tableName +
-          " for which stats gathering is requested doesn't exist.");
-      }
-
-      MStorageDescriptor mSDS = mTable.getSd();
-      List<MFieldSchema> colList = mSDS.getCD().getCols();
-
-      for(MFieldSchema mCol:colList) {
-        if (mCol.getName().equals(mStatsObj.getColName().trim())) {
-          foundCol = true;
-          break;
-        }
-      }
-
-      if (!foundCol) {
-        throw new
-          NoSuchObjectException("Column " + colName +
-          " for which stats gathering is requested doesn't exist.");
-      }
-
-      MTableColumnStatistics oldStatsObj = getMTableColumnStatistics(dbName, tableName, colName);
-
-      if (oldStatsObj != null) {
-       oldStatsObj.setAvgColLen(mStatsObj.getAvgColLen());
-       oldStatsObj.setLongHighValue(mStatsObj.getLongHighValue());
-       oldStatsObj.setDoubleHighValue(mStatsObj.getDoubleHighValue());
-       oldStatsObj.setLastAnalyzed(mStatsObj.getLastAnalyzed());
-       oldStatsObj.setLongLowValue(mStatsObj.getLongLowValue());
-       oldStatsObj.setDoubleLowValue(mStatsObj.getDoubleLowValue());
-       oldStatsObj.setMaxColLen(mStatsObj.getMaxColLen());
-       oldStatsObj.setNumDVs(mStatsObj.getNumDVs());
-       oldStatsObj.setNumFalses(mStatsObj.getNumFalses());
-       oldStatsObj.setNumTrues(mStatsObj.getNumTrues());
-       oldStatsObj.setNumNulls(mStatsObj.getNumNulls());
-      } else {
-        pm.makePersistent(mStatsObj);
-      }
-   }
-
-  private ColumnStatisticsObj getPartitionColumnStatisticsObj(MPartitionColumnStatistics mStatsObj)
-  {
-    ColumnStatisticsObj statsObj = new ColumnStatisticsObj();
-    statsObj.setColType(mStatsObj.getColType());
-    statsObj.setColName(mStatsObj.getColName());
-    String colType = mStatsObj.getColType().toLowerCase();
-    ColumnStatisticsData colStatsData = new ColumnStatisticsData();
-
-    if (colType.equals("boolean")) {
-      BooleanColumnStatsData boolStats = new BooleanColumnStatsData();
-      boolStats.setNumFalses(mStatsObj.getNumFalses());
-      boolStats.setNumTrues(mStatsObj.getNumTrues());
-      boolStats.setNumNulls(mStatsObj.getNumNulls());
-      colStatsData.setBooleanStats(boolStats);
-    } else if (colType.equals("string") ||
-        colType.startsWith("varchar") || colType.startsWith("char")) {
-      StringColumnStatsData stringStats = new StringColumnStatsData();
-      stringStats.setNumNulls(mStatsObj.getNumNulls());
-      stringStats.setAvgColLen(mStatsObj.getAvgColLen());
-      stringStats.setMaxColLen(mStatsObj.getMaxColLen());
-      stringStats.setNumDVs(mStatsObj.getNumDVs());
-      colStatsData.setStringStats(stringStats);
-    } else if (colType.equals("binary")) {
-      BinaryColumnStatsData binaryStats = new BinaryColumnStatsData();
-      binaryStats.setNumNulls(mStatsObj.getNumNulls());
-      binaryStats.setAvgColLen(mStatsObj.getAvgColLen());
-      binaryStats.setMaxColLen(mStatsObj.getMaxColLen());
-      colStatsData.setBinaryStats(binaryStats);
-    } else if (colType.equals("tinyint") || colType.equals("smallint") ||
-        colType.equals("int") || colType.equals("bigint") ||
-        colType.equals("timestamp")) {
-      LongColumnStatsData longStats = new LongColumnStatsData();
-      longStats.setNumNulls(mStatsObj.getNumNulls());
-      longStats.setHighValue(mStatsObj.getLongHighValue());
-      longStats.setLowValue(mStatsObj.getLongLowValue());
-      longStats.setNumDVs(mStatsObj.getNumDVs());
-      colStatsData.setLongStats(longStats);
-   } else if (colType.equals("double") || colType.equals("float")) {
-     DoubleColumnStatsData doubleStats = new DoubleColumnStatsData();
-     doubleStats.setNumNulls(mStatsObj.getNumNulls());
-     doubleStats.setHighValue(mStatsObj.getDoubleHighValue());
-     doubleStats.setLowValue(mStatsObj.getDoubleLowValue());
-     doubleStats.setNumDVs(mStatsObj.getNumDVs());
-     colStatsData.setDoubleStats(doubleStats);
-   }
-   statsObj.setStatsData(colStatsData);
-   return statsObj;
-  }
-
-  private ColumnStatisticsDesc getPartitionColumnStatisticsDesc(
-    MPartitionColumnStatistics mStatsObj) {
-    ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc();
-    statsDesc.setIsTblLevel(false);
-    statsDesc.setDbName(mStatsObj.getDbName());
-    statsDesc.setTableName(mStatsObj.getTableName());
-    statsDesc.setPartName(mStatsObj.getPartitionName());
-    statsDesc.setLastAnalyzed(mStatsObj.getLastAnalyzed());
-    return statsDesc;
-  }
-
-  private void writeMPartitionColumnStatistics(MPartitionColumnStatistics mStatsObj,
-    List<String> partVal) throws NoSuchObjectException, MetaException, InvalidObjectException,
-    InvalidInputException
-  {
+  private void writeMPartitionColumnStatistics(Table table, Partition partition,
+      MPartitionColumnStatistics mStatsObj) throws NoSuchObjectException,
+        MetaException, InvalidObjectException, InvalidInputException {
     String dbName = mStatsObj.getDbName();
     String tableName = mStatsObj.getTableName();
     String partName = mStatsObj.getPartitionName();
@@ -5235,29 +4989,10 @@ public class ObjectStore implements RawS
     LOG.info("Updating partition level column statistics for db=" + dbName + " tableName=" +
       tableName + " partName=" + partName + " colName=" + colName);
 
-    MTable mTable = getMTable(mStatsObj.getDbName(), mStatsObj.getTableName());
     boolean foundCol = false;
-
-    if (mTable == null) {
-      throw new
-        NoSuchObjectException("Table " + tableName +
-        " for which stats gathering is requested doesn't exist.");
-    }
-
-    MPartition mPartition =
-                 getMPartition(mStatsObj.getDbName(), mStatsObj.getTableName(), partVal);
-
-    if (mPartition == null) {
-      throw new
-        NoSuchObjectException("Partition " + partName +
-        " for which stats gathering is requested doesn't exist");
-    }
-
-    MStorageDescriptor mSDS = mPartition.getSd();
-    List<MFieldSchema> colList = mSDS.getCD().getCols();
-
-    for(MFieldSchema mCol:colList) {
-      if (mCol.getName().equals(mStatsObj.getColName().trim())) {
+    List<FieldSchema> colList = partition.getSd().getCols();
+    for (FieldSchema col : colList) {
+      if (col.getName().equals(mStatsObj.getColName().trim())) {
         foundCol = true;
         break;
       }
@@ -5269,38 +5004,33 @@ public class ObjectStore implements RawS
         " for which stats gathering is requested doesn't exist.");
     }
 
-    MPartitionColumnStatistics oldStatsObj = getMPartitionColumnStatistics(dbName, tableName,
-                                                               partName, partVal, colName);
-    if (oldStatsObj != null) {
-      oldStatsObj.setAvgColLen(mStatsObj.getAvgColLen());
-      oldStatsObj.setLongHighValue(mStatsObj.getLongHighValue());
-      oldStatsObj.setDoubleHighValue(mStatsObj.getDoubleHighValue());
-      oldStatsObj.setLastAnalyzed(mStatsObj.getLastAnalyzed());
-      oldStatsObj.setLongLowValue(mStatsObj.getLongLowValue());
-      oldStatsObj.setDoubleLowValue(mStatsObj.getDoubleLowValue());
-      oldStatsObj.setMaxColLen(mStatsObj.getMaxColLen());
-      oldStatsObj.setNumDVs(mStatsObj.getNumDVs());
-      oldStatsObj.setNumFalses(mStatsObj.getNumFalses());
-      oldStatsObj.setNumTrues(mStatsObj.getNumTrues());
-      oldStatsObj.setNumNulls(mStatsObj.getNumNulls());
+    List<MPartitionColumnStatistics> oldStats = getMPartitionColumnStatistics(
+        table, Lists.newArrayList(partName), Lists.newArrayList(colName));
+    if (!oldStats.isEmpty()) {
+      assert oldStats.size() == 1;
+      StatObjectConverter.setFieldsIntoOldStats(mStatsObj, oldStats.get(0));
     } else {
       pm.makePersistent(mStatsObj);
     }
- }
+  }
 
   public boolean updateTableColumnStatistics(ColumnStatistics colStats)
-    throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException
-  {
+    throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException {
     boolean committed = false;
 
+    openTransaction();
     try {
-      openTransaction();
       List<ColumnStatisticsObj> statsObjs = colStats.getStatsObj();
       ColumnStatisticsDesc statsDesc = colStats.getStatsDesc();
 
+      // DataNucleus objects get detached all over the place for no (real) reason.
+      // So let's not use them anywhere unless absolutely necessary.
+      Table table = ensureGetTable(statsDesc.getDbName(), statsDesc.getTableName());
       for (ColumnStatisticsObj statsObj:statsObjs) {
-          MTableColumnStatistics mStatsObj = convertToMTableColumnStatistics(statsDesc, statsObj);
-          writeMTableColumnStatistics(mStatsObj);
+        // We have to get mtable again because DataNucleus.
+        MTableColumnStatistics mStatsObj = StatObjectConverter.convertToMTableColumnStatistics(
+            ensureGetMTable(statsDesc.getDbName(), statsDesc.getTableName()), statsDesc, statsObj);
+        writeMTableColumnStatistics(table, mStatsObj);
       }
       committed = commitTransaction();
       return committed;
@@ -5309,22 +5039,29 @@ public class ObjectStore implements RawS
         rollbackTransaction();
       }
     }
- }
+  }
 
   public boolean updatePartitionColumnStatistics(ColumnStatistics colStats, List<String> partVals)
-    throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException
-  {
+    throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException {
     boolean committed = false;
 
     try {
     openTransaction();
     List<ColumnStatisticsObj> statsObjs = colStats.getStatsObj();
     ColumnStatisticsDesc statsDesc = colStats.getStatsDesc();
-
+    Table table = ensureGetTable(statsDesc.getDbName(), statsDesc.getTableName());
+    Partition partition = convertToPart(getMPartition(
+        statsDesc.getDbName(), statsDesc.getTableName(), partVals));
     for (ColumnStatisticsObj statsObj:statsObjs) {
-        MPartitionColumnStatistics mStatsObj =
-            convertToMPartitionColumnStatistics(statsDesc, statsObj, partVals);
-        writeMPartitionColumnStatistics(mStatsObj, partVals);
+      // We have to get partition again because DataNucleus
+      MPartition mPartition = getMPartition(
+          statsDesc.getDbName(), statsDesc.getTableName(), partVals);
+      if (partition == null) {
+        throw new NoSuchObjectException("Partition for which stats is gathered doesn't exist.");
+      }
+      MPartitionColumnStatistics mStatsObj =
+          StatObjectConverter.convertToMPartitionColumnStatistics(mPartition, statsDesc, statsObj);
+      writeMPartitionColumnStatistics(table, partition, mStatsObj);
     }
     committed = commitTransaction();
     return committed;
@@ -5335,190 +5072,199 @@ public class ObjectStore implements RawS
     }
   }
 
-  private MTableColumnStatistics getMTableColumnStatistics(String dbName, String tableName,
-    String colName) throws NoSuchObjectException, InvalidInputException, MetaException
-  {
+  private List<MTableColumnStatistics> getMTableColumnStatistics(
+      Table table, List<String> colNames) throws MetaException {
     boolean committed = false;
-
-    if (dbName == null) {
-      dbName = MetaStoreUtils.DEFAULT_DATABASE_NAME;
-    }
-
-    if (tableName == null || colName == null) {
-      throw new InvalidInputException("TableName/ColName passed to get_table_column_statistics " +
-      "is null");
-    }
-
+    openTransaction();
     try {
-      openTransaction();
-      MTableColumnStatistics mStatsObj = null;
-      MTable mTable = getMTable(dbName.trim(), tableName.trim());
-      boolean foundCol = false;
+      List<MTableColumnStatistics> result = null;
+      validateTableCols(table, colNames);
 
-      if (mTable == null) {
-        throw new NoSuchObjectException("Table " + tableName +
-        " for which stats is requested doesn't exist.");
-      }
-
-      MStorageDescriptor mSDS = mTable.getSd();
-      List<MFieldSchema> colList = mSDS.getCD().getCols();
-
-      for(MFieldSchema mCol:colList) {
-        if (mCol.getName().equals(colName.trim())) {
-          foundCol = true;
-          break;
-        }
+      Query query = pm.newQuery(MTableColumnStatistics.class);
+      String filter = "tableName == t1 && dbName == t2 && (";
+      String paramStr = "java.lang.String t1, java.lang.String t2";
+      Object[] params = new Object[colNames.size() + 2];
+      params[0] = table.getTableName();
+      params[1] = table.getDbName();
+      for (int i = 0; i < colNames.size(); ++i) {
+        filter += ((i == 0) ? "" : " || ") + "colName == c" + i;
+        paramStr += ", java.lang.String c" + i;
+        params[i + 2] = colNames.get(i);
       }
-
-      if (!foundCol) {
-        throw new NoSuchObjectException("Column " + colName +
-        " for which stats is requested doesn't exist.");
+      filter += ")";
+      query.setFilter(filter);
+      query.declareParameters(paramStr);
+      result = (List<MTableColumnStatistics>) query.executeWithArray(params);
+      pm.retrieveAll(result);
+      if (result.size() > colNames.size()) {
+        throw new MetaException(
+            "Unexpected " + result.size() + " statistics for " + colNames.size() + " columns");
       }
-
-      Query query = pm.newQuery(MTableColumnStatistics.class);
-      query.setFilter("table.tableName == t1 && " +
-        "dbName == t2 && " + "colName == t3");
-      query
-      .declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3");
-      query.setUnique(true);
-
-      mStatsObj = (MTableColumnStatistics) query.execute(tableName.trim(),
-                                                        dbName.trim(), colName.trim());
-      pm.retrieve(mStatsObj);
       committed = commitTransaction();
-      return mStatsObj;
+      return result;
+    } catch (Exception ex) {
+      LOG.error("Error retrieving statistics via jdo", ex);
+      if (ex instanceof MetaException) {
+        throw (MetaException)ex;
+      }
+      throw new MetaException(ex.getMessage());
     } finally {
       if (!committed) {
         rollbackTransaction();
-        return null;
+        return Lists.newArrayList();
       }
     }
   }
 
- public ColumnStatistics getTableColumnStatistics(String dbName, String tableName, String colName)
-   throws MetaException, NoSuchObjectException, InvalidInputException
-  {
-    ColumnStatistics statsObj;
-    MTableColumnStatistics mStatsObj = getMTableColumnStatistics(dbName, tableName, colName);
-
-    if (mStatsObj == null) {
-      throw new NoSuchObjectException("Statistics for dbName=" + dbName + " tableName=" + tableName
-        + " columnName=" + colName + " doesn't exist.");
+  private void validateTableCols(Table table, List<String> colNames) throws MetaException {
+    List<FieldSchema> colList = table.getSd().getCols();
+    for (String colName : colNames) {
+      boolean foundCol = false;
+      for (FieldSchema mCol : colList) {
+        if (mCol.getName().equals(colName.trim())) {
+          foundCol = true;
+          break;
+        }
+      }
+      if (!foundCol) {
+        throw new MetaException("Column " + colName + " doesn't exist.");
+      }
     }
-
-    statsObj = convertToTableColumnStatistics(mStatsObj);
-    return statsObj;
   }
 
-  public ColumnStatistics getPartitionColumnStatistics(String dbName, String tableName,
-    String partName, List<String> partVal, String colName)
-    throws MetaException, NoSuchObjectException, InvalidInputException
-  {
-    ColumnStatistics statsObj;
-    MPartitionColumnStatistics mStatsObj =
-          getMPartitionColumnStatistics(dbName, tableName, partName, partVal, colName);
-
-    if (mStatsObj == null) {
-      throw new NoSuchObjectException("Statistics for dbName=" + dbName + " tableName=" + tableName
-          + " partName= " + partName + " columnName=" + colName + " doesn't exist.");
-    }
-    statsObj = convertToPartColumnStatistics(mStatsObj);
-    return statsObj;
+  public ColumnStatistics getTableColumnStatistics(String dbName, String tableName,
+      List<String> colNames) throws MetaException, NoSuchObjectException {
+    return getTableColumnStatisticsInternal(dbName, tableName, colNames, true, true);
+  }
+
+  protected ColumnStatistics getTableColumnStatisticsInternal(
+      String dbName, String tableName, final List<String> colNames, boolean allowSql,
+      boolean allowJdo) throws MetaException, NoSuchObjectException {
+    return new GetStatHelper(dbName.toLowerCase(), tableName.toLowerCase(), allowSql, allowJdo) {
+      protected ColumnStatistics getSqlResult(GetHelper<ColumnStatistics> ctx) throws MetaException {
+        return directSql.getTableStats(dbName, tblName, colNames);
+      }
+      protected ColumnStatistics getJdoResult(
+          GetHelper<ColumnStatistics> ctx) throws MetaException, NoSuchObjectException {
+        List<MTableColumnStatistics> mStats = getMTableColumnStatistics(getTable(), colNames);
+        if (mStats.isEmpty()) return null;
+        // LastAnalyzed is stored per column, but thrift object has it per multiple columns.
+        // Luckily, nobody actually uses it, so we will set to lowest value of all columns for now.
+        ColumnStatisticsDesc desc = StatObjectConverter.getTableColumnStatisticsDesc(mStats.get(0));
+        List<ColumnStatisticsObj> statObjs = new ArrayList<ColumnStatisticsObj>(mStats.size());
+        for (MTableColumnStatistics mStat : mStats) {
+          if (desc.getLastAnalyzed() > mStat.getLastAnalyzed()) {
+            desc.setLastAnalyzed(mStat.getLastAnalyzed());
+          }
+          statObjs.add(StatObjectConverter.getTableColumnStatisticsObj(mStat));
+        }
+        return new ColumnStatistics(desc, statObjs);
+      }
+    }.run(true);
   }
 
-  private ColumnStatistics convertToPartColumnStatistics(MPartitionColumnStatistics mStatsObj)
-  {
-    if (mStatsObj == null) {
-      return null;
-    }
-
-    ColumnStatisticsDesc statsDesc = getPartitionColumnStatisticsDesc(mStatsObj);
-    ColumnStatisticsObj statsObj = getPartitionColumnStatisticsObj(mStatsObj);
-    List<ColumnStatisticsObj> statsObjs = new ArrayList<ColumnStatisticsObj>();
-    statsObjs.add(statsObj);
-
-    ColumnStatistics colStats = new ColumnStatistics();
-    colStats.setStatsDesc(statsDesc);
-    colStats.setStatsObj(statsObjs);
-    return colStats;
+  public List<ColumnStatistics> getPartitionColumnStatistics(String dbName, String tableName,
+      List<String> partNames, List<String> colNames) throws MetaException, NoSuchObjectException {
+    return getPartitionColumnStatisticsInternal(
+        dbName, tableName, partNames, colNames, true, true);
+  }
+
+  protected List<ColumnStatistics> getPartitionColumnStatisticsInternal(
+      String dbName, String tableName, final List<String> partNames, final List<String> colNames,
+      boolean allowSql, boolean allowJdo) throws MetaException, NoSuchObjectException {
+    return new GetListHelper<ColumnStatistics>(dbName, tableName, allowSql, allowJdo) {
+      protected List<ColumnStatistics> getSqlResult(
+          GetHelper<List<ColumnStatistics>> ctx) throws MetaException {
+        return directSql.getPartitionStats(dbName, tblName, partNames, colNames);
+      }
+      protected List<ColumnStatistics> getJdoResult(
+          GetHelper<List<ColumnStatistics>> ctx) throws MetaException, NoSuchObjectException {
+        List<MPartitionColumnStatistics> mStats =
+            getMPartitionColumnStatistics(getTable(), partNames, colNames);
+        List<ColumnStatistics> result = new ArrayList<ColumnStatistics>(
+            Math.min(mStats.size(), partNames.size()));
+        String lastPartName = null;
+        List<ColumnStatisticsObj> curList = null;
+        ColumnStatisticsDesc csd = null;
+        for (int i = 0; i <= mStats.size(); ++i) {
+          boolean isLast = i == mStats.size();
+          MPartitionColumnStatistics mStatsObj = isLast ? null : mStats.get(i);
+          String partName = isLast ? null : (String)mStatsObj.getPartitionName();
+          if (isLast || !partName.equals(lastPartName)) {
+            if (i != 0) {
+              result.add(new ColumnStatistics(csd, curList));
+            }
+            if (isLast) {
+              continue;
+            }
+            csd = StatObjectConverter.getPartitionColumnStatisticsDesc(mStatsObj);
+            curList = new ArrayList<ColumnStatisticsObj>(colNames.size());
+          }
+          curList.add(StatObjectConverter.getPartitionColumnStatisticsObj(mStatsObj));
+          lastPartName = partName;
+        }
+        return result;
+      }
+    }.run(true);
   }
 
-  private MPartitionColumnStatistics getMPartitionColumnStatistics(String dbName, String tableName,
-    String partName, List<String> partVal, String colName) throws NoSuchObjectException,
-    InvalidInputException, MetaException
-  {
+  private List<MPartitionColumnStatistics> getMPartitionColumnStatistics(
+      Table table, List<String> partNames, List<String> colNames)
+          throws NoSuchObjectException, MetaException {
     boolean committed = false;
     MPartitionColumnStatistics mStatsObj = null;
-
-    if (dbName == null) {
-      dbName = MetaStoreUtils.DEFAULT_DATABASE_NAME;
-    }
-
-    if (tableName == null || partVal == null || colName == null) {
-      throw new InvalidInputException("TableName/PartName/ColName passed to " +
-        " get_partition_column_statistics is null");
-    }
-
     try {
       openTransaction();
-      MTable mTable = getMTable(dbName.trim(), tableName.trim());
+      // We are not going to verify SD for each partition. Just verify for the table.
+      validateTableCols(table, colNames);
       boolean foundCol = false;
-
-      if (mTable == null) {
-        throw new NoSuchObjectException("Table "  + tableName +
-          " for which stats is requested doesn't exist.");
-      }
-
-      MPartition mPartition =
-                  getMPartition(dbName, tableName, partVal);
-
-      if (mPartition == null) {
-        throw new
-          NoSuchObjectException("Partition " + partName +
-          " for which stats is requested doesn't exist");
-      }
-
-      MStorageDescriptor mSDS = mPartition.getSd();
-      List<MFieldSchema> colList = mSDS.getCD().getCols();
-
-      for(MFieldSchema mCol:colList) {
-        if (mCol.getName().equals(colName.trim())) {
-          foundCol = true;
-          break;
-        }
-      }
-
-      if (!foundCol) {
-        throw new NoSuchObjectException("Column " + colName +
-        " for which stats is requested doesn't exist.");
-      }
-
       Query query = pm.newQuery(MPartitionColumnStatistics.class);
-      query.setFilter("partition.partitionName == t1 && " +
-        "dbName == t2 && " + "tableName == t3 && " + "colName == t4");
-      query
-      .declareParameters("java.lang.String t1, java.lang.String t2, " +
-         "java.lang.String t3, java.lang.String t4");
-      query.setUnique(true);
-
-      mStatsObj = (MPartitionColumnStatistics) query.executeWithArray(partName.trim(),
-                                                        dbName.trim(), tableName.trim(),
-                                                        colName.trim());
-      pm.retrieve(mStatsObj);
+      String paramStr = "java.lang.String t1, java.lang.String t2";
+      String filter = "tableName == t1 && dbName == t2 && (";
+      Object[] params = new Object[colNames.size() + partNames.size() + 2];
+      int i = 0;
+      params[i++] = table.getTableName();
+      params[i++] = table.getDbName();
+      int firstI = i;
+      for (String s : partNames) {
+        filter += ((i == firstI) ? "" : " || ") + "partitionName == p" + i;
+        paramStr += ", java.lang.String p" + i;
+        params[i++] = s;
+      }
+      filter += ") && (";
+      firstI = i;
+      for (String s : colNames) {
+        filter += ((i == firstI) ? "" : " || ") + "colName == c" + i;
+        paramStr += ", java.lang.String c" + i;
+        params[i++] = s;
+      }
+      filter += ")";
+      query.setFilter(filter);
+      query.declareParameters(paramStr);
+      query.setOrdering("partitionName ascending");
+      @SuppressWarnings("unchecked")
+      List<MPartitionColumnStatistics> result =
+          (List<MPartitionColumnStatistics>) query.executeWithArray(params);
+      pm.retrieveAll(result);
       committed = commitTransaction();
-      return mStatsObj;
-
+      return result;
+    } catch (Exception ex) {
+      LOG.error("Error retrieving statistics via jdo", ex);
+      if (ex instanceof MetaException) {
+        throw (MetaException)ex;
+      }
+      throw new MetaException(ex.getMessage());
     } finally {
       if (!committed) {
         rollbackTransaction();
-       }
+        return Lists.newArrayList();
+      }
     }
   }
 
   public boolean deletePartitionColumnStatistics(String dbName, String tableName,
     String partName, List<String> partVals,String colName)
-    throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException
-  {
+    throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException {
     boolean ret = false;
 
     if (dbName == null) {

Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java?rev=1562653&r1=1562652&r2=1562653&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java Thu Jan 30 00:24:28 2014
@@ -137,7 +137,7 @@ public interface RawStore extends Config
       InvalidInputException;
 
   public abstract List<Partition> getPartitions(String dbName,
-      String tableName, int max) throws MetaException;
+      String tableName, int max) throws MetaException, NoSuchObjectException;
 
   public abstract void alterTable(String dbname, String name, Table newTable)
       throws InvalidObjectException, MetaException;
@@ -378,30 +378,15 @@ public interface RawStore extends Config
    *
    */
   public abstract ColumnStatistics getTableColumnStatistics(String dbName, String tableName,
-    String colName) throws MetaException, NoSuchObjectException, InvalidInputException,
-    InvalidObjectException;
+    List<String> colName) throws MetaException, NoSuchObjectException;
 
   /**
-   * Returns the relevant column statistics for a given column in a given partition in a given
+   * Returns the relevant column statistics for given columns in given partitions in a given
    * table in a given database if such statistics exist.
-   * @param partName
-   *
-   * @param The name of the database, defaults to current database
-   * @param The name of the table
-   * @param The name of the partition
-   * @param List of partVals for the partition
-   * @param The name of the column for which statistics is requested
-   * @return Relevant column statistics for the column for the given partition in a given table
-   * @throws NoSuchObjectException
-   * @throws MetaException
-   * @throws InvalidInputException
-   * @throws InvalidObjectException
-   *
    */
-
-  public abstract ColumnStatistics getPartitionColumnStatistics(String dbName, String tableName,
-    String partName, List<String> partVals, String colName)
-    throws MetaException, NoSuchObjectException, InvalidInputException, InvalidObjectException;
+  public abstract List<ColumnStatistics> getPartitionColumnStatistics(
+     String dbName, String tblName, List<String> partNames, List<String> colNames)
+      throws MetaException, NoSuchObjectException;
 
   /**
    * Deletes column statistics if present associated with a given db, table, partition and col. If

Added: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/StatObjectConverter.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/StatObjectConverter.java?rev=1562653&view=auto
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/StatObjectConverter.java (added)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/StatObjectConverter.java Thu Jan 30 00:24:28 2014
@@ -0,0 +1,320 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
+import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
+import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
+import org.apache.hadoop.hive.metastore.model.MFieldSchema;
+import org.apache.hadoop.hive.metastore.model.MPartition;
+import org.apache.hadoop.hive.metastore.model.MPartitionColumnStatistics;
+import org.apache.hadoop.hive.metastore.model.MStorageDescriptor;
+import org.apache.hadoop.hive.metastore.model.MTable;
+import org.apache.hadoop.hive.metastore.model.MTableColumnStatistics;
+
+/**
+ * This class contains conversion logic that creates Thrift stat objects from
+ * JDO stat objects and plain arrays from DirectSQL.
+ * It is hidden here so that we wouldn't have to look at it in elsewhere.
+ */
+public class StatObjectConverter {
+  // JDO
+  public static MTableColumnStatistics convertToMTableColumnStatistics(MTable table,
+      ColumnStatisticsDesc statsDesc, ColumnStatisticsObj statsObj)
+          throws NoSuchObjectException, MetaException, InvalidObjectException {
+     if (statsObj == null || statsDesc == null) {
+       throw new InvalidObjectException("Invalid column stats object");
+     }
+
+     MTableColumnStatistics mColStats = new MTableColumnStatistics();
+     mColStats.setTable(table);
+     mColStats.setDbName(statsDesc.getDbName());
+     mColStats.setTableName(statsDesc.getTableName());
+     mColStats.setLastAnalyzed(statsDesc.getLastAnalyzed());
+     mColStats.setColName(statsObj.getColName());
+     mColStats.setColType(statsObj.getColType());
+
+     if (statsObj.getStatsData().isSetBooleanStats()) {
+       BooleanColumnStatsData boolStats = statsObj.getStatsData().getBooleanStats();
+       mColStats.setBooleanStats(boolStats.getNumTrues(), boolStats.getNumFalses(),
+           boolStats.getNumNulls());
+     } else if (statsObj.getStatsData().isSetLongStats()) {
+       LongColumnStatsData longStats = statsObj.getStatsData().getLongStats();
+       mColStats.setLongStats(longStats.getNumNulls(), longStats.getNumDVs(),
+           longStats.getLowValue(), longStats.getHighValue());
+     } else if (statsObj.getStatsData().isSetDoubleStats()) {
+       DoubleColumnStatsData doubleStats = statsObj.getStatsData().getDoubleStats();
+       mColStats.setDoubleStats(doubleStats.getNumNulls(), doubleStats.getNumDVs(),
+           doubleStats.getLowValue(), doubleStats.getHighValue());
+     } else if (statsObj.getStatsData().isSetStringStats()) {
+       StringColumnStatsData stringStats = statsObj.getStatsData().getStringStats();
+       mColStats.setStringStats(stringStats.getNumNulls(), stringStats.getNumDVs(),
+         stringStats.getMaxColLen(), stringStats.getAvgColLen());
+     } else if (statsObj.getStatsData().isSetBinaryStats()) {
+       BinaryColumnStatsData binaryStats = statsObj.getStatsData().getBinaryStats();
+       mColStats.setBinaryStats(binaryStats.getNumNulls(), binaryStats.getMaxColLen(),
+         binaryStats.getAvgColLen());
+     }
+     return mColStats;
+  }
+
+  public static void setFieldsIntoOldStats(
+      MTableColumnStatistics mStatsObj, MTableColumnStatistics oldStatsObj) {
+    oldStatsObj.setAvgColLen(mStatsObj.getAvgColLen());
+    oldStatsObj.setLongHighValue(mStatsObj.getLongHighValue());
+    oldStatsObj.setDoubleHighValue(mStatsObj.getDoubleHighValue());
+    oldStatsObj.setLastAnalyzed(mStatsObj.getLastAnalyzed());
+    oldStatsObj.setLongLowValue(mStatsObj.getLongLowValue());
+    oldStatsObj.setDoubleLowValue(mStatsObj.getDoubleLowValue());
+    oldStatsObj.setMaxColLen(mStatsObj.getMaxColLen());
+    oldStatsObj.setNumDVs(mStatsObj.getNumDVs());
+    oldStatsObj.setNumFalses(mStatsObj.getNumFalses());
+    oldStatsObj.setNumTrues(mStatsObj.getNumTrues());
+    oldStatsObj.setNumNulls(mStatsObj.getNumNulls());
+  }
+
+  public static void setFieldsIntoOldStats(
+      MPartitionColumnStatistics mStatsObj, MPartitionColumnStatistics oldStatsObj) {
+    oldStatsObj.setAvgColLen(mStatsObj.getAvgColLen());
+    oldStatsObj.setLongHighValue(mStatsObj.getLongHighValue());
+    oldStatsObj.setDoubleHighValue(mStatsObj.getDoubleHighValue());
+    oldStatsObj.setLastAnalyzed(mStatsObj.getLastAnalyzed());
+    oldStatsObj.setLongLowValue(mStatsObj.getLongLowValue());
+    oldStatsObj.setDoubleLowValue(mStatsObj.getDoubleLowValue());
+    oldStatsObj.setMaxColLen(mStatsObj.getMaxColLen());
+    oldStatsObj.setNumDVs(mStatsObj.getNumDVs());
+    oldStatsObj.setNumFalses(mStatsObj.getNumFalses());
+    oldStatsObj.setNumTrues(mStatsObj.getNumTrues());
+    oldStatsObj.setNumNulls(mStatsObj.getNumNulls());
+  }
+
+  public static ColumnStatisticsObj getTableColumnStatisticsObj(
+      MTableColumnStatistics mStatsObj) {
+    ColumnStatisticsObj statsObj = new ColumnStatisticsObj();
+    statsObj.setColType(mStatsObj.getColType());
+    statsObj.setColName(mStatsObj.getColName());
+    String colType = mStatsObj.getColType().toLowerCase();
+    ColumnStatisticsData colStatsData = new ColumnStatisticsData();
+
+    if (colType.equals("boolean")) {
+      BooleanColumnStatsData boolStats = new BooleanColumnStatsData();
+      boolStats.setNumFalses(mStatsObj.getNumFalses());
+      boolStats.setNumTrues(mStatsObj.getNumTrues());
+      boolStats.setNumNulls(mStatsObj.getNumNulls());
+      colStatsData.setBooleanStats(boolStats);
+    } else if (colType.equals("string") ||
+        colType.startsWith("varchar") || colType.startsWith("char")) {
+      StringColumnStatsData stringStats = new StringColumnStatsData();
+      stringStats.setNumNulls(mStatsObj.getNumNulls());
+      stringStats.setAvgColLen(mStatsObj.getAvgColLen());
+      stringStats.setMaxColLen(mStatsObj.getMaxColLen());
+      stringStats.setNumDVs(mStatsObj.getNumDVs());
+      colStatsData.setStringStats(stringStats);
+    } else if (colType.equals("binary")) {
+      BinaryColumnStatsData binaryStats = new BinaryColumnStatsData();
+      binaryStats.setNumNulls(mStatsObj.getNumNulls());
+      binaryStats.setAvgColLen(mStatsObj.getAvgColLen());
+      binaryStats.setMaxColLen(mStatsObj.getMaxColLen());
+      colStatsData.setBinaryStats(binaryStats);
+    } else if (colType.equals("bigint") || colType.equals("int") ||
+        colType.equals("smallint") || colType.equals("tinyint") ||
+        colType.equals("timestamp")) {
+      LongColumnStatsData longStats = new LongColumnStatsData();
+      longStats.setNumNulls(mStatsObj.getNumNulls());
+      longStats.setHighValue(mStatsObj.getLongHighValue());
+      longStats.setLowValue(mStatsObj.getLongLowValue());
+      longStats.setNumDVs(mStatsObj.getNumDVs());
+      colStatsData.setLongStats(longStats);
+    } else if (colType.equals("double") || colType.equals("float")) {
+      DoubleColumnStatsData doubleStats = new DoubleColumnStatsData();
+      doubleStats.setNumNulls(mStatsObj.getNumNulls());
+      doubleStats.setHighValue(mStatsObj.getDoubleHighValue());
+      doubleStats.setLowValue(mStatsObj.getDoubleLowValue());
+      doubleStats.setNumDVs(mStatsObj.getNumDVs());
+      colStatsData.setDoubleStats(doubleStats);
+    }
+    statsObj.setStatsData(colStatsData);
+    return statsObj;
+  }
+
+  public static ColumnStatisticsDesc getTableColumnStatisticsDesc(
+      MTableColumnStatistics mStatsObj) {
+    ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc();
+    statsDesc.setIsTblLevel(true);
+    statsDesc.setDbName(mStatsObj.getDbName());
+    statsDesc.setTableName(mStatsObj.getTableName());
+    statsDesc.setLastAnalyzed(mStatsObj.getLastAnalyzed());
+    return statsDesc;
+  }
+
+  public static MPartitionColumnStatistics convertToMPartitionColumnStatistics(
+      MPartition partition, ColumnStatisticsDesc statsDesc, ColumnStatisticsObj statsObj)
+          throws MetaException, NoSuchObjectException {
+    if (statsDesc == null || statsObj == null) {
+      return null;
+    }
+
+    MPartitionColumnStatistics mColStats = new MPartitionColumnStatistics();
+    mColStats.setPartition(partition);
+    mColStats.setDbName(statsDesc.getDbName());
+    mColStats.setTableName(statsDesc.getTableName());
+    mColStats.setPartitionName(statsDesc.getPartName());
+    mColStats.setLastAnalyzed(statsDesc.getLastAnalyzed());
+    mColStats.setColName(statsObj.getColName());
+    mColStats.setColType(statsObj.getColType());
+
+    if (statsObj.getStatsData().isSetBooleanStats()) {
+      BooleanColumnStatsData boolStats = statsObj.getStatsData().getBooleanStats();
+      mColStats.setBooleanStats(boolStats.getNumTrues(), boolStats.getNumFalses(),
+          boolStats.getNumNulls());
+    } else if (statsObj.getStatsData().isSetLongStats()) {
+      LongColumnStatsData longStats = statsObj.getStatsData().getLongStats();
+      mColStats.setLongStats(longStats.getNumNulls(), longStats.getNumDVs(),
+          longStats.getLowValue(), longStats.getHighValue());
+    } else if (statsObj.getStatsData().isSetDoubleStats()) {
+      DoubleColumnStatsData doubleStats = statsObj.getStatsData().getDoubleStats();
+      mColStats.setDoubleStats(doubleStats.getNumNulls(), doubleStats.getNumDVs(),
+          doubleStats.getLowValue(), doubleStats.getHighValue());
+    } else if (statsObj.getStatsData().isSetStringStats()) {
+      StringColumnStatsData stringStats = statsObj.getStatsData().getStringStats();
+      mColStats.setStringStats(stringStats.getNumNulls(), stringStats.getNumDVs(),
+        stringStats.getMaxColLen(), stringStats.getAvgColLen());
+    } else if (statsObj.getStatsData().isSetBinaryStats()) {
+      BinaryColumnStatsData binaryStats = statsObj.getStatsData().getBinaryStats();
+      mColStats.setBinaryStats(binaryStats.getNumNulls(), binaryStats.getMaxColLen(),
+        binaryStats.getAvgColLen());
+    }
+    return mColStats;
+  }
+
+  public static ColumnStatisticsObj getPartitionColumnStatisticsObj(
+      MPartitionColumnStatistics mStatsObj) {
+    ColumnStatisticsObj statsObj = new ColumnStatisticsObj();
+    statsObj.setColType(mStatsObj.getColType());
+    statsObj.setColName(mStatsObj.getColName());
+    String colType = mStatsObj.getColType().toLowerCase();
+    ColumnStatisticsData colStatsData = new ColumnStatisticsData();
+
+    if (colType.equals("boolean")) {
+      BooleanColumnStatsData boolStats = new BooleanColumnStatsData();
+      boolStats.setNumFalses(mStatsObj.getNumFalses());
+      boolStats.setNumTrues(mStatsObj.getNumTrues());
+      boolStats.setNumNulls(mStatsObj.getNumNulls());
+      colStatsData.setBooleanStats(boolStats);
+    } else if (colType.equals("string") ||
+        colType.startsWith("varchar") || colType.startsWith("char")) {
+      StringColumnStatsData stringStats = new StringColumnStatsData();
+      stringStats.setNumNulls(mStatsObj.getNumNulls());
+      stringStats.setAvgColLen(mStatsObj.getAvgColLen());
+      stringStats.setMaxColLen(mStatsObj.getMaxColLen());
+      stringStats.setNumDVs(mStatsObj.getNumDVs());
+      colStatsData.setStringStats(stringStats);
+    } else if (colType.equals("binary")) {
+      BinaryColumnStatsData binaryStats = new BinaryColumnStatsData();
+      binaryStats.setNumNulls(mStatsObj.getNumNulls());
+      binaryStats.setAvgColLen(mStatsObj.getAvgColLen());
+      binaryStats.setMaxColLen(mStatsObj.getMaxColLen());
+      colStatsData.setBinaryStats(binaryStats);
+    } else if (colType.equals("tinyint") || colType.equals("smallint") ||
+        colType.equals("int") || colType.equals("bigint") ||
+        colType.equals("timestamp")) {
+      LongColumnStatsData longStats = new LongColumnStatsData();
+      longStats.setNumNulls(mStatsObj.getNumNulls());
+      longStats.setHighValue(mStatsObj.getLongHighValue());
+      longStats.setLowValue(mStatsObj.getLongLowValue());
+      longStats.setNumDVs(mStatsObj.getNumDVs());
+      colStatsData.setLongStats(longStats);
+   } else if (colType.equals("double") || colType.equals("float")) {
+     DoubleColumnStatsData doubleStats = new DoubleColumnStatsData();
+     doubleStats.setNumNulls(mStatsObj.getNumNulls());
+     doubleStats.setHighValue(mStatsObj.getDoubleHighValue());
+     doubleStats.setLowValue(mStatsObj.getDoubleLowValue());
+     doubleStats.setNumDVs(mStatsObj.getNumDVs());
+     colStatsData.setDoubleStats(doubleStats);
+   }
+   statsObj.setStatsData(colStatsData);
+   return statsObj;
+  }
+
+  public static ColumnStatisticsDesc getPartitionColumnStatisticsDesc(
+    MPartitionColumnStatistics mStatsObj) {
+    ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc();
+    statsDesc.setIsTblLevel(false);
+    statsDesc.setDbName(mStatsObj.getDbName());
+    statsDesc.setTableName(mStatsObj.getTableName());
+    statsDesc.setPartName(mStatsObj.getPartitionName());
+    statsDesc.setLastAnalyzed(mStatsObj.getLastAnalyzed());
+    return statsDesc;
+  }
+
+  // SQL
+  public static void fillColumnStatisticsData(String colType, ColumnStatisticsData data,
+      Object llow, Object lhigh, Object dlow, Object dhigh, Object nulls, Object dist,
+      Object avglen, Object maxlen, Object trues, Object falses) {
+    if (colType.equals("boolean")) {
+      BooleanColumnStatsData boolStats = new BooleanColumnStatsData();
+      boolStats.setNumFalses((Long)falses);
+      boolStats.setNumTrues((Long)trues);
+      boolStats.setNumNulls((Long)nulls);
+      data.setBooleanStats(boolStats);
+    } else if (colType.equals("string") ||
+        colType.startsWith("varchar") || colType.startsWith("char")) {
+      StringColumnStatsData stringStats = new StringColumnStatsData();
+      stringStats.setNumNulls((Long)nulls);
+      stringStats.setAvgColLen((Double)avglen);
+      stringStats.setMaxColLen((Long)maxlen);
+      stringStats.setNumDVs((Long)dist);
+      data.setStringStats(stringStats);
+    } else if (colType.equals("binary")) {
+      BinaryColumnStatsData binaryStats = new BinaryColumnStatsData();
+      binaryStats.setNumNulls((Long)nulls);
+      binaryStats.setAvgColLen((Double)avglen);
+      binaryStats.setMaxColLen((Long)maxlen);
+      data.setBinaryStats(binaryStats);
+    } else if (colType.equals("bigint") || colType.equals("int") ||
+        colType.equals("smallint") || colType.equals("tinyint") ||
+        colType.equals("timestamp")) {
+      LongColumnStatsData longStats = new LongColumnStatsData();
+      longStats.setNumNulls((Long)nulls);
+      longStats.setHighValue((Long)lhigh);
+      longStats.setLowValue((Long)llow);
+      longStats.setNumDVs((Long)dist);
+      data.setLongStats(longStats);
+    } else if (colType.equals("double") || colType.equals("float")) {
+      DoubleColumnStatsData doubleStats = new DoubleColumnStatsData();
+      doubleStats.setNumNulls((Long)nulls);
+      doubleStats.setHighValue((Double)dhigh);
+      doubleStats.setLowValue((Double)dlow);
+      doubleStats.setNumDVs((Long)dist);
+      data.setDoubleStats(doubleStats);
+    }
+  }
+}

Modified: hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java?rev=1562653&r1=1562652&r2=1562653&view=diff
==============================================================================
--- hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java (original)
+++ hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java Thu Jan 30 00:24:28 2014
@@ -200,7 +200,7 @@ public class DummyRawStoreControlledComm
 
   @Override
   public List<Partition> getPartitions(String dbName, String tableName, int max)
-      throws MetaException {
+      throws MetaException, NoSuchObjectException {
     return objectStore.getPartitions(dbName, tableName, max);
   }
 
@@ -494,9 +494,8 @@ public class DummyRawStoreControlledComm
 
   @Override
   public ColumnStatistics getTableColumnStatistics(String dbName, String tableName,
-      String colName)
-      throws MetaException, NoSuchObjectException, InvalidInputException {
-    return objectStore.getTableColumnStatistics(dbName, tableName, colName);
+      List<String> colNames) throws MetaException, NoSuchObjectException {
+    return objectStore.getTableColumnStatistics(dbName, tableName, colNames);
   }
 
   @Override
@@ -516,15 +515,6 @@ public class DummyRawStoreControlledComm
   }
 
   @Override
-  public ColumnStatistics getPartitionColumnStatistics(String dbName, String tableName,
-      String partName, List<String> partVal, String colName)
-      throws MetaException, NoSuchObjectException, InvalidInputException,
-      InvalidObjectException  {
-    return objectStore.getPartitionColumnStatistics(dbName, tableName, partName,
-        partVal, colName);
-  }
-
-  @Override
   public boolean updateTableColumnStatistics(ColumnStatistics statsObj)
       throws NoSuchObjectException, MetaException, InvalidObjectException,
       InvalidInputException {
@@ -585,6 +575,13 @@ public class DummyRawStoreControlledComm
   }
 
   @Override
+  public List<ColumnStatistics> getPartitionColumnStatistics(String dbName,
+      String tblName, List<String> colNames, List<String> partNames)
+      throws MetaException, NoSuchObjectException {
+    return objectStore.getPartitionColumnStatistics(dbName, tblName  , colNames, partNames);
+  }
+
+  @Override
   public boolean doesPartitionExist(String dbName, String tableName,
       List<String> partVals) throws MetaException, NoSuchObjectException {
     return objectStore.doesPartitionExist(dbName, tableName, partVals);
@@ -595,5 +592,4 @@ public class DummyRawStoreControlledComm
       throws InvalidObjectException, MetaException {
     return objectStore.addPartitions(dbName, tblName, parts);
   }
-
 }

Modified: hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java?rev=1562653&r1=1562652&r2=1562653&view=diff
==============================================================================
--- hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java (original)
+++ hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java Thu Jan 30 00:24:28 2014
@@ -562,12 +562,11 @@ public class DummyRawStoreForJdoConnecti
   }
 
   @Override
-  public ColumnStatistics getTableColumnStatistics(String dbName, String tableName, String colName)
-      throws MetaException, NoSuchObjectException {
+  public  ColumnStatistics getTableColumnStatistics(String dbName, String tableName,
+      List<String> colName) throws MetaException, NoSuchObjectException {
     return null;
   }
 
-
   @Override
   public boolean deleteTableColumnStatistics(String dbName, String tableName,
                                               String colName)
@@ -585,13 +584,6 @@ public class DummyRawStoreForJdoConnecti
   }
 
   @Override
-  public ColumnStatistics getPartitionColumnStatistics(String dbName, String tableName,
-    String partName, List<String> partVal, String colName) throws MetaException,
-    NoSuchObjectException, InvalidInputException, InvalidObjectException  {
-    return null;
-  }
-
-  @Override
   public boolean updateTableColumnStatistics(ColumnStatistics statsObj)
       throws NoSuchObjectException, MetaException, InvalidObjectException {
     return false;
@@ -616,6 +608,13 @@ public class DummyRawStoreForJdoConnecti
   }
 
   @Override
+  public List<ColumnStatistics> getPartitionColumnStatistics(String dbName,
+      String tblName, List<String> colNames, List<String> partNames)
+      throws MetaException, NoSuchObjectException {
+    return null;
+  }
+
+  @Override
   public boolean doesPartitionExist(String dbName, String tableName,
       List<String> partVals) throws MetaException, NoSuchObjectException {
     return false;

Modified: hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/VerifyingObjectStore.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/VerifyingObjectStore.java?rev=1562653&r1=1562652&r2=1562653&view=diff
==============================================================================
--- hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/VerifyingObjectStore.java (original)
+++ hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/VerifyingObjectStore.java Thu Jan 30 00:24:28 2014
@@ -35,6 +35,7 @@ import org.apache.commons.lang.ClassUtil
 import org.apache.commons.lang.builder.EqualsBuilder;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
 import org.apache.hadoop.hive.metastore.api.MetaException;
 import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
 import org.apache.hadoop.hive.metastore.api.Partition;
@@ -55,7 +56,7 @@ class VerifyingObjectStore extends Objec
         dbName, tblName, filter, maxParts, true, false);
     List<Partition> ormResults = getPartitionsByFilterInternal(
         dbName, tblName, filter, maxParts, false, true);
-    verifyParts(sqlResults, ormResults);
+    verifyLists(sqlResults, ormResults, Partition.class);
     return sqlResults;
   }
 
@@ -66,7 +67,7 @@ class VerifyingObjectStore extends Objec
         dbName, tblName, partNames, true, false);
     List<Partition> ormResults = getPartitionsByNamesInternal(
         dbName, tblName, partNames, false, true);
-    verifyParts(sqlResults, ormResults);
+    verifyLists(sqlResults, ormResults, Partition.class);
     return sqlResults;
   }
 
@@ -83,21 +84,59 @@ class VerifyingObjectStore extends Objec
       LOG.error(msg);
       throw new MetaException(msg);
     }
-    verifyParts(result, ormParts);
+    verifyLists(result, ormParts, Partition.class);
     return sqlResult;
   }
 
   @Override
   public List<Partition> getPartitions(
-      String dbName, String tableName, int maxParts) throws MetaException {
+      String dbName, String tableName, int maxParts) throws MetaException, NoSuchObjectException {
     List<Partition> sqlResults = getPartitionsInternal(dbName, tableName, maxParts, true, false);
     List<Partition> ormResults = getPartitionsInternal(dbName, tableName, maxParts, false, true);
-    verifyParts(sqlResults, ormResults);
+    verifyLists(sqlResults, ormResults, Partition.class);
     return sqlResults;
-  };
+  }
+
+  @Override
+  public ColumnStatistics getTableColumnStatistics(String dbName,
+      String tableName, List<String> colNames) throws MetaException, NoSuchObjectException {
+    ColumnStatistics sqlResult = getTableColumnStatisticsInternal(
+        dbName, tableName, colNames, true, false);
+    ColumnStatistics jdoResult = getTableColumnStatisticsInternal(
+        dbName, tableName, colNames, false, true);
+    verifyObjects(sqlResult, jdoResult, ColumnStatistics.class);
+    return sqlResult;
+  }
+
+  @Override
+  public List<ColumnStatistics> getPartitionColumnStatistics(String dbName,
+      String tableName, List<String> partNames, List<String> colNames)
+      throws MetaException, NoSuchObjectException {
+    List<ColumnStatistics> sqlResult = getPartitionColumnStatisticsInternal(
+        dbName, tableName, partNames, colNames, true, false);
+    List<ColumnStatistics> jdoResult = getPartitionColumnStatisticsInternal(
+        dbName, tableName, partNames, colNames,  false, true);
+    verifyLists(sqlResult, jdoResult, ColumnStatistics.class);
+    return sqlResult;
+  }
+
+  private void verifyObjects(
+      Object sqlResult, Object jdoResult, Class<?> clazz) throws MetaException {
+    if (EqualsBuilder.reflectionEquals(sqlResult, jdoResult)) return;
+    StringBuilder errorStr = new StringBuilder("Objects are different: \n");
+    try {
+      dumpObject(errorStr, "SQL", sqlResult, clazz, 0);
+      errorStr.append("\n");
+      dumpObject(errorStr, "ORM", jdoResult, clazz, 0);
+    } catch (Throwable t) {
+      errorStr.append("Error getting the diff: " + t);
+    }
+    LOG.error("Different results: \n" + errorStr.toString());
+    throw new MetaException("Different results from SQL and ORM, see log for details");
+  }
 
-  private void verifyParts(Collection<Partition> sqlResults, Collection<Partition> ormResults)
-      throws MetaException {
+  private <T> void verifyLists(Collection<T> sqlResults, Collection<T> ormResults,
+      Class<?> clazz) throws MetaException {
     final int MAX_DIFFS = 5;
     if (sqlResults.size() != ormResults.size()) {
       String msg = "Lists are not the same size: SQL " + sqlResults.size()
@@ -106,18 +145,18 @@ class VerifyingObjectStore extends Objec
       throw new MetaException(msg);
     }
 
-    Iterator<Partition> sqlIter = sqlResults.iterator(), ormIter = ormResults.iterator();
+    Iterator<T> sqlIter = sqlResults.iterator(), ormIter = ormResults.iterator();
     StringBuilder errorStr = new StringBuilder();
     int errors = 0;
     for (int partIx = 0; partIx < sqlResults.size(); ++partIx) {
       assert sqlIter.hasNext() && ormIter.hasNext();
-      Partition p1 = sqlIter.next(), p2 = ormIter.next();
+      T p1 = sqlIter.next(), p2 = ormIter.next();
       if (EqualsBuilder.reflectionEquals(p1, p2)) continue;
       errorStr.append("Results are different at list index " + partIx + ": \n");
       try {
-        dumpObject(errorStr, "SQL", p1, Partition.class, 0);
+        dumpObject(errorStr, "SQL", p1, clazz, 0);
         errorStr.append("\n");
-        dumpObject(errorStr, "ORM", p2, Partition.class, 0);
+        dumpObject(errorStr, "ORM", p2, clazz, 0);
         errorStr.append("\n\n");
       } catch (Throwable t) {
         String msg = "Error getting the diff at list index " + partIx;
@@ -136,8 +175,8 @@ class VerifyingObjectStore extends Objec
     }
   }
 
-  private void dumpObject(StringBuilder errorStr, String name, Object p, Class<?> c, int level)
-      throws IllegalAccessException {
+  private static void dumpObject(StringBuilder errorStr, String name, Object p,
+      Class<?> c, int level) throws IllegalAccessException {
     String offsetStr = repeat("  ", level);
     if (p == null || c == String.class || c.isPrimitive()
         || ClassUtils.wrapperToPrimitive(c) != null) {

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java?rev=1562653&r1=1562652&r2=1562653&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java Thu Jan 30 00:24:28 2014
@@ -65,6 +65,7 @@ import org.apache.hadoop.hive.metastore.
 import org.apache.hadoop.hive.metastore.Warehouse;
 import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
 import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
 import org.apache.hadoop.hive.metastore.api.Database;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.metastore.api.HiveObjectPrivilege;
@@ -2472,26 +2473,25 @@ private void constructOneLBLocationMap(F
     }
   }
 
-  public ColumnStatistics getTableColumnStatistics(String dbName, String tableName, String colName)
-    throws HiveException {
+  public List<ColumnStatisticsObj> getTableColumnStatistics(
+      String dbName, String tableName, List<String> colNames) throws HiveException {
     try {
-      return getMSC().getTableColumnStatistics(dbName, tableName, colName);
+      return getMSC().getTableColumnStatistics(dbName, tableName, colNames);
     } catch (Exception e) {
       LOG.debug(StringUtils.stringifyException(e));
       throw new HiveException(e);
     }
-
   }
 
-  public ColumnStatistics getPartitionColumnStatistics(String dbName, String tableName,
-    String partName, String colName) throws HiveException {
+  public Map<String, List<ColumnStatisticsObj>> getPartitionColumnStatistics(String dbName,
+      String tableName, List<String> partNames, List<String> colNames) throws HiveException {
       try {
-        return getMSC().getPartitionColumnStatistics(dbName, tableName, partName, colName);
-      } catch (Exception e) {
-        LOG.debug(StringUtils.stringifyException(e));
-        throw new HiveException(e);
-      }
+      return getMSC().getPartitionColumnStatistics(dbName, tableName, partNames, colNames);
+    } catch (Exception e) {
+      LOG.debug(StringUtils.stringifyException(e));
+      throw new HiveException(e);
     }
+  }
 
   public boolean deleteTableColumnStatistics(String dbName, String tableName, String colName)
     throws HiveException {



Mime
View raw message