hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject hive git commit: HIVE-16871: CachedStore.get_aggr_stats_for has side affect (Daniel Dai, reviewed by Thejas Nair)
Date Fri, 09 Jun 2017 21:04:17 GMT
Repository: hive
Updated Branches:
  refs/heads/master c18ee4320 -> 8877e9821


HIVE-16871: CachedStore.get_aggr_stats_for has side affect (Daniel Dai, reviewed by Thejas
Nair)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8877e982
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8877e982
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8877e982

Branch: refs/heads/master
Commit: 8877e98217e076075bf156becf28e50b7173fe4a
Parents: c18ee43
Author: Daniel Dai <daijy@hortonworks.com>
Authored: Fri Jun 9 14:04:00 2017 -0700
Committer: Daniel Dai <daijy@hortonworks.com>
Committed: Fri Jun 9 14:04:00 2017 -0700

----------------------------------------------------------------------
 .../hive/metastore/cache/SharedCache.java       |  4 +-
 .../hive/metastore/cache/TestCachedStore.java   | 67 ++++++++++++++++++++
 2 files changed, 69 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/8877e982/metastore/src/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java b/metastore/src/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java
index 6b6355b..b454372 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java
@@ -137,7 +137,7 @@ public class SharedCache {
   }
 
   public static synchronized ColumnStatisticsObj getCachedTableColStats(String colStatsCacheKey)
{
-    return tableColStatsCache.get(colStatsCacheKey);
+    return tableColStatsCache.get(colStatsCacheKey).deepCopy();
   }
 
   public static synchronized void removeTableColStatsFromCache(String dbName, String tblName)
{
@@ -426,7 +426,7 @@ public class SharedCache {
   }
 
   public static synchronized ColumnStatisticsObj getCachedPartitionColStats(String key) {
-    return partitionColStatsCache.get(key);
+    return partitionColStatsCache.get(key).deepCopy();
   }
 
   public static synchronized void addPartitionColStatsToCache(String dbName, String tableName,

http://git-wip-us.apache.org/repos/asf/hive/blob/8877e982/metastore/src/test/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java
b/metastore/src/test/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java
index 7a3ec09..5a3c78d 100644
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.ObjectStore;
 import org.apache.hadoop.hive.metastore.TableType;
 import org.apache.hadoop.hive.metastore.TestObjectStore.MockPartitionExpressionProxy;
+import org.apache.hadoop.hive.metastore.api.AggrStats;
 import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
 import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
 import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
@@ -673,4 +674,70 @@ public class TestCachedStore {
     Assert.assertEquals(SharedCache.getCachedPartitionCount(), 2);
     Assert.assertEquals(SharedCache.getSdCache().size(), 2);
   }
+
+  @Test
+  public void testAggrStatsRepeatedRead() throws Exception {
+    String dbName = "testTableColStatsOps";
+    String tblName = "tbl";
+    String colName = "col";
+
+    Database db = new Database(dbName, null, "some_location", null);
+    cachedStore.createDatabase(db);
+
+    List<FieldSchema> cols = new ArrayList<FieldSchema>();
+    cols.add(new FieldSchema("f1", "int", null));
+    List<FieldSchema> partCols = new ArrayList<FieldSchema>();
+    partCols.add(new FieldSchema("col", "int", null));
+    StorageDescriptor sd =
+        new StorageDescriptor(cols, null, "input", "output", false, 0, new SerDeInfo("serde",
"seriallib", new HashMap<String, String>()),
+            null, null, null);
+
+    Table tbl =
+        new Table(tblName, dbName, null, 0, 0, 0, sd, partCols, new HashMap<String, String>(),
+            null, null, TableType.MANAGED_TABLE.toString());
+    cachedStore.createTable(tbl);
+
+    List<String> partVals1 = new ArrayList<String>();
+    partVals1.add("1");
+    List<String> partVals2 = new ArrayList<String>();
+    partVals2.add("2");
+
+    Partition ptn1 =
+        new Partition(partVals1, dbName, tblName, 0, 0, sd, new HashMap<String, String>());
+    cachedStore.addPartition(ptn1);
+    Partition ptn2 =
+        new Partition(partVals2, dbName, tblName, 0, 0, sd, new HashMap<String, String>());
+    cachedStore.addPartition(ptn2);
+
+    ColumnStatistics stats = new ColumnStatistics();
+    ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc(true, dbName, tblName);
+    statsDesc.setPartName("col");
+    List<ColumnStatisticsObj> colStatObjs = new ArrayList<ColumnStatisticsObj>();
+
+    ColumnStatisticsData data = new ColumnStatisticsData();
+    ColumnStatisticsObj colStats = new ColumnStatisticsObj(colName, "int", data);
+    LongColumnStatsData longStats = new LongColumnStatsData();
+    longStats.setLowValue(0);
+    longStats.setHighValue(100);
+    longStats.setNumNulls(50);
+    longStats.setNumDVs(30);
+    data.setLongStats(longStats);
+    colStatObjs.add(colStats);
+
+    stats.setStatsDesc(statsDesc);
+    stats.setStatsObj(colStatObjs);
+
+    cachedStore.updatePartitionColumnStatistics(stats.deepCopy(), partVals1);
+    cachedStore.updatePartitionColumnStatistics(stats.deepCopy(), partVals2);
+
+    List<String> colNames = new ArrayList<String>();
+    colNames.add(colName);
+    List<String> aggrPartVals = new ArrayList<String>();
+    aggrPartVals.add("1");
+    aggrPartVals.add("2");
+    AggrStats aggrStats = cachedStore.get_aggr_stats_for(dbName, tblName, aggrPartVals, colNames);
+    Assert.assertEquals(aggrStats.getColStats().get(0).getStatsData().getLongStats().getNumNulls(),
100);
+    aggrStats = cachedStore.get_aggr_stats_for(dbName, tblName, aggrPartVals, colNames);
+    Assert.assertEquals(aggrStats.getColStats().get(0).getStatsData().getLongStats().getNumNulls(),
100);
+  }
 }


Mime
View raw message