hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1574255 - in /hive/trunk/ql/src: java/org/apache/hadoop/hive/ql/stats/fs/ test/queries/clientpositive/
Date Tue, 04 Mar 2014 23:41:03 GMT
Author: hashutosh
Date: Tue Mar  4 23:41:02 2014
New Revision: 1574255

URL: http://svn.apache.org/r1574255
Log:
HIVE-6539 : Couple of issues in fs based stats collection (Ashutosh Chauhan via Gunther Hagleitner)

Modified:
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/fs/FSStatsAggregator.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/fs/FSStatsPublisher.java
    hive/trunk/ql/src/test/queries/clientpositive/metadata_only_queries.q
    hive/trunk/ql/src/test/queries/clientpositive/metadata_only_queries_with_filters.q
    hive/trunk/ql/src/test/queries/clientpositive/stats_only_null.q

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/fs/FSStatsAggregator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/fs/FSStatsAggregator.java?rev=1574255&r1=1574254&r2=1574255&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/fs/FSStatsAggregator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/fs/FSStatsAggregator.java Tue Mar
 4 23:41:02 2014
@@ -80,9 +80,17 @@ public class FSStatsAggregator implement
   @Override
   public String aggregateStats(String partID, String statType) {
     long counter = 0;
+    LOG.debug("Part ID: " + partID + "\t" + statType);
     for (Map<String,Map<String,String>> statsMap : statsList) {
-      String statVal = statsMap.get(partID).get(statType);
-      counter += Long.valueOf(statVal == null ? "0" : statVal);
+      Map<String,String> partStat = statsMap.get(partID);
+      if (null == partStat) { // not all partitions are scanned in all mappers, so this could
be null.
+        continue;
+      }
+      String statVal = partStat.get(statType);
+      if (null == statVal) { // partition was found, but was empty.
+        continue;
+      }
+      counter += Long.valueOf(statVal);
     }
     LOG.info("Read stats for : " + partID + "\t" + statType + "\t" + counter);
 

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/fs/FSStatsPublisher.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/fs/FSStatsPublisher.java?rev=1574255&r1=1574254&r2=1574255&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/fs/FSStatsPublisher.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/fs/FSStatsPublisher.java Tue Mar
 4 23:41:02 2014
@@ -69,7 +69,9 @@ public class FSStatsPublisher implements
 
   @Override
   public boolean publishStat(String partKV, Map<String, String> stats) {
-    statsMap.put(partKV, stats);
+    LOG.debug("Putting in map : " + partKV + "\t" + stats);
+    // we need to do new hashmap, since stats object is reused across calls.
+    statsMap.put(partKV, new HashMap<String, String>(stats));
     return true;
   }
 
@@ -81,6 +83,7 @@ public class FSStatsPublisher implements
       LOG.debug("About to create stats file for this task : " + statsFile);
       Output output = new Output(statsFile.getFileSystem(conf).create(statsFile,true));
       LOG.info("Created file : " + statsFile);
+      LOG.info("Writing stats in it : " + statsMap);
       Utilities.runtimeSerializationKryo.get().writeObject(output, statsMap);
       output.close();
       return true;

Modified: hive/trunk/ql/src/test/queries/clientpositive/metadata_only_queries.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/metadata_only_queries.q?rev=1574255&r1=1574254&r2=1574255&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/metadata_only_queries.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/metadata_only_queries.q Tue Mar  4 23:41:02
2014
@@ -1,3 +1,4 @@
+set hive.stats.dbclass=fs;
 set hive.compute.query.using.stats=true;
 set hive.stats.autogather=true;
 create table over10k(
@@ -73,3 +74,4 @@ drop table stats_tbl;
 drop table stats_tbl_part;
 
 set hive.compute.query.using.stats=false;
+set hive.stats.dbclass=jdbc:derby;

Modified: hive/trunk/ql/src/test/queries/clientpositive/metadata_only_queries_with_filters.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/metadata_only_queries_with_filters.q?rev=1574255&r1=1574254&r2=1574255&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/metadata_only_queries_with_filters.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/metadata_only_queries_with_filters.q Tue
Mar  4 23:41:02 2014
@@ -1,3 +1,4 @@
+set hive.stats.dbclass=fs;
 set hive.compute.query.using.stats=true;
 create table over10k(
            t tinyint,
@@ -47,3 +48,4 @@ select count(*), count(1), sum(1), sum(2
 
 drop table stats_tbl_part;
 set hive.compute.query.using.stats=false;
+set hive.stats.dbclass=jdbc:derby;

Modified: hive/trunk/ql/src/test/queries/clientpositive/stats_only_null.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/stats_only_null.q?rev=1574255&r1=1574254&r2=1574255&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/stats_only_null.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/stats_only_null.q Tue Mar  4 23:41:02 2014
@@ -1,3 +1,4 @@
+set hive.stats.dbclass=fs;
 set hive.compute.query.using.stats=true;
 set hive.stats.autogather=true;
 CREATE TABLE temps_null(a double, b int, c STRING, d smallint) STORED AS TEXTFILE; 
@@ -37,3 +38,4 @@ drop table stats_null;
 drop table stats_null_part;
 drop table temps_null;
 set hive.compute.query.using.stats=false;
+set hive.stats.dbclass=jdbc:derby;



Mime
View raw message