hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ser...@apache.org
Subject [27/31] hive git commit: HIVE-15665 : LLAP: OrcFileMetadata objects in cache can impact heap usage (Sergey Shelukhin, reviewed by Prasanth Jayachandran)
Date Fri, 15 Sep 2017 19:52:28 GMT
http://git-wip-us.apache.org/repos/asf/hive/blob/50fb6f3c/ql/src/test/results/clientpositive/llap/orc_ppd_schema_evol_3a.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/orc_ppd_schema_evol_3a.q.out b/ql/src/test/results/clientpositive/llap/orc_ppd_schema_evol_3a.q.out
index b799527..1157edb 100644
--- a/ql/src/test/results/clientpositive/llap/orc_ppd_schema_evol_3a.q.out
+++ b/ql/src/test/results/clientpositive/llap/orc_ppd_schema_evol_3a.q.out
@@ -215,9 +215,10 @@ Stage-1 HIVE COUNTERS:
    RECORDS_OUT_0: 1
    RECORDS_OUT_INTERMEDIATE_Map_1: 0
 Stage-1 LLAP IO COUNTERS:
-   ALLOCATED_BYTES: 524288
-   ALLOCATED_USED_BYTES: 269
-   CACHE_MISS_BYTES: 249
+   ALLOCATED_BYTES: 1048576
+   ALLOCATED_USED_BYTES: 382
+   CACHE_HIT_BYTES: 0
+   CACHE_MISS_BYTES: 359
    METADATA_CACHE_MISS: 2
    NUM_DECODED_BATCHES: 3
    NUM_VECTOR_BATCHES: 3
@@ -255,7 +256,7 @@ Stage-1 HIVE COUNTERS:
    RECORDS_OUT_0: 1
    RECORDS_OUT_INTERMEDIATE_Map_1: 8
 Stage-1 LLAP IO COUNTERS:
-   CACHE_HIT_BYTES: 249
+   CACHE_HIT_BYTES: 359
    CACHE_MISS_BYTES: 0
    METADATA_CACHE_HIT: 2
    NUM_DECODED_BATCHES: 3
@@ -268,9 +269,9 @@ PREHOOK: type: QUERY
 PREHOOK: Input: default@orc_ppd
 #### A masked pattern was here ####
 Stage-1 FILE SYSTEM COUNTERS:
-   HDFS_BYTES_READ: 0
+   HDFS_BYTES_READ: 720
    HDFS_BYTES_WRITTEN: 101
-   HDFS_READ_OPS: 2
+   HDFS_READ_OPS: 3
    HDFS_LARGE_READ_OPS: 0
    HDFS_WRITE_OPS: 2
 Stage-1 HIVE COUNTERS:
@@ -280,8 +281,10 @@ Stage-1 HIVE COUNTERS:
    RECORDS_OUT_0: 1
    RECORDS_OUT_INTERMEDIATE_Map_1: 8
 Stage-1 LLAP IO COUNTERS:
-   CACHE_HIT_BYTES: 249
-   CACHE_MISS_BYTES: 0
+   ALLOCATED_BYTES: 262144
+   ALLOCATED_USED_BYTES: 2376
+   CACHE_HIT_BYTES: 359
+   CACHE_MISS_BYTES: 720
    METADATA_CACHE_HIT: 2
    NUM_DECODED_BATCHES: 1
    NUM_VECTOR_BATCHES: 1
@@ -305,7 +308,7 @@ Stage-1 HIVE COUNTERS:
    RECORDS_OUT_0: 1
    RECORDS_OUT_INTERMEDIATE_Map_1: 18
 Stage-1 LLAP IO COUNTERS:
-   CACHE_HIT_BYTES: 249
+   CACHE_HIT_BYTES: 359
    CACHE_MISS_BYTES: 0
    METADATA_CACHE_HIT: 2
    NUM_DECODED_BATCHES: 3
@@ -330,7 +333,7 @@ Stage-1 HIVE COUNTERS:
    RECORDS_OUT_0: 1
    RECORDS_OUT_INTERMEDIATE_Map_1: 18
 Stage-1 LLAP IO COUNTERS:
-   CACHE_HIT_BYTES: 249
+   CACHE_HIT_BYTES: 1079
    CACHE_MISS_BYTES: 0
    METADATA_CACHE_HIT: 2
    NUM_DECODED_BATCHES: 2
@@ -359,7 +362,7 @@ Stage-1 HIVE COUNTERS:
    RECORDS_OUT_0: 1
    RECORDS_OUT_INTERMEDIATE_Map_1: 0
 Stage-1 LLAP IO COUNTERS:
-   CACHE_HIT_BYTES: 249
+   CACHE_HIT_BYTES: 359
    CACHE_MISS_BYTES: 0
    METADATA_CACHE_HIT: 2
    NUM_DECODED_BATCHES: 3
@@ -398,7 +401,7 @@ Stage-1 HIVE COUNTERS:
    RECORDS_OUT_0: 1
    RECORDS_OUT_INTERMEDIATE_Map_1: 8
 Stage-1 LLAP IO COUNTERS:
-   CACHE_HIT_BYTES: 249
+   CACHE_HIT_BYTES: 359
    CACHE_MISS_BYTES: 0
    METADATA_CACHE_HIT: 2
    NUM_DECODED_BATCHES: 3
@@ -423,7 +426,7 @@ Stage-1 HIVE COUNTERS:
    RECORDS_OUT_0: 1
    RECORDS_OUT_INTERMEDIATE_Map_1: 8
 Stage-1 LLAP IO COUNTERS:
-   CACHE_HIT_BYTES: 249
+   CACHE_HIT_BYTES: 1079
    CACHE_MISS_BYTES: 0
    METADATA_CACHE_HIT: 2
    NUM_DECODED_BATCHES: 1
@@ -448,7 +451,7 @@ Stage-1 HIVE COUNTERS:
    RECORDS_OUT_0: 1
    RECORDS_OUT_INTERMEDIATE_Map_1: 18
 Stage-1 LLAP IO COUNTERS:
-   CACHE_HIT_BYTES: 249
+   CACHE_HIT_BYTES: 359
    CACHE_MISS_BYTES: 0
    METADATA_CACHE_HIT: 2
    NUM_DECODED_BATCHES: 3
@@ -473,7 +476,7 @@ Stage-1 HIVE COUNTERS:
    RECORDS_OUT_0: 1
    RECORDS_OUT_INTERMEDIATE_Map_1: 18
 Stage-1 LLAP IO COUNTERS:
-   CACHE_HIT_BYTES: 249
+   CACHE_HIT_BYTES: 1079
    CACHE_MISS_BYTES: 0
    METADATA_CACHE_HIT: 2
    NUM_DECODED_BATCHES: 2
@@ -502,7 +505,7 @@ Stage-1 HIVE COUNTERS:
    RECORDS_OUT_0: 1
    RECORDS_OUT_INTERMEDIATE_Map_1: 0
 Stage-1 LLAP IO COUNTERS:
-   CACHE_HIT_BYTES: 249
+   CACHE_HIT_BYTES: 359
    CACHE_MISS_BYTES: 0
    METADATA_CACHE_HIT: 2
    NUM_DECODED_BATCHES: 3
@@ -541,7 +544,7 @@ Stage-1 HIVE COUNTERS:
    RECORDS_OUT_0: 1
    RECORDS_OUT_INTERMEDIATE_Map_1: 8
 Stage-1 LLAP IO COUNTERS:
-   CACHE_HIT_BYTES: 249
+   CACHE_HIT_BYTES: 359
    CACHE_MISS_BYTES: 0
    METADATA_CACHE_HIT: 2
    NUM_DECODED_BATCHES: 3
@@ -566,7 +569,7 @@ Stage-1 HIVE COUNTERS:
    RECORDS_OUT_0: 1
    RECORDS_OUT_INTERMEDIATE_Map_1: 8
 Stage-1 LLAP IO COUNTERS:
-   CACHE_HIT_BYTES: 249
+   CACHE_HIT_BYTES: 1079
    CACHE_MISS_BYTES: 0
    METADATA_CACHE_HIT: 2
    NUM_DECODED_BATCHES: 1
@@ -591,7 +594,7 @@ Stage-1 HIVE COUNTERS:
    RECORDS_OUT_0: 1
    RECORDS_OUT_INTERMEDIATE_Map_1: 18
 Stage-1 LLAP IO COUNTERS:
-   CACHE_HIT_BYTES: 249
+   CACHE_HIT_BYTES: 359
    CACHE_MISS_BYTES: 0
    METADATA_CACHE_HIT: 2
    NUM_DECODED_BATCHES: 3
@@ -616,7 +619,7 @@ Stage-1 HIVE COUNTERS:
    RECORDS_OUT_0: 1
    RECORDS_OUT_INTERMEDIATE_Map_1: 18
 Stage-1 LLAP IO COUNTERS:
-   CACHE_HIT_BYTES: 249
+   CACHE_HIT_BYTES: 1079
    CACHE_MISS_BYTES: 0
    METADATA_CACHE_HIT: 2
    NUM_DECODED_BATCHES: 2
@@ -645,7 +648,7 @@ Stage-1 HIVE COUNTERS:
    RECORDS_OUT_0: 1
    RECORDS_OUT_INTERMEDIATE_Map_1: 0
 Stage-1 LLAP IO COUNTERS:
-   CACHE_HIT_BYTES: 249
+   CACHE_HIT_BYTES: 359
    CACHE_MISS_BYTES: 0
    METADATA_CACHE_HIT: 2
    NUM_DECODED_BATCHES: 3
@@ -684,7 +687,7 @@ Stage-1 HIVE COUNTERS:
    RECORDS_OUT_0: 1
    RECORDS_OUT_INTERMEDIATE_Map_1: 8
 Stage-1 LLAP IO COUNTERS:
-   CACHE_HIT_BYTES: 249
+   CACHE_HIT_BYTES: 359
    CACHE_MISS_BYTES: 0
    METADATA_CACHE_HIT: 2
    NUM_DECODED_BATCHES: 3
@@ -709,7 +712,7 @@ Stage-1 HIVE COUNTERS:
    RECORDS_OUT_0: 1
    RECORDS_OUT_INTERMEDIATE_Map_1: 8
 Stage-1 LLAP IO COUNTERS:
-   CACHE_HIT_BYTES: 249
+   CACHE_HIT_BYTES: 1079
    CACHE_MISS_BYTES: 0
    METADATA_CACHE_HIT: 2
    NUM_DECODED_BATCHES: 1
@@ -734,7 +737,7 @@ Stage-1 HIVE COUNTERS:
    RECORDS_OUT_0: 1
    RECORDS_OUT_INTERMEDIATE_Map_1: 18
 Stage-1 LLAP IO COUNTERS:
-   CACHE_HIT_BYTES: 249
+   CACHE_HIT_BYTES: 359
    CACHE_MISS_BYTES: 0
    METADATA_CACHE_HIT: 2
    NUM_DECODED_BATCHES: 3
@@ -759,7 +762,7 @@ Stage-1 HIVE COUNTERS:
    RECORDS_OUT_0: 1
    RECORDS_OUT_INTERMEDIATE_Map_1: 18
 Stage-1 LLAP IO COUNTERS:
-   CACHE_HIT_BYTES: 249
+   CACHE_HIT_BYTES: 1079
    CACHE_MISS_BYTES: 0
    METADATA_CACHE_HIT: 2
    NUM_DECODED_BATCHES: 2
@@ -890,9 +893,9 @@ PREHOOK: type: QUERY
 PREHOOK: Input: default@orc_ppd
 #### A masked pattern was here ####
 Stage-1 FILE SYSTEM COUNTERS:
-   HDFS_BYTES_READ: 5287
+   HDFS_BYTES_READ: 4912
    HDFS_BYTES_WRITTEN: 101
-   HDFS_READ_OPS: 4
+   HDFS_READ_OPS: 3
    HDFS_LARGE_READ_OPS: 0
    HDFS_WRITE_OPS: 2
 Stage-1 HIVE COUNTERS:
@@ -902,12 +905,11 @@ Stage-1 HIVE COUNTERS:
    RECORDS_OUT_0: 1
    RECORDS_OUT_INTERMEDIATE_Map_1: 2
 Stage-1 LLAP IO COUNTERS:
-   ALLOCATED_BYTES: 262144
-   ALLOCATED_USED_BYTES: 8400
-   CACHE_HIT_BYTES: 0
-   CACHE_MISS_BYTES: 4809
-   METADATA_CACHE_HIT: 1
-   METADATA_CACHE_MISS: 1
+   ALLOCATED_BYTES: 524288
+   ALLOCATED_USED_BYTES: 8527
+   CACHE_HIT_BYTES: 24
+   CACHE_MISS_BYTES: 4912
+   METADATA_CACHE_HIT: 2
    NUM_DECODED_BATCHES: 3
    NUM_VECTOR_BATCHES: 3
    ROWS_EMITTED: 2100
@@ -918,25 +920,27 @@ PREHOOK: type: QUERY
 PREHOOK: Input: default@orc_ppd
 #### A masked pattern was here ####
 Stage-1 FILE SYSTEM COUNTERS:
-   HDFS_BYTES_READ: 0
+   HDFS_BYTES_READ: 1751
    HDFS_BYTES_WRITTEN: 101
-   HDFS_READ_OPS: 2
+   HDFS_READ_OPS: 3
    HDFS_LARGE_READ_OPS: 0
    HDFS_WRITE_OPS: 2
 Stage-1 HIVE COUNTERS:
    CREATED_FILES: 1
    DESERIALIZE_ERRORS: 0
-   RECORDS_IN_Map_1: 2100
+   RECORDS_IN_Map_1: 100
    RECORDS_OUT_0: 1
    RECORDS_OUT_INTERMEDIATE_Map_1: 2
 Stage-1 LLAP IO COUNTERS:
-   CACHE_HIT_BYTES: 4809
-   CACHE_MISS_BYTES: 0
+   ALLOCATED_BYTES: 262144
+   ALLOCATED_USED_BYTES: 2376
+   CACHE_HIT_BYTES: 4936
+   CACHE_MISS_BYTES: 1751
    METADATA_CACHE_HIT: 2
-   NUM_DECODED_BATCHES: 3
-   NUM_VECTOR_BATCHES: 3
-   ROWS_EMITTED: 2100
-   SELECTED_ROWGROUPS: 3
+   NUM_DECODED_BATCHES: 1
+   NUM_VECTOR_BATCHES: 1
+   ROWS_EMITTED: 100
+   SELECTED_ROWGROUPS: 1
 2
 PREHOOK: query: alter table orc_ppd change column f f double
 PREHOOK: type: ALTERTABLE_RENAMECOL
@@ -1027,9 +1031,9 @@ PREHOOK: type: QUERY
 PREHOOK: Input: default@orc_ppd
 #### A masked pattern was here ####
 Stage-1 FILE SYSTEM COUNTERS:
-   HDFS_BYTES_READ: 4577
+   HDFS_BYTES_READ: 4099
    HDFS_BYTES_WRITTEN: 101
-   HDFS_READ_OPS: 4
+   HDFS_READ_OPS: 3
    HDFS_LARGE_READ_OPS: 0
    HDFS_WRITE_OPS: 2
 Stage-1 HIVE COUNTERS:
@@ -1039,12 +1043,11 @@ Stage-1 HIVE COUNTERS:
    RECORDS_OUT_0: 1
    RECORDS_OUT_INTERMEDIATE_Map_1: 6
 Stage-1 LLAP IO COUNTERS:
-   ALLOCATED_BYTES: 786432
-   ALLOCATED_USED_BYTES: 11299
-   CACHE_HIT_BYTES: 0
-   CACHE_MISS_BYTES: 3980
-   METADATA_CACHE_HIT: 1
-   METADATA_CACHE_MISS: 1
+   ALLOCATED_BYTES: 1048576
+   ALLOCATED_USED_BYTES: 11420
+   CACHE_HIT_BYTES: 24
+   CACHE_MISS_BYTES: 4099
+   METADATA_CACHE_HIT: 2
    NUM_DECODED_BATCHES: 3
    NUM_VECTOR_BATCHES: 3
    ROWS_EMITTED: 2100
@@ -1055,9 +1058,9 @@ PREHOOK: type: QUERY
 PREHOOK: Input: default@orc_ppd
 #### A masked pattern was here ####
 Stage-1 FILE SYSTEM COUNTERS:
-   HDFS_BYTES_READ: 0
+   HDFS_BYTES_READ: 1592
    HDFS_BYTES_WRITTEN: 101
-   HDFS_READ_OPS: 2
+   HDFS_READ_OPS: 3
    HDFS_LARGE_READ_OPS: 0
    HDFS_WRITE_OPS: 2
 Stage-1 HIVE COUNTERS:
@@ -1067,8 +1070,10 @@ Stage-1 HIVE COUNTERS:
    RECORDS_OUT_0: 1
    RECORDS_OUT_INTERMEDIATE_Map_1: 6
 Stage-1 LLAP IO COUNTERS:
-   CACHE_HIT_BYTES: 3980
-   CACHE_MISS_BYTES: 0
+   ALLOCATED_BYTES: 262144
+   ALLOCATED_USED_BYTES: 2376
+   CACHE_HIT_BYTES: 4123
+   CACHE_MISS_BYTES: 1592
    METADATA_CACHE_HIT: 2
    NUM_DECODED_BATCHES: 3
    NUM_VECTOR_BATCHES: 3
@@ -1138,7 +1143,7 @@ Stage-1 HIVE COUNTERS:
    RECORDS_OUT_0: 1
    RECORDS_OUT_INTERMEDIATE_Map_1: 6
 Stage-1 LLAP IO COUNTERS:
-   CACHE_HIT_BYTES: 3980
+   CACHE_HIT_BYTES: 4123
    CACHE_MISS_BYTES: 0
    METADATA_CACHE_HIT: 2
    NUM_DECODED_BATCHES: 3
@@ -1163,7 +1168,7 @@ Stage-1 HIVE COUNTERS:
    RECORDS_OUT_0: 1
    RECORDS_OUT_INTERMEDIATE_Map_1: 6
 Stage-1 LLAP IO COUNTERS:
-   CACHE_HIT_BYTES: 3980
+   CACHE_HIT_BYTES: 5715
    CACHE_MISS_BYTES: 0
    METADATA_CACHE_HIT: 2
    NUM_DECODED_BATCHES: 3
@@ -1234,7 +1239,7 @@ Stage-1 HIVE COUNTERS:
    RECORDS_OUT_0: 1
    RECORDS_OUT_INTERMEDIATE_Map_1: 6
 Stage-1 LLAP IO COUNTERS:
-   CACHE_HIT_BYTES: 3980
+   CACHE_HIT_BYTES: 4123
    CACHE_MISS_BYTES: 0
    METADATA_CACHE_HIT: 2
    NUM_DECODED_BATCHES: 3
@@ -1259,7 +1264,7 @@ Stage-1 HIVE COUNTERS:
    RECORDS_OUT_0: 1
    RECORDS_OUT_INTERMEDIATE_Map_1: 6
 Stage-1 LLAP IO COUNTERS:
-   CACHE_HIT_BYTES: 3980
+   CACHE_HIT_BYTES: 5715
    CACHE_MISS_BYTES: 0
    METADATA_CACHE_HIT: 2
    NUM_DECODED_BATCHES: 3
@@ -1276,9 +1281,9 @@ PREHOOK: type: QUERY
 PREHOOK: Input: default@orc_ppd
 #### A masked pattern was here ####
 Stage-1 FILE SYSTEM COUNTERS:
-   HDFS_BYTES_READ: 2780
+   HDFS_BYTES_READ: 2183
    HDFS_BYTES_WRITTEN: 101
-   HDFS_READ_OPS: 4
+   HDFS_READ_OPS: 3
    HDFS_LARGE_READ_OPS: 0
    HDFS_WRITE_OPS: 2
 Stage-1 HIVE COUNTERS:
@@ -1288,12 +1293,11 @@ Stage-1 HIVE COUNTERS:
    RECORDS_OUT_0: 1
    RECORDS_OUT_INTERMEDIATE_Map_1: 4
 Stage-1 LLAP IO COUNTERS:
-   ALLOCATED_BYTES: 524288
-   ALLOCATED_USED_BYTES: 4201
-   CACHE_HIT_BYTES: 0
-   CACHE_MISS_BYTES: 2098
-   METADATA_CACHE_HIT: 1
-   METADATA_CACHE_MISS: 1
+   ALLOCATED_BYTES: 786432
+   ALLOCATED_USED_BYTES: 4293
+   CACHE_HIT_BYTES: 24
+   CACHE_MISS_BYTES: 2183
+   METADATA_CACHE_HIT: 2
    NUM_DECODED_BATCHES: 3
    NUM_VECTOR_BATCHES: 3
    ROWS_EMITTED: 2100
@@ -1323,25 +1327,27 @@ PREHOOK: type: QUERY
 PREHOOK: Input: default@orc_ppd
 #### A masked pattern was here ####
 Stage-1 FILE SYSTEM COUNTERS:
-   HDFS_BYTES_READ: 0
+   HDFS_BYTES_READ: 1217
    HDFS_BYTES_WRITTEN: 101
-   HDFS_READ_OPS: 2
+   HDFS_READ_OPS: 3
    HDFS_LARGE_READ_OPS: 0
    HDFS_WRITE_OPS: 2
 Stage-1 HIVE COUNTERS:
    CREATED_FILES: 1
    DESERIALIZE_ERRORS: 0
-   RECORDS_IN_Map_1: 2100
+   RECORDS_IN_Map_1: 1000
    RECORDS_OUT_0: 1
    RECORDS_OUT_INTERMEDIATE_Map_1: 4
 Stage-1 LLAP IO COUNTERS:
-   CACHE_HIT_BYTES: 2098
-   CACHE_MISS_BYTES: 0
+   ALLOCATED_BYTES: 262144
+   ALLOCATED_USED_BYTES: 2376
+   CACHE_HIT_BYTES: 2207
+   CACHE_MISS_BYTES: 1217
    METADATA_CACHE_HIT: 2
-   NUM_DECODED_BATCHES: 3
-   NUM_VECTOR_BATCHES: 3
-   ROWS_EMITTED: 2100
-   SELECTED_ROWGROUPS: 3
+   NUM_DECODED_BATCHES: 1
+   NUM_VECTOR_BATCHES: 1
+   ROWS_EMITTED: 1000
+   SELECTED_ROWGROUPS: 1
 4
 PREHOOK: query: select count(*) from orc_ppd where si = 442 or boo is not null or boo = false
 PREHOOK: type: QUERY

http://git-wip-us.apache.org/repos/asf/hive/blob/50fb6f3c/storage-api/src/java/org/apache/hadoop/hive/common/io/FileMetadataCache.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/io/FileMetadataCache.java
b/storage-api/src/java/org/apache/hadoop/hive/common/io/FileMetadataCache.java
index 403c3ad..e684ece 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/common/io/FileMetadataCache.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/common/io/FileMetadataCache.java
@@ -18,6 +18,8 @@
 
 package org.apache.hadoop.hive.common.io;
 
+
+import java.nio.ByteBuffer;
 import java.io.IOException;
 import java.io.InputStream;
 
@@ -30,8 +32,6 @@ public interface FileMetadataCache {
    */
   MemoryBufferOrBuffers getFileMetadata(Object fileKey);
 
-  // TODO: add BB put method(s) when merging with ORC off-heap metadata cache
-
   /**
    * Puts the metadata for a given file (e.g. a footer buffer into cache).
    * @param fileKey The file key.
@@ -43,9 +43,11 @@ public interface FileMetadataCache {
   MemoryBufferOrBuffers putFileMetadata(
       Object fileKey, int length, InputStream is) throws IOException;
 
+  MemoryBufferOrBuffers putFileMetadata(Object fileKey, ByteBuffer tailBuffer);
+
   /**
    * Releases the buffer returned from getFileMetadata or putFileMetadata method.
    * @param buffer The buffer to release.
    */
   void decRefBuffer(MemoryBufferOrBuffers buffer);
-}
\ No newline at end of file
+} 
\ No newline at end of file


Mime
View raw message