kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From liy...@apache.org
Subject [12/12] kylin git commit: add a toggle for dumpping storage partitions
Date Tue, 27 Dec 2016 14:16:28 GMT
add a toggle for dumpping storage partitions

add more log

temp for dumpped partitions

minor fix


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/24db7c16
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/24db7c16
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/24db7c16

Branch: refs/heads/mhb-master
Commit: 24db7c167b5d8f6c1797169ae90d1d573e2c737c
Parents: 9f2d114
Author: Hongbin Ma <mahongbin@apache.org>
Authored: Fri Dec 23 13:56:01 2016 +0800
Committer: Hongbin Ma <mahongbin@apache.org>
Committed: Tue Dec 27 22:15:44 2016 +0800

----------------------------------------------------------------------
 .../kylin/common/debug/BackdoorToggles.java     | 30 ++++++++++++++++++++
 .../gtrecord/StorageResponseGTScatter.java      |  2 ++
 2 files changed, 32 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/24db7c16/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index ca4a19c..8109cf2 100644
--- a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++ b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -62,6 +62,14 @@ public class BackdoorToggles {
         return getBoolean(DEBUG_TOGGLE_LOCAL_COPROCESSOR);
     }
 
+    public static String getPartitionDumpDir() {
+        return getString(DEBUG_TOGGLE_PARTITION_DUMP_DIR);
+    }
+
+    public static String getDumpedPartitionDir() {
+        return getString(DEBUG_TOGGLE_DUMPED_PARTITION_DIR);
+    }
+
     public static int getQueryTimeout() {
         String v = getString(DEBUG_TOGGLE_QUERY_TIMEOUT);
         if (v == null)
@@ -190,6 +198,28 @@ public class BackdoorToggles {
      */
     public final static String DEBUG_TOGGLE_SHARD_ASSIGNMENT = "DEBUG_TOGGLE_SHARD_ASSIGNMENT";
 
+    /**
+     * set DEBUG_TOGGLE_PARTITION_DUMP_DIR="dir" to dump the partitions from storage.
+     * The dumped partitions are used for performance profiling, for example.
+     *
+     example:(put it into request body)
+     "backdoorToggles": {
+     "DEBUG_TOGGLE_PARTITION_DUMP_DIR": "/tmp/dumping"
+     }
+     */
+    public final static String DEBUG_TOGGLE_PARTITION_DUMP_DIR = "DEBUG_TOGGLE_PARTITION_DUMP_DIR";
+
+    /**
+     * set DEBUG_TOGGLE_DUMPED_PARTITION_DIR="dir" to specify the dir to retrieve previously
dumped partitions
+     * it's a companion toggle with DEBUG_TOGGLE_PARTITION_DUMP_DIR
+     *
+     example:(put it into request body)
+     "backdoorToggles": {
+     "DEBUG_TOGGLE_DUMPED_PARTITION_DIR": "/tmp/dumped"
+     }
+     */
+    public final static String DEBUG_TOGGLE_DUMPED_PARTITION_DIR = "DEBUG_TOGGLE_DUMPED_PARTITION_DIR";
+
     // properties on statement may go with this "channel" too
     /**
      * set ATTR_STATEMENT_MAX_ROWS="maxRows" to statement's max rows property

http://git-wip-us.apache.org/repos/asf/kylin/blob/24db7c16/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
index 6283340..dc8746f 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java
@@ -77,8 +77,10 @@ public class StorageResponseGTScatter implements IGTScanner {
     public Iterator<GTRecord> iterator() {
         Iterator<Iterator<GTRecord>> shardSubsets = Iterators.transform(blocks,
new EndpointResponseGTScatterFunc());
         if (storagePushDownLimit != Integer.MAX_VALUE) {
+            logger.info("Using SortedIteratorMergerWithLimit to merge partitions");
             return new SortedIteratorMergerWithLimit<GTRecord>(shardSubsets, storagePushDownLimit,
GTRecord.getPrimaryKeyComparator()).getIterator();
         } else {
+            logger.info("Using Iterators.concat to merge partitions");
             return Iterators.concat(shardSubsets);
         }
     }


Mime
View raw message