kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kangkai...@apache.org
Subject kylin git commit: KYLIN-2377 Add kylin client query timeout
Date Wed, 08 Feb 2017 07:52:35 GMT
Repository: kylin
Updated Branches:
  refs/heads/master 0dcce15f7 -> 8263752a4


KYLIN-2377 Add kylin client query timeout


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8263752a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8263752a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8263752a

Branch: refs/heads/master
Commit: 8263752a499158342e0588fda851a4006e8b1669
Parents: 0dcce15
Author: kangkaisen <kangkaisen@live.com>
Authored: Tue Jan 10 14:02:18 2017 +0800
Committer: kangkaisen <kangkaisen@163.com>
Committed: Wed Feb 8 15:51:30 2017 +0800

----------------------------------------------------------------------
 .../org/apache/kylin/common/KylinConfigBase.java    |  4 ++++
 .../java/org/apache/kylin/cube/CubeInstance.java    |  1 +
 .../kylin/metadata/realization/IRealization.java    |  5 ++++-
 .../org/apache/kylin/storage/StorageContext.java    | 16 +++++++++++++++-
 .../storage/gtrecord/GTCubeStorageQueryBase.java    |  2 ++
 .../gtrecord/SequentialCubeTupleIterator.java       |  5 +++++
 .../apache/kylin/storage/hybrid/HybridInstance.java |  1 +
 7 files changed, 32 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/8263752a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index fe15b1e..ebd9dfc 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -877,6 +877,10 @@ abstract public class KylinConfigBase implements Serializable {
         return udfMap;
     }
 
+    public int getQueryTimeoutSeconds() {
+        return Integer.parseInt(this.getOptional("kylin.query.timeout-seconds", "0"));
+    }
+
     // ============================================================================
     // SERVER
     // ============================================================================

http://git-wip-us.apache.org/repos/asf/kylin/blob/8263752a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 1d60575..fb9a7a7 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -178,6 +178,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
         return sizeRecordSize;
     }
 
+    @Override
     public KylinConfig getConfig() {
         return config;
     }

http://git-wip-us.apache.org/repos/asf/kylin/blob/8263752a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
index a0243f4..aafc0f0 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
@@ -21,6 +21,7 @@ package org.apache.kylin.metadata.realization;
 import java.util.List;
 import java.util.Set;
 
+import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.metadata.model.ColumnDesc;
 import org.apache.kylin.metadata.model.DataModelDesc;
 import org.apache.kylin.metadata.model.IStorageAware;
@@ -42,7 +43,7 @@ public interface IRealization extends IStorageAware {
     public DataModelDesc getModel();
 
     public Set<TblColRef> getAllColumns();
-    
+
     public Set<ColumnDesc> getAllColumnDescs();
 
     public List<TblColRef> getAllDimensions();
@@ -60,4 +61,6 @@ public interface IRealization extends IStorageAware {
     public long getDateRangeEnd();
 
     public boolean supportsLimitPushDown();
+
+    public KylinConfig getConfig();
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/8263752a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
index ec46f83..ab0ea73 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
@@ -41,6 +41,7 @@ public class StorageContext {
     private int finalPushDownLimit = Integer.MAX_VALUE;
     private boolean hasSort = false;
     private boolean acceptPartialResult = false;
+    private long deadline;
 
     private boolean exactAggregation = false;
     private boolean needStorageAggregation = false;
@@ -123,6 +124,19 @@ public class StorageContext {
         }
     }
 
+    public long getDeadline() {
+        return this.deadline;
+    }
+
+    public void setDeadline(IRealization realization) {
+        int timeout = realization.getConfig().getQueryTimeoutSeconds() * 1000;
+        if (timeout == 0) {
+            this.deadline = Long.MAX_VALUE;
+        } else {
+            this.deadline = timeout + System.currentTimeMillis();
+        }
+    }
+
     public void markSort() {
         this.hasSort = true;
     }
@@ -202,5 +216,5 @@ public class StorageContext {
     public void setStorageQuery(IStorageQuery storageQuery) {
         this.storageQuery = storageQuery;
     }
-    
+
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/8263752a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index 4fcfad1..4dbdf94 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -120,6 +120,8 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery
{
 
         // set limit push down
         enableStorageLimitIfPossible(cuboid, groups, derivedPostAggregation, groupsD, filter,
loosenedColumnD, sqlDigest.aggregations, context);
+        // set query deadline
+        context.setDeadline(cubeInstance);
         // set cautious threshold to prevent out of memory
         setThresholdIfNecessary(dimensionsD, metrics, context);
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/8263752a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
index c621215..c6b2c6c 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
@@ -28,6 +28,7 @@ import javax.annotation.Nullable;
 
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.cube.cuboid.Cuboid;
+import org.apache.kylin.gridtable.GTScanTimeoutException;
 import org.apache.kylin.metadata.model.FunctionDesc;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.metadata.tuple.ITuple;
@@ -141,6 +142,10 @@ public class SequentialCubeTupleIterator implements ITupleIterator {
 
     @Override
     public ITuple next() {
+        if (scanCount % 100 == 1 && System.currentTimeMillis() > context.getDeadline())
{
+            throw new GTScanTimeoutException("Query Timeout!");
+        }
+
         // prevent the big query to make the Query Server OOM
         if (scanCount++ > SCAN_THRESHOLD) {
             throw new ScanOutOfLimitException("Scan count exceed the scan threshold: " +
SCAN_THRESHOLD);

http://git-wip-us.apache.org/repos/asf/kylin/blob/8263752a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
index 4f0e446..1b113ee 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
@@ -251,6 +251,7 @@ public class HybridInstance extends RootPersistentEntity implements IRealization
         return getType() + "[name=" + name + "]";
     }
 
+    @Override
     public KylinConfig getConfig() {
         return config;
     }


Mime
View raw message