Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 105A6200C15 for ; Wed, 8 Feb 2017 08:52:37 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 0ECF9160B5A; Wed, 8 Feb 2017 07:52:37 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 30ED7160B4E for ; Wed, 8 Feb 2017 08:52:36 +0100 (CET) Received: (qmail 7823 invoked by uid 500); 8 Feb 2017 07:52:35 -0000 Mailing-List: contact commits-help@kylin.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@kylin.apache.org Delivered-To: mailing list commits@kylin.apache.org Received: (qmail 7814 invoked by uid 99); 8 Feb 2017 07:52:35 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 08 Feb 2017 07:52:35 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 3CB37DFC47; Wed, 8 Feb 2017 07:52:35 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: kangkaisen@apache.org To: commits@kylin.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: kylin git commit: KYLIN-2377 Add kylin client query timeout Date: Wed, 8 Feb 2017 07:52:35 +0000 (UTC) archived-at: Wed, 08 Feb 2017 07:52:37 -0000 Repository: kylin Updated Branches: refs/heads/master 0dcce15f7 -> 8263752a4 KYLIN-2377 Add kylin client query timeout Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8263752a Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8263752a Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8263752a Branch: refs/heads/master Commit: 8263752a499158342e0588fda851a4006e8b1669 Parents: 0dcce15 Author: kangkaisen Authored: Tue Jan 10 14:02:18 2017 +0800 Committer: kangkaisen Committed: Wed Feb 8 15:51:30 2017 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/common/KylinConfigBase.java | 4 ++++ .../java/org/apache/kylin/cube/CubeInstance.java | 1 + .../kylin/metadata/realization/IRealization.java | 5 ++++- .../org/apache/kylin/storage/StorageContext.java | 16 +++++++++++++++- .../storage/gtrecord/GTCubeStorageQueryBase.java | 2 ++ .../gtrecord/SequentialCubeTupleIterator.java | 5 +++++ .../apache/kylin/storage/hybrid/HybridInstance.java | 1 + 7 files changed, 32 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/8263752a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index fe15b1e..ebd9dfc 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -877,6 +877,10 @@ abstract public class KylinConfigBase implements Serializable { return udfMap; } + public int getQueryTimeoutSeconds() { + return Integer.parseInt(this.getOptional("kylin.query.timeout-seconds", "0")); + } + // ============================================================================ // SERVER // ============================================================================ http://git-wip-us.apache.org/repos/asf/kylin/blob/8263752a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java index 1d60575..fb9a7a7 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java @@ -178,6 +178,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization, return sizeRecordSize; } + @Override public KylinConfig getConfig() { return config; } http://git-wip-us.apache.org/repos/asf/kylin/blob/8263752a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java index a0243f4..aafc0f0 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java @@ -21,6 +21,7 @@ package org.apache.kylin.metadata.realization; import java.util.List; import java.util.Set; +import org.apache.kylin.common.KylinConfig; import org.apache.kylin.metadata.model.ColumnDesc; import org.apache.kylin.metadata.model.DataModelDesc; import org.apache.kylin.metadata.model.IStorageAware; @@ -42,7 +43,7 @@ public interface IRealization extends IStorageAware { public DataModelDesc getModel(); public Set getAllColumns(); - + public Set getAllColumnDescs(); public List getAllDimensions(); @@ -60,4 +61,6 @@ public interface IRealization extends IStorageAware { public long getDateRangeEnd(); public boolean supportsLimitPushDown(); + + public KylinConfig getConfig(); } http://git-wip-us.apache.org/repos/asf/kylin/blob/8263752a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java index ec46f83..ab0ea73 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java @@ -41,6 +41,7 @@ public class StorageContext { private int finalPushDownLimit = Integer.MAX_VALUE; private boolean hasSort = false; private boolean acceptPartialResult = false; + private long deadline; private boolean exactAggregation = false; private boolean needStorageAggregation = false; @@ -123,6 +124,19 @@ public class StorageContext { } } + public long getDeadline() { + return this.deadline; + } + + public void setDeadline(IRealization realization) { + int timeout = realization.getConfig().getQueryTimeoutSeconds() * 1000; + if (timeout == 0) { + this.deadline = Long.MAX_VALUE; + } else { + this.deadline = timeout + System.currentTimeMillis(); + } + } + public void markSort() { this.hasSort = true; } @@ -202,5 +216,5 @@ public class StorageContext { public void setStorageQuery(IStorageQuery storageQuery) { this.storageQuery = storageQuery; } - + } http://git-wip-us.apache.org/repos/asf/kylin/blob/8263752a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java index 4fcfad1..4dbdf94 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java @@ -120,6 +120,8 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery { // set limit push down enableStorageLimitIfPossible(cuboid, groups, derivedPostAggregation, groupsD, filter, loosenedColumnD, sqlDigest.aggregations, context); + // set query deadline + context.setDeadline(cubeInstance); // set cautious threshold to prevent out of memory setThresholdIfNecessary(dimensionsD, metrics, context); http://git-wip-us.apache.org/repos/asf/kylin/blob/8263752a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java index c621215..c6b2c6c 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java @@ -28,6 +28,7 @@ import javax.annotation.Nullable; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.cube.cuboid.Cuboid; +import org.apache.kylin.gridtable.GTScanTimeoutException; import org.apache.kylin.metadata.model.FunctionDesc; import org.apache.kylin.metadata.model.TblColRef; import org.apache.kylin.metadata.tuple.ITuple; @@ -141,6 +142,10 @@ public class SequentialCubeTupleIterator implements ITupleIterator { @Override public ITuple next() { + if (scanCount % 100 == 1 && System.currentTimeMillis() > context.getDeadline()) { + throw new GTScanTimeoutException("Query Timeout!"); + } + // prevent the big query to make the Query Server OOM if (scanCount++ > SCAN_THRESHOLD) { throw new ScanOutOfLimitException("Scan count exceed the scan threshold: " + SCAN_THRESHOLD); http://git-wip-us.apache.org/repos/asf/kylin/blob/8263752a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java index 4f0e446..1b113ee 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java @@ -251,6 +251,7 @@ public class HybridInstance extends RootPersistentEntity implements IRealization return getType() + "[name=" + name + "]"; } + @Override public KylinConfig getConfig() { return config; }