Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id D343F200BE3 for ; Thu, 22 Dec 2016 15:15:42 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id D1F19160B27; Thu, 22 Dec 2016 14:15:42 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 007DB160B1F for ; Thu, 22 Dec 2016 15:15:41 +0100 (CET) Received: (qmail 19754 invoked by uid 500); 22 Dec 2016 14:15:41 -0000 Mailing-List: contact commits-help@kylin.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@kylin.apache.org Delivered-To: mailing list commits@kylin.apache.org Received: (qmail 19744 invoked by uid 99); 22 Dec 2016 14:15:41 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 22 Dec 2016 14:15:41 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 0516EDFCF4; Thu, 22 Dec 2016 14:15:40 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: liyang@apache.org To: commits@kylin.apache.org Date: Thu, 22 Dec 2016 14:15:40 -0000 Message-Id: <283784ba006d4129aeb17b7592cb404e@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [1/2] kylin git commit: temp archived-at: Thu, 22 Dec 2016 14:15:43 -0000 Repository: kylin Updated Branches: refs/heads/mhb22 [created] 1f1a7a869 temp Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8f6fbd47 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8f6fbd47 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8f6fbd47 Branch: refs/heads/mhb22 Commit: 8f6fbd478430170090203c79ee4d256c838b31b8 Parents: 85313ee Author: Hongbin Ma Authored: Thu Dec 22 09:59:43 2016 +0800 Committer: Hongbin Ma Committed: Thu Dec 22 20:33:35 2016 +0800 ---------------------------------------------------------------------- .../apache/kylin/common/KylinConfigBase.java | 6 +++- .../gtrecord/StorageResponseGTScatter.java | 3 +- .../apache/kylin/rest/service/QueryService.java | 30 +++++--------------- 3 files changed, 14 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/8f6fbd47/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index a1c9050..ceda95c 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -782,7 +782,7 @@ abstract public class KylinConfigBase implements Serializable { return Boolean.valueOf(getOptional("kylin.query.skip-empty-segments", "true")); } - @Deprecated//Limit is good even it's large. This config is meaning less since we already have scan threshold + @Deprecated //Limit is good even it's large. This config is meaning less since we already have scan threshold public int getStoragePushDownLimitMax() { return Integer.parseInt(getOptional("kylin.query.max-limit-pushdown", "10000")); } @@ -791,6 +791,10 @@ abstract public class KylinConfigBase implements Serializable { return Integer.parseInt(getOptional("kylin.query.scan-threshold", "10000000")); } + public int getLargeQueryThreshold() { + return Integer.parseInt(getOptional("kylin.query.large-query-threshold", String.valueOf((int) (getScanThreshold() * 0.1)))); + } + public int getDerivedInThreshold() { return Integer.parseInt(getOptional("kylin.query.derived-filter-translation-threshold", "20")); } http://git-wip-us.apache.org/repos/asf/kylin/blob/8f6fbd47/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java index fe1afd3..88116b5 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/StorageResponseGTScatter.java @@ -67,7 +67,8 @@ public class StorageResponseGTScatter implements IGTScanner { @Override public void close() throws IOException { - //do nothing + //If upper consumer failed while consuming the GTRecords, the consumer should call IGTScanner's close method to ensure releasing resource + Iterators. } @Override http://git-wip-us.apache.org/repos/asf/kylin/blob/8f6fbd47/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java index 8810c85..82bc407 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java @@ -19,27 +19,8 @@ package org.apache.kylin.rest.service; import java.io.IOException; -import java.sql.Connection; -import java.sql.DatabaseMetaData; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.SQLException; -import java.sql.Statement; -import java.sql.Time; -import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.UUID; +import java.sql.*; +import java.util.*; import javax.annotation.PostConstruct; import javax.sql.DataSource; @@ -360,8 +341,11 @@ public class QueryService extends BasicService { sqlResponse.setDuration(System.currentTimeMillis() - startTime); logger.info("Stats of SQL response: isException: {}, duration: {}, total scan count {}", // String.valueOf(sqlResponse.getIsException()), String.valueOf(sqlResponse.getDuration()), String.valueOf(sqlResponse.getTotalScanCount())); - if (queryCacheEnabled && !sqlResponse.getIsException() // - && (sqlResponse.getDuration() > durationThreshold || sqlResponse.getTotalScanCount() > scancountThreshold)) { + if (queryCacheEnabled && // + !sqlResponse.getIsException() && // + (sqlResponse.getDuration() > durationThreshold || sqlResponse.getTotalScanCount() > scancountThreshold) && // + (sqlResponse.getResults().size() < kylinConfig.getLargeQueryThreshold())) //don't cache too large response + { cacheManager.getCache(SUCCESS_QUERY_CACHE).put(new Element(sqlRequest, sqlResponse)); } } else {