Return-Path: X-Original-To: apmail-tajo-commits-archive@minotaur.apache.org Delivered-To: apmail-tajo-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 4E11B173F6 for ; Mon, 13 Apr 2015 14:23:23 +0000 (UTC) Received: (qmail 61909 invoked by uid 500); 13 Apr 2015 14:23:23 -0000 Delivered-To: apmail-tajo-commits-archive@tajo.apache.org Received: (qmail 61814 invoked by uid 500); 13 Apr 2015 14:23:23 -0000 Mailing-List: contact commits-help@tajo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@tajo.apache.org Delivered-To: mailing list commits@tajo.apache.org Received: (qmail 61740 invoked by uid 99); 13 Apr 2015 14:23:23 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 13 Apr 2015 14:23:23 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 0A493E0A9E; Mon, 13 Apr 2015 14:23:23 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jihoonson@apache.org To: commits@tajo.apache.org Date: Mon, 13 Apr 2015 14:23:24 -0000 Message-Id: <356bf42ddaee4212b27303f48d6343e0@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [3/4] tajo git commit: TAJO-1430: Improve SQLAnalyzer by session-based parsing-result caching. TAJO-1430: Improve SQLAnalyzer by session-based parsing-result caching. Closes #442 Signed-off-by: Jihoon Son Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/7d720888 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/7d720888 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/7d720888 Branch: refs/heads/index_support Commit: 7d720888fd02b44ed18738ac4da36866fac46fa7 Parents: d57b16f Author: Dongjoon Hyun Authored: Mon Apr 13 22:49:01 2015 +0900 Committer: Jihoon Son Committed: Mon Apr 13 22:49:24 2015 +0900 ---------------------------------------------------------------------- CHANGES | 3 ++ .../org/apache/tajo/algebra/Aggregation.java | 16 ++++--- .../org/apache/tajo/algebra/AlterTable.java | 8 +++- .../apache/tajo/algebra/BetweenPredicate.java | 2 +- .../org/apache/tajo/algebra/CreateTable.java | 46 +++++++++++++------- .../java/org/apache/tajo/algebra/Insert.java | 4 +- .../org/apache/tajo/algebra/UnaryOperator.java | 4 +- .../org/apache/tajo/algebra/ValueListExpr.java | 2 +- .../java/org/apache/tajo/conf/TajoConf.java | 1 + .../org/apache/tajo/master/GlobalEngine.java | 44 +++++++++++++++++-- .../java/org/apache/tajo/session/Session.java | 11 +++++ 11 files changed, 110 insertions(+), 31 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/7d720888/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index d7977bb..a18ae88 100644 --- a/CHANGES +++ b/CHANGES @@ -12,6 +12,9 @@ Release 0.11.0 - unreleased TAJO-1135: Implement queryable virtual table for cluster information. (jihun) + TAJO-1430: Improve SQLAnalyzer by session-based parsing-result caching. + (Contributed by Dongjoon Hyun, Committed by jihoon) + IMPROVEMENT TAJO-1509: Use dedicated thread to release resource allocated to container. http://git-wip-us.apache.org/repos/asf/tajo/blob/7d720888/tajo-algebra/src/main/java/org/apache/tajo/algebra/Aggregation.java ---------------------------------------------------------------------- diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/Aggregation.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/Aggregation.java index edb523b..8bcd7e2 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/Aggregation.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/Aggregation.java @@ -71,14 +71,18 @@ public class Aggregation extends UnaryOperator { public Object clone() throws CloneNotSupportedException { Aggregation aggregation = (Aggregation) super.clone(); - aggregation.namedExprs = new NamedExpr[namedExprs.length]; - for (int i = 0; i < namedExprs.length; i++) { - aggregation.namedExprs[i] = (NamedExpr) namedExprs[i].clone(); + if (namedExprs != null) { + aggregation.namedExprs = new NamedExpr[namedExprs.length]; + for (int i = 0; i < namedExprs.length; i++) { + aggregation.namedExprs[i] = (NamedExpr) namedExprs[i].clone(); + } } - aggregation.groups = new GroupElement[groups.length]; - for (int i = 0; i < groups.length; i++) { - aggregation.groups[i] = (GroupElement) groups[i].clone(); + if (groups != null) { + aggregation.groups = new GroupElement[groups.length]; + for (int i = 0; i < groups.length; i++) { + aggregation.groups[i] = (GroupElement) groups[i].clone(); + } } return aggregation; } http://git-wip-us.apache.org/repos/asf/tajo/blob/7d720888/tajo-algebra/src/main/java/org/apache/tajo/algebra/AlterTable.java ---------------------------------------------------------------------- diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/AlterTable.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/AlterTable.java index 3ca9a7b..9440257 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/AlterTable.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/AlterTable.java @@ -168,12 +168,16 @@ public class AlterTable extends Expr { alter.newTableName = newTableName; alter.columnName = columnName; alter.newColumnName = newColumnName; - alter.addNewColumn = (ColumnDefinition) addNewColumn.clone(); + if (addNewColumn != null) { + alter.addNewColumn = (ColumnDefinition) addNewColumn.clone(); + } alter.alterTableOpType = alterTableOpType; alter.columns = columns; alter.values = values; alter.location = location; - alter.params = new HashMap(params); + if (params != null) { + alter.params = new HashMap(params); + } return alter; } } http://git-wip-us.apache.org/repos/asf/tajo/blob/7d720888/tajo-algebra/src/main/java/org/apache/tajo/algebra/BetweenPredicate.java ---------------------------------------------------------------------- diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/BetweenPredicate.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/BetweenPredicate.java index 4b17bdb..e368061 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/BetweenPredicate.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/BetweenPredicate.java @@ -81,7 +81,7 @@ public class BetweenPredicate extends Expr { between.not = not; between.symmetric = symmetric; between.predicand = (Expr) predicand.clone(); - between.begin = (Expr) between.clone(); + between.begin = (Expr) begin.clone(); between.end = (Expr) end.clone(); return between; } http://git-wip-us.apache.org/repos/asf/tajo/blob/7d720888/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java ---------------------------------------------------------------------- diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java index 4056ed3..2d4a241 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/CreateTable.java @@ -196,8 +196,12 @@ public class CreateTable extends Expr { createTable.storageType = storageType; createTable.location = location; createTable.subquery = subquery; - createTable.params = new HashMap(params); - createTable.partition = (PartitionMethodDescExpr) partition.clone(); + if (params != null) { + createTable.params = new HashMap(params); + } + if (partition != null) { + createTable.partition = (PartitionMethodDescExpr) partition.clone(); + } createTable.ifNotExists = ifNotExists; return createTable; } @@ -305,11 +309,13 @@ public class CreateTable extends Expr { @Override public Object clone() throws CloneNotSupportedException { RangePartition range = (RangePartition) super.clone(); - range.columns = new ColumnReferenceExpr[columns.length]; - for (int i = 0; i < columns.length; i++) { - range.columns[i] = (ColumnReferenceExpr) columns[i].clone(); + if (columns != null) { + range.columns = new ColumnReferenceExpr[columns.length]; + for (int i = 0; i < columns.length; i++) { + range.columns[i] = (ColumnReferenceExpr) columns[i].clone(); + } } - if (range.specifiers != null) { + if (specifiers != null) { range.specifiers = new ArrayList(); for (int i = 0; i < specifiers.size(); i++) { range.specifiers.add(specifiers.get(i)); @@ -376,9 +382,11 @@ public class CreateTable extends Expr { @Override public Object clone() throws CloneNotSupportedException { HashPartition hash = (HashPartition) super.clone(); - hash.columns = new ColumnReferenceExpr[columns.length]; - for (int i = 0; i < columns.length; i++) { - hash.columns[i] = (ColumnReferenceExpr) columns[i].clone(); + if (columns != null) { + hash.columns = new ColumnReferenceExpr[columns.length]; + for (int i = 0; i < columns.length; i++) { + hash.columns[i] = (ColumnReferenceExpr) columns[i].clone(); + } } hash.quantity = quantity; if (specifiers != null) { @@ -428,9 +436,11 @@ public class CreateTable extends Expr { @Override public Object clone() throws CloneNotSupportedException { ListPartition listPartition = (ListPartition) super.clone(); - listPartition.columns = new ColumnReferenceExpr[columns.length]; - for (int i = 0; i < columns.length; i++) { - listPartition.columns[i] = (ColumnReferenceExpr) columns[i].clone(); + if (columns != null) { + listPartition.columns = new ColumnReferenceExpr[columns.length]; + for (int i = 0; i < columns.length; i++) { + listPartition.columns[i] = (ColumnReferenceExpr) columns[i].clone(); + } } if (specifiers != null) { listPartition.specifiers = new ArrayList(); @@ -472,9 +482,11 @@ public class CreateTable extends Expr { @Override public Object clone() throws CloneNotSupportedException { ColumnPartition columnPartition = (ColumnPartition) super.clone(); - columnPartition.columns = new ColumnDefinition[columns.length]; - for (int i = 0; i < columns.length; i++) { - columnPartition.columns[i] = (ColumnDefinition) columns[i].clone(); + if (columns != null) { + columnPartition.columns = new ColumnDefinition[columns.length]; + for (int i = 0; i < columns.length; i++) { + columnPartition.columns[i] = (ColumnDefinition) columns[i].clone(); + } } return columnPartition; } @@ -524,7 +536,9 @@ public class CreateTable extends Expr { @Override public Object clone() throws CloneNotSupportedException { RangePartitionSpecifier specifier = (RangePartitionSpecifier) super.clone(); - specifier.end = (Expr) end.clone(); + if (end != null) { + specifier.end = (Expr) end.clone(); + } specifier.maxValue = maxValue; return specifier; } http://git-wip-us.apache.org/repos/asf/tajo/blob/7d720888/tajo-algebra/src/main/java/org/apache/tajo/algebra/Insert.java ---------------------------------------------------------------------- diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/Insert.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/Insert.java index ce9b703..0826d90 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/Insert.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/Insert.java @@ -151,7 +151,9 @@ public class Insert extends Expr { insert.storageType = storageType; insert.location = location; insert.subquery = (Expr) subquery.clone(); - insert.params = new HashMap(params); + if (params != null) { + insert.params = new HashMap(params); + } return insert; } } http://git-wip-us.apache.org/repos/asf/tajo/blob/7d720888/tajo-algebra/src/main/java/org/apache/tajo/algebra/UnaryOperator.java ---------------------------------------------------------------------- diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/UnaryOperator.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/UnaryOperator.java index b85d58c..46f03eb 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/UnaryOperator.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/UnaryOperator.java @@ -49,7 +49,9 @@ public abstract class UnaryOperator extends Expr { @Override public Object clone() throws CloneNotSupportedException { UnaryOperator unaryOperator = (UnaryOperator) super.clone(); - unaryOperator.child = (Expr) child.clone(); + if (child != null) { + unaryOperator.child = (Expr) child.clone(); + } return unaryOperator; } } http://git-wip-us.apache.org/repos/asf/tajo/blob/7d720888/tajo-algebra/src/main/java/org/apache/tajo/algebra/ValueListExpr.java ---------------------------------------------------------------------- diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/ValueListExpr.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/ValueListExpr.java index 32a5e82..7923e31 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/ValueListExpr.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/ValueListExpr.java @@ -52,7 +52,7 @@ public class ValueListExpr extends Expr { ValueListExpr valueListExpr = (ValueListExpr) super.clone(); valueListExpr.values = new Expr[values.length]; for (int i = 0; i < values.length; i++) { - valueListExpr.values = (Expr[]) values[i].clone(); + valueListExpr.values[i] = (Expr) values[i].clone(); } return valueListExpr; } http://git-wip-us.apache.org/repos/asf/tajo/blob/7d720888/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java index e74d842..8e2b3d2 100644 --- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java +++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java @@ -203,6 +203,7 @@ public class TajoConf extends Configuration { // Query Configuration QUERY_SESSION_TIMEOUT("tajo.query.session.timeout-sec", 60, Validators.min("0")), + QUERY_SESSION_QUERY_CACHE_SIZE("tajo.query.session.query-cache-size-kb", 1024, Validators.min("0")), // Shuffle Configuration -------------------------------------------------- PULLSERVER_PORT("tajo.pullserver.port", 0, Validators.range("0", "65535")), http://git-wip-us.apache.org/repos/asf/tajo/blob/7d720888/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java index 9d853a5..08403ff 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java @@ -19,6 +19,10 @@ package org.apache.tajo.master; import com.google.common.annotations.VisibleForTesting; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.cache.Weigher; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.service.AbstractService; @@ -32,7 +36,9 @@ import org.apache.tajo.catalog.CatalogService; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; +import org.apache.tajo.conf.TajoConf; import org.apache.tajo.engine.parser.SQLAnalyzer; +import org.apache.tajo.engine.parser.SQLSyntaxError; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.ipc.ClientProtos; import org.apache.tajo.master.TajoMaster.MasterContext; @@ -54,6 +60,7 @@ import org.apache.tajo.util.CommonTestingUtil; import java.io.IOException; import java.sql.SQLException; +import java.util.concurrent.TimeUnit; import static org.apache.tajo.ipc.ClientProtos.SubmitQueryResponse; @@ -143,6 +150,25 @@ public class GlobalEngine extends AbstractService { newQueryContext.putAll(CommonTestingUtil.getSessionVarsForTest()); } + // Set queryCache in session + int queryCacheSize = context.getConf().getIntVar(TajoConf.ConfVars.QUERY_SESSION_QUERY_CACHE_SIZE); + if (queryCacheSize > 0 && session.getQueryCache() == null) { + Weigher weighByLength = new Weigher() { + public int weigh(String key, Expr expr) { + return key.length(); + } + }; + LoadingCache cache = CacheBuilder.newBuilder() + .maximumWeight(queryCacheSize * 1024) + .weigher(weighByLength) + .expireAfterAccess(1, TimeUnit.HOURS) + .build(new CacheLoader() { + public Expr load(String sql) throws SQLSyntaxError { + return analyzer.parse(sql); + } + }); + session.setQueryCache(cache); + } return newQueryContext; } @@ -155,7 +181,7 @@ public class GlobalEngine extends AbstractService { if (isJson) { planningContext = buildExpressionFromJson(query); } else { - planningContext = buildExpressionFromSql(query); + planningContext = buildExpressionFromSql(query, session); } String jsonExpr = planningContext.toJson(); @@ -184,10 +210,22 @@ public class GlobalEngine extends AbstractService { return JsonHelper.fromJson(json, Expr.class); } - public Expr buildExpressionFromSql(String sql) throws InterruptedException, IOException, + public Expr buildExpressionFromSql(String sql, Session session) throws InterruptedException, IOException, IllegalQueryStatusException { context.getSystemMetrics().counter("Query", "totalQuery").inc(); - return analyzer.parse(sql); + try { + if (session.getQueryCache() == null) { + return analyzer.parse(sql); + } else { + return (Expr) session.getQueryCache().get(sql.trim()).clone(); + } + } catch (Exception e) { + if (e.getCause() instanceof SQLSyntaxError) { + throw (SQLSyntaxError) e.getCause(); + } else { + throw new SQLSyntaxError(e.getCause().getMessage()); + } + } } public QueryId updateQuery(QueryContext queryContext, String sql, boolean isJson) throws IOException, http://git-wip-us.apache.org/repos/asf/tajo/blob/7d720888/tajo-core/src/main/java/org/apache/tajo/session/Session.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/session/Session.java b/tajo-core/src/main/java/org/apache/tajo/session/Session.java index 7ac4f85..ea7d29a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/session/Session.java +++ b/tajo-core/src/main/java/org/apache/tajo/session/Session.java @@ -18,11 +18,13 @@ package org.apache.tajo.session; +import com.google.common.cache.LoadingCache; import com.google.common.collect.ImmutableMap; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.tajo.QueryId; import org.apache.tajo.SessionVars; +import org.apache.tajo.algebra.Expr; import org.apache.tajo.master.exec.NonForwardQueryResultScanner; import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.common.ProtoObject; @@ -40,6 +42,7 @@ public class Session implements SessionConstants, ProtoObject, Clo private String currentDatabase; private final Map sessionVariables; private final Map nonForwardQueryMap = new HashMap(); + private LoadingCache cache; // transient status private volatile long lastAccessTime; @@ -121,6 +124,14 @@ public class Session implements SessionConstants, ProtoObject, Clo return currentDatabase; } + public synchronized void setQueryCache(LoadingCache cache) { + this.cache = cache; + } + + public synchronized LoadingCache getQueryCache() { + return cache; + } + @Override public SessionProto getProto() { SessionProto.Builder builder = SessionProto.newBuilder();