Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 29E6C200BE3 for ; Thu, 8 Dec 2016 06:49:52 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 28A6E160B2F; Thu, 8 Dec 2016 05:49:52 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id F03C9160B2B for ; Thu, 8 Dec 2016 06:49:50 +0100 (CET) Received: (qmail 3894 invoked by uid 500); 8 Dec 2016 05:49:50 -0000 Mailing-List: contact commits-help@kylin.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@kylin.apache.org Delivered-To: mailing list commits@kylin.apache.org Received: (qmail 3801 invoked by uid 99); 8 Dec 2016 05:49:50 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 08 Dec 2016 05:49:50 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id E2FB1F2152; Thu, 8 Dec 2016 05:49:49 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: liyang@apache.org To: commits@kylin.apache.org Date: Thu, 08 Dec 2016 05:49:51 -0000 Message-Id: <71d6bb7f589045e497e4654409eb3bff@git.apache.org> In-Reply-To: <3497744d60834e6e97c7c430f9fced5e@git.apache.org> References: <3497744d60834e6e97c7c430f9fced5e@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [03/18] kylin git commit: KYLIN-1875 fix new instance of ParameterDesc & FunctionDesc archived-at: Thu, 08 Dec 2016 05:49:52 -0000 KYLIN-1875 fix new instance of ParameterDesc & FunctionDesc Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/64d9b8b8 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/64d9b8b8 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/64d9b8b8 Branch: refs/heads/master-hbase1.x Commit: 64d9b8b89f84cc31f9931e5a6cdf89bdfa13925a Parents: 227bbf4 Author: Li Yang Authored: Thu Dec 1 14:10:44 2016 +0800 Committer: Li Yang Committed: Wed Dec 7 17:53:37 2016 +0800 ---------------------------------------------------------------------- .../apache/kylin/cube/RawQueryLastHacker.java | 8 +--- .../metadata/measure/MeasureCodecTest.java | 3 +- .../kylin/measure/raw/RawMeasureType.java | 12 ++--- .../kylin/metadata/model/FunctionDesc.java | 26 ++++------- .../kylin/metadata/model/ParameterDesc.java | 47 +++++++++++++++----- .../apache/kylin/storage/StorageMockUtils.java | 32 +++---------- .../kylin/query/relnode/OLAPAggregateRel.java | 12 ++--- 7 files changed, 59 insertions(+), 81 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/64d9b8b8/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java b/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java index b0a4823..682e48c 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java @@ -57,12 +57,8 @@ public class RawQueryLastHacker { sqlDigest.groupbyColumns.add(col); } else { // For measure columns, take them as metric columns with aggregation function SUM(). - ParameterDesc colParameter = new ParameterDesc(); - colParameter.setType("column"); - colParameter.setValue(col.getName()); - FunctionDesc sumFunc = new FunctionDesc(); - sumFunc.setExpression("SUM"); - sumFunc.setParameter(colParameter); + ParameterDesc parameter = ParameterDesc.newInstance(col); + FunctionDesc sumFunc = FunctionDesc.newInstance("SUM", parameter, null); boolean measureHasSum = false; for (MeasureDesc colMeasureDesc : cubeDesc.getMeasures()) { http://git-wip-us.apache.org/repos/asf/kylin/blob/64d9b8b8/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java index 6bb71e9..18680ec 100644 --- a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java +++ b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java @@ -83,8 +83,7 @@ public class MeasureCodecTest extends LocalFileMetadataTestCase { private MeasureDesc measure(String returnType) { MeasureDesc desc = new MeasureDesc(); - FunctionDesc func = new FunctionDesc(); - func.setReturnType(returnType); + FunctionDesc func = FunctionDesc.newInstance(null, null, returnType); desc.setFunction(func); return desc; } http://git-wip-us.apache.org/repos/asf/kylin/blob/64d9b8b8/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java index 3a49d31..a5bb06b 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java @@ -197,18 +197,12 @@ public class RawMeasureType extends MeasureType> { if (sqlDigest.isRawQuery) { for (MeasureDesc measureDesc : measureDescs) { TblColRef col = this.getRawColumn(measureDesc.getFunction()); - ParameterDesc colParameter = new ParameterDesc(); - colParameter.setType("column"); - colParameter.setValue(col.getName()); - FunctionDesc rawFunc = new FunctionDesc(); - rawFunc.setExpression("RAW"); - rawFunc.setParameter(colParameter); + ParameterDesc colParameter = ParameterDesc.newInstance(col); + FunctionDesc rawFunc = FunctionDesc.newInstance("RAW", colParameter, null); if (sqlDigest.allColumns.contains(col)) { if (measureDesc.getFunction().equals(rawFunc)) { - FunctionDesc sumFunc = new FunctionDesc(); - sumFunc.setExpression("SUM"); - sumFunc.setParameter(colParameter); + FunctionDesc sumFunc = FunctionDesc.newInstance("SUM", colParameter, null); sqlDigest.aggregations.remove(sumFunc); sqlDigest.aggregations.add(rawFunc); logger.info("Add RAW measure on column " + col); http://git-wip-us.apache.org/repos/asf/kylin/blob/64d9b8b8/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java index b9e5543..832cb4a 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java @@ -40,6 +40,15 @@ import com.google.common.collect.Sets; @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) public class FunctionDesc { + public static FunctionDesc newInstance(String expression, ParameterDesc param, String returnType) { + FunctionDesc r = new FunctionDesc(); + r.expression = (expression == null) ? null : expression.toUpperCase(); + r.parameter = param; + r.returnType = returnType; + r.returnDataType = DataType.getType(returnType); + return r; + } + public static final String FUNC_SUM = "SUM"; public static final String FUNC_MIN = "MIN"; public static final String FUNC_MAX = "MAX"; @@ -202,18 +211,10 @@ public class FunctionDesc { return expression; } - public void setExpression(String expression) { - this.expression = expression; - } - public ParameterDesc getParameter() { return parameter; } - public void setParameter(ParameterDesc parameter) { - this.parameter = parameter; - } - public int getParameterCount() { int count = 0; for (ParameterDesc p = parameter; p != null; p = p.getNextParameter()) { @@ -230,19 +231,10 @@ public class FunctionDesc { return returnDataType; } - public void setReturnType(String returnType) { - this.returnType = returnType; - this.returnDataType = DataType.getType(returnType); - } - public Map getConfiguration() { return configuration; } - public void setConfiguration(Map configurations) { - this.configuration = configurations; - } - @Override public int hashCode() { final int prime = 31; http://git-wip-us.apache.org/repos/asf/kylin/blob/64d9b8b8/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java index 4a95fea..c14d061 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java @@ -19,6 +19,7 @@ package org.apache.kylin.metadata.model; import java.io.UnsupportedEncodingException; +import java.util.Arrays; import java.util.List; import com.fasterxml.jackson.annotation.JsonAutoDetect; @@ -26,12 +27,42 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; /** */ @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) public class ParameterDesc { + public static ParameterDesc newInstance(Object... objs) { + if (objs.length == 0) + throw new IllegalArgumentException(); + + ParameterDesc r = new ParameterDesc(); + + Object obj = objs[0]; + if (obj instanceof TblColRef) { + TblColRef col = (TblColRef) obj; + r.type = FunctionDesc.PARAMETER_TYPE_COLUMN; + r.value = col.getIdentity(); + r.colRefs = ImmutableList.of(col); + } else { + r.type = FunctionDesc.PARAMETER_TYPE_CONSTANT; + r.value = (String) obj; + } + + if (objs.length >= 2) { + r.nextParameter = newInstance(Arrays.copyOfRange(objs, 1, objs.length)); + if (r.nextParameter.colRefs.size() > 0) { + if (r.colRefs.isEmpty()) + r.colRefs = r.nextParameter.colRefs; + else + r.colRefs = ImmutableList.copyOf(Iterables.concat(r.colRefs, r.nextParameter.colRefs)); + } + } + return r; + } + @JsonProperty("type") private String type; @JsonProperty("value") @@ -47,10 +78,6 @@ public class ParameterDesc { return type; } - public void setType(String type) { - this.type = type; - } - public byte[] getBytes() throws UnsupportedEncodingException { return value.getBytes("UTF-8"); } @@ -58,16 +85,16 @@ public class ParameterDesc { public String getValue() { return value; } - - public void setValue(String value) { + + void setValue(String value) { this.value = value; } public List getColRefs() { return colRefs; } - - public void setColRefs(List colRefs) { + + void setColRefs(List colRefs) { this.colRefs = colRefs; } @@ -75,10 +102,6 @@ public class ParameterDesc { return nextParameter; } - public void setNextParameter(ParameterDesc nextParameter) { - this.nextParameter = nextParameter; - } - public boolean isColumnType() { return FunctionDesc.PARAMETER_TYPE_COLUMN.equals(type); } http://git-wip-us.apache.org/repos/asf/kylin/blob/64d9b8b8/core-storage/src/test/java/org/apache/kylin/storage/StorageMockUtils.java ---------------------------------------------------------------------- diff --git a/core-storage/src/test/java/org/apache/kylin/storage/StorageMockUtils.java b/core-storage/src/test/java/org/apache/kylin/storage/StorageMockUtils.java index 0786f32..0706713 100644 --- a/core-storage/src/test/java/org/apache/kylin/storage/StorageMockUtils.java +++ b/core-storage/src/test/java/org/apache/kylin/storage/StorageMockUtils.java @@ -36,8 +36,6 @@ import org.apache.kylin.metadata.model.TableRef; import org.apache.kylin.metadata.model.TblColRef; import org.apache.kylin.metadata.tuple.TupleInfo; -import com.google.common.collect.ImmutableList; - /** */ public class StorageMockUtils { @@ -83,14 +81,8 @@ public class StorageMockUtils { TblColRef priceCol = model.findColumn("DEFAULT.TEST_KYLIN_FACTPRICE"); - FunctionDesc f1 = new FunctionDesc(); - f1.setExpression("SUM"); - ParameterDesc p1 = new ParameterDesc(); - p1.setType("column"); - p1.setValue("PRICE"); - p1.setColRefs(ImmutableList.of(priceCol)); - f1.setParameter(p1); - f1.setReturnType("decimal(19,4)"); + FunctionDesc f1 = FunctionDesc.newInstance("SUM", // + ParameterDesc.newInstance(priceCol), "decimal(19,4)"); functions.add(f1); return functions; @@ -102,24 +94,12 @@ public class StorageMockUtils { TblColRef priceCol = model.findColumn("DEFAULT.TEST_KYLIN_FACT.PRICE"); TblColRef sellerCol = model.findColumn("DEFAULT.TEST_KYLIN_FACT.SELLER_ID"); - FunctionDesc f1 = new FunctionDesc(); - f1.setExpression("SUM"); - ParameterDesc p1 = new ParameterDesc(); - p1.setType("column"); - p1.setValue("PRICE"); - p1.setColRefs(ImmutableList.of(priceCol)); - f1.setParameter(p1); - f1.setReturnType("decimal(19,4)"); + FunctionDesc f1 = FunctionDesc.newInstance("SUM", // + ParameterDesc.newInstance(priceCol), "decimal(19,4)"); functions.add(f1); - FunctionDesc f2 = new FunctionDesc(); - f2.setExpression("COUNT_DISTINCT"); - ParameterDesc p2 = new ParameterDesc(); - p2.setType("column"); - p2.setValue("SELLER_ID"); - p2.setColRefs(ImmutableList.of(sellerCol)); - f2.setParameter(p2); - f2.setReturnType("hllc(10)"); + FunctionDesc f2 = FunctionDesc.newInstance("COUNT_DISTINCT", // + ParameterDesc.newInstance(sellerCol), "hllc(10)"); functions.add(f2); return functions; http://git-wip-us.apache.org/repos/asf/kylin/blob/64d9b8b8/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java index 62351d3..24933f5 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java @@ -19,7 +19,6 @@ package org.apache.kylin.query.relnode; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -238,16 +237,11 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel { int index = aggCall.getArgList().get(0); TblColRef column = inputColumnRowType.getColumnByIndex(index); if (!column.isInnerColumn()) { - parameter = new ParameterDesc(); - parameter.setValue(column.getName()); - parameter.setType(FunctionDesc.PARAMETER_TYPE_COLUMN); - parameter.setColRefs(Arrays.asList(column)); + parameter = ParameterDesc.newInstance(column); } } - FunctionDesc aggFunc = new FunctionDesc(); - String funcName = getAggrFuncName(aggCall); - aggFunc.setExpression(funcName); - aggFunc.setParameter(parameter); + String expression = getAggrFuncName(aggCall); + FunctionDesc aggFunc = FunctionDesc.newInstance(expression, parameter, null); this.aggregations.add(aggFunc); } }