kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mahong...@apache.org
Subject [1/3] kylin git commit: KYLIN-2201 fix combination check overflow error
Date Thu, 17 Nov 2016 08:49:49 GMT
Repository: kylin
Updated Branches:
  refs/heads/yang21 ac4e9ec56 -> d87b7a512


KYLIN-2201 fix combination check overflow error

Signed-off-by: Hongbin Ma <mahongbin@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9bd6a2ab
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9bd6a2ab
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9bd6a2ab

Branch: refs/heads/yang21
Commit: 9bd6a2ab6b5fd8f11a80198012a519bfe4edac08
Parents: 02e723a
Author: Roger Shi <rogershijicheng@hotmail.com>
Authored: Thu Nov 17 11:07:17 2016 +0800
Committer: Hongbin Ma <mahongbin@apache.org>
Committed: Thu Nov 17 11:50:09 2016 +0800

----------------------------------------------------------------------
 .../org/apache/kylin/cube/model/CubeDesc.java   |   6 +-
 .../kylin/cube/AggregationGroupRuleTest.java    |  11 +
 .../org/apache/kylin/cube/CubeDescTest.java     |   7 +
 .../ut_cube_desc_combination_int_overflow.json  | 398 +++++++++++++++++++
 4 files changed, 419 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/9bd6a2ab/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 4195451..c914ebb 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -580,7 +580,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware
{
                 throw new IllegalStateException("Aggregation group " + index + " select rule
field not set");
             }
 
-            int combination = 1;
+            long combination = 1;
             Set<String> includeDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
             getDims(includeDims, agg.getIncludes());
 
@@ -598,7 +598,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware
{
             Set<String> jointDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
             getDims(jointDimsList, jointDims, agg.getSelectRule().joint_dims);
             if (jointDimsList.size() > 0) {
-                combination = combination * (1 << jointDimsList.size());
+                combination = combination * (1L << jointDimsList.size());
             }
 
             if (!includeDims.containsAll(mandatoryDims) || !includeDims.containsAll(hierarchyDims)
|| !includeDims.containsAll(jointDims)) {
@@ -612,7 +612,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware
{
             normalDims.removeAll(hierarchyDims);
             normalDims.removeAll(jointDims);
 
-            combination = combination * (1 << normalDims.size());
+            combination = combination * (1L << normalDims.size());
 
             if (combination > config.getCubeAggrGroupMaxCombination()) {
                 String msg = "Aggregation group " + index + " has too many combinations,
use 'mandatory'/'hierarchy'/'joint' to optimize; or update 'kylin.cube.aggrgroup.max.combination'
to a bigger value.";

http://git-wip-us.apache.org/repos/asf/kylin/blob/9bd6a2ab/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
index 6c3d544..7fea440 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
@@ -23,6 +23,7 @@ import static org.junit.Assert.assertTrue;
 
 import java.io.File;
 import java.io.FileInputStream;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.Arrays;
 
@@ -115,6 +116,16 @@ public class AggregationGroupRuleTest {
         assertEquals("Aggregation group 0 joint columns overlap with more than 1 dim in same
hierarchy", (vContext.getResults()[0].getMessage()));
     }
 
+    @Test
+    public void testCombinationIntOverflow() throws IOException {
+        ValidateContext vContext = new ValidateContext();
+        CubeDesc desc = JsonUtil.readValue(new FileInputStream(LocalFileMetadataTestCase.LOCALMETA_TEST_DATA
+ "/cube_desc/ut_cube_desc_combination_int_overflow.json"), CubeDesc.class);
+
+        IValidatorRule<CubeDesc> rule = getAggregationGroupRule();
+        rule.validate(desc, vContext);
+        assertEquals(1, vContext.getResults().length);
+    }
+
     public AggregationGroupRule getAggregationGroupRule() {
         AggregationGroupRule rule = new AggregationGroupRule() {
             @Override

http://git-wip-us.apache.org/repos/asf/kylin/blob/9bd6a2ab/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
index 98cff43..597121b 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
@@ -194,6 +194,13 @@ public class CubeDescTest extends LocalFileMetadataTestCase {
     }
 
     @Test
+    public void testCombinationIntOverflow() throws  Exception {
+        thrown.expect(IllegalStateException.class);
+        CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("ut_cube_desc_combination_int_overflow");
+        cubeDesc.init(getTestConfig());
+    }
+
+    @Test
     public void testSerialize() throws Exception {
         CubeDesc desc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("test_kylin_cube_with_slr_desc");
         String str = JsonUtil.writeValueAsIndentString(desc);

http://git-wip-us.apache.org/repos/asf/kylin/blob/9bd6a2ab/examples/test_case_data/localmeta/cube_desc/ut_cube_desc_combination_int_overflow.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/ut_cube_desc_combination_int_overflow.json
b/examples/test_case_data/localmeta/cube_desc/ut_cube_desc_combination_int_overflow.json
new file mode 100644
index 0000000..ec6f8f4
--- /dev/null
+++ b/examples/test_case_data/localmeta/cube_desc/ut_cube_desc_combination_int_overflow.json
@@ -0,0 +1,398 @@
+{
+  "uuid" : "9e89e128-f13e-4209-82d2-973985114793",
+  "last_modified" : 1479207711845,
+  "name" : "ut_cube_desc_combination_int_overflow",
+  "model_name" : "ut_large_dimension_number",
+  "description" : "",
+  "null_string" : null,
+  "dimensions" : [ {
+    "name" : "DEFAULT.WIDE_TABLE.A",
+    "table" : "DEFAULT.WIDE_TABLE",
+    "column" : "A",
+    "derived" : null
+  },{
+    "name" : "DEFAULT.WIDE_TABLE.B",
+    "table" : "DEFAULT.WIDE_TABLE",
+    "column" : "B",
+    "derived" : null
+  },{
+    "name" : "DEFAULT.WIDE_TABLE.C",
+    "table" : "DEFAULT.WIDE_TABLE",
+    "column" : "C",
+    "derived" : null
+  },{
+    "name" : "DEFAULT.WIDE_TABLE.D",
+    "table" : "DEFAULT.WIDE_TABLE",
+    "column" : "D",
+    "derived" : null
+  },{
+    "name" : "DEFAULT.WIDE_TABLE.E",
+    "table" : "DEFAULT.WIDE_TABLE",
+    "column" : "E",
+    "derived" : null
+  },{
+    "name" : "DEFAULT.WIDE_TABLE.F",
+    "table" : "DEFAULT.WIDE_TABLE",
+    "column" : "F",
+    "derived" : null
+  },{
+    "name" : "DEFAULT.WIDE_TABLE.G",
+    "table" : "DEFAULT.WIDE_TABLE",
+    "column" : "G",
+    "derived" : null
+  },{
+    "name" : "DEFAULT.WIDE_TABLE.H",
+    "table" : "DEFAULT.WIDE_TABLE",
+    "column" : "H",
+    "derived" : null
+  },{
+    "name" : "DEFAULT.WIDE_TABLE.I",
+    "table" : "DEFAULT.WIDE_TABLE",
+    "column" : "I",
+    "derived" : null
+  },{
+    "name" : "DEFAULT.WIDE_TABLE.J",
+    "table" : "DEFAULT.WIDE_TABLE",
+    "column" : "J",
+    "derived" : null
+  },{
+    "name" : "DEFAULT.WIDE_TABLE.K",
+    "table" : "DEFAULT.WIDE_TABLE",
+    "column" : "K",
+    "derived" : null
+  },{
+    "name" : "DEFAULT.WIDE_TABLE.L",
+    "table" : "DEFAULT.WIDE_TABLE",
+    "column" : "L",
+    "derived" : null
+  },{
+    "name" : "DEFAULT.WIDE_TABLE.M",
+    "table" : "DEFAULT.WIDE_TABLE",
+    "column" : "M",
+    "derived" : null
+  },{
+    "name" : "DEFAULT.WIDE_TABLE.N",
+    "table" : "DEFAULT.WIDE_TABLE",
+    "column" : "N",
+    "derived" : null
+  },{
+    "name" : "DEFAULT.WIDE_TABLE.O",
+    "table" : "DEFAULT.WIDE_TABLE",
+    "column" : "O",
+    "derived" : null
+  },{
+    "name" : "DEFAULT.WIDE_TABLE.P",
+    "table" : "DEFAULT.WIDE_TABLE",
+    "column" : "P",
+    "derived" : null
+  },{
+    "name" : "DEFAULT.WIDE_TABLE.Q",
+    "table" : "DEFAULT.WIDE_TABLE",
+    "column" : "Q",
+    "derived" : null
+  },{
+    "name" : "DEFAULT.WIDE_TABLE.R",
+    "table" : "DEFAULT.WIDE_TABLE",
+    "column" : "R",
+    "derived" : null
+  },{
+    "name" : "DEFAULT.WIDE_TABLE.S",
+    "table" : "DEFAULT.WIDE_TABLE",
+    "column" : "S",
+    "derived" : null
+  },{
+    "name" : "DEFAULT.WIDE_TABLE.T",
+    "table" : "DEFAULT.WIDE_TABLE",
+    "column" : "T",
+    "derived" : null
+  },{
+    "name" : "DEFAULT.WIDE_TABLE.U",
+    "table" : "DEFAULT.WIDE_TABLE",
+    "column" : "U",
+    "derived" : null
+  },{
+    "name" : "DEFAULT.WIDE_TABLE.V",
+    "table" : "DEFAULT.WIDE_TABLE",
+    "column" : "V",
+    "derived" : null
+  },{
+    "name" : "DEFAULT.WIDE_TABLE.W",
+    "table" : "DEFAULT.WIDE_TABLE",
+    "column" : "W",
+    "derived" : null
+  },{
+    "name" : "DEFAULT.WIDE_TABLE.X",
+    "table" : "DEFAULT.WIDE_TABLE",
+    "column" : "X",
+    "derived" : null
+  },{
+    "name" : "DEFAULT.WIDE_TABLE.Y",
+    "table" : "DEFAULT.WIDE_TABLE",
+    "column" : "Y",
+    "derived" : null
+  },{
+    "name" : "DEFAULT.WIDE_TABLE.Z",
+    "table" : "DEFAULT.WIDE_TABLE",
+    "column" : "Z",
+    "derived" : null
+  },{
+    "name" : "DEFAULT.WIDE_TABLE.AA",
+    "table" : "DEFAULT.WIDE_TABLE",
+    "column" : "AA",
+    "derived" : null
+  },{
+    "name" : "DEFAULT.WIDE_TABLE.AB",
+    "table" : "DEFAULT.WIDE_TABLE",
+    "column" : "AB",
+    "derived" : null
+  },{
+    "name" : "DEFAULT.WIDE_TABLE.AC",
+    "table" : "DEFAULT.WIDE_TABLE",
+    "column" : "AC",
+    "derived" : null
+  },{
+    "name" : "DEFAULT.WIDE_TABLE.AD",
+    "table" : "DEFAULT.WIDE_TABLE",
+    "column" : "AD",
+    "derived" : null
+  },{
+    "name" : "DEFAULT.WIDE_TABLE.AE",
+    "table" : "DEFAULT.WIDE_TABLE",
+    "column" : "AE",
+    "derived" : null
+  },{
+    "name" : "DEFAULT.WIDE_TABLE.AF",
+    "table" : "DEFAULT.WIDE_TABLE",
+    "column" : "AF",
+    "derived" : null
+  },{
+    "name" : "DEFAULT.WIDE_TABLE.AG",
+    "table" : "DEFAULT.WIDE_TABLE",
+    "column" : "AG",
+    "derived" : null
+  },{
+    "name" : "DEFAULT.WIDE_TABLE.AH",
+    "table" : "DEFAULT.WIDE_TABLE",
+    "column" : "AH",
+    "derived" : null
+  } ],
+  "measures" : [ {
+    "name" : "_COUNT_",
+    "function" : {
+      "expression" : "COUNT",
+      "parameter" : {
+        "type" : "constant",
+        "value" : "1",
+        "next_parameter" : null
+      },
+      "returntype" : "bigint"
+    },
+    "dependent_measure_ref" : null
+  } ],
+  "dictionaries" : [ ],
+  "rowkey" : {
+    "rowkey_columns" : [ {
+      "column" : "A",
+      "encoding" : "dict",
+      "isShardBy" : false,
+      "index" : "eq"
+    }, {
+      "column" : "B",
+      "encoding" : "dict",
+      "isShardBy" : false,
+      "index" : "eq"
+    }, {
+      "column" : "C",
+      "encoding" : "dict",
+      "isShardBy" : false,
+      "index" : "eq"
+    }, {
+      "column" : "D",
+      "encoding" : "dict",
+      "isShardBy" : false,
+      "index" : "eq"
+    }, {
+      "column" : "E",
+      "encoding" : "dict",
+      "isShardBy" : false,
+      "index" : "eq"
+    }, {
+      "column" : "F",
+      "encoding" : "dict",
+      "isShardBy" : false,
+      "index" : "eq"
+    }, {
+      "column" : "G",
+      "encoding" : "dict",
+      "isShardBy" : false,
+      "index" : "eq"
+    }, {
+      "column" : "H",
+      "encoding" : "dict",
+      "isShardBy" : false,
+      "index" : "eq"
+    }, {
+      "column" : "I",
+      "encoding" : "dict",
+      "isShardBy" : false,
+      "index" : "eq"
+    }, {
+      "column" : "J",
+      "encoding" : "dict",
+      "isShardBy" : false,
+      "index" : "eq"
+    }, {
+      "column" : "K",
+      "encoding" : "dict",
+      "isShardBy" : false,
+      "index" : "eq"
+    }, {
+      "column" : "L",
+      "encoding" : "dict",
+      "isShardBy" : false,
+      "index" : "eq"
+    }, {
+      "column" : "M",
+      "encoding" : "dict",
+      "isShardBy" : false,
+      "index" : "eq"
+    }, {
+      "column" : "N",
+      "encoding" : "dict",
+      "isShardBy" : false,
+      "index" : "eq"
+    }, {
+      "column" : "O",
+      "encoding" : "dict",
+      "isShardBy" : false,
+      "index" : "eq"
+    }, {
+      "column" : "P",
+      "encoding" : "dict",
+      "isShardBy" : false,
+      "index" : "eq"
+    }, {
+      "column" : "Q",
+      "encoding" : "dict",
+      "isShardBy" : false,
+      "index" : "eq"
+    }, {
+      "column" : "R",
+      "encoding" : "dict",
+      "isShardBy" : false,
+      "index" : "eq"
+    }, {
+      "column" : "S",
+      "encoding" : "dict",
+      "isShardBy" : false,
+      "index" : "eq"
+    }, {
+      "column" : "T",
+      "encoding" : "dict",
+      "isShardBy" : false,
+      "index" : "eq"
+    }, {
+      "column" : "U",
+      "encoding" : "dict",
+      "isShardBy" : false,
+      "index" : "eq"
+    }, {
+      "column" : "V",
+      "encoding" : "dict",
+      "isShardBy" : false,
+      "index" : "eq"
+    }, {
+      "column" : "W",
+      "encoding" : "dict",
+      "isShardBy" : false,
+      "index" : "eq"
+    }, {
+      "column" : "X",
+      "encoding" : "dict",
+      "isShardBy" : false,
+      "index" : "eq"
+    }, {
+      "column" : "Y",
+      "encoding" : "dict",
+      "isShardBy" : false,
+      "index" : "eq"
+    }, {
+      "column" : "Z",
+      "encoding" : "dict",
+      "isShardBy" : false,
+      "index" : "eq"
+    }, {
+      "column" : "AA",
+      "encoding" : "dict",
+      "isShardBy" : false,
+      "index" : "eq"
+    }, {
+      "column" : "AB",
+      "encoding" : "dict",
+      "isShardBy" : false,
+      "index" : "eq"
+    }, {
+      "column" : "AC",
+      "encoding" : "dict",
+      "isShardBy" : false,
+      "index" : "eq"
+    }, {
+      "column" : "AD",
+      "encoding" : "dict",
+      "isShardBy" : false,
+      "index" : "eq"
+    }, {
+      "column" : "AE",
+      "encoding" : "dict",
+      "isShardBy" : false,
+      "index" : "eq"
+    }, {
+      "column" : "AF",
+      "encoding" : "dict",
+      "isShardBy" : false,
+      "index" : "eq"
+    }, {
+      "column" : "AG",
+      "encoding" : "dict",
+      "isShardBy" : false,
+      "index" : "eq"
+    }, {
+      "column" : "AH",
+      "encoding" : "dict",
+      "isShardBy" : false,
+      "index" : "eq"
+    }]
+  },
+  "hbase_mapping" : {
+    "column_family" : [ {
+      "name" : "F1",
+      "columns" : [ {
+        "qualifier" : "M",
+        "measure_refs" : [ "_COUNT_" ]
+      } ]
+    } ]
+  },
+  "aggregation_groups" : [ {
+    "includes" : [ "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N",
"O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z", "AA", "AB", "AC", "AD", "AE",
"AF", "AG", "AH"],
+    "select_rule" : {
+      "hierarchy_dims" : [ ],
+      "mandatory_dims" : [ ],
+      "joint_dims" : [ ]
+    }
+  }],
+  "signature" : "4NqJVcVIYRF5PkJ0jS+9Rg==",
+  "notify_list" : [ ],
+  "status_need_notify" : [ "ERROR", "DISCARDED", "SUCCEED" ],
+  "partition_date_start" : 1469923200000,
+  "partition_date_end" : 3153600000000,
+  "auto_merge_time_ranges" : [ 604800000, 2419200000 ],
+  "retention_range" : 0,
+  "engine_type" : 100,
+  "storage_type" : 100,
+  "override_kylin_properties" : {
+    "kylin.cube.algorithm" : "auto",
+    "kylin.cube.aggrgroup.max.combination" : "4096",
+    "kylin.job.cubing.inmem.sampling.percent" : "100",
+    "kylin.hbase.default.compression.codec" : "none"
+  }
+}
\ No newline at end of file


Mime
View raw message