kylin-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "ASF GitHub Bot (JIRA)" <j...@apache.org>
Subject [jira] [Commented] (KYLIN-3694) Kylin On Druid Storage
Date Mon, 26 Nov 2018 10:40:00 GMT

    [ https://issues.apache.org/jira/browse/KYLIN-3694?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16698734#comment-16698734
] 

ASF GitHub Bot commented on KYLIN-3694:
---------------------------------------

shaofengshi closed pull request #357: KYLIN-3694 Bug fix and revert guava version
URL: https://github.com/apache/kylin/pull/357
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/assembly/pom.xml b/assembly/pom.xml
index 6dfe165cbe..1e546cb313 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -34,11 +34,6 @@
     </properties>
 
     <dependencies>
-        <dependency>
-            <groupId>com.google.guava</groupId>
-            <artifactId>guava</artifactId>
-            <scope>compile</scope>
-        </dependency>
         <dependency>
             <groupId>org.apache.kylin</groupId>
             <artifactId>kylin-source-hive</artifactId>
@@ -228,10 +223,6 @@
                                     <pattern>org.apache.commons.jocl</pattern>
                                     <shadedPattern>${shadeBase}.org.apache.commons.jocl</shadedPattern>
                                 </relocation>
-                                <relocation>
-                                    <pattern>com.google.guava</pattern>
-                                    <shadedPattern>${shadeBase}.com.google.guava</shadedPattern>
-                                </relocation>
                             </relocations>
                             <filters>
                                 <filter>
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java b/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
index 8f4a1f6919..f78e92b530 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
@@ -87,7 +87,7 @@
                 Hasher hc = hf.newHasher();
                 final String cell = row.get(flatDesc.getRowKeyColumnIndexes()[i]);
                 if (cell != null) {
-                    row_hashcodes[i] = hc.putUnencodedChars(cell).hash().asBytes();
+                    row_hashcodes[i] = hc.putString(cell).hash().asBytes();
                 } else {
                     row_hashcodes[i] = hc.putInt(0).hash().asBytes();
                 }
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtilTest.java
b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtilTest.java
index b9fb9841e1..c3642cb22d 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtilTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtilTest.java
@@ -147,7 +147,7 @@ public void createDirectChildrenCacheTest() {
 
     @Test
     public void createDirectChildrenCacheStressTest() {
-        Stopwatch sw = Stopwatch.createStarted();
+        Stopwatch sw = new Stopwatch();
         Set<Long> cuboidSet = generateMassCuboidSet();
         System.out.println("Time elapsed for creating sorted cuboid list: " + sw.elapsed(TimeUnit.MILLISECONDS));
         sw.reset();
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
index 81fe71f9f6..0e15687d7f 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
@@ -167,7 +167,8 @@ public void testEstimateBitmapMemSize() {
         }
     }
 
-    @Test
+    // Ignore because of OOM
+    // @Test
     public void testEstimateMemSize() throws InterruptedException {
         int scale = Integer.parseInt(System.getProperty("scale", "1"));
         scale = Math.max(1, Math.min(10, scale));
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CalculateStatsFromBaseCuboidMapper.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CalculateStatsFromBaseCuboidMapper.java
index 0814281d87..1b32944228 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CalculateStatsFromBaseCuboidMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CalculateStatsFromBaseCuboidMapper.java
@@ -146,7 +146,7 @@ public void putRowKeyToHLLOld(String[] row) {
             Hasher hc = hf.newHasher();
             String colValue = row[i];
             if (colValue != null) {
-                rowHashCodes[i] = hc.putUnencodedChars(colValue).hash().asBytes();
+                rowHashCodes[i] = hc.putString(colValue).hash().asBytes();
             } else {
                 rowHashCodes[i] = hc.putInt(0).hash().asBytes();
             }
@@ -170,7 +170,7 @@ private void putRowKeyToHLLNew(String[] row) {
             String colValue = row[i];
             if (colValue == null)
                 colValue = "0";
-            byte[] bytes = hc.putUnencodedChars(colValue).hash().asBytes();
+            byte[] bytes = hc.putString(colValue).hash().asBytes();
             rowHashCodesLong[i] = (Bytes.toLong(bytes) + i);//add column ordinal to the hash
value to distinguish between (a,b) and (b,a)
         }
 
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
index 6c5edda752..7bffce7bcf 100755
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
@@ -350,7 +350,7 @@ private void putRowKeyToHLLOld(String[] row) {
                 Hasher hc = hf.newHasher();
                 String colValue = row[rowkeyColIndex[i]];
                 if (colValue != null) {
-                    rowHashCodes[i] = hc.putUnencodedChars(colValue).hash().asBytes();
+                    rowHashCodes[i] = hc.putString(colValue).hash().asBytes();
                 } else {
                     rowHashCodes[i] = hc.putInt(0).hash().asBytes();
                 }
@@ -374,7 +374,7 @@ private void putRowKeyToHLLNew(String[] row) {
                 String colValue = row[rowkeyColIndex[i]];
                 if (colValue == null)
                     colValue = "0";
-                byte[] bytes = hc.putUnencodedChars(colValue).hash().asBytes();
+                byte[] bytes = hc.putString(colValue).hash().asBytes();
                 rowHashCodesLong[i] = (Bytes.toLong(bytes) + i);//add column ordinal to the
hash value to distinguish between (a,b) and (b,a)
             }
 
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java
index fcf50c7608..916f9fad8c 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java
@@ -105,7 +105,7 @@ private void putRowKeyToHLL(List<String> row) {
         int x = 0;
         for (String field : row) {
             Hasher hc = hf.newHasher();
-            row_index[x++] = hc.putUnencodedChars(field).hash().asBytes();
+            row_index[x++] = hc.putString(field).hash().asBytes();
         }
 
         for (int i = 0, n = allCuboidsBitSet.length; i < n; i++) {
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NewCubeSamplingMethodTest.java
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NewCubeSamplingMethodTest.java
index a82f73d958..97e356d08e 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NewCubeSamplingMethodTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NewCubeSamplingMethodTest.java
@@ -113,7 +113,7 @@ public void run() throws Exception {
                     int x = 0;
                     for (String field : row) {
                         Hasher hc = hf.newHasher();
-                        colHashValues[x++] = hc.putUnencodedChars(field).hash().asBytes();
+                        colHashValues[x++] = hc.putString(field).hash().asBytes();
                     }
 
                     Hasher hc = hf.newHasher();
@@ -139,7 +139,7 @@ public void run() throws Exception {
                     int x = 0;
                     for (String field : row) {
                         Hasher hc = hf2.newHasher();
-                        byte[] bytes = hc.putUnencodedChars(x + field).hash().asBytes();
+                        byte[] bytes = hc.putString(x + field).hash().asBytes();
                         valueHashLong[x++] = Bytes.toLong(bytes);
                     }
 
@@ -213,7 +213,7 @@ private void putRowKeyToHLL(List<String> row, byte[][] colHashValues,
HLLCounter
         int x = 0;
         for (String field : row) {
             Hasher hc = hashFunction.newHasher();
-            colHashValues[x++] = hc.putUnencodedChars(field).hash().asBytes();
+            colHashValues[x++] = hc.putString(field).hash().asBytes();
         }
 
         for (int i = 0, n = allCuboidsBitSet.length; i < n; i++) {
@@ -230,7 +230,7 @@ private void putRowKeyToHLLNew(List<String> row, long[] hashValuesLong,
HLLCount
         int x = 0;
         for (String field : row) {
             Hasher hc = hashFunction.newHasher();
-            byte[] bytes = hc.putUnencodedChars(x + field).hash().asBytes();
+            byte[] bytes = hc.putString(x + field).hash().asBytes();
             hashValuesLong[x++] = Bytes.toLong(bytes);
         }
 
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkFactDistinct.java
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkFactDistinct.java
index 878ae7f5a4..31cadaa3d8 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkFactDistinct.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkFactDistinct.java
@@ -528,7 +528,7 @@ private void putRowKeyToHLLOld(String[] row) {
                 Hasher hc = hf.newHasher();
                 String colValue = row[rowkeyColIndex[i]];
                 if (colValue != null) {
-                    rowHashCodes[i] = hc.putUnencodedChars(colValue).hash().asBytes();
+                    rowHashCodes[i] = hc.putString(colValue).hash().asBytes();
                 } else {
                     rowHashCodes[i] = hc.putInt(0).hash().asBytes();
                 }
@@ -551,8 +551,8 @@ private void putRowKeyToHLLNew(String[] row) {
                 Hasher hc = hf.newHasher();
                 String colValue = row[rowkeyColIndex[i]];
                 if (colValue == null)
-                    colValue = "0";
-                byte[] bytes = hc.putUnencodedChars(colValue).hash().asBytes();
+                    colValue = "0";//
+                byte[] bytes = hc.putString(colValue).hash().asBytes();
                 rowHashCodesLong[i] = (Bytes.toLong(bytes) + i);//add column ordinal to the
hash value to distinguish between (a,b) and (b,a)
             }
 
diff --git a/pom.xml b/pom.xml
index 7b91a5d8ee..d957f0718d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -82,7 +82,7 @@
     <zookeeper.version>3.4.13</zookeeper.version>
     <curator.version>2.12.0</curator.version>
     <jsr305.version>3.0.1</jsr305.version>
-    <guava.version>16.0.1</guava.version>
+    <guava.version>14.0</guava.version>
     <jsch.version>0.1.54</jsch.version>
     <commons-cli.version>1.2</commons-cli.version>
     <commons-lang.version>2.6</commons-lang.version>
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/KylinAuthenticationProvider.java
b/server-base/src/main/java/org/apache/kylin/rest/security/KylinAuthenticationProvider.java
index 92e25c82b4..dd9cbad0a8 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/KylinAuthenticationProvider.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/KylinAuthenticationProvider.java
@@ -78,7 +78,7 @@ public KylinAuthenticationProvider(AuthenticationProvider authenticationProvider
     @Override
     public Authentication authenticate(Authentication authentication) throws AuthenticationException
{
 
-        byte[] hashKey = hf.hashUnencodedChars(authentication.getName() + authentication.getCredentials()).asBytes();
+        byte[] hashKey = hf.hashString(authentication.getName() + authentication.getCredentials()).asBytes();
         String userKey = Arrays.toString(hashKey);
 
         if (userService.isEvictCacheFlag()) {
diff --git a/storage-druid/src/main/java/org/apache/kylin/storage/druid/DruidSchema.java b/storage-druid/src/main/java/org/apache/kylin/storage/druid/DruidSchema.java
index 54c85919d1..8f473c4bdb 100644
--- a/storage-druid/src/main/java/org/apache/kylin/storage/druid/DruidSchema.java
+++ b/storage-druid/src/main/java/org/apache/kylin/storage/druid/DruidSchema.java
@@ -99,7 +99,7 @@ public static Interval segmentInterval(CubeSegment segment) {
         if (!segment.getCubeDesc().getModel().getPartitionDesc().isPartitioned()) {
             return Intervals.utc(0, segment.getCreateTimeUTC());
         }
-        return Intervals.utc(segment.getTSRange().start.v, segment.getTSRange().start.v);
+        return Intervals.utc(segment.getTSRange().start.v, segment.getTSRange().end.v);
     }
 
     public static String getDataSource(CubeDesc cubeDesc) {
diff --git a/storage-druid/src/main/java/org/apache/kylin/storage/druid/read/cursor/HttpDruidClient.java
b/storage-druid/src/main/java/org/apache/kylin/storage/druid/read/cursor/HttpDruidClient.java
index 9af7c636c4..6bf9d5d5d4 100644
--- a/storage-druid/src/main/java/org/apache/kylin/storage/druid/read/cursor/HttpDruidClient.java
+++ b/storage-druid/src/main/java/org/apache/kylin/storage/druid/read/cursor/HttpDruidClient.java
@@ -26,6 +26,7 @@
 import com.fasterxml.jackson.dataformat.smile.SmileFactory;
 import com.google.common.base.Throwables;
 import com.google.common.io.ByteSource;
+import com.google.common.io.ByteStreams;
 import com.google.common.util.concurrent.ListenableFuture;
 import io.druid.java.util.common.IAE;
 import io.druid.java.util.common.RE;
@@ -162,7 +163,8 @@ public InputStream nextElement() {
                     try {
                         // An empty byte array is put at the end to give the SequenceInputStream.close()
as something to close out
                         // after done is set to true, regardless of the rest of the stream's
state.
-                        queue.put(ByteSource.empty().openStream());
+                        ByteSource empty = ByteStreams.asByteSource(new byte[]{});
+                        queue.put(empty.openStream());
                     } catch (InterruptedException e) {
                         Thread.currentThread().interrupt();
                         throw Throwables.propagate(e);


 

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


> Kylin On Druid Storage
> ----------------------
>
>                 Key: KYLIN-3694
>                 URL: https://issues.apache.org/jira/browse/KYLIN-3694
>             Project: Kylin
>          Issue Type: New Feature
>          Components: Job Engine, Metadata, Query Engine
>    Affects Versions: v2.5.0
>            Reporter: kangkaisen
>            Assignee: kangkaisen
>            Priority: Major
>         Attachments: Kylin On Druid Storage.pdf
>
>
> Meituan Kylin team has implemented a new storage engine for Kylin: Druid 
> Storage Engine. 
> The attach file is the Kylin On Druid Storage Engine architecture design 
> doc. 
> We would like to contribute the feature to community, please let us know if 
> you have any concern. [^Kylin On Druid Storage.pdf]



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

Mime
View raw message