kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From liy...@apache.org
Subject [38/50] incubator-kylin git commit: KYLIN-671 use guava 12, reinvent some wheels
Date Sun, 26 Apr 2015 04:27:20 GMT
KYLIN-671 use guava 12, reinvent some wheels


Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/52706058
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/52706058
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/52706058

Branch: refs/heads/streaming-localdict
Commit: 527060589f6d2565a3325a56ee8215da8552fa83
Parents: d6218b4
Author: honma <honma@ebay.com>
Authored: Thu Apr 16 19:30:43 2015 +0800
Committer: honma <honma@ebay.com>
Committed: Wed Apr 22 16:46:02 2015 +0800

----------------------------------------------------------------------
 .../org/apache/kylin/common/util/RangeUtil.java |  88 ++++++++++-
 .../org/apache/kylin/common/util/BasicTest.java |   6 +-
 .../org/apache/kylin/common/util/RangeTest.java | 149 +++++++++++++++++--
 .../kylin/job/hadoop/invertedindex/IITest.java  |  23 +--
 pom.xml                                         |   2 +-
 .../apache/kylin/query/test/KylinQueryTest.java |   2 +-
 .../endpoint/TsConditionExtractor.java          |  13 +-
 .../storage/hybrid/HybridStorageEngine.java     |  11 +-
 .../storage/hbase/InvertedIndexHBaseTest.java   |   2 +-
 .../invertedindex/IIStreamBuilder.java          |   5 +-
 10 files changed, 255 insertions(+), 46 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/52706058/common/src/main/java/org/apache/kylin/common/util/RangeUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/RangeUtil.java b/common/src/main/java/org/apache/kylin/common/util/RangeUtil.java
index 2dd2032..31b61a2 100644
--- a/common/src/main/java/org/apache/kylin/common/util/RangeUtil.java
+++ b/common/src/main/java/org/apache/kylin/common/util/RangeUtil.java
@@ -1,5 +1,6 @@
 package org.apache.kylin.common.util;
 
+import java.util.Collections;
 import java.util.List;
 
 import com.google.common.collect.*;
@@ -13,14 +14,91 @@ public class RangeUtil {
      * @return
      */
     public static <C extends Comparable<?>> List<Range<C>> remove(Range<C>
self, Range<C> other) {
-        RangeSet<C> rangeSet = TreeRangeSet.create();
-        rangeSet.add(self);
-        rangeSet.remove(other);
-        return Lists.newArrayList(rangeSet.asRanges());
+        // mimic the following logic in guava 18:
+        //        RangeSet<C> rangeSet = TreeRangeSet.create();
+        //        rangeSet.add(self);
+        //        rangeSet.remove(other);
+        //        return Lists.newArrayList(rangeSet.asRanges());
+
+        if (!self.isConnected(other)) {
+            return Collections.singletonList(self);
+        }
+        Range<C> share = self.intersection(other);
+        if (share.isEmpty()) {
+            return Collections.singletonList(self);
+        }
+
+        List<Range<C>> ret = Lists.newArrayList();
+
+        //see left part
+        if (!self.hasLowerBound()) {
+            if (share.hasLowerBound()) {
+                if (share.lowerBoundType() == BoundType.CLOSED) {
+                    ret.add(Ranges.lessThan(share.lowerEndpoint()));
+                } else {
+                    ret.add(Ranges.atMost(share.lowerEndpoint()));
+                }
+            }
+        } else {
+            if (self.lowerEndpoint() != share.lowerEndpoint()) {
+                if (self.lowerBoundType() == BoundType.CLOSED) {
+                    if (share.lowerBoundType() == BoundType.CLOSED) {
+                        ret.add(Ranges.closedOpen(self.lowerEndpoint(), share.lowerEndpoint()));
+                    } else {
+                        ret.add(Ranges.closed(self.lowerEndpoint(), share.lowerEndpoint()));
+                    }
+                } else {
+                    if (share.lowerBoundType() == BoundType.CLOSED) {
+                        ret.add(Ranges.open(self.lowerEndpoint(), share.lowerEndpoint()));
+                    } else {
+                        ret.add(Ranges.openClosed(self.lowerEndpoint(), share.lowerEndpoint()));
+                    }
+                }
+            } else {
+                if (self.lowerBoundType() == BoundType.CLOSED && share.lowerBoundType()
== BoundType.OPEN) {
+                    ret.add(Ranges.closed(self.lowerEndpoint(), share.lowerEndpoint()));
+                }
+            }
+        }
+
+        //see right part 
+        if (!self.hasUpperBound()) {
+            if (share.hasUpperBound()) {
+                if (share.upperBoundType() == BoundType.CLOSED) {
+                    ret.add(Ranges.greaterThan(share.upperEndpoint()));
+                } else {
+                    ret.add(Ranges.atLeast(share.upperEndpoint()));
+                }
+            }
+        } else {
+            if (self.upperEndpoint() != share.upperEndpoint()) {
+                if (self.upperBoundType() == BoundType.CLOSED) {
+                    if (share.upperBoundType() == BoundType.CLOSED) {
+                        ret.add(Ranges.openClosed(share.upperEndpoint(), self.upperEndpoint()));
+                    } else {
+                        ret.add(Ranges.closed(share.upperEndpoint(), self.upperEndpoint()));
+                    }
+                } else {
+                    if (share.upperBoundType() == BoundType.CLOSED) {
+                        ret.add(Ranges.open(share.upperEndpoint(), self.upperEndpoint()));
+                    } else {
+                        ret.add(Ranges.closedOpen(share.upperEndpoint(), self.upperEndpoint()));
+                    }
+                }
+            } else {
+                if (self.upperBoundType() == BoundType.CLOSED && share.upperBoundType()
== BoundType.OPEN) {
+                    ret.add(Ranges.closed(self.upperEndpoint(), share.upperEndpoint()));
+                }
+            }
+        }
+
+
+        return ret;
+
     }
 
     public static String formatTsRange(Range<Long> tsRange) {
-        if(tsRange == null)
+        if (tsRange == null)
             return null;
 
         StringBuilder sb = new StringBuilder();

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/52706058/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/BasicTest.java b/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
index a178311..25e4d44 100644
--- a/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
+++ b/common/src/test/java/org/apache/kylin/common/util/BasicTest.java
@@ -31,7 +31,7 @@ import org.junit.Test;
 import org.slf4j.LoggerFactory;
 
 import com.google.common.collect.Lists;
-import com.google.common.collect.Range;
+import com.google.common.collect.Ranges;
 
 /**
 * Created by honma on 10/17/14.
@@ -66,8 +66,8 @@ public class BasicTest {
     public void test1() throws Exception {
 
 
-        System.out.println(Range.open(3, 5).isConnected(Range.open(4, 10)));
-        System.out.println(Range.open(4, 10).isConnected(Range.open(3,5)));
+        System.out.println(Ranges.open(3, 5).isConnected(Ranges.open(4, 10)));
+        System.out.println(Ranges.open(4, 10).isConnected(Ranges.open(3,5)));
 
 
         String bb = "\\x00\\x00\\x00\\x00\\x01\\x3F\\xD0\\x2D\\58\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00";//2013/07/12
07:59:37

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/52706058/common/src/test/java/org/apache/kylin/common/util/RangeTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/RangeTest.java b/common/src/test/java/org/apache/kylin/common/util/RangeTest.java
index 59eb9d7..b222bb7 100644
--- a/common/src/test/java/org/apache/kylin/common/util/RangeTest.java
+++ b/common/src/test/java/org/apache/kylin/common/util/RangeTest.java
@@ -1,26 +1,149 @@
 package org.apache.kylin.common.util;
 
-import com.google.common.collect.Range;
 import org.junit.Assert;
 import org.junit.Test;
 
+import com.google.common.collect.Lists;
+import com.google.common.collect.Range;
+import com.google.common.collect.Ranges;
+
 /**
  * Created by Hongbin Ma(Binmahone) on 4/14/15.
  */
 public class RangeTest {
+
     @Test
-    public void basicTest() {
-        Range a0 = Range.closed(2, 5);
-        Range a1 = Range.open(2, 5);
-        Range a2 = Range.open(2, 5);
-        Range a3 = Range.open(2, 3);
-        Range a4 = Range.open(2, 9);
-
-        Assert.assertTrue(RangeUtil.remove(a0, a1).size() == 2);
-        Assert.assertTrue(RangeUtil.remove(a1, a2).size() == 0);
-        Assert.assertTrue(RangeUtil.remove(a1, a3).size() == 1);
-        Assert.assertTrue(RangeUtil.remove(a1, a3).get(0).equals(Range.closedOpen(3, 5)));
-        Assert.assertTrue(RangeUtil.remove(a1, a4).size() == 0);
+    public void testClosed() {
+
+        Range anull = Ranges.closedOpen(0, 0);
+
+        Range r1 = Ranges.closed(2, 5);
+
+        Range a1 = Ranges.open(1, 2);
+        Range a2 = Ranges.open(1, 3);
+        Range a3 = Ranges.open(1, 5);
+        Range a4 = Ranges.open(1, 6);
+        Range a5 = Ranges.open(6, 7);
+        Range a6 = Ranges.open(5, 7);
+        Range a7 = Ranges.open(4, 7);
+        Range a8 = Ranges.open(2, 7);
+        Range a9 = Ranges.open(1, 7);
+
+        Range b1 = Ranges.closed(1, 2);
+        Range b2 = Ranges.closed(1, 3);
+        Range b3 = Ranges.closed(1, 5);
+        Range b4 = Ranges.closed(1, 6);
+        Range b5 = Ranges.closed(6, 7);
+        Range b6 = Ranges.closed(5, 7);
+        Range b7 = Ranges.closed(4, 7);
+        Range b8 = Ranges.closed(2, 7);
+        Range b9 = Ranges.closed(1, 7);
+
+        Range c1 = Ranges.open(2, 3);
+        Range c2 = Ranges.open(3, 4);
+        Range c3 = Ranges.open(4, 5);
+
+        Range d1 = Ranges.closed(2, 3);
+        Range d2 = Ranges.closed(3, 4);
+        Range d3 = Ranges.closed(4, 5);
+
+
+        Assert.assertTrue(RangeUtil.remove(r1, anull).equals(Lists.newArrayList(r1)));
+        Assert.assertTrue(RangeUtil.remove(r1, a1).equals(Lists.newArrayList(r1)));
+        Assert.assertTrue(RangeUtil.remove(r1, a2).equals(Lists.newArrayList(Ranges.closed(3,
5))));
+        Assert.assertTrue(RangeUtil.remove(r1, a3).equals(Lists.newArrayList(Ranges.closed(5,
5))));
+        Assert.assertTrue(RangeUtil.remove(r1, a4).equals(Lists.newArrayList()));
+        Assert.assertTrue(RangeUtil.remove(r1, a5).equals(Lists.newArrayList(r1)));
+        Assert.assertTrue(RangeUtil.remove(r1, a6).equals(Lists.newArrayList(r1)));
+        Assert.assertTrue(RangeUtil.remove(r1, a7).equals(Lists.newArrayList(Ranges.closed(2,
4))));
+        Assert.assertTrue(RangeUtil.remove(r1, a8).equals(Lists.newArrayList(Ranges.closed(2,
2))));
+        Assert.assertTrue(RangeUtil.remove(r1, a9).equals(Lists.newArrayList()));
+
+        Assert.assertTrue(RangeUtil.remove(r1, b1).equals(Lists.newArrayList(Ranges.openClosed(2,
5))));
+        Assert.assertTrue(RangeUtil.remove(r1, b2).equals(Lists.newArrayList(Ranges.openClosed(3,
5))));
+        Assert.assertTrue(RangeUtil.remove(r1, b3).equals(Lists.newArrayList()));
+        Assert.assertTrue(RangeUtil.remove(r1, b4).equals(Lists.newArrayList()));
+        Assert.assertTrue(RangeUtil.remove(r1, b5).equals(Lists.newArrayList(r1)));
+        Assert.assertTrue(RangeUtil.remove(r1, b6).equals(Lists.newArrayList(Ranges.closedOpen(2,
5))));
+        Assert.assertTrue(RangeUtil.remove(r1, b7).equals(Lists.newArrayList(Ranges.closedOpen(2,
4))));
+        Assert.assertTrue(RangeUtil.remove(r1, b8).equals(Lists.newArrayList()));
+        Assert.assertTrue(RangeUtil.remove(r1, b9).equals(Lists.newArrayList()));
+
+        Assert.assertTrue(RangeUtil.remove(r1, c1).equals(Lists.newArrayList(Ranges.closed(2,
2), Ranges.closed(3, 5))));
+        Assert.assertTrue(RangeUtil.remove(r1, c2).equals(Lists.newArrayList(Ranges.closed(2,
3), Ranges.closed(4, 5))));
+        Assert.assertTrue(RangeUtil.remove(r1, c3).equals(Lists.newArrayList(Ranges.closed(2,
4), Ranges.closed(5, 5))));
+
+        Assert.assertTrue(RangeUtil.remove(r1, d1).equals(Lists.newArrayList(Ranges.openClosed(3,
5))));
+        Assert.assertTrue(RangeUtil.remove(r1, d2).equals(Lists.newArrayList(Ranges.closedOpen(2,
3), Ranges.openClosed(4, 5))));
+        Assert.assertTrue(RangeUtil.remove(r1, d3).equals(Lists.newArrayList(Ranges.closedOpen(2,
4))));
+
+    }
+
+    @Test
+    public void testOpen() {
+
+        Range anull = Ranges.closedOpen(0, 0);
+
+        Range r1 = Ranges.open(2, 5);
+
+        Range a1 = Ranges.open(1, 2);
+        Range a2 = Ranges.open(1, 3);
+        Range a3 = Ranges.open(1, 5);
+        Range a4 = Ranges.open(1, 6);
+        Range a5 = Ranges.open(6, 7);
+        Range a6 = Ranges.open(5, 7);
+        Range a7 = Ranges.open(4, 7);
+        Range a8 = Ranges.open(2, 7);
+        Range a9 = Ranges.open(1, 7);
+
+        Range b1 = Ranges.closed(1, 2);
+        Range b2 = Ranges.closed(1, 3);
+        Range b3 = Ranges.closed(1, 5);
+        Range b4 = Ranges.closed(1, 6);
+        Range b5 = Ranges.closed(6, 7);
+        Range b6 = Ranges.closed(5, 7);
+        Range b7 = Ranges.closed(4, 7);
+        Range b8 = Ranges.closed(2, 7);
+        Range b9 = Ranges.closed(1, 7);
+
+        Range c1 = Ranges.open(2, 3);
+        Range c2 = Ranges.open(3, 4);
+        Range c3 = Ranges.open(4, 5);
+
+        Range d1 = Ranges.closed(2, 3);
+        Range d2 = Ranges.closed(3, 4);
+        Range d3 = Ranges.closed(4, 5);
+
+
+        Assert.assertTrue(RangeUtil.remove(r1, anull).equals(Lists.newArrayList(r1)));
+
+        Assert.assertTrue(RangeUtil.remove(r1, a1).equals(Lists.newArrayList(r1)));
+        Assert.assertTrue(RangeUtil.remove(r1, a2).equals(Lists.newArrayList(Ranges.closedOpen(3,
5))));
+        Assert.assertTrue(RangeUtil.remove(r1, a3).equals(Lists.newArrayList()));
+        Assert.assertTrue(RangeUtil.remove(r1, a4).equals(Lists.newArrayList()));
+        Assert.assertTrue(RangeUtil.remove(r1, a5).equals(Lists.newArrayList(r1)));
+        Assert.assertTrue(RangeUtil.remove(r1, a6).equals(Lists.newArrayList(r1)));
+        Assert.assertTrue(RangeUtil.remove(r1, a7).equals(Lists.newArrayList(Ranges.openClosed(2,
4))));
+        Assert.assertTrue(RangeUtil.remove(r1, a8).equals(Lists.newArrayList()));
+        Assert.assertTrue(RangeUtil.remove(r1, a9).equals(Lists.newArrayList()));
+
+        Assert.assertTrue(RangeUtil.remove(r1, b1).equals(Lists.newArrayList(Ranges.open(2,
5))));
+        Assert.assertTrue(RangeUtil.remove(r1, b2).equals(Lists.newArrayList(Ranges.open(3,
5))));
+        Assert.assertTrue(RangeUtil.remove(r1, b3).equals(Lists.newArrayList()));
+        Assert.assertTrue(RangeUtil.remove(r1, b4).equals(Lists.newArrayList()));
+        Assert.assertTrue(RangeUtil.remove(r1, b5).equals(Lists.newArrayList(r1)));
+        Assert.assertTrue(RangeUtil.remove(r1, b6).equals(Lists.newArrayList(Ranges.open(2,
5))));
+        Assert.assertTrue(RangeUtil.remove(r1, b7).equals(Lists.newArrayList(Ranges.open(2,
4))));
+        Assert.assertTrue(RangeUtil.remove(r1, b8).equals(Lists.newArrayList()));
+        Assert.assertTrue(RangeUtil.remove(r1, b9).equals(Lists.newArrayList()));
+
+        Assert.assertTrue(RangeUtil.remove(r1, c1).equals(Lists.newArrayList( Ranges.closedOpen(3,
5))));
+        Assert.assertTrue(RangeUtil.remove(r1, c2).equals(Lists.newArrayList(Ranges.openClosed(2,
3), Ranges.closedOpen(4, 5))));
+        Assert.assertTrue(RangeUtil.remove(r1, c3).equals(Lists.newArrayList(Ranges.openClosed(2,
4))));
+
+        Assert.assertTrue(RangeUtil.remove(r1, d1).equals(Lists.newArrayList(Ranges.open(3,
5))));
+        Assert.assertTrue(RangeUtil.remove(r1, d2).equals(Lists.newArrayList(Ranges.open(2,
3), Ranges.open(4, 5))));
+        Assert.assertTrue(RangeUtil.remove(r1, d3).equals(Lists.newArrayList(Ranges.open(2,
4))));
 
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/52706058/job/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java b/job/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
index 03756a8..dc91dfd 100644
--- a/job/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
+++ b/job/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
@@ -1,9 +1,11 @@
 package org.apache.kylin.job.hadoop.invertedindex;
 
-import com.google.common.base.Function;
-import com.google.common.collect.Collections2;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
+import java.io.IOException;
+import java.util.*;
+
+import javax.annotation.Nullable;
+
+import org.apache.commons.lang.NotImplementedException;
 import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.client.Result;
@@ -39,12 +41,15 @@ import org.apache.kylin.storage.hbase.coprocessor.endpoint.generated.IIProtos;
 import org.apache.kylin.streaming.StreamMessage;
 import org.apache.kylin.streaming.StringStreamParser;
 import org.apache.kylin.streaming.invertedindex.SliceBuilder;
-import org.junit.*;
-import sun.reflect.generics.reflectiveObjects.NotImplementedException;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
 
-import javax.annotation.Nullable;
-import java.io.IOException;
-import java.util.*;
+import com.google.common.base.Function;
+import com.google.common.collect.Collections2;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
 
 /**
  * Created by Hongbin Ma(Binmahone) on 3/26/15.

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/52706058/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 6b36e1e..e750154 100644
--- a/pom.xml
+++ b/pom.xml
@@ -73,7 +73,7 @@
         <log4j.version>1.2.17</log4j.version>
         <slf4j.version>1.6.4</slf4j.version>
         <jackson.version>2.2.3</jackson.version>
-        <guava.version>18.0</guava.version>
+        <guava.version>12.0</guava.version>
         <jsch.version>0.1.51</jsch.version>
         <xerces.version>2.9.1</xerces.version>
         <xalan.version>2.7.1</xalan.version>

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/52706058/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java b/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
index 4475575..25fcc43 100644
--- a/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
+++ b/query/src/test/java/org/apache/kylin/query/test/KylinQueryTest.java
@@ -95,7 +95,7 @@ public class KylinQueryTest extends KylinTestBase {
     @Test
     public void testSingleRunQuery() throws Exception {
 
-        String queryFileName = "src/test/resources/query/sql/query54.sql";
+        String queryFileName = "src/test/resources/query/sql/query11.sql";
 
         File sqlFile = new File(queryFileName);
         runSQL(sqlFile, true, true);

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/52706058/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/TsConditionExtractor.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/TsConditionExtractor.java
b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/TsConditionExtractor.java
index 0b36d4e..00e566a 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/TsConditionExtractor.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/TsConditionExtractor.java
@@ -1,5 +1,6 @@
 package org.apache.kylin.storage.hbase.coprocessor.endpoint;
 
+import com.google.common.collect.Ranges;
 import org.apache.kylin.metadata.filter.CompareTupleFilter;
 import org.apache.kylin.metadata.filter.LogicalTupleFilter;
 import org.apache.kylin.metadata.filter.TupleFilter;
@@ -21,7 +22,7 @@ public class TsConditionExtractor {
 
         if (filter instanceof LogicalTupleFilter) {
             if (filter.getOperator() == TupleFilter.FilterOperatorEnum.AND) {
-                Range ret = Range.all();
+                Range ret = Ranges.all();
                 for (TupleFilter child : filter.getChildren()) {
                     Range childRange = extractTsConditionInternal(child, colRef);
                     if (childRange != null) {
@@ -45,19 +46,19 @@ public class TsConditionExtractor {
                 switch (compareTupleFilter.getOperator()) {
                 case EQ:
                     t = DateFormat.stringToMillis((String) firstValue);
-                    return Range.closed(t, t);
+                    return Ranges.closed(t, t);
                 case LT:
                     t = DateFormat.stringToMillis((String) firstValue);
-                    return Range.lessThan(t);
+                    return Ranges.lessThan(t);
                 case LTE:
                     t = DateFormat.stringToMillis((String) firstValue);
-                    return Range.atMost(t);
+                    return Ranges.atMost(t);
                 case GT:
                     t = DateFormat.stringToMillis((String) firstValue);
-                    return Range.greaterThan(t);
+                    return Ranges.greaterThan(t);
                 case GTE:
                     t = DateFormat.stringToMillis((String) firstValue);
-                    return Range.atLeast(t);
+                    return Ranges.atLeast(t);
                 case NEQ:
                 case IN://not handled for now
                     break;

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/52706058/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java
b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java
index 3ac77b3..8667ded 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hybrid/HybridStorageEngine.java
@@ -1,8 +1,7 @@
 package org.apache.kylin.storage.hybrid;
 
-import com.google.common.base.Function;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Range;
+import javax.annotation.Nullable;
+
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.metadata.MetadataManager;
 import org.apache.kylin.metadata.model.DataModelDesc;
@@ -18,7 +17,9 @@ import org.apache.kylin.storage.StorageEngineFactory;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import javax.annotation.Nullable;
+import com.google.common.base.Function;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Ranges;
 
 /**
  * Created by shaoshi on 2/13/15.
@@ -50,7 +51,7 @@ public class HybridStorageEngine implements IStorageEngine {
         TblColRef partitionColRef = modelDesc.getPartitionDesc().getPartitionDateColumnRef();
 
         ITupleIterator iterator2 = SQLDigestUtil.appendTsFilterToExecute(sqlDigest, partitionColRef,
//
-                Range.atLeast(hybridInstance.getHistoryRealizationInstance().getDateRangeEnd()),//
+                Ranges.atLeast(hybridInstance.getHistoryRealizationInstance().getDateRangeEnd()),//
                 new Function<Void, ITupleIterator>() {
                     @Nullable
                     @Override

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/52706058/storage/src/test/java/org/apache/kylin/storage/hbase/InvertedIndexHBaseTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/hbase/InvertedIndexHBaseTest.java
b/storage/src/test/java/org/apache/kylin/storage/hbase/InvertedIndexHBaseTest.java
index a0553de..fd24487 100644
--- a/storage/src/test/java/org/apache/kylin/storage/hbase/InvertedIndexHBaseTest.java
+++ b/storage/src/test/java/org/apache/kylin/storage/hbase/InvertedIndexHBaseTest.java
@@ -88,7 +88,7 @@ public class InvertedIndexHBaseTest extends HBaseMetadataTestCase {
         }
 
         List<TableRecord> records = iterateRecords(slices);
-        dump(records);
+        //dump(records);
         System.out.println(records.size() + " records");
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/52706058/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
b/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
index 5ed05e8..9593c1e 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamBuilder.java
@@ -100,14 +100,15 @@ public class IIStreamBuilder extends StreamBuilder {
                 return;
             }
             logger.info("stream build start, size:" + streamsToBuild.size());
-            Stopwatch stopwatch = Stopwatch.createStarted();
+            Stopwatch stopwatch = new Stopwatch();
+            stopwatch.start();
             final Slice slice = sliceBuilder.buildSlice(streamsToBuild, getStreamParser());
             logger.info("slice info, shard:" + slice.getShard() + " timestamp:" + slice.getTimestamp()
+ " record count:" + slice.getRecordCount());
 
             loadToHBase(hTable, slice, new IIKeyValueCodec(slice.getInfo()));
             submitOffset(offset);
             stopwatch.stop();
-            logger.info("stream build finished, size:" + streamsToBuild.size() + " elapsed
time:" + stopwatch.elapsed(TimeUnit.MILLISECONDS) + " " + TimeUnit.MILLISECONDS);
+            logger.info("stream build finished, size:" + streamsToBuild.size() + " elapsed
time:" + stopwatch.elapsedTime(TimeUnit.MILLISECONDS) + " " + TimeUnit.MILLISECONDS);
         } else {
             logger.info("nothing to build, skip building");
         }


Mime
View raw message