kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From lid...@apache.org
Subject [4/4] kylin git commit: KYLIN-2233 Support HBase 1.0.2
Date Tue, 11 Apr 2017 08:28:46 GMT
KYLIN-2233 Support HBase 1.0.2

Signed-off-by: Yang Li <liyang@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/294eb196
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/294eb196
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/294eb196

Branch: refs/heads/yang22-hbase102
Commit: 294eb196f9d3c20f6c11e164f19aabd783554239
Parents: 20dc839
Author: Cheng Wang <cheng.wang@kyligence.io>
Authored: Wed Oct 12 14:39:24 2016 +0800
Committer: lidongsjtu <lidong@apache.org>
Committed: Tue Apr 11 16:26:49 2017 +0800

----------------------------------------------------------------------
 pom.xml                                               |  2 +-
 .../storage/hbase/cube/v1/RegionScannerAdapter.java   | 13 +++----------
 .../coprocessor/observer/AggregateRegionObserver.java |  4 ++--
 .../v1/coprocessor/observer/AggregationScanner.java   | 14 ++++----------
 .../observer/ObserverAggregationCache.java            | 10 ++--------
 .../observer/AggregateRegionObserverTest.java         | 10 ++--------
 .../cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java  |  3 +--
 7 files changed, 15 insertions(+), 41 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/294eb196/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index dc23a2f..099316a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -54,7 +54,7 @@
         <hive-hcatalog.version>1.2.1</hive-hcatalog.version>
 
         <!-- HBase versions -->
-        <hbase-hadoop2.version>1.1.1</hbase-hadoop2.version>
+        <hbase-hadoop2.version>1.0.2</hbase-hadoop2.version>
         <kafka.version>0.10.0.0</kafka.version>
 
         <!-- Hadoop deps, keep compatible with hadoop2.version -->

http://git-wip-us.apache.org/repos/asf/kylin/blob/294eb196/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
index 3d30767..c07d47f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
@@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.hadoop.hbase.regionserver.ScannerContext;
 
 /**
  * @author yangli9
@@ -51,7 +50,7 @@ public class RegionScannerAdapter implements RegionScanner {
     }
 
     @Override
-    public boolean next(List<Cell> result, ScannerContext scannerContext) throws IOException
{
+    public boolean next(List<Cell> result, int limit) throws IOException {
         return next(result);
     }
 
@@ -61,7 +60,7 @@ public class RegionScannerAdapter implements RegionScanner {
     }
 
     @Override
-    public boolean nextRaw(List<Cell> result, ScannerContext scannerContext) throws
IOException {
+    public boolean nextRaw(List<Cell> result, int limit) throws IOException {
         return next(result);
     }
 
@@ -94,10 +93,4 @@ public class RegionScannerAdapter implements RegionScanner {
     public long getMvccReadPoint() {
         return Long.MAX_VALUE;
     }
-
-    @Override
-    public int getBatch() {
-        return -1;
-    }
-
-}
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/294eb196/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
index 7e25e4c..7139ca7 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
@@ -26,7 +26,7 @@ import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.regionserver.Region;
+import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
 import org.apache.kylin.gridtable.StorageSideBehavior;
@@ -99,7 +99,7 @@ public class AggregateRegionObserver extends BaseRegionObserver {
         // start/end region operation & sync on scanner is suggested by the
         // javadoc of RegionScanner.nextRaw()
         // FIXME: will the lock still work when a iterator is returned? is it safe? Is readonly
attribute helping here? by mhb
-        Region region = ctxt.getEnvironment().getRegion();
+        HRegion region = ctxt.getEnvironment().getRegion();
         region.startRegionOperation();
         try {
             synchronized (innerScanner) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/294eb196/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
index d64f48f..a900ea1 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
@@ -25,7 +25,6 @@ import java.util.List;
 import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.hadoop.hbase.regionserver.ScannerContext;
 import org.apache.kylin.gridtable.StorageSideBehavior;
 import org.apache.kylin.measure.MeasureAggregator;
 import org.apache.kylin.storage.hbase.common.coprocessor.AggrKey;
@@ -117,8 +116,8 @@ public class AggregationScanner implements RegionScanner {
     }
 
     @Override
-    public boolean next(List<Cell> result, ScannerContext scannerContext) throws IOException
{
-        return outerScanner.next(result, scannerContext);
+    public boolean next(List<Cell> result, int limit) throws IOException {
+        return outerScanner.next(result, limit);
     }
 
     @Override
@@ -127,8 +126,8 @@ public class AggregationScanner implements RegionScanner {
     }
 
     @Override
-    public boolean nextRaw(List<Cell> result, ScannerContext scannerContext) throws
IOException {
-        return outerScanner.nextRaw(result, scannerContext);
+    public boolean nextRaw(List<Cell> result, int limit) throws IOException {
+        return outerScanner.nextRaw(result, limit);
     }
 
     @Override
@@ -161,11 +160,6 @@ public class AggregationScanner implements RegionScanner {
         return outerScanner.getMvccReadPoint();
     }
 
-    @Override
-    public int getBatch() {
-        return outerScanner.getBatch();
-    }
-
     private static class Stats {
         long inputRows = 0;
         long inputBytes = 0;

http://git-wip-us.apache.org/repos/asf/kylin/blob/294eb196/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java
index 331e34d..8404262 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java
@@ -30,7 +30,6 @@ import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.KeyValue.Type;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.hadoop.hbase.regionserver.ScannerContext;
 import org.apache.kylin.measure.MeasureAggregator;
 import org.apache.kylin.storage.hbase.common.coprocessor.AggrKey;
 import org.apache.kylin.storage.hbase.common.coprocessor.AggregationCache;
@@ -113,7 +112,7 @@ public class ObserverAggregationCache extends AggregationCache {
         }
 
         @Override
-        public boolean next(List<Cell> result, ScannerContext scannerContext) throws
IOException {
+        public boolean next(List<Cell> result, int limit) throws IOException {
             return next(result);
         }
 
@@ -123,7 +122,7 @@ public class ObserverAggregationCache extends AggregationCache {
         }
 
         @Override
-        public boolean nextRaw(List<Cell> result, ScannerContext scannerContext) throws
IOException {
+        public boolean nextRaw(List<Cell> result, int limit) throws IOException {
             return next(result);
         }
 
@@ -162,11 +161,6 @@ public class ObserverAggregationCache extends AggregationCache {
             // AggregateRegionObserver.LOG.info("Kylin Scanner getMvccReadPoint()");
             return Long.MAX_VALUE;
         }
-
-        @Override
-        public int getBatch() {
-            return innerScanner.getBatch();
-        }
     }
 
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/294eb196/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
index 4695353..5ec0238 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
@@ -35,7 +35,6 @@ import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.KeyValue.Type;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.hadoop.hbase.regionserver.ScannerContext;
 import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.gridtable.StorageSideBehavior;
 import org.apache.kylin.metadata.datatype.LongMutable;
@@ -231,7 +230,7 @@ public class AggregateRegionObserverTest {
         }
 
         @Override
-        public boolean next(List<Cell> result, ScannerContext scannerContext) throws
IOException {
+        public boolean next(List<Cell> result, int limit) throws IOException {
             return next(result);
         }
 
@@ -300,11 +299,6 @@ public class AggregateRegionObserverTest {
             return 0;
         }
 
-        @Override
-        public int getBatch() {
-            return 0;
-        }
-
         /*
          * (non-Javadoc)
          * 
@@ -322,7 +316,7 @@ public class AggregateRegionObserverTest {
         }
 
         @Override
-        public boolean nextRaw(List<Cell> list, ScannerContext scannerContext) throws
IOException {
+        public boolean nextRaw(List<Cell> list, int limit) throws IOException {
             return false;
         }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/294eb196/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
index 04e2e8b..4e87093 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
@@ -44,7 +44,6 @@ import org.apache.hadoop.hbase.filter.FilterList;
 import org.apache.hadoop.hbase.filter.FilterList.Operator;
 import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
 import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.Region;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Pair;
@@ -225,7 +224,7 @@ public class TestFuzzyRowFilterV2EndToEnd {
         scan.addFamily(cf.getBytes());
         scan.setFilter(filter);
         List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(table.getBytes());
-        Region first = regions.get(0);
+        HRegion first = regions.get(0);
         first.getScanner(scan);
         RegionScanner scanner = first.getScanner(scan);
         List<Cell> results = new ArrayList<Cell>();


Mime
View raw message