kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From liy...@apache.org
Subject incubator-kylin git commit: KYLIN-625, refactor IGTStore to take GTRecord as pk start/end
Date Sat, 23 May 2015 23:22:26 GMT
Repository: incubator-kylin
Updated Branches:
  refs/heads/0.8.0 ce775594a -> 7d8ae6e38


KYLIN-625, refactor IGTStore to take GTRecord as pk start/end


Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/7d8ae6e3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/7d8ae6e3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/7d8ae6e3

Branch: refs/heads/0.8.0
Commit: 7d8ae6e3811ca53364437e0d5ccdb08f1a68d885
Parents: ce77559
Author: Yang Li <liyang@apache.org>
Authored: Sun May 24 07:22:07 2015 +0800
Committer: Yang Li <liyang@apache.org>
Committed: Sun May 24 07:22:07 2015 +0800

----------------------------------------------------------------------
 .../storage/cube/CubeHBaseReadonlyStore.java    |  9 ++++--
 .../kylin/storage/gridtable/GTComboStore.java   | 11 +++----
 .../kylin/storage/gridtable/GTRawScanner.java   |  7 +----
 .../kylin/storage/gridtable/GTRecord.java       | 25 +++++++++++++++
 .../kylin/storage/gridtable/GTRowBlock.java     |  2 +-
 .../kylin/storage/gridtable/IGTStore.java       |  2 +-
 .../apache/kylin/storage/gridtable/ScanKey.java | 33 --------------------
 .../gridtable/diskstore/GTDiskStore.java        |  4 +--
 .../gridtable/memstore/GTSimpleMemStore.java    |  4 +--
 9 files changed, 43 insertions(+), 54 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7d8ae6e3/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseReadonlyStore.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseReadonlyStore.java
b/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseReadonlyStore.java
index cb67fc5..cd0904b 100644
--- a/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseReadonlyStore.java
+++ b/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseReadonlyStore.java
@@ -23,6 +23,7 @@ import org.apache.kylin.cube.model.HBaseColumnDesc;
 import org.apache.kylin.cube.model.HBaseColumnFamilyDesc;
 import org.apache.kylin.cube.model.HBaseMappingDesc;
 import org.apache.kylin.storage.gridtable.GTInfo;
+import org.apache.kylin.storage.gridtable.GTRecord;
 import org.apache.kylin.storage.gridtable.GTRowBlock;
 import org.apache.kylin.storage.gridtable.GTRowBlock.Writer;
 import org.apache.kylin.storage.gridtable.GTScanRequest;
@@ -64,7 +65,7 @@ public class CubeHBaseReadonlyStore implements IGTStore {
     }
 
     @Override
-    public IGTStoreScanner scan(ByteArray pkStart, ByteArray pkEnd, BitSet selectedColumnBlocks,
GTScanRequest additionalPushDown) throws IOException {
+    public IGTStoreScanner scan(GTRecord pkStart, GTRecord pkEnd, BitSet selectedColumnBlocks,
GTScanRequest additionalPushDown) throws IOException {
         // TODO enable coprocessor
 
         // primary key (also the 0th column block) is always selected
@@ -122,7 +123,7 @@ public class CubeHBaseReadonlyStore implements IGTStore {
         };
     }
 
-    private Scan buildScan(ByteArray pkStart, ByteArray pkEnd, List<Pair<byte[], byte[]>>
selectedColumns) {
+    private Scan buildScan(GTRecord pkStart, GTRecord pkEnd, List<Pair<byte[], byte[]>>
selectedColumns) {
         Scan scan = new Scan();
         scan.setCaching(SCAN_CACHE);
         scan.setCacheBlocks(true);
@@ -140,8 +141,10 @@ public class CubeHBaseReadonlyStore implements IGTStore {
         return scan;
     }
 
-    private byte[] makeRowKeyToScan(ByteArray pk, byte fill) {
+    private byte[] makeRowKeyToScan(GTRecord pkRec, byte fill) {
+        ByteArray pk = GTRecord.exportScanKey(pkRec);
         int pkMaxLen = info.getMaxColumnLength(info.getPrimaryKey());
+        
         byte[] buf = new byte[pkMaxLen + RowConstants.ROWKEY_CUBOIDID_LEN];
         Arrays.fill(buf, fill);
         

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7d8ae6e3/storage/src/main/java/org/apache/kylin/storage/gridtable/GTComboStore.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTComboStore.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTComboStore.java
index c0d56c3..92707ed 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTComboStore.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTComboStore.java
@@ -1,14 +1,13 @@
 package org.apache.kylin.storage.gridtable;
 
-import org.apache.kylin.common.util.ByteArray;
+import java.io.IOException;
+import java.util.BitSet;
+
 import org.apache.kylin.storage.gridtable.diskstore.GTDiskStore;
 import org.apache.kylin.storage.gridtable.memstore.GTSimpleMemStore;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.IOException;
-import java.util.BitSet;
-
 /**
  */
 public class GTComboStore implements IGTStore {
@@ -18,7 +17,7 @@ public class GTComboStore implements IGTStore {
     private final GTInfo gtInfo;
 
     private void convert(IGTStore input, IGTStore output) throws IOException {
-        final IGTStoreScanner scanner = input.scan(ScanKey.makeScanKey(gtInfo, new GTRecord(gtInfo)),
ScanKey.makeScanKey(gtInfo, new GTRecord(gtInfo)), null, null);
+        final IGTStoreScanner scanner = input.scan(null, null, null, null);
         final IGTStoreWriter writer = output.rebuild(-1);
         while (scanner.hasNext()) {
             writer.write(scanner.next());
@@ -101,7 +100,7 @@ public class GTComboStore implements IGTStore {
     }
 
     @Override
-    public IGTStoreScanner scan(ByteArray pkStart, ByteArray pkEnd, BitSet selectedColBlocks,
GTScanRequest additionalPushDown) throws IOException {
+    public IGTStoreScanner scan(GTRecord pkStart, GTRecord pkEnd, BitSet selectedColBlocks,
GTScanRequest additionalPushDown) throws IOException {
         return getCurrent().scan(pkStart, pkEnd, selectedColBlocks, additionalPushDown);
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7d8ae6e3/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRawScanner.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRawScanner.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRawScanner.java
index ff97cd5..1e934b5 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRawScanner.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRawScanner.java
@@ -5,7 +5,6 @@ import java.util.BitSet;
 import java.util.Iterator;
 import java.util.NoSuchElementException;
 
-import org.apache.kylin.common.util.ByteArray;
 import org.apache.kylin.storage.gridtable.IGTStore.IGTStoreScanner;
 
 public class GTRawScanner implements IGTScanner {
@@ -23,12 +22,8 @@ public class GTRawScanner implements IGTScanner {
 
     public GTRawScanner(GTInfo info, IGTStore store, GTScanRequest req) throws IOException
{
         this.info = info;
-
-        ByteArray start = ScanKey.makeScanKey(info, req.getPkStart());
-        ByteArray end = ScanKey.makeScanKey(info, req.getPkEnd());
         this.selectedColBlocks = info.selectColumnBlocks(req.getColumns());
-
-        this.storeScanner = store.scan(start, end, selectedColBlocks, req);
+        this.storeScanner = store.scan(req.getPkStart(), req.getPkEnd(), selectedColBlocks,
req);
         this.oneRecord = new GTRecord(info);
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7d8ae6e3/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java
index 61ff25d..3d0f9bb 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java
@@ -233,4 +233,29 @@ public class GTRecord implements Comparable<GTRecord> {
         }
     }
 
+    /* similar to export(primaryKey) but will stop at the first null value */
+    public static ByteArray exportScanKey(GTRecord rec) {
+        if (rec == null)
+            return null;
+        
+        GTInfo info = rec.getInfo();
+        
+        BitSet selectedColumns = new BitSet();
+        int len = 0;
+        for (int i = info.primaryKey.nextSetBit(0); i >= 0; i = info.primaryKey.nextSetBit(i
+ 1)) {
+            if (rec.cols[i].array() == null) {
+                break;
+            }
+            selectedColumns.set(i);
+            len += rec.cols[i].length();
+        }
+        
+        if (selectedColumns.cardinality() == 0)
+            return null;
+
+        ByteArray buf = ByteArray.allocate(len);
+        rec.exportColumns(selectedColumns, buf);
+        return buf;
+    }
+    
 }

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7d8ae6e3/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlock.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlock.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlock.java
index ec24da6..4a68659 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlock.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlock.java
@@ -192,7 +192,7 @@ public class GTRowBlock {
         }
     }
 
-    public void export(DataOutputStream dataOutputStream, ByteArray array) throws IOException
{
+    private void export(DataOutputStream dataOutputStream, ByteArray array) throws IOException
{
         dataOutputStream.writeInt(array.length());
         dataOutputStream.write(array.array(), array.offset(), array.length());
     }

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7d8ae6e3/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTStore.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTStore.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTStore.java
index 4defa24..76bf76a 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTStore.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTStore.java
@@ -15,7 +15,7 @@ public interface IGTStore {
     
     IGTStoreWriter append(int shard, GTRowBlock.Writer fillLast) throws IOException;
     
-    IGTStoreScanner scan(ByteArray pkStart, ByteArray pkEnd, BitSet selectedColBlocks, GTScanRequest
additionalPushDown) throws IOException;
+    IGTStoreScanner scan(GTRecord pkStart, GTRecord pkEnd, BitSet selectedColBlocks, GTScanRequest
additionalPushDown) throws IOException;
 
     interface IGTStoreWriter extends Closeable {
         void write(GTRowBlock block) throws IOException;

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7d8ae6e3/storage/src/main/java/org/apache/kylin/storage/gridtable/ScanKey.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/ScanKey.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/ScanKey.java
deleted file mode 100644
index d8edf45..0000000
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/ScanKey.java
+++ /dev/null
@@ -1,33 +0,0 @@
-package org.apache.kylin.storage.gridtable;
-
-import org.apache.kylin.common.util.ByteArray;
-
-import java.util.BitSet;
-
-/**
- */
-public final class ScanKey {
-
-    private ScanKey() {
-    }
-
-    static ByteArray makeScanKey(GTInfo info, GTRecord rec) {
-        int firstPKCol = info.primaryKey.nextSetBit(0);
-        if (rec == null || rec.cols[firstPKCol].array() == null)
-            return null;
-
-        BitSet selectedColumns = new BitSet();
-        int len = 0;
-        for (int i = info.primaryKey.nextSetBit(0); i >= 0; i = info.primaryKey.nextSetBit(i
+ 1)) {
-            if (rec.cols[i].array() == null) {
-                break;
-            }
-            selectedColumns.set(i);
-            len += rec.cols[i].length();
-        }
-
-        ByteArray buf = ByteArray.allocate(len);
-        rec.exportColumns(selectedColumns, buf);
-        return buf;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7d8ae6e3/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/GTDiskStore.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/GTDiskStore.java
b/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/GTDiskStore.java
index 9684d8d..b6f7d95 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/GTDiskStore.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/GTDiskStore.java
@@ -10,8 +10,8 @@ import java.nio.ByteBuffer;
 import java.util.BitSet;
 import java.util.UUID;
 
-import org.apache.kylin.common.util.ByteArray;
 import org.apache.kylin.storage.gridtable.GTInfo;
+import org.apache.kylin.storage.gridtable.GTRecord;
 import org.apache.kylin.storage.gridtable.GTRowBlock;
 import org.apache.kylin.storage.gridtable.GTScanRequest;
 import org.apache.kylin.storage.gridtable.IGTStore;
@@ -151,7 +151,7 @@ public class GTDiskStore implements IGTStore {
     }
 
     @Override
-    public IGTStoreScanner scan(ByteArray pkStart, ByteArray pkEnd, BitSet selectedColBlocks,
GTScanRequest additionalPushDown) throws IOException {
+    public IGTStoreScanner scan(GTRecord pkStart, GTRecord pkEnd, BitSet selectedColBlocks,
GTScanRequest additionalPushDown) throws IOException {
         return new DiskStoreScanner(fileSystem.getReader(getRowBlockFile(identifier)));
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7d8ae6e3/storage/src/main/java/org/apache/kylin/storage/gridtable/memstore/GTSimpleMemStore.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/memstore/GTSimpleMemStore.java
b/storage/src/main/java/org/apache/kylin/storage/gridtable/memstore/GTSimpleMemStore.java
index c59ee04..a4d0b8d 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/memstore/GTSimpleMemStore.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/memstore/GTSimpleMemStore.java
@@ -6,8 +6,8 @@ import java.util.BitSet;
 import java.util.Iterator;
 import java.util.List;
 
-import org.apache.kylin.common.util.ByteArray;
 import org.apache.kylin.storage.gridtable.GTInfo;
+import org.apache.kylin.storage.gridtable.GTRecord;
 import org.apache.kylin.storage.gridtable.GTRowBlock;
 import org.apache.kylin.storage.gridtable.GTScanRequest;
 import org.apache.kylin.storage.gridtable.IGTStore;
@@ -78,7 +78,7 @@ public class GTSimpleMemStore implements IGTStore {
     }
 
     @Override
-    public IGTStoreScanner scan(ByteArray pkStart, ByteArray pkEnd, BitSet selectedColBlocks,
GTScanRequest additionalPushDown) {
+    public IGTStoreScanner scan(GTRecord pkStart, GTRecord pkEnd, BitSet selectedColBlocks,
GTScanRequest additionalPushDown) {
 
         return new IGTStoreScanner() {
             Iterator<GTRowBlock> it = rowBlockList.iterator();


Mime
View raw message