Return-Path: X-Original-To: apmail-kylin-commits-archive@minotaur.apache.org Delivered-To: apmail-kylin-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 622C318BAF for ; Sat, 23 May 2015 23:22:35 +0000 (UTC) Received: (qmail 98999 invoked by uid 500); 23 May 2015 23:22:35 -0000 Delivered-To: apmail-kylin-commits-archive@kylin.apache.org Received: (qmail 98972 invoked by uid 500); 23 May 2015 23:22:35 -0000 Mailing-List: contact commits-help@kylin.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@kylin.incubator.apache.org Delivered-To: mailing list commits@kylin.incubator.apache.org Received: (qmail 98963 invoked by uid 99); 23 May 2015 23:22:35 -0000 Received: from Unknown (HELO spamd4-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 23 May 2015 23:22:35 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd4-us-west.apache.org (ASF Mail Server at spamd4-us-west.apache.org) with ESMTP id CEFD7C06F7 for ; Sat, 23 May 2015 23:22:34 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd4-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: 1.771 X-Spam-Level: * X-Spam-Status: No, score=1.771 tagged_above=-999 required=6.31 tests=[KAM_ASCII_DIVIDERS=0.8, KAM_LAZY_DOMAIN_SECURITY=1, RCVD_IN_MSPIKE_H3=-0.01, RCVD_IN_MSPIKE_WL=-0.01, T_RP_MATCHES_RCVD=-0.01, URIBL_BLOCKED=0.001] autolearn=disabled Received: from mx1-us-west.apache.org ([10.40.0.8]) by localhost (spamd4-us-west.apache.org [10.40.0.11]) (amavisd-new, port 10024) with ESMTP id j-GNlwLbsEv3 for ; Sat, 23 May 2015 23:22:27 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx1-us-west.apache.org (ASF Mail Server at mx1-us-west.apache.org) with SMTP id 3C74621246 for ; Sat, 23 May 2015 23:22:27 +0000 (UTC) Received: (qmail 98958 invoked by uid 99); 23 May 2015 23:22:27 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 23 May 2015 23:22:27 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id C75A8E065F; Sat, 23 May 2015 23:22:26 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: liyang@apache.org To: commits@kylin.incubator.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: incubator-kylin git commit: KYLIN-625, refactor IGTStore to take GTRecord as pk start/end Date: Sat, 23 May 2015 23:22:26 +0000 (UTC) Repository: incubator-kylin Updated Branches: refs/heads/0.8.0 ce775594a -> 7d8ae6e38 KYLIN-625, refactor IGTStore to take GTRecord as pk start/end Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/7d8ae6e3 Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/7d8ae6e3 Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/7d8ae6e3 Branch: refs/heads/0.8.0 Commit: 7d8ae6e3811ca53364437e0d5ccdb08f1a68d885 Parents: ce77559 Author: Yang Li Authored: Sun May 24 07:22:07 2015 +0800 Committer: Yang Li Committed: Sun May 24 07:22:07 2015 +0800 ---------------------------------------------------------------------- .../storage/cube/CubeHBaseReadonlyStore.java | 9 ++++-- .../kylin/storage/gridtable/GTComboStore.java | 11 +++---- .../kylin/storage/gridtable/GTRawScanner.java | 7 +---- .../kylin/storage/gridtable/GTRecord.java | 25 +++++++++++++++ .../kylin/storage/gridtable/GTRowBlock.java | 2 +- .../kylin/storage/gridtable/IGTStore.java | 2 +- .../apache/kylin/storage/gridtable/ScanKey.java | 33 -------------------- .../gridtable/diskstore/GTDiskStore.java | 4 +-- .../gridtable/memstore/GTSimpleMemStore.java | 4 +-- 9 files changed, 43 insertions(+), 54 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7d8ae6e3/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseReadonlyStore.java ---------------------------------------------------------------------- diff --git a/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseReadonlyStore.java b/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseReadonlyStore.java index cb67fc5..cd0904b 100644 --- a/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseReadonlyStore.java +++ b/storage/src/main/java/org/apache/kylin/storage/cube/CubeHBaseReadonlyStore.java @@ -23,6 +23,7 @@ import org.apache.kylin.cube.model.HBaseColumnDesc; import org.apache.kylin.cube.model.HBaseColumnFamilyDesc; import org.apache.kylin.cube.model.HBaseMappingDesc; import org.apache.kylin.storage.gridtable.GTInfo; +import org.apache.kylin.storage.gridtable.GTRecord; import org.apache.kylin.storage.gridtable.GTRowBlock; import org.apache.kylin.storage.gridtable.GTRowBlock.Writer; import org.apache.kylin.storage.gridtable.GTScanRequest; @@ -64,7 +65,7 @@ public class CubeHBaseReadonlyStore implements IGTStore { } @Override - public IGTStoreScanner scan(ByteArray pkStart, ByteArray pkEnd, BitSet selectedColumnBlocks, GTScanRequest additionalPushDown) throws IOException { + public IGTStoreScanner scan(GTRecord pkStart, GTRecord pkEnd, BitSet selectedColumnBlocks, GTScanRequest additionalPushDown) throws IOException { // TODO enable coprocessor // primary key (also the 0th column block) is always selected @@ -122,7 +123,7 @@ public class CubeHBaseReadonlyStore implements IGTStore { }; } - private Scan buildScan(ByteArray pkStart, ByteArray pkEnd, List> selectedColumns) { + private Scan buildScan(GTRecord pkStart, GTRecord pkEnd, List> selectedColumns) { Scan scan = new Scan(); scan.setCaching(SCAN_CACHE); scan.setCacheBlocks(true); @@ -140,8 +141,10 @@ public class CubeHBaseReadonlyStore implements IGTStore { return scan; } - private byte[] makeRowKeyToScan(ByteArray pk, byte fill) { + private byte[] makeRowKeyToScan(GTRecord pkRec, byte fill) { + ByteArray pk = GTRecord.exportScanKey(pkRec); int pkMaxLen = info.getMaxColumnLength(info.getPrimaryKey()); + byte[] buf = new byte[pkMaxLen + RowConstants.ROWKEY_CUBOIDID_LEN]; Arrays.fill(buf, fill); http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7d8ae6e3/storage/src/main/java/org/apache/kylin/storage/gridtable/GTComboStore.java ---------------------------------------------------------------------- diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTComboStore.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTComboStore.java index c0d56c3..92707ed 100644 --- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTComboStore.java +++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTComboStore.java @@ -1,14 +1,13 @@ package org.apache.kylin.storage.gridtable; -import org.apache.kylin.common.util.ByteArray; +import java.io.IOException; +import java.util.BitSet; + import org.apache.kylin.storage.gridtable.diskstore.GTDiskStore; import org.apache.kylin.storage.gridtable.memstore.GTSimpleMemStore; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.util.BitSet; - /** */ public class GTComboStore implements IGTStore { @@ -18,7 +17,7 @@ public class GTComboStore implements IGTStore { private final GTInfo gtInfo; private void convert(IGTStore input, IGTStore output) throws IOException { - final IGTStoreScanner scanner = input.scan(ScanKey.makeScanKey(gtInfo, new GTRecord(gtInfo)), ScanKey.makeScanKey(gtInfo, new GTRecord(gtInfo)), null, null); + final IGTStoreScanner scanner = input.scan(null, null, null, null); final IGTStoreWriter writer = output.rebuild(-1); while (scanner.hasNext()) { writer.write(scanner.next()); @@ -101,7 +100,7 @@ public class GTComboStore implements IGTStore { } @Override - public IGTStoreScanner scan(ByteArray pkStart, ByteArray pkEnd, BitSet selectedColBlocks, GTScanRequest additionalPushDown) throws IOException { + public IGTStoreScanner scan(GTRecord pkStart, GTRecord pkEnd, BitSet selectedColBlocks, GTScanRequest additionalPushDown) throws IOException { return getCurrent().scan(pkStart, pkEnd, selectedColBlocks, additionalPushDown); } http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7d8ae6e3/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRawScanner.java ---------------------------------------------------------------------- diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRawScanner.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRawScanner.java index ff97cd5..1e934b5 100644 --- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRawScanner.java +++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRawScanner.java @@ -5,7 +5,6 @@ import java.util.BitSet; import java.util.Iterator; import java.util.NoSuchElementException; -import org.apache.kylin.common.util.ByteArray; import org.apache.kylin.storage.gridtable.IGTStore.IGTStoreScanner; public class GTRawScanner implements IGTScanner { @@ -23,12 +22,8 @@ public class GTRawScanner implements IGTScanner { public GTRawScanner(GTInfo info, IGTStore store, GTScanRequest req) throws IOException { this.info = info; - - ByteArray start = ScanKey.makeScanKey(info, req.getPkStart()); - ByteArray end = ScanKey.makeScanKey(info, req.getPkEnd()); this.selectedColBlocks = info.selectColumnBlocks(req.getColumns()); - - this.storeScanner = store.scan(start, end, selectedColBlocks, req); + this.storeScanner = store.scan(req.getPkStart(), req.getPkEnd(), selectedColBlocks, req); this.oneRecord = new GTRecord(info); } http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7d8ae6e3/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java ---------------------------------------------------------------------- diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java index 61ff25d..3d0f9bb 100644 --- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java +++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRecord.java @@ -233,4 +233,29 @@ public class GTRecord implements Comparable { } } + /* similar to export(primaryKey) but will stop at the first null value */ + public static ByteArray exportScanKey(GTRecord rec) { + if (rec == null) + return null; + + GTInfo info = rec.getInfo(); + + BitSet selectedColumns = new BitSet(); + int len = 0; + for (int i = info.primaryKey.nextSetBit(0); i >= 0; i = info.primaryKey.nextSetBit(i + 1)) { + if (rec.cols[i].array() == null) { + break; + } + selectedColumns.set(i); + len += rec.cols[i].length(); + } + + if (selectedColumns.cardinality() == 0) + return null; + + ByteArray buf = ByteArray.allocate(len); + rec.exportColumns(selectedColumns, buf); + return buf; + } + } http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7d8ae6e3/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlock.java ---------------------------------------------------------------------- diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlock.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlock.java index ec24da6..4a68659 100644 --- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlock.java +++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRowBlock.java @@ -192,7 +192,7 @@ public class GTRowBlock { } } - public void export(DataOutputStream dataOutputStream, ByteArray array) throws IOException { + private void export(DataOutputStream dataOutputStream, ByteArray array) throws IOException { dataOutputStream.writeInt(array.length()); dataOutputStream.write(array.array(), array.offset(), array.length()); } http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7d8ae6e3/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTStore.java ---------------------------------------------------------------------- diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTStore.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTStore.java index 4defa24..76bf76a 100644 --- a/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTStore.java +++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTStore.java @@ -15,7 +15,7 @@ public interface IGTStore { IGTStoreWriter append(int shard, GTRowBlock.Writer fillLast) throws IOException; - IGTStoreScanner scan(ByteArray pkStart, ByteArray pkEnd, BitSet selectedColBlocks, GTScanRequest additionalPushDown) throws IOException; + IGTStoreScanner scan(GTRecord pkStart, GTRecord pkEnd, BitSet selectedColBlocks, GTScanRequest additionalPushDown) throws IOException; interface IGTStoreWriter extends Closeable { void write(GTRowBlock block) throws IOException; http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7d8ae6e3/storage/src/main/java/org/apache/kylin/storage/gridtable/ScanKey.java ---------------------------------------------------------------------- diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/ScanKey.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/ScanKey.java deleted file mode 100644 index d8edf45..0000000 --- a/storage/src/main/java/org/apache/kylin/storage/gridtable/ScanKey.java +++ /dev/null @@ -1,33 +0,0 @@ -package org.apache.kylin.storage.gridtable; - -import org.apache.kylin.common.util.ByteArray; - -import java.util.BitSet; - -/** - */ -public final class ScanKey { - - private ScanKey() { - } - - static ByteArray makeScanKey(GTInfo info, GTRecord rec) { - int firstPKCol = info.primaryKey.nextSetBit(0); - if (rec == null || rec.cols[firstPKCol].array() == null) - return null; - - BitSet selectedColumns = new BitSet(); - int len = 0; - for (int i = info.primaryKey.nextSetBit(0); i >= 0; i = info.primaryKey.nextSetBit(i + 1)) { - if (rec.cols[i].array() == null) { - break; - } - selectedColumns.set(i); - len += rec.cols[i].length(); - } - - ByteArray buf = ByteArray.allocate(len); - rec.exportColumns(selectedColumns, buf); - return buf; - } -} http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7d8ae6e3/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/GTDiskStore.java ---------------------------------------------------------------------- diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/GTDiskStore.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/GTDiskStore.java index 9684d8d..b6f7d95 100644 --- a/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/GTDiskStore.java +++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/diskstore/GTDiskStore.java @@ -10,8 +10,8 @@ import java.nio.ByteBuffer; import java.util.BitSet; import java.util.UUID; -import org.apache.kylin.common.util.ByteArray; import org.apache.kylin.storage.gridtable.GTInfo; +import org.apache.kylin.storage.gridtable.GTRecord; import org.apache.kylin.storage.gridtable.GTRowBlock; import org.apache.kylin.storage.gridtable.GTScanRequest; import org.apache.kylin.storage.gridtable.IGTStore; @@ -151,7 +151,7 @@ public class GTDiskStore implements IGTStore { } @Override - public IGTStoreScanner scan(ByteArray pkStart, ByteArray pkEnd, BitSet selectedColBlocks, GTScanRequest additionalPushDown) throws IOException { + public IGTStoreScanner scan(GTRecord pkStart, GTRecord pkEnd, BitSet selectedColBlocks, GTScanRequest additionalPushDown) throws IOException { return new DiskStoreScanner(fileSystem.getReader(getRowBlockFile(identifier))); } http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7d8ae6e3/storage/src/main/java/org/apache/kylin/storage/gridtable/memstore/GTSimpleMemStore.java ---------------------------------------------------------------------- diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/memstore/GTSimpleMemStore.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/memstore/GTSimpleMemStore.java index c59ee04..a4d0b8d 100644 --- a/storage/src/main/java/org/apache/kylin/storage/gridtable/memstore/GTSimpleMemStore.java +++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/memstore/GTSimpleMemStore.java @@ -6,8 +6,8 @@ import java.util.BitSet; import java.util.Iterator; import java.util.List; -import org.apache.kylin.common.util.ByteArray; import org.apache.kylin.storage.gridtable.GTInfo; +import org.apache.kylin.storage.gridtable.GTRecord; import org.apache.kylin.storage.gridtable.GTRowBlock; import org.apache.kylin.storage.gridtable.GTScanRequest; import org.apache.kylin.storage.gridtable.IGTStore; @@ -78,7 +78,7 @@ public class GTSimpleMemStore implements IGTStore { } @Override - public IGTStoreScanner scan(ByteArray pkStart, ByteArray pkEnd, BitSet selectedColBlocks, GTScanRequest additionalPushDown) { + public IGTStoreScanner scan(GTRecord pkStart, GTRecord pkEnd, BitSet selectedColBlocks, GTScanRequest additionalPushDown) { return new IGTStoreScanner() { Iterator it = rowBlockList.iterator();