kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From shaofeng...@apache.org
Subject [06/50] [abbrv] kylin git commit: KYLIN-1465 log improvement on others
Date Wed, 23 Mar 2016 09:14:50 GMT
KYLIN-1465 log improvement on others


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b9a3418b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b9a3418b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b9a3418b

Branch: refs/heads/1.5.x-HBase1.1.3
Commit: b9a3418b7f2121745090bb6463e7be754babfb73
Parents: b41c446
Author: Hongbin Ma <mahongbin@apache.org>
Authored: Fri Mar 4 15:59:37 2016 +0800
Committer: Hongbin Ma <mahongbin@apache.org>
Committed: Fri Mar 4 16:01:04 2016 +0800

----------------------------------------------------------------------
 .../kylin/cube/gridtable/CubeGridTable.java     |  6 ++---
 .../gridtable/NotEnoughGTInfoException.java     | 23 --------------------
 .../kylin/gridtable/GTScanReqSerDerTest.java    | 11 +++++-----
 .../kylin/storage/translate/HBaseKeyRange.java  |  4 +---
 .../streaming/cube/StreamingCubeBuilder.java    |  6 +++--
 .../hbase/cube/v1/CubeSegmentTupleIterator.java | 14 ++++++------
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 ++---
 .../hbase/cube/v2/CubeHBaseEndpointRPC.java     | 18 +++++++--------
 .../storage/hbase/cube/v2/CubeHBaseRPC.java     |  8 +++----
 .../hbase/cube/v2/CubeSegmentScanner.java       |  3 +--
 .../storage/hbase/cube/v2/CubeStorageQuery.java | 10 +++------
 .../kylin/storage/hbase/cube/v2/RawScan.java    |  2 +-
 12 files changed, 41 insertions(+), 70 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a3418b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java
index 05fc8a5..5f0bb07 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java
@@ -29,18 +29,18 @@ public class CubeGridTable {
                 dictionaryMap.put(col, dictionary);
             }
         }
-        
+
         return dictionaryMap;
     }
 
-    public static GTInfo newGTInfo(CubeSegment cubeSeg, long cuboidId) throws NotEnoughGTInfoException
{
+    public static GTInfo newGTInfo(CubeSegment cubeSeg, long cuboidId) {
         Map<TblColRef, Dictionary<String>> dictionaryMap = getDimensionToDictionaryMap(cubeSeg,
cuboidId);
         Cuboid cuboid = Cuboid.findById(cubeSeg.getCubeDesc(), cuboidId);
         for (TblColRef dim : cuboid.getColumns()) {
             if (cubeSeg.getCubeDesc().getRowkey().isUseDictionary(dim)) {
                 Dictionary dict = dictionaryMap.get(dim);
                 if (dict == null) {
-                    throw new NotEnoughGTInfoException();
+                    throw new RuntimeException("Dictionary for " + dim + " is not found");
                 }
             }
         }

http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a3418b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/NotEnoughGTInfoException.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/NotEnoughGTInfoException.java
b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/NotEnoughGTInfoException.java
deleted file mode 100644
index 9bbcf75..0000000
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/NotEnoughGTInfoException.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.gridtable;
-
-@SuppressWarnings("serial")
-public class NotEnoughGTInfoException extends Exception {
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a3418b/core-cube/src/test/java/org/apache/kylin/gridtable/GTScanReqSerDerTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/GTScanReqSerDerTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/GTScanReqSerDerTest.java
index 6642d95..77cc2d8 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/GTScanReqSerDerTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/GTScanReqSerDerTest.java
@@ -29,7 +29,6 @@ import org.apache.kylin.cube.CubeManager;
 import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.cube.cuboid.Cuboid;
 import org.apache.kylin.cube.gridtable.CubeGridTable;
-import org.apache.kylin.cube.gridtable.NotEnoughGTInfoException;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -73,7 +72,7 @@ public class GTScanReqSerDerTest extends LocalFileMetadataTestCase {
         buffer.flip();
 
         GTInfo sInfo = GTInfo.serializer.deserialize(buffer);
-        this.compareTwoGTInfo(info,sInfo);
+        this.compareTwoGTInfo(info, sInfo);
     }
 
     @Test
@@ -83,11 +82,11 @@ public class GTScanReqSerDerTest extends LocalFileMetadataTestCase {
         buffer.flip();
 
         GTInfo sInfo = GTInfo.serializer.deserialize(buffer);
-        this.compareTwoGTInfo(info,sInfo);
+        this.compareTwoGTInfo(info, sInfo);
     }
 
     @Test
-    public void testGTInfo() throws NotEnoughGTInfoException {
+    public void testGTInfo() {
         CubeInstance cube = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube("test_kylin_cube_with_slr_ready");
         CubeSegment segment = cube.getFirstSegment();
 
@@ -95,8 +94,8 @@ public class GTScanReqSerDerTest extends LocalFileMetadataTestCase {
         GTInfo.serializer.serialize(info, buffer);
         buffer.flip();
 
-        GTInfo sInfo = GTInfo.serializer.deserialize(buffer); 
-        this.compareTwoGTInfo(info,sInfo);
+        GTInfo sInfo = GTInfo.serializer.deserialize(buffer);
+        this.compareTwoGTInfo(info, sInfo);
     }
 
     private void compareTwoGTInfo(GTInfo info, GTInfo sInfo) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a3418b/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java
b/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java
index a5ca62b..4f39cf6 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java
@@ -45,9 +45,7 @@ import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 
 /**
- * 
  * @author xjiang
- * 
  */
 public class HBaseKeyRange implements Comparable<HBaseKeyRange> {
 
@@ -147,7 +145,7 @@ public class HBaseKeyRange implements Comparable<HBaseKeyRange>
{
             buf.append(BytesUtil.toHex(fuzzyKey.getFirst()));
             buf.append(" ");
             buf.append(BytesUtil.toHex(fuzzyKey.getSecond()));
-            buf.append(System.lineSeparator());
+            buf.append(";");
         }
         this.fuzzyKeyString = buf.toString();
     }

http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a3418b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
----------------------------------------------------------------------
diff --git a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
index d7056cf..c4f2b7e 100644
--- a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
+++ b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
@@ -73,6 +73,7 @@ public class StreamingCubeBuilder implements StreamingBatchBuilder {
     private static final Logger logger = LoggerFactory.getLogger(StreamingCubeBuilder.class);
 
     private final String cubeName;
+    private int processedRowCount = 0;
 
     public StreamingCubeBuilder(String cubeName) {
         this.cubeName = cubeName;
@@ -81,16 +82,16 @@ public class StreamingCubeBuilder implements StreamingBatchBuilder {
     @Override
     public void build(StreamingBatch streamingBatch, Map<TblColRef, Dictionary<String>>
dictionaryMap, ICuboidWriter cuboidWriter) {
         try {
-
             CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
             final CubeInstance cubeInstance = cubeManager.reloadCubeLocal(cubeName);
             LinkedBlockingQueue<List<String>> blockingQueue = new LinkedBlockingQueue<List<String>>();
             InMemCubeBuilder inMemCubeBuilder = new InMemCubeBuilder(cubeInstance.getDescriptor(),
dictionaryMap);
             final Future<?> future = Executors.newCachedThreadPool().submit(inMemCubeBuilder.buildAsRunnable(blockingQueue,
cuboidWriter));
+            processedRowCount = streamingBatch.getMessages().size();
             for (StreamingMessage streamingMessage : streamingBatch.getMessages()) {
                 blockingQueue.put(streamingMessage.getData());
             }
-            blockingQueue.put(Collections.<String> emptyList());
+            blockingQueue.put(Collections.<String>emptyList());
             future.get();
             cuboidWriter.flush();
 
@@ -157,6 +158,7 @@ public class StreamingCubeBuilder implements StreamingBatchBuilder {
     public void commit(IBuildable buildable) {
         CubeSegment cubeSegment = (CubeSegment) buildable;
         cubeSegment.setStatus(SegmentStatusEnum.READY);
+        cubeSegment.setInputRecords(processedRowCount);
         CubeUpdate cubeBuilder = new CubeUpdate(cubeSegment.getCubeInstance());
         cubeBuilder.setToAddSegs(cubeSegment);
         try {

http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a3418b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
index 5e842f7..909de39 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
@@ -210,26 +210,26 @@ public class CubeSegmentTupleIterator implements ITupleIterator {
 
     private void logScan(HBaseKeyRange keyRange) {
         StringBuilder info = new StringBuilder();
-        info.append("\nScan hbase table ").append(tableName).append(": ");
+        info.append(" Scan hbase table ").append(tableName).append(": ");
         if (keyRange.getCuboid().requirePostAggregation()) {
-            info.append("cuboid require post aggregation, from ");
+            info.append(" cuboid require post aggregation, from ");
         } else {
-            info.append("cuboid exact match, from ");
+            info.append(" cuboid exact match, from ");
         }
         info.append(keyRange.getCuboid().getInputID());
         info.append(" to ");
         info.append(keyRange.getCuboid().getId());
-        info.append("\nStart: ");
+        info.append(" Start: ");
         info.append(keyRange.getStartKeyAsString());
         info.append(" - ");
         info.append(Bytes.toStringBinary(keyRange.getStartKey()));
-        info.append("\nStop:  ");
+        info.append(" Stop:  ");
         info.append(keyRange.getStopKeyAsString());
         info.append(" - ");
         info.append(Bytes.toStringBinary(keyRange.getStopKey()));
         if (this.scan.getFilter() != null) {
-            info.append("\nFuzzy key counts: " + keyRange.getFuzzyKeys().size());
-            info.append("\nFuzzy: ");
+            info.append(" Fuzzy key counts: " + keyRange.getFuzzyKeys().size());
+            info.append(" Fuzzy: ");
             info.append(keyRange.getFuzzyKeyAsString());
         }
         logger.info(info.toString());

http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a3418b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
index 1b8b586..3d7f620 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
@@ -493,7 +493,7 @@ public class CubeStorageQuery implements ICachableStorageQuery {
         List<Collection<ColumnValueRange>> result = Lists.newArrayList();
 
         if (flatFilter == null) {
-            result.add(Collections.<ColumnValueRange> emptyList());
+            result.add(Collections.<ColumnValueRange>emptyList());
             return result;
         }
 
@@ -535,7 +535,7 @@ public class CubeStorageQuery implements ICachableStorageQuery {
         }
         if (globalAlwaysTrue) {
             orAndRanges.clear();
-            orAndRanges.add(Collections.<ColumnValueRange> emptyList());
+            orAndRanges.add(Collections.<ColumnValueRange>emptyList());
         }
         return orAndRanges;
     }
@@ -762,7 +762,7 @@ public class CubeStorageQuery implements ICachableStorageQuery {
     private void setLimit(TupleFilter filter, StorageContext context) {
         boolean goodAggr = context.isExactAggregation();
         boolean goodFilter = filter == null || (TupleFilter.isEvaluableRecursively(filter)
&& context.isCoprocessorEnabled());
-        boolean goodSort = context.hasSort() == false;
+        boolean goodSort = !context.hasSort();
         if (goodAggr && goodFilter && goodSort) {
             logger.info("Enable limit " + context.getLimit());
             context.enableLimit();

http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a3418b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index f22964f..e6f9ac1 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -293,15 +293,15 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
 
         final AtomicInteger totalScannedCount = new AtomicInteger(0);
         final ExpectedSizeIterator epResultItr = new ExpectedSizeIterator(scanRequests.size()
* shardNum);
+        final String currentThreadName = Thread.currentThread().getName();
 
         for (final Pair<byte[], byte[]> epRange : getEPKeyRanges(cuboidBaseShard, shardNum,
totalShards)) {
             executorService.submit(new Runnable() {
                 @Override
                 public void run() {
                     for (int i = 0; i < scanRequests.size(); ++i) {
-                        int scanIndex = i;
                         CubeVisitProtos.CubeVisitRequest.Builder builder = CubeVisitProtos.CubeVisitRequest.newBuilder();
-                        builder.setGtScanRequest(scanRequestByteStrings.get(scanIndex)).setHbaseRawScan(rawScanByteStrings.get(scanIndex));
+                        builder.setGtScanRequest(scanRequestByteStrings.get(i)).setHbaseRawScan(rawScanByteStrings.get(i));
                         for (IntList intList : hbaseColumnsToGTIntList) {
                             builder.addHbaseColumnsToGT(intList);
                         }
@@ -317,7 +317,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
 
                         for (Map.Entry<byte[], CubeVisitProtos.CubeVisitResponse> result
: results.entrySet()) {
                             totalScannedCount.addAndGet(result.getValue().getStats().getScannedRowCount());
-                            logger.info(getStatsString(result));
+                            logger.info("<spawned by " + currentThreadName + ">" +
getStatsString(result));
                             try {
                                 epResultItr.append(CompressionUtils.decompress(HBaseZeroCopyByteString.zeroCopyGetBytes(result.getValue().getCompressedRows())));
                             } catch (IOException | DataFormatException e) {
@@ -335,12 +335,12 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
     private String getStatsString(Map.Entry<byte[], CubeVisitProtos.CubeVisitResponse>
result) {
         StringBuilder sb = new StringBuilder();
         Stats stats = result.getValue().getStats();
-        sb.append("Endpoint RPC returned from HTable " + cubeSeg.getStorageLocationIdentifier()
+ " Shard " + BytesUtil.toHex(result.getKey()) + " on host: " + stats.getHostname() + ".");
-        sb.append("Total scanned row: " + stats.getScannedRowCount() + ". ");
-        sb.append("Total filtered/aggred row: " + stats.getAggregatedRowCount() + ". ");
-        sb.append("Time elapsed in EP: " + (stats.getServiceEndTime() - stats.getServiceStartTime())
+ "(ms). ");
-        sb.append("Server CPU usage: " + stats.getSystemCpuLoad() + ", server physical mem
left: " + stats.getFreePhysicalMemorySize() + ", server swap mem left:" + stats.getFreeSwapSpaceSize()
+ ".");
-        sb.append("Etc message: " + stats.getEtcMsg() + ".");
+        sb.append("Endpoint RPC returned from HTable ").append(cubeSeg.getStorageLocationIdentifier()).append("
Shard ").append(BytesUtil.toHex(result.getKey())).append(" on host: ").append(stats.getHostname()).append(".");
+        sb.append("Total scanned row: ").append(stats.getScannedRowCount()).append(". ");
+        sb.append("Total filtered/aggred row: ").append(stats.getAggregatedRowCount()).append(".
");
+        sb.append("Time elapsed in EP: ").append(stats.getServiceEndTime() - stats.getServiceStartTime()).append("(ms).
");
+        sb.append("Server CPU usage: ").append(stats.getSystemCpuLoad()).append(", server
physical mem left: ").append(stats.getFreePhysicalMemorySize()).append(", server swap mem
left:").append(stats.getFreeSwapSpaceSize()).append(".");
+        sb.append("Etc message: ").append(stats.getEtcMsg()).append(".");
         return sb.toString();
 
     }

http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a3418b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
index db39455..eb5ac9f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
@@ -273,20 +273,20 @@ public abstract class CubeHBaseRPC {
         info.append(cuboid.getInputID());
         info.append(" to ");
         info.append(cuboid.getId());
-        info.append("\nStart: ");
+        info.append(" Start: ");
         info.append(rawScan.getStartKeyAsString());
         info.append(" (");
         info.append(Bytes.toStringBinary(rawScan.startKey) + ")");
-        info.append("\nStop:  ");
+        info.append(" Stop:  ");
         info.append(rawScan.getEndKeyAsString());
         info.append(" (");
         info.append(Bytes.toStringBinary(rawScan.endKey) + ")");
         if (rawScan.fuzzyKeys != null && rawScan.fuzzyKeys.size() != 0) {
-            info.append("\nFuzzy key counts: " + rawScan.fuzzyKeys.size());
+            info.append(" Fuzzy key counts: " + rawScan.fuzzyKeys.size());
             info.append(". Fuzzy keys : ");
             info.append(rawScan.getFuzzyKeyAsString());
         } else {
-            info.append("\nNo Fuzzy Key");
+            info.append(", No Fuzzy Key");
         }
         logger.info(info.toString());
     }

http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a3418b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
index 3f00566..ee5ae96 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
@@ -21,7 +21,6 @@ import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.cube.cuboid.Cuboid;
 import org.apache.kylin.cube.gridtable.CubeGridTable;
 import org.apache.kylin.cube.gridtable.CuboidToGridTableMapping;
-import org.apache.kylin.cube.gridtable.NotEnoughGTInfoException;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.dict.TupleFilterFunctionTransformer;
 import org.apache.kylin.gridtable.EmptyGTScanner;
@@ -56,7 +55,7 @@ public class CubeSegmentScanner implements IGTScanner {
     final Cuboid cuboid;
 
     public CubeSegmentScanner(CubeSegment cubeSeg, Cuboid cuboid, Set<TblColRef> dimensions,
Set<TblColRef> groups, //
-            Collection<FunctionDesc> metrics, TupleFilter filter, boolean allowPreAggregate)
throws NotEnoughGTInfoException {
+            Collection<FunctionDesc> metrics, TupleFilter filter, boolean allowPreAggregate)
 {
         this.cuboid = cuboid;
         this.cubeSeg = cubeSeg;
         this.info = CubeGridTable.newGTInfo(cubeSeg, cuboid.getId());

http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a3418b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
index ab8c80f..df0bb84 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
@@ -11,7 +11,6 @@ import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeManager;
 import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.cube.gridtable.NotEnoughGTInfoException;
 import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
 import org.apache.kylin.dict.lookup.LookupStringTable;
@@ -96,14 +95,11 @@ public class CubeStorageQuery implements ICachableStorageQuery {
         List<CubeSegmentScanner> scanners = Lists.newArrayList();
         for (CubeSegment cubeSeg : cubeInstance.getSegments(SegmentStatusEnum.READY)) {
             CubeSegmentScanner scanner;
-            try {
-                scanner = new CubeSegmentScanner(cubeSeg, cuboid, dimensionsD, groupsD, metrics,
filterD, !isExactAggregation);
-            } catch (NotEnoughGTInfoException e) {
-                //deal with empty cube segment
-                logger.info("Cannot construct Segment {}'s GTInfo, this may due to empty
segment or broken metadata", cubeSeg);
-                logger.info("error stack", e);
+            if (cubeSeg.getInputRecords() == 0) {
+                logger.info("Skip cube segment {} because its input record is 0", cubeSeg);
                 continue;
             }
+            scanner = new CubeSegmentScanner(cubeSeg, cuboid, dimensionsD, groupsD, metrics,
filterD, !isExactAggregation);
             scanners.add(scanner);
         }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/b9a3418b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/RawScan.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/RawScan.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/RawScan.java
index 361b1dd..c2ffdba 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/RawScan.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/RawScan.java
@@ -61,7 +61,7 @@ public class RawScan {
             buf.append(BytesUtil.toHex(fuzzyKey.getFirst()));
             buf.append(" ");
             buf.append(BytesUtil.toHex(fuzzyKey.getSecond()));
-            buf.append(System.lineSeparator());
+            buf.append(";");
         }
         return buf.toString();
     }


Mime
View raw message