kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From shaofeng...@apache.org
Subject incubator-kylin git commit: KYLIN-760 Fix the error in cube build
Date Tue, 19 May 2015 02:54:50 GMT
Repository: incubator-kylin
Updated Branches:
  refs/heads/0.8.0 fff4df99b -> da67e9555


KYLIN-760 Fix the error in cube build

Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/da67e955
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/da67e955
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/da67e955

Branch: refs/heads/0.8.0
Commit: da67e9555b013c8dd9dd79d504ff2861750ce3a3
Parents: fff4df9
Author: shaofengshi <shaofengshi@apache.org>
Authored: Tue May 19 10:54:31 2015 +0800
Committer: shaofengshi <shaofengshi@apache.org>
Committed: Tue May 19 10:54:31 2015 +0800

----------------------------------------------------------------------
 .../job/hadoop/cubev2/InMemCubeBuilder.java     | 29 +++++++++------
 .../hadoop/cubev2/MapContextGTRecordWriter.java | 39 +++++++++++++++++---
 2 files changed, 50 insertions(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/da67e955/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/InMemCubeBuilder.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/InMemCubeBuilder.java b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/InMemCubeBuilder.java
index 392ac23..cff3474 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/InMemCubeBuilder.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/InMemCubeBuilder.java
@@ -86,6 +86,7 @@ public class InMemCubeBuilder implements Runnable {
     private int[] hbaseMeasureRefIndex;
     private MeasureDesc[] measureDescs;
     private int measureCount;
+    private boolean hasDependentMeasure = false;
 
     protected IGTRecordWriter gtRecordWriter;
 
@@ -147,7 +148,9 @@ public class InMemCubeBuilder implements Runnable {
             }
         }
 
-        measureDescs = desc.getMeasures().toArray(new MeasureDesc[measureCount]);
+        this.hasDependentMeasure = dependentMeasures.size() > 0;
+
+        this.measureDescs = desc.getMeasures().toArray(new MeasureDesc[measureCount]);
     }
 
 
@@ -212,21 +215,23 @@ public class InMemCubeBuilder implements Runnable {
                     newRecord.set(index, record.get(i));
                 }
 
-                // update measures which have 'dependent_measure_ref'
-                newRecord.getValues(dependentMetrics, hllObjects);
+                if(hasDependentMeasure) {
+                    // update measures which have 'dependent_measure_ref'
+                    newRecord.getValues(dependentMetrics, hllObjects);
 
-                for (Integer i : dependentMeasures.keySet()) {
-                    for (int index = 0, c = dependentMetrics.nextSetBit(0); c >= 0; index++,
c = dependentMetrics.nextSetBit(c + 1)) {
-                        if (c == allNeededColumns.cardinality() - measureCount + dependentMeasures.get(i))
{
-                            assert hllObjects[index] instanceof HyperLogLogPlusCounter; //
currently only HLL is allowed
+                    for (Integer i : dependentMeasures.keySet()) {
+                        for (int index = 0, c = dependentMetrics.nextSetBit(0); c >= 0;
index++, c = dependentMetrics.nextSetBit(c + 1)) {
+                            if (c == allNeededColumns.cardinality() - measureCount + dependentMeasures.get(i))
{
+                                assert hllObjects[index] instanceof HyperLogLogPlusCounter;
// currently only HLL is allowed
 
-                            byteBuffer.clear();
-                            BytesUtil.writeVLong(((HyperLogLogPlusCounter) hllObjects[index]).getCountEstimate(),
byteBuffer);
-                            byteArray.set(byteBuffer.array(), 0, byteBuffer.position());
-                            newRecord.set(allNeededColumns.cardinality() - measureCount +
i, byteArray);
+                                byteBuffer.clear();
+                                BytesUtil.writeVLong(((HyperLogLogPlusCounter) hllObjects[index]).getCountEstimate(),
byteBuffer);
+                                byteArray.set(byteBuffer.array(), 0, byteBuffer.position());
+                                newRecord.set(allNeededColumns.cardinality() - measureCount
+ i, byteArray);
+                            }
                         }
-                    }
 
+                    }
                 }
 
                 builder.write(newRecord);

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/da67e955/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MapContextGTRecordWriter.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MapContextGTRecordWriter.java
b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MapContextGTRecordWriter.java
index 299143f..008819e 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MapContextGTRecordWriter.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cubev2/MapContextGTRecordWriter.java
@@ -1,21 +1,26 @@
 package org.apache.kylin.job.hadoop.cubev2;
 
+import com.google.common.collect.Lists;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.kylin.common.util.Bytes;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapreduce.MapContext;
+import org.apache.kylin.common.util.Bytes;
 import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.cube.cuboid.Cuboid;
 import org.apache.kylin.cube.kv.RowConstants;
 import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.HBaseColumnDesc;
+import org.apache.kylin.cube.model.HBaseColumnFamilyDesc;
+import org.apache.kylin.metadata.model.MeasureDesc;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.apache.kylin.storage.gridtable.GTRecord;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.BitSet;
+import java.util.List;
 
 /**
  */
@@ -29,17 +34,39 @@ public class MapContextGTRecordWriter implements IGTRecordWriter {
 
     private int bytesLength;
     private int dimensions;
+    private int measureCount;
     private byte[] keyBuf;
     private int[] measureColumnsIndex;
     private ByteBuffer valueBuf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
     private ImmutableBytesWritable outputKey = new ImmutableBytesWritable();
     private Text outputValue = new Text();
-    long cuboidRowCount = 0;
+    private long cuboidRowCount = 0;
+    private int[] hbaseMeasureRefIndex;
 
     public MapContextGTRecordWriter(MapContext<?, ?, ImmutableBytesWritable, Text>
mapContext, CubeDesc cubeDesc, CubeSegment cubeSegment) {
         this.mapContext = mapContext;
         this.cubeDesc = cubeDesc;
         this.cubeSegment = cubeSegment;
+        this.measureCount = cubeDesc.getMeasures().size();
+        hbaseMeasureRefIndex = new int[measureCount];
+
+        List<MeasureDesc> hbaseMeasureList = Lists.newArrayList();
+        for (HBaseColumnFamilyDesc familyDesc : cubeDesc.getHbaseMapping().getColumnFamily())
{
+            for (HBaseColumnDesc hbaseColDesc : familyDesc.getColumns()) {
+                for (MeasureDesc measure : hbaseColDesc.getMeasures()) {
+                    hbaseMeasureList.add(measure);
+                }
+            }
+        }
+
+        for (int i = 0; i < measureCount; i++) {
+            for (int j = 0; j < measureCount; j++) {
+                if (cubeDesc.getMeasures().get(i).equals(hbaseMeasureList.get(j))) {
+                    hbaseMeasureRefIndex[i] = j;
+                    break;
+                }
+            }
+        }
 
     }
 
@@ -49,7 +76,7 @@ public class MapContextGTRecordWriter implements IGTRecordWriter {
         if (lastCuboidId == null || !lastCuboidId.equals(cuboidId)) {
             // output another cuboid
             initVariables(cuboidId);
-            if(lastCuboidId != null) {
+            if (lastCuboidId != null) {
                 logger.info("Cuboid " + lastCuboidId + " has " + cuboidRowCount + " rows");
                 cuboidRowCount = 0;
             }
@@ -84,9 +111,9 @@ public class MapContextGTRecordWriter implements IGTRecordWriter {
 
         keyBuf = new byte[bytesLength];
         dimensions = BitSet.valueOf(new long[]{cuboidId}).cardinality();
-        measureColumnsIndex = new int[cubeDesc.getMeasures().size()];
-        for(int i =0; i< cubeDesc.getMeasures().size(); i++) {
-            measureColumnsIndex[i] = dimensions + i;
+        measureColumnsIndex = new int[measureCount];
+        for (int i = 0; i < measureCount; i++) {
+            measureColumnsIndex[i] = dimensions + hbaseMeasureRefIndex[i];
         }
 
         System.arraycopy(Bytes.toBytes(cuboidId), 0, keyBuf, 0, RowConstants.ROWKEY_CUBOIDID_LEN);


Mime
View raw message