KYLIN-2790 Add extending point for column family
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d62cb58b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d62cb58b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d62cb58b
Branch: refs/heads/master
Commit: d62cb58bde55766dbe87bbad29953dd1243c2014
Parents: 4782289
Author: auphyroc99 <454530524@qq.com>
Authored: Wed Aug 16 10:34:02 2017 +0800
Committer: Roger Shi <rogershijicheng@gmail.com>
Committed: Wed Aug 30 18:13:52 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 4 ++++
.../org/apache/kylin/cube/model/CubeDesc.java | 18 ++++++++++++++--
.../kylin/cube/model/HBaseMappingDesc.java | 18 ++++++++++++++++
.../org/apache/kylin/gridtable/GTRecord.java | 22 ++++++++++++++++++++
4 files changed, 60 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d62cb58b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index a113327..47806a5 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -286,6 +286,10 @@ abstract public class KylinConfigBase implements Serializable {
"org.apache.kylin.storage.hbase.util.ZookeeperDistributedLock$Factory");
return (DistributedLockFactory) ClassUtil.newInstance(clsName);
}
+
+ public String getHBaseMappingAdapter() {
+ return getOptional("kylin.metadata.hbasemapping-adapter");
+ }
// ============================================================================
// DICTIONARY & SNAPSHOT
http://git-wip-us.apache.org/repos/asf/kylin/blob/d62cb58b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 0e22587..d263e1f 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -22,6 +22,7 @@ import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
+import java.lang.reflect.Method;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
@@ -603,8 +604,21 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware
{
validateAggregationGroups(); // check if aggregation group is valid
validateAggregationGroupsCombination();
- if (hbaseMapping != null) {
- hbaseMapping.init(this);
+ String hbaseMappingAdapterName = config.getHBaseMappingAdapter();
+
+ if (hbaseMappingAdapterName != null) {
+ try {
+ Class<?> hbaseMappingAdapterClass = Class.forName(hbaseMappingAdapterName);
+ Method initMethod = hbaseMappingAdapterClass.getMethod("initHBaseMapping",
CubeDesc.class);
+ initMethod.invoke(null, this);
+ } catch (Exception e) {
+ logger.error("Wrong configuration for kylin.metadata.hbasemapping-adapter:
class "
+ + hbaseMappingAdapterName + " not found. ");
+ }
+ } else {
+ if (hbaseMapping != null) {
+ hbaseMapping.init(this);
+ }
}
initMeasureReferenceToColumnFamily();
http://git-wip-us.apache.org/repos/asf/kylin/blob/d62cb58b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java
index d1e5829..77b4b14 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/HBaseMappingDesc.java
@@ -87,6 +87,24 @@ public class HBaseMappingDesc implements java.io.Serializable {
}
}
+ public void initAsSeparatedColumns(CubeDesc cubeDesc) {
+ cubeRef = cubeDesc;
+
+ int cfNum = cubeDesc.getMeasures().size();
+ columnFamily = new HBaseColumnFamilyDesc[cfNum];
+
+ for (int i = 0; i < cfNum; i++) {
+ HBaseColumnFamilyDesc cf = new HBaseColumnFamilyDesc();
+ HBaseColumnDesc col = new HBaseColumnDesc();
+ String measureRef = cubeDesc.getMeasures().get(i).getName();
+ col.setMeasureRefs(new String[] { measureRef });
+ col.setQualifier("M");
+ cf.setColumns(new HBaseColumnDesc[] { col });
+ cf.setName("F" + (i + 1));
+ columnFamily[i] = cf;
+ }
+ }
+
@Override
public String toString() {
return "HBaseMappingDesc [columnFamily=" + Arrays.toString(columnFamily) + "]";
http://git-wip-us.apache.org/repos/asf/kylin/blob/d62cb58b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
index 3e62ea7..862fb99 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTRecord.java
@@ -295,5 +295,27 @@ public class GTRecord implements Comparable<GTRecord>, Cloneable
{
buf.position(pos);
}
}
+
+ public void loadColumnsFromColumnBlocks(ImmutableBitSet[] selectedColumnBlocks, ImmutableBitSet
selectedCols, ByteBuffer buf) {
+
+ int pos = buf.position();
+
+ for (ImmutableBitSet selectedColBlock : selectedColumnBlocks) {
+
+ for (int i = 0; i < selectedColBlock.trueBitCount(); i++) {
+
+ int c = selectedColBlock.trueBitAt(i);
+
+ int len = info.codeSystem.codeLength(c, buf);
+ if(selectedCols.get(c)) {
+ cols[c].set(buf.array(), buf.arrayOffset() + pos, len);
+ }
+ pos += len;
+ buf.position(pos);
+ }
+
+ }
+
+ }
}
|