kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From liy...@apache.org
Subject incubator-kylin git commit: KYLIN-815 IPartitionConditionBuilder, flexible range condition to match hive partitions
Date Sun, 07 Jun 2015 08:58:54 GMT
Repository: incubator-kylin
Updated Branches:
  refs/heads/0.8.0 98e1b2986 -> c04847264


KYLIN-815 IPartitionConditionBuilder, flexible range condition to match hive partitions


Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/c0484726
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/c0484726
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/c0484726

Branch: refs/heads/0.8.0
Commit: c048472646e6bf995d9214a0ca312eca4a9cd84e
Parents: 98e1b29
Author: Yang Li <liyang@apache.org>
Authored: Sun Jun 7 16:58:47 2015 +0800
Committer: Yang Li <liyang@apache.org>
Committed: Sun Jun 7 16:58:47 2015 +0800

----------------------------------------------------------------------
 .../org/apache/kylin/common/util/ClassUtil.java |  8 ++
 .../org/apache/kylin/job/JoinedFlatTable.java   | 55 +++++---------
 .../kylin/metadata/model/PartitionDesc.java     | 77 ++++++++++++++++----
 3 files changed, 86 insertions(+), 54 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c0484726/common/src/main/java/org/apache/kylin/common/util/ClassUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/ClassUtil.java b/common/src/main/java/org/apache/kylin/common/util/ClassUtil.java
index 296e81c..63a06ed 100644
--- a/common/src/main/java/org/apache/kylin/common/util/ClassUtil.java
+++ b/common/src/main/java/org/apache/kylin/common/util/ClassUtil.java
@@ -62,4 +62,12 @@ public class ClassUtil {
         return (Class<? extends T>) ret;
     }
 
+    public static Object newInstance(String clz) {
+        try {
+            return forName(clz, Object.class).newInstance();
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c0484726/job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java b/job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index fb33440..283f462 100644
--- a/job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -20,29 +20,30 @@ package org.apache.kylin.job;
 
 import java.io.File;
 import java.io.IOException;
-import java.text.SimpleDateFormat;
-import java.util.*;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
 
 import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.DocumentBuilderFactory;
 import javax.xml.parsers.ParserConfigurationException;
 
-import org.w3c.dom.Document;
-import org.w3c.dom.NodeList;
-import org.xml.sax.SAXException;
-
-import org.apache.kylin.common.util.StringUtil;
 import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.job.engine.JobEngineConfig;
 import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
-import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
-import org.apache.kylin.metadata.model.IntermediateColumnDesc;
+import org.apache.kylin.job.engine.JobEngineConfig;
 import org.apache.kylin.job.hadoop.hive.SqlHiveDataTypeMapping;
 import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
+import org.apache.kylin.metadata.model.IntermediateColumnDesc;
 import org.apache.kylin.metadata.model.JoinDesc;
 import org.apache.kylin.metadata.model.LookupDesc;
+import org.apache.kylin.metadata.model.PartitionDesc;
 import org.apache.kylin.metadata.model.TblColRef;
+import org.w3c.dom.Document;
+import org.w3c.dom.NodeList;
+import org.xml.sax.SAXException;
 
 /**
  * @author George Song (ysong1)
@@ -202,36 +203,23 @@ public class JoinedFlatTable {
         whereBuilder.append("WHERE");
 
         CubeDesc cubeDesc = desc.getCubeDesc();
-
-        if (cubeDesc.getModel().getFilterCondition() != null && cubeDesc.getModel().getFilterCondition().equals("")
== false) {
-            whereBuilder.append(" (").append(cubeDesc.getModel().getFilterCondition()).append(")
");
+        DataModelDesc model = cubeDesc.getModel();
+        
+        if (model.getFilterCondition() != null && model.getFilterCondition().equals("")
== false) {
+            whereBuilder.append(" (").append(model.getFilterCondition()).append(") ");
             hasCondition = true;
         }
 
         CubeSegment cubeSegment = desc.getCubeSegment();
 
         if (null != cubeSegment) {
+            PartitionDesc partDesc = model.getPartitionDesc();
             long dateStart = cubeSegment.getDateRangeStart();
             long dateEnd = cubeSegment.getDateRangeEnd();
 
             if (!(dateStart == 0 && dateEnd == Long.MAX_VALUE)) {
-                String partitionColumnName = cubeDesc.getModel().getPartitionDesc().getPartitionDateColumn();
-                int indexOfDot = partitionColumnName.lastIndexOf(".");
-
-                // convert to use table alias;
-                if (indexOfDot > 0) {
-                    String partitionTableName = partitionColumnName.substring(0, indexOfDot);
-                    String columeOnly = partitionColumnName.substring(indexOfDot);
-                    String partitionTableAlias = tableAliasMap.get(partitionTableName);
-                    partitionColumnName = partitionTableAlias + columeOnly;
-                }
-
                 whereBuilder.append(hasCondition ? " AND (" : " (");
-                if (dateStart > 0) {
-                    whereBuilder.append(partitionColumnName + " >= '" + formatDateTimeInWhereClause(dateStart)
+ "' ");
-                    whereBuilder.append("AND ");
-                }
-                whereBuilder.append(partitionColumnName + " < '" + formatDateTimeInWhereClause(dateEnd)
+ "'");
+                whereBuilder.append(partDesc.getPartitionConditionBuilder().buildDateRangeCondition(partDesc,
dateStart, dateEnd, tableAliasMap));
                 whereBuilder.append(")\n");
                 hasCondition = true;
             }
@@ -242,15 +230,6 @@ public class JoinedFlatTable {
         }
     }
 
-    private static String formatDateTimeInWhereClause(long datetime) {
-        SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
-        f.setTimeZone(TimeZone.getTimeZone("GMT"));
-        Date date = new Date(datetime);
-        String str = f.format(date);
-        // note "2014-10-01" >= "2014-10-01 00:00:00" is FALSE
-        return StringUtil.dropSuffix(str, " 00:00:00");
-    }
-    
     private static String colName(String canonicalColName) {
         return canonicalColName.replace(".", "_");
     }

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c0484726/metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java b/metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
index cf0a8e3..cb60c6a 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
@@ -20,11 +20,14 @@ package org.apache.kylin.metadata.model;
 
 import java.util.Map;
 
+import org.apache.commons.lang3.StringUtils;
+import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.common.util.StringSplitter;
+
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
 import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
 import com.fasterxml.jackson.annotation.JsonProperty;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.kylin.common.util.StringSplitter;
 
 /**
  * @author xduo
@@ -44,31 +47,37 @@ public class PartitionDesc {
     private long partitionDateStart = 0L;
     @JsonProperty("partition_type")
     private PartitionType partitionType = PartitionType.APPEND;
+    @JsonProperty("partition_condition_builder")
+    private String partitionConditionBuilderClz = DefaultPartitionConditionBuilder.class.getName();
 
     private TblColRef partitionDateColumnRef;
+    private IPartitionConditionBuilder partitionConditionBuilder;
 
     public void init(Map<String, TableDesc> tables) {
-        if (StringUtils.isNotEmpty(partitionDateColumn)) {
-            partitionDateColumn = partitionDateColumn.toUpperCase();
+        if (StringUtils.isEmpty(partitionDateColumn))
+            return;
+
+        partitionDateColumn = partitionDateColumn.toUpperCase();
 
-            String[] columns = StringSplitter.split(partitionDateColumn, ".");
+        String[] columns = StringSplitter.split(partitionDateColumn, ".");
 
-            if (null != columns && columns.length == 3) {
-                String tableName = columns[0].toUpperCase() + "." + columns[1].toUpperCase();
+        if (null != columns && columns.length == 3) {
+            String tableName = columns[0].toUpperCase() + "." + columns[1].toUpperCase();
 
-                TableDesc table = tables.get(tableName);
-                ColumnDesc col = table.findColumnByName(columns[2]);
-                if (col != null) {
-                    partitionDateColumnRef = new TblColRef(col);
-                } else {
-                    throw new IllegalStateException("The column '" + partitionDateColumn
+ "' provided in 'partition_date_column' doesn't exist.");
-                }
+            TableDesc table = tables.get(tableName);
+            ColumnDesc col = table.findColumnByName(columns[2]);
+            if (col != null) {
+                partitionDateColumnRef = new TblColRef(col);
             } else {
-                throw new IllegalStateException("The 'partition_date_column' format is invalid:
" + partitionDateColumn + ", it should be {db}.{table}.{column}.");
+                throw new IllegalStateException("The column '" + partitionDateColumn + "'
provided in 'partition_date_column' doesn't exist.");
             }
+        } else {
+            throw new IllegalStateException("The 'partition_date_column' format is invalid:
" + partitionDateColumn + ", it should be {db}.{table}.{column}.");
         }
+
+        partitionConditionBuilder = (IPartitionConditionBuilder) ClassUtil.newInstance(partitionConditionBuilderClz);
     }
-    
+
     public boolean isPartitioned() {
         return partitionDateColumnRef != null;
     }
@@ -97,8 +106,44 @@ public class PartitionDesc {
         this.partitionType = partitionType;
     }
 
+    public IPartitionConditionBuilder getPartitionConditionBuilder() {
+        return partitionConditionBuilder;
+    }
+
     public TblColRef getPartitionDateColumnRef() {
         return partitionDateColumnRef;
     }
 
+    // ============================================================================
+
+    public static interface IPartitionConditionBuilder {
+        String buildDateRangeCondition(PartitionDesc partDesc, long startInclusive, long
endExclusive, Map<String, String> tableAlias);
+    }
+
+    public static class DefaultPartitionConditionBuilder implements IPartitionConditionBuilder
{
+
+        @Override
+        public String buildDateRangeCondition(PartitionDesc partDesc, long startInclusive,
long endExclusive, Map<String, String> tableAlias) {
+            String partitionColumnName = partDesc.getPartitionDateColumn();
+
+            // convert to use table alias
+            int indexOfDot = partitionColumnName.lastIndexOf(".");
+            if (indexOfDot > 0) {
+                String partitionTableName = partitionColumnName.substring(0, indexOfDot);
+                if (tableAlias != null && tableAlias.containsKey(partitionTableName))
+                    partitionColumnName = tableAlias.get(partitionTableName) + partitionColumnName.substring(indexOfDot);
+            }
+
+            StringBuilder builder = new StringBuilder();
+
+            if (startInclusive > 0) {
+                builder.append(partitionColumnName + " >= '" + DateFormat.formatToDateStr(startInclusive)
+ "' ");
+                builder.append("AND ");
+            }
+            builder.append(partitionColumnName + " < '" + DateFormat.formatToDateStr(endExclusive)
+ "'");
+
+            return builder.toString();
+        }
+
+    }
 }


Mime
View raw message