kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mahong...@apache.org
Subject incubator-kylin git commit: KYLIN-817 fix bugs
Date Tue, 09 Jun 2015 07:01:24 GMT
Repository: incubator-kylin
Updated Branches:
  refs/heads/0.8.0 4b435b370 -> 4219fdac1


KYLIN-817 fix bugs


Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/4219fdac
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/4219fdac
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/4219fdac

Branch: refs/heads/0.8.0
Commit: 4219fdac1105e7736e8d51e1ab73da275b308b79
Parents: 4b435b3
Author: honma <honma@ebay.com>
Authored: Tue Jun 9 15:00:35 2015 +0800
Committer: honma <honma@ebay.com>
Committed: Tue Jun 9 15:00:35 2015 +0800

----------------------------------------------------------------------
 .../metadata/filter/DateConditionModifier.java  | 32 ++++++++++++++------
 .../apache/kylin/metadata/model/DataType.java   | 17 ++++++++---
 .../apache/kylin/rest/service/CacheService.java |  2 +-
 .../kylin/streaming/KafkaClusterConfig.java     |  1 -
 4 files changed, 36 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4219fdac/metadata/src/main/java/org/apache/kylin/metadata/filter/DateConditionModifier.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/filter/DateConditionModifier.java
b/metadata/src/main/java/org/apache/kylin/metadata/filter/DateConditionModifier.java
index 8d5c10c..08be3de 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/filter/DateConditionModifier.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/filter/DateConditionModifier.java
@@ -3,6 +3,7 @@ package org.apache.kylin.metadata.filter;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
 import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.metadata.model.DataType;
 import org.apache.kylin.metadata.model.TblColRef;
 
 import java.util.Collection;
@@ -11,21 +12,17 @@ import java.util.List;
 import java.util.Set;
 
 /**
+ * Calcite passed down all time family constants as GregorianCalendar,
+ * we'll have to reformat it to date/time according to column definition
  */
 public class DateConditionModifier implements TupleFilterSerializer.Decorator {
 
-    private IdentityHashMap<TupleFilter, Boolean> dateCompareTupleChildren;
+    private IdentityHashMap<TupleFilter, DataType> dateCompareTupleChildren;
 
     public DateConditionModifier(TupleFilter root) {
         this.dateCompareTupleChildren = Maps.newIdentityHashMap();
     }
 
-    /**
-     * replace filter on timestamp column to null, so that two tuple filter trees can
-     * be compared regardless of the filter condition on timestamp column (In top level where
conditions concatenated by ANDs)
-     * @param filter
-     * @return
-     */
     @Override
     public TupleFilter onSerialize(TupleFilter filter) {
 
@@ -38,24 +35,39 @@ public class DateConditionModifier implements TupleFilterSerializer.Decorator
{
             }
 
             TblColRef col = cfilter.getColumn();
-            if (col == null || (!"date".equals(col.getDatatype()))) {
+            if (col == null || !col.getType().isDateTimeFamily()) {
                 return cfilter;
             }
 
             for (TupleFilter child : filter.getChildren()) {
-                dateCompareTupleChildren.put(child, true);
+                dateCompareTupleChildren.put(child, col.getType());
             }
         }
 
         if (filter instanceof ConstantTupleFilter && dateCompareTupleChildren.containsKey(filter))
{
             ConstantTupleFilter constantTupleFilter = (ConstantTupleFilter) filter;
             Set<String> newValues = Sets.newHashSet();
+            DataType columnType = dateCompareTupleChildren.get(filter);
 
             for (String value : (Collection<String>) constantTupleFilter.getValues())
{
-                newValues.add(DateFormat.formatToDateStr(Long.valueOf(value)));
+                newValues.add(formatTime(Long.valueOf(value), columnType));
             }
             return new ConstantTupleFilter(newValues);
         }
         return filter;
     }
+
+    private String formatTime(long millis, DataType dataType) {
+        if (dataType.isDatetime() || dataType.isTime()) {
+            throw new RuntimeException("Datetime and time type are not supported yet");
+        }
+
+        if (dataType.isTimestamp()) {
+            return DateFormat.formatToTimeStr(millis);
+        } else if (dataType.isDate()) {
+            return DateFormat.formatToDateStr(millis);
+        } else {
+            throw new RuntimeException("Unknown type " + dataType + " to formatTime");
+        }
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4219fdac/metadata/src/main/java/org/apache/kylin/metadata/model/DataType.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/model/DataType.java b/metadata/src/main/java/org/apache/kylin/metadata/model/DataType.java
index 607b547..027b348 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/model/DataType.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/model/DataType.java
@@ -89,7 +89,6 @@ public class DataType {
 
     private static final ConcurrentMap<DataType, DataType> CACHE = new ConcurrentHashMap<DataType,
DataType>();
 
-
     public static DataType getInstance(String type) {
         if (type == null)
             return null;
@@ -144,7 +143,6 @@ public class DataType {
             }
         }
 
-
         // FIXME 256 for unknown string precision
         if ((name.equals("char") || name.equals("varchar")) && precision == -1) {
             precision = 256; // to save memory at frontend, e.g. tableau will
@@ -203,11 +201,22 @@ public class DataType {
         return DATETIME_FAMILY.contains(name);
     }
 
-    public boolean isDate()
-    {
+    public boolean isDate() {
         return name.equals("date");
     }
 
+    public boolean isTime() {
+        return name.equals("time");
+    }
+
+    public boolean isTimestamp() {
+        return name.equals("timestamp");
+    }
+
+    public boolean isDatetime() {
+        return name.equals("datetime");
+    }
+
     public boolean isTinyInt() {
         return name.equals("tinyint");
     }

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4219fdac/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
index 04680d1..e580f3a 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
@@ -164,7 +164,7 @@ public class CacheService extends BasicService {
         if (Constant.SERVER_MODE_JOB.equals(serverMode.toLowerCase()) || Constant.SERVER_MODE_ALL.equals(serverMode.toLowerCase()))
{
             logger.debug("This is the job engine node, will check whether auto merge is needed
on cube " + cubeName);
             CubeSegment newSeg = null;
-            synchronized (getCubeManager().getCube(cubeName)) {
+            synchronized (CacheService.class) {
                 CubeInstance cube = getCubeManager().getCube(cubeName);
                 try {
                     newSeg = getCubeManager().autoMergeCubeSegments(cube);

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4219fdac/streaming/src/main/java/org/apache/kylin/streaming/KafkaClusterConfig.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/KafkaClusterConfig.java b/streaming/src/main/java/org/apache/kylin/streaming/KafkaClusterConfig.java
index 0661498..ff34dd9 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/KafkaClusterConfig.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/KafkaClusterConfig.java
@@ -14,7 +14,6 @@ import javax.annotation.Nullable;
 import java.util.List;
 
 /**
- * Created by Hongbin Ma(Binmahone) on 6/3/15.
  */
 @JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE,
isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE)
 public class KafkaClusterConfig extends RootPersistentEntity {


Mime
View raw message