kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From liy...@apache.org
Subject [1/2] kylin git commit: KYLIN-1766 Alter LookupTable to hold millis instead of yyyymmdd
Date Sun, 19 Jun 2016 06:29:19 GMT
Repository: kylin
Updated Branches:
  refs/heads/master a82662bcd -> 11e20adbd


KYLIN-1766 Alter LookupTable to hold millis instead of yyyymmdd


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8a96b57f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8a96b57f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8a96b57f

Branch: refs/heads/master
Commit: 8a96b57f066ed58d55339d3b8e18bbf51c45ec34
Parents: a82662b
Author: Yang Li <liyang@apache.org>
Authored: Sat Jun 18 22:34:04 2016 +0800
Committer: Yang Li <liyang@apache.org>
Committed: Sun Jun 19 14:15:51 2016 +0800

----------------------------------------------------------------------
 .../java/org/apache/kylin/cube/CubeManager.java |  2 -
 .../java/org/apache/kylin/cube/CubeSegment.java |  3 +-
 .../apache/kylin/cube/kv/RowKeyColumnOrder.java |  2 +-
 .../apache/kylin/cube/model/RowKeyColDesc.java  |  4 +-
 .../kylin/gridtable/GTScanRangePlanner.java     |  1 -
 .../kylin/dict/lookup/LookupBytesTable.java     | 56 ---------------
 .../kylin/dict/lookup/LookupStringTable.java    | 71 +++++++++++++++----
 .../apache/kylin/dict/lookup/LookupTable.java   | 18 +++--
 .../kylin/dict/lookup/LookupTableTest.java      | 19 +++--
 .../metadata/filter/ConstantTupleFilter.java    |  5 +-
 .../apache/kylin/engine/mr/LookupTableTest.java | 74 --------------------
 .../resources/query/sql_derived/query11.sql     | 26 +++++++
 .../kylin/query/enumerator/OLAPEnumerator.java  | 25 ++-----
 .../kylin/query/relnode/OLAPFilterRel.java      | 25 ++++++-
 storage-hbase/pom.xml                           |  4 +-
 .../storage/hbase/cube/v1/CubeStorageQuery.java |  6 +-
 16 files changed, 157 insertions(+), 184 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/8a96b57f/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index a33cf00..caa2765 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -201,8 +201,6 @@ public class CubeManager implements IRealizationProvider {
         DictionaryInfo info = null;
         try {
             DictionaryManager dictMgr = getDictionaryManager();
-            // logger.info("Using metadata url " + metadataUrl +
-            // " for DictionaryManager");
             String dictResPath = cubeSeg.getDictResPath(col);
             if (dictResPath == null)
                 return null;

http://git-wip-us.apache.org/repos/asf/kylin/blob/8a96b57f/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index f79e06d..b9195fd 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -305,7 +305,8 @@ public class CubeSegment implements Comparable<CubeSegment>, IRealizationSegment
 
     public Dictionary<String> getDictionary(TblColRef col) {
         TblColRef reuseCol = getCubeDesc().getDictionaryReuseColumn(col);
-        return CubeManager.getInstance(this.getCubeInstance().getConfig()).getDictionary(this,
reuseCol);
+        CubeManager cubeMgr = CubeManager.getInstance(this.getCubeInstance().getConfig());
+        return cubeMgr.getDictionary(this, reuseCol);
     }
 
     public CubeDimEncMap getDimensionEncodingMap() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/8a96b57f/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnOrder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnOrder.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnOrder.java
index fea3736..23c8f6a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnOrder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/RowKeyColumnOrder.java
@@ -32,7 +32,7 @@ abstract public class RowKeyColumnOrder implements Comparator<String>
{
     public static final StringOrder STRING_ORDER = new StringOrder();
 
     public static RowKeyColumnOrder getInstance(DataType type) {
-        if (type.isNumberFamily())
+        if (type.isNumberFamily() || type.isDateTimeFamily())
             return NUMBER_ORDER;
         else
             return STRING_ORDER;

http://git-wip-us.apache.org/repos/asf/kylin/blob/8a96b57f/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
index c4174a6..d4cfcb7 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
@@ -79,10 +79,10 @@ public class RowKeyColDesc {
         if (DictionaryDimEnc.ENCODING_NAME.equals(encodingName)) {
             DataType type = colRef.getType();
             if (type.isDate()) {
-                encodingName = DateDimEnc.ENCODING_NAME;
+                encoding = encodingName = DateDimEnc.ENCODING_NAME;
             }
             if (type.isTime() || type.isTimestamp() || type.isDatetime()) {
-                encodingName = TimeDimEnc.ENCODING_NAME;
+                encoding = encodingName = TimeDimEnc.ENCODING_NAME;
             }
         }
     }

http://git-wip-us.apache.org/repos/asf/kylin/blob/8a96b57f/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
index baf3428..7d31bf7 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRangePlanner.java
@@ -156,7 +156,6 @@ public class GTScanRangePlanner {
         //start key GTRecord compare to stop key GTRecord
         this.rangeStartEndComparator = getRangeStartEndComparator(comp);
 
-
         this.gtFilter = gtFilter;
         this.gtStartAndEnd = gtStartAndEnd;
         this.gtPartitionCol = gtPartitionCol;

http://git-wip-us.apache.org/repos/asf/kylin/blob/8a96b57f/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupBytesTable.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupBytesTable.java
b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupBytesTable.java
deleted file mode 100644
index 0758edc..0000000
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupBytesTable.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.dict.lookup;
-
-import java.io.IOException;
-
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.source.ReadableTable;
-
-/**
- * @author yangli9
- * 
- */
-public class LookupBytesTable extends LookupTable<ByteArray> {
-
-    public LookupBytesTable(TableDesc tableDesc, String[] keyColumns, ReadableTable table)
throws IOException {
-        super(tableDesc, keyColumns, table);
-    }
-
-    @Override
-    protected ByteArray[] convertRow(String[] cols) {
-        ByteArray[] r = new ByteArray[cols.length];
-        for (int i = 0; i < cols.length; i++) {
-            r[i] = cols[i] == null ? null : new ByteArray(Bytes.toBytes(cols[i]));
-        }
-        return r;
-    }
-
-    @Override
-    protected String toString(ByteArray cell) {
-        return cell.toString();
-    }
-
-    public Class<?> getType() {
-        return ByteArray.class;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/8a96b57f/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupStringTable.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupStringTable.java
b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupStringTable.java
index 9f6346a..fd88fb0 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupStringTable.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupStringTable.java
@@ -19,8 +19,10 @@
 package org.apache.kylin.dict.lookup;
 
 import java.io.IOException;
+import java.util.Comparator;
 
 import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.metadata.datatype.DataType;
 import org.apache.kylin.metadata.model.ColumnDesc;
 import org.apache.kylin.metadata.model.TableDesc;
 import org.apache.kylin.source.ReadableTable;
@@ -30,37 +32,80 @@ import org.apache.kylin.source.ReadableTable;
  * 
  */
 public class LookupStringTable extends LookupTable<String> {
+    
+    private static final Comparator<String> dateStrComparator = new Comparator<String>()
{
+        @Override
+        public int compare(String o1, String o2) {
+            long l1 = Long.parseLong(o1);
+            long l2 = Long.parseLong(o2);
+            return Long.compare(l1, l2);
+        }
+    };
 
-    int[] keyIndexOfDates;
+    private static final Comparator<String> numStrComparator = new Comparator<String>()
{
+        @Override
+        public int compare(String o1, String o2) {
+            double d1 = Double.parseDouble(o1);
+            double d2 = Double.parseDouble(o2);
+            return Double.compare(d1, d2);
+        }
+    };
+    
+    private static final Comparator<String> defaultStrComparator = new Comparator<String>()
{
+        @Override
+        public int compare(String o1, String o2) {
+            return o1.compareTo(o2);
+        }
+    };
+    
+    boolean[] colIsDateTime;
+    boolean[] colIsNumber;
     
     public LookupStringTable(TableDesc tableDesc, String[] keyColumns, ReadableTable table)
throws IOException {
         super(tableDesc, keyColumns, table);
     }
+    
+    @Override
+    protected void init() throws IOException {
+        ColumnDesc[] cols = tableDesc.getColumns();
+        colIsDateTime = new boolean[cols.length];
+        colIsNumber = new boolean[cols.length];
+        for (int i = 0; i < cols.length; i++) {
+            DataType t = cols[i].getType();
+            colIsDateTime[i] = t.isDateTimeFamily();
+            colIsNumber[i] = t.isNumberFamily();
+        }
+        
+        super.init();
+    }
 
     @Override
     protected String[] convertRow(String[] cols) {
-        if (keyIndexOfDates == null) {
-            keyIndexOfDates = new int[keyColumns.length];
-            for (int i = 0; i < keyColumns.length; i++) {
-                ColumnDesc col = tableDesc.findColumnByName(keyColumns[i]);
-                keyIndexOfDates[i] = col.getType().isDateTimeFamily() ? col.getZeroBasedIndex()
: -1;
+        for (int i = 0; i < cols.length; i++) {
+            if (colIsDateTime[i]) {
+                cols[i] = String.valueOf(DateFormat.stringToMillis(cols[i]));
             }
         }
-        
-        for (int i = 0; i < keyIndexOfDates.length; i++) {
-            int c = keyIndexOfDates[i];
-            if (c >= 0)
-                cols[c] = String.valueOf(DateFormat.stringToMillis(cols[c]));
-        }
         return cols;
     }
 
     @Override
+    protected Comparator<String> getComparator(int idx) {
+        if (colIsDateTime[idx])
+            return dateStrComparator;
+        else if (colIsNumber[idx])
+            return numStrComparator;
+        else
+            return defaultStrComparator;
+    }
+    
+    @Override
     protected String toString(String cell) {
         return cell;
     }
 
     public Class<?> getType() {
         return String.class;
-    }
+    }
+
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/8a96b57f/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
index 9abd28c..5a96df6 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
@@ -21,6 +21,7 @@ package org.apache.kylin.dict.lookup;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Comparator;
 import java.util.List;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
@@ -39,7 +40,7 @@ import com.google.common.collect.Sets;
  *
  * @author yangli9
  */
-abstract public class LookupTable<T extends Comparable<T>> {
+abstract public class LookupTable<T> {
 
     protected TableDesc tableDesc;
     protected String[] keyColumns;
@@ -109,15 +110,18 @@ abstract public class LookupTable<T extends Comparable<T>>
{
     public Pair<T, T> mapRange(String col, T beginValue, T endValue, String returnCol)
{
         int colIdx = tableDesc.findColumnByName(col).getZeroBasedIndex();
         int returnIdx = tableDesc.findColumnByName(returnCol).getZeroBasedIndex();
+        Comparator<T> colComp = getComparator(colIdx);
+        Comparator<T> returnComp = getComparator(returnIdx);
+        
         T returnBegin = null;
         T returnEnd = null;
         for (T[] row : data.values()) {
-            if (between(beginValue, row[colIdx], endValue)) {
+            if (between(beginValue, row[colIdx], endValue, colComp)) {
                 T returnValue = row[returnIdx];
-                if (returnBegin == null || returnValue.compareTo(returnBegin) < 0) {
+                if (returnBegin == null || returnComp.compare(returnValue, returnBegin) <
0) {
                     returnBegin = returnValue;
                 }
-                if (returnEnd == null || returnValue.compareTo(returnEnd) > 0) {
+                if (returnEnd == null || returnComp.compare(returnValue, returnEnd) >
0) {
                     returnEnd = returnValue;
                 }
             }
@@ -140,10 +144,12 @@ abstract public class LookupTable<T extends Comparable<T>>
{
         return result;
     }
 
-    private boolean between(T beginValue, T v, T endValue) {
-        return (beginValue == null || beginValue.compareTo(v) <= 0) && (endValue
== null || v.compareTo(endValue) <= 0);
+    private boolean between(T beginValue, T v, T endValue, Comparator<T> comp) {
+        return (beginValue == null || comp.compare(beginValue, v) <= 0) && (endValue
== null || comp.compare(v, endValue) <= 0);
     }
 
+    abstract protected Comparator<T> getComparator(int colIdx);
+    
     public String toString() {
         return "LookupTable [path=" + table + "]";
     }

http://git-wip-us.apache.org/repos/asf/kylin/blob/8a96b57f/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java
b/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java
index 980a263..3f67dcd 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java
@@ -65,7 +65,7 @@ public class LookupTableTest extends LocalFileMetadataTestCase {
         for (String i : results) {
             System.out.println(i);
 
-            Assert.assertEquals("2012-01-01", i);
+            Assert.assertEquals(millis("2012-01-01"), i);
         }
     }
 
@@ -77,11 +77,22 @@ public class LookupTableTest extends LocalFileMetadataTestCase {
         System.out.println("The first qtr_beg_dt is " + results.getFirst());
         System.out.println("The last qtr_beg_dt is " + results.getSecond());
 
-        Assert.assertEquals("2012-01-01", results.getFirst());
-        Assert.assertEquals("2012-10-01", results.getSecond());
+        Assert.assertEquals(millis("2012-01-01"), results.getFirst());
+        Assert.assertEquals(millis("2012-10-01"), results.getSecond());
     }
 
     @Test
+    public void testMapRange2() throws Exception {
+        Pair<String, String> results = lookupTable.mapRange("WEEK_BEG_DT", millis("2013-05-01"),
millis("2013-08-01"), "CAL_DT");
+
+        System.out.println(DateFormat.formatToDateStr(Long.parseLong(results.getFirst())));
+        System.out.println(DateFormat.formatToDateStr(Long.parseLong(results.getSecond())));
+        
+        Assert.assertEquals(millis("2013-05-05"), results.getFirst());
+        Assert.assertEquals(millis("2013-08-03"), results.getSecond());
+    }
+    
+    @Test
     public void testMapValues() throws Exception {
         Set<String> values = new HashSet<String>();
         values.add(millis("2012-01-24"));
@@ -92,7 +103,7 @@ public class LookupTableTest extends LocalFileMetadataTestCase {
         for (String i : results) {
             System.out.println(i);
 
-            Assert.assertEquals("2012-01-01", i);
+            Assert.assertEquals(millis("2012-01-01"), i);
         }
     }
     

http://git-wip-us.apache.org/repos/asf/kylin/blob/8a96b57f/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
index db3eb4f..61d87e8 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
@@ -21,13 +21,12 @@ package org.apache.kylin.metadata.filter;
 import java.nio.ByteBuffer;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.TreeSet;
 
-import com.google.common.collect.Lists;
-import org.apache.commons.collections.comparators.NullComparator;
 import org.apache.kylin.common.util.BytesUtil;
 import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
 
+import com.google.common.collect.Lists;
+
 /**
  * 
  * @author xjiang

http://git-wip-us.apache.org/repos/asf/kylin/blob/8a96b57f/engine-mr/src/test/java/org/apache/kylin/engine/mr/LookupTableTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/LookupTableTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/LookupTableTest.java
deleted file mode 100644
index 87f9133..0000000
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/LookupTableTest.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.engine.mr;
-
-import java.io.File;
-
-import org.apache.kylin.common.util.Array;
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.dict.lookup.LookupBytesTable;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * @author yangli9
- */
-public class LookupTableTest extends LocalFileMetadataTestCase {
-
-    @Before
-    public void setup() throws Exception {
-        createTestMetadata();
-    }
-
-    @After
-    public void after() throws Exception {
-        cleanupTestMetadata();
-    }
-
-    @Test
-    public void testBasic() throws Exception {
-        TableDesc siteTable = MetadataManager.getInstance(getTestConfig()).getTableDesc("EDW.TEST_SITES");
-        TableDesc categoryTable = MetadataManager.getInstance(getTestConfig()).getTableDesc("DEFAULT.test_category_groupings");
-        LookupBytesTable lookup;
-
-        System.out.println("============================================================================");
-
-        File f = new File(LOCALMETA_TEST_DATA + "/data/EDW.TEST_SITES.csv");
-        lookup = new LookupBytesTable(siteTable, new String[] { "SITE_ID" }, new DFSFileTable("file://"
+ f.getAbsolutePath(), 10));
-        lookup.dump();
-
-        System.out.println("============================================================================");
-
-        f = new File(LOCALMETA_TEST_DATA + "/data/DEFAULT.TEST_CATEGORY_GROUPINGS.csv");
-        lookup = new LookupBytesTable(categoryTable, new String[] { "leaf_categ_id", "site_id"
}, new DFSFileTable("file://" + f.getAbsolutePath(), 36));
-        lookup.dump();
-
-        System.out.println("============================================================================");
-
-        ByteArray k1 = new ByteArray(Bytes.toBytes("533"));
-        ByteArray k2 = new ByteArray(Bytes.toBytes("0"));
-        Array<ByteArray> key = new Array<ByteArray>(new ByteArray[] { k1, k2
});
-        System.out.println(lookup.getRow(key));
-    }
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/8a96b57f/kylin-it/src/test/resources/query/sql_derived/query11.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_derived/query11.sql b/kylin-it/src/test/resources/query/sql_derived/query11.sql
new file mode 100644
index 0000000..41157c5
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_derived/query11.sql
@@ -0,0 +1,26 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select test_cal_dt.week_beg_dt, test_kylin_fact.lstg_format_name, test_category_groupings.meta_categ_name,
sum(test_kylin_fact.price) as gmv, count(*) as trans_cnt 
+ from test_kylin_fact 
+ inner JOIN edw.test_cal_dt as test_cal_dt 
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt 
+ inner JOIN test_category_groupings 
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id
= test_category_groupings.site_id 
+ where test_cal_dt.week_beg_dt between DATE '2013-05-01' and DATE '2013-10-01' 
+ group by test_cal_dt.week_beg_dt, test_kylin_fact.lstg_format_name, test_category_groupings.meta_categ_name


http://git-wip-us.apache.org/repos/asf/kylin/blob/8a96b57f/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
index 07f72b1..ef4ddf8 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
@@ -24,11 +24,9 @@ import java.util.Properties;
 import org.apache.calcite.DataContext;
 import org.apache.calcite.jdbc.CalciteConnection;
 import org.apache.calcite.linq4j.Enumerator;
-import org.apache.kylin.dict.DictCodeSystem;
+import org.apache.kylin.common.util.DateFormat;
 import org.apache.kylin.metadata.filter.CompareTupleFilter;
-import org.apache.kylin.metadata.filter.TimeConditionLiteralsReplacer;
 import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilterSerializer;
 import org.apache.kylin.metadata.model.FunctionDesc;
 import org.apache.kylin.metadata.model.MeasureDesc;
 import org.apache.kylin.metadata.model.ParameterDesc;
@@ -112,11 +110,8 @@ public class OLAPEnumerator implements Enumerator<Object[]> {
         // bind dynamic variables
         bindVariable(olapContext.filter);
 
-        // modify date condition
-        olapContext.filter = modifyTimeLiterals(olapContext.filter);
-        olapContext.resetSQLDigest();
-
         // cube don't have correct result for simple query without group by, but let's try
to return something makes sense
+        olapContext.resetSQLDigest();
         SQLDigest sqlDigest = olapContext.getSQLDigest();
         hackNoGroupByAggregation(sqlDigest);
 
@@ -130,16 +125,6 @@ public class OLAPEnumerator implements Enumerator<Object[]> {
         return iterator;
     }
 
-    /**
-     * Calcite passed down all time family constants as GregorianCalendar,
-     * we'll have to reformat it to date/time according to column definition
-     */
-    private TupleFilter modifyTimeLiterals(TupleFilter filter) {
-        TimeConditionLiteralsReplacer filterDecorator = new TimeConditionLiteralsReplacer(filter);
-        byte[] bytes = TupleFilterSerializer.serialize(filter, filterDecorator, DictCodeSystem.INSTANCE);
-        return TupleFilterSerializer.deserialize(bytes, DictCodeSystem.INSTANCE);
-    }
-
     private void bindVariable(TupleFilter filter) {
         if (filter == null) {
             return;
@@ -155,7 +140,11 @@ public class OLAPEnumerator implements Enumerator<Object[]> {
                 String variable = entry.getKey();
                 Object value = optiqContext.get(variable);
                 if (value != null) {
-                    compFilter.bindVariable(variable, value.toString());
+                    String str = value.toString();
+                    if (compFilter.getColumn().getType().isDateTimeFamily())
+                        str = String.valueOf(DateFormat.stringToMillis(str));
+                    
+                    compFilter.bindVariable(variable, str);
                 }
 
             }

http://git-wip-us.apache.org/repos/asf/kylin/blob/8a96b57f/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
index 2806b4b..ae6b265 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
@@ -50,7 +50,9 @@ import org.apache.calcite.rex.RexProgramBuilder;
 import org.apache.calcite.rex.RexVisitorImpl;
 import org.apache.calcite.sql.SqlKind;
 import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.type.SqlTypeFamily;
 import org.apache.calcite.util.NlsString;
+import org.apache.kylin.common.util.DateFormat;
 import org.apache.kylin.metadata.filter.CaseTupleFilter;
 import org.apache.kylin.metadata.filter.ColumnTupleFilter;
 import org.apache.kylin.metadata.filter.CompareTupleFilter;
@@ -64,6 +66,7 @@ import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
 import org.apache.kylin.metadata.model.TblColRef;
 
 import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 
 /**
@@ -144,7 +147,7 @@ public class OLAPFilterRel extends Filter implements OLAPRel {
             for (RexNode operand : call.operands) {
                 TupleFilter childFilter = operand.accept(this);
                 if (filter == null) {
-                    filter = childFilter;
+                    filter = cast(childFilter, call.type);
                 } else {
                     filter.addChild(childFilter);
                 }
@@ -159,6 +162,26 @@ public class OLAPFilterRel extends Filter implements OLAPRel {
             return filter;
         }
 
+        private TupleFilter cast(TupleFilter filter, RelDataType type) {
+            if ((filter instanceof ConstantTupleFilter) == false) {
+                return filter;
+            }
+
+            ConstantTupleFilter constFilter = (ConstantTupleFilter) filter;
+            
+            if (type.getFamily() == SqlTypeFamily.DATE || type.getFamily() == SqlTypeFamily.DATETIME
|| type.getFamily() == SqlTypeFamily.TIMESTAMP) {
+                List<String> newValues = Lists.newArrayList();
+                for (Object v : constFilter.getValues()) {
+                    if (v == null)
+                        newValues.add(null);
+                    else
+                        newValues.add(String.valueOf(DateFormat.stringToMillis(v.toString())));
+                }
+                constFilter = new ConstantTupleFilter(newValues);
+            }
+            return constFilter;
+        }
+
         private CompareTupleFilter mergeToInClause(TupleFilter filter) {
             List<? extends TupleFilter> children = filter.getChildren();
             TblColRef inColumn = null;

http://git-wip-us.apache.org/repos/asf/kylin/blob/8a96b57f/storage-hbase/pom.xml
----------------------------------------------------------------------
diff --git a/storage-hbase/pom.xml b/storage-hbase/pom.xml
index 6c9e284..943b2d4 100644
--- a/storage-hbase/pom.xml
+++ b/storage-hbase/pom.xml
@@ -137,8 +137,10 @@
                                     <include>org.apache.kylin:kylin-core-cube</include>
                                     <include>org.apache.kylin:kylin-invertedindex</include>
                                     <include>com.ning:compress-lzf</include>
-                                    <include>com.n3twork.druid:extendedset</include>
                                     <include>org.roaringbitmap:RoaringBitmap</include>
+                                    <!-- below for inverted index only -->
+                                    <include>com.n3twork.druid:extendedset</include>
+                                    <include>org.apache.commons:commons-lang3</include>
                                 </includes>
                             </artifactSet>
                             <filters>

http://git-wip-us.apache.org/repos/asf/kylin/blob/8a96b57f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
index 4b3cb67..672bcbe 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
@@ -48,6 +48,7 @@ import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
 import org.apache.kylin.cube.model.HBaseColumnDesc;
 import org.apache.kylin.cube.model.HBaseMappingDesc;
+import org.apache.kylin.cube.model.RowKeyDesc;
 import org.apache.kylin.dict.lookup.LookupStringTable;
 import org.apache.kylin.measure.MeasureType;
 import org.apache.kylin.metadata.filter.ColumnTupleFilter;
@@ -543,10 +544,13 @@ public class CubeStorageQuery implements IStorageQuery {
         }
 
         // a little pre-evaluation to remove invalid EQ/IN values and round start/end according
to dictionary
+        RowKeyDesc rowkey = cubeSegment.getCubeDesc().getRowkey();
         Iterator<ColumnValueRange> it = rangeMap.values().iterator();
         while (it.hasNext()) {
             ColumnValueRange range = it.next();
-            range.preEvaluateWithDict((Dictionary<String>) cubeSegment.getDictionary(range.getColumn()));
+            if (rowkey.isUseDictionary(range.getColumn())) {
+                range.preEvaluateWithDict((Dictionary<String>) cubeSegment.getDictionary(range.getColumn()));
+            }
             if (range.satisfyAll())
                 it.remove();
             else if (range.satisfyNone())


Mime
View raw message