lens-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amareshw...@apache.org
Subject lens git commit: LENS-1466 : Fix for 'Rewritten queries not resolving time_range_in clause'
Date Fri, 01 Sep 2017 09:53:59 GMT
Repository: lens
Updated Branches:
  refs/heads/current-release-line 10eef27a0 -> ff4f3726a


LENS-1466 : Fix for 'Rewritten queries not resolving time_range_in clause'


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/ff4f3726
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/ff4f3726
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/ff4f3726

Branch: refs/heads/current-release-line
Commit: ff4f3726a25db684ff7623fea68e954712dd925f
Parents: 10eef27
Author: Rajitha R <rajitha.407@gmail.com>
Authored: Fri Sep 1 13:02:32 2017 +0530
Committer: Amareshwari Sriramadasu <amareshwari@apache.org>
Committed: Fri Sep 1 15:23:45 2017 +0530

----------------------------------------------------------------------
 .../lens/cube/parse/StorageCandidate.java       |  2 +-
 .../apache/lens/cube/metadata/DateFactory.java  |  6 ++
 .../apache/lens/cube/parse/CubeTestSetup.java   | 47 +++++++++++--
 .../lens/cube/parse/TestCubeRewriter.java       | 15 ++++-
 .../resources/schema/cubes/base/testcube.xml    |  2 +
 .../test/resources/schema/facts/summary5.xml    | 70 ++++++++++++++++++++
 .../src/test/resources/schema/storages/c98.xml  | 26 ++++++++
 7 files changed, 159 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lens/blob/ff4f3726/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
index 2b1add3..27835b3 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
@@ -533,9 +533,9 @@ public class StorageCandidate implements Candidate, CandidateTable {
           log.debug("Adding non existing partition {}", part);
           if (addNonExistingParts) {
             // Add non existing partitions for all cases of whether we populate all non existing
or not.
-            this.participatingUpdatePeriods.add(maxInterval);
             missingPartitions.add(part);
             if (!failOnPartialData) {
+              this.participatingUpdatePeriods.add(maxInterval);
               partitions.add(part);
               part.getStorageTables().add(storageTable);
             }

http://git-wip-us.apache.org/repos/asf/lens/blob/ff4f3726/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java
index 855f54a..75f2abd 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java
@@ -150,6 +150,7 @@ public class DateFactory {
 
   // Time Instances as Date Type
   public static final Date NOW;
+  public static final Date ONEDAY_BACK;
   public static final Date TWODAYS_BACK;
   public static final Date TWO_MONTHS_BACK;
   public static final Date THIS_MONTH_TRUNCATED;
@@ -168,6 +169,7 @@ public class DateFactory {
   public static final String TWO_DAYS_RANGE_TTD2;
   public static final String TWO_DAYS_RANGE_TTD2_BEFORE_4_DAYS;
   public static final String TWO_DAYS_RANGE_IT;
+  public static final String ONE_DAY_RANGE_IT;
   public static final String THIS_YEAR_RANGE;
   public static final String LAST_YEAR_RANGE;
   public static final String TWO_MONTHS_RANGE_UPTO_MONTH;
@@ -189,6 +191,9 @@ public class DateFactory {
     // Figure out if current hour is 0th hour
     zerothHour = getDateStringWithOffset(HOURLY, 0).endsWith("-00");
 
+    ONEDAY_BACK = getDateWithOffset(DAILY, -1);
+    System.out.println("Test ONEDAY_BACK:" + ONEDAY_BACK);
+
     TWODAYS_BACK = getDateWithOffset(DAILY, -2);
     System.out.println("Test TWODAYS_BACK:" + TWODAYS_BACK);
 
@@ -214,6 +219,7 @@ public class DateFactory {
     TWO_DAYS_RANGE_TTD2 = getTimeRangeString("test_time_dim2", DAILY, -2, 0, HOURLY);
     TWO_DAYS_RANGE_TTD2_BEFORE_4_DAYS = getTimeRangeString("test_time_dim2", DAILY, -6, -4,
HOURLY);
     TWO_DAYS_RANGE_IT = getTimeRangeString("it", DAILY, -2, 0, HOURLY);
+    ONE_DAY_RANGE_IT = getTimeRangeString("it", DAILY, -1, 0, DAILY);
     THIS_YEAR_RANGE = getTimeRangeString(YEARLY, 0, 1);
     LAST_YEAR_RANGE = getTimeRangeString(YEARLY, -1, 0);
     TWO_MONTHS_RANGE_UPTO_MONTH = getTimeRangeString(MONTHLY, -2, 0);

http://git-wip-us.apache.org/repos/asf/lens/blob/ff4f3726/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
index 0366e56..5608b39 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
@@ -22,12 +22,7 @@ package org.apache.lens.cube.parse;
 import static java.util.Calendar.DAY_OF_MONTH;
 import static java.util.Calendar.HOUR_OF_DAY;
 
-import static org.apache.lens.cube.metadata.DateFactory.BEFORE_4_DAYS;
-import static org.apache.lens.cube.metadata.DateFactory.BEFORE_6_DAYS;
-import static org.apache.lens.cube.metadata.DateFactory.NOW;
-import static org.apache.lens.cube.metadata.DateFactory.TWODAYS_BACK;
-import static org.apache.lens.cube.metadata.DateFactory.TWO_MONTHS_BACK;
-import static org.apache.lens.cube.metadata.DateFactory.isZerothHour;
+import static org.apache.lens.cube.metadata.DateFactory.*;
 import static org.apache.lens.cube.metadata.UpdatePeriod.DAILY;
 import static org.apache.lens.cube.metadata.UpdatePeriod.HOURLY;
 import static org.apache.lens.cube.metadata.UpdatePeriod.MINUTELY;
@@ -132,6 +127,7 @@ public class CubeTestSetup {
   private static String c3 = "C3";
   private static String c4 = "C4";
   private static String c5 = "C5";
+  private static String c98 = "C98";
   private static String c99 = "C99";
   private static Map<String, String> factValidityProperties = Maps.newHashMap();
   @Getter
@@ -881,10 +877,13 @@ public class CubeTestSetup {
     CubeFactTable fact3 = client.getCubeFactTable(factName);
     createPIEParts(client, fact3, c2);
 
-
     factName = "summary4";
     CubeFactTable fact4 = client.getCubeFactTable(factName);
     createPIEParts(client, fact4, c2);
+
+    factName = "summary5";
+    CubeFactTable fact5 = client.getCubeFactTable(factName);
+    createPIParts(client, fact5, c98);
   }
 
   private void createBaseCubeFactPartitions(CubeMetastoreClient client) throws HiveException,
LensException {
@@ -994,6 +993,7 @@ public class CubeTestSetup {
           pTimes.get(HOURLY).add(ptime);
           iTimes.get(HOURLY).add(itime);
           client.addPartition(sPartSpec, storageName, CubeTableType.FACT);
+
           pcal.add(HOUR_OF_DAY, 1);
           ical.add(HOUR_OF_DAY, 1);
         }
@@ -1015,6 +1015,39 @@ public class CubeTestSetup {
     }
   }
 
+
+  private void createPIParts(CubeMetastoreClient client, CubeFactTable fact, String storageName)
+    throws Exception {
+    // Add partitions in PI storage
+    //daily partition registered  for pt=day1, it = day1
+    //hourly partitions registered for pt=day1-hours[0-23] it=day1-hours[0-23]
+    Calendar pcal = Calendar.getInstance();
+    pcal.setTime(ONEDAY_BACK);
+    pcal.set(HOUR_OF_DAY, 0);
+    Calendar ical = Calendar.getInstance();
+    ical.setTime(ONEDAY_BACK);
+    ical.set(HOUR_OF_DAY, 0);
+
+    Date ptime = pcal.getTime();
+    Date itime = ical.getTime();
+    Map<String, Date> timeParts = new HashMap<String, Date>();
+
+    timeParts.put("pt", ptime);
+    timeParts.put("it", itime);
+    StoragePartitionDesc sPartSpec = new StoragePartitionDesc(fact.getName(), timeParts,
null, DAILY);
+    client.addPartition(sPartSpec, storageName, CubeTableType.FACT);
+    for (int i = 0; i < 24; i++) {
+      ptime = pcal.getTime();
+      itime = ical.getTime();
+      timeParts.put("pt", ptime);
+      timeParts.put("it", itime);
+      sPartSpec = new StoragePartitionDesc(fact.getName(), timeParts, null, HOURLY);
+      client.addPartition(sPartSpec, storageName, CubeTableType.FACT);
+      pcal.add(HOUR_OF_DAY, 1);
+      ical.add(HOUR_OF_DAY, 1);
+    }
+  }
+
   public static void printQueryAST(String query, String label) throws LensException {
     System.out.println("--" + label + "--AST--");
     System.out.println("--query- " + query);

http://git-wip-us.apache.org/repos/asf/lens/blob/ff4f3726/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
index bc0fa2e..9b54957 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
@@ -65,7 +65,7 @@ public class TestCubeRewriter extends TestQueryRewrite {
   @BeforeTest
   public void setupDriver() throws Exception {
     conf = LensServerAPITestUtil.getConfiguration(
-      DRIVER_SUPPORTED_STORAGES, "C0,C1,C2",
+      DRIVER_SUPPORTED_STORAGES, "C0,C1,C2,C98",
       DISABLE_AUTO_JOINS, true,
       ENABLE_SELECT_TO_GROUPBY, true,
       ENABLE_GROUP_BY_TO_SELECT, true,
@@ -1470,6 +1470,19 @@ public class TestCubeRewriter extends TestQueryRewrite {
   }
 
   @Test
+  public void testTimeRangeIn() throws Exception {
+    //check whether time_range_in is resolving in cube rewrite
+    Configuration conf = getConf();
+    conf.set(CubeQueryConfUtil.PROCESS_TIME_PART_COL, "pt");
+    conf.set(CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, "true");
+    conf.setClass(CubeQueryConfUtil.TIME_RANGE_WRITER_CLASS, AbridgedTimeRangeWriter.class,
TimeRangeWriter.class);
+    CubeQueryContext ctx = rewriteCtx("select dim1, sum(msr23)" + " from testCube" + " where
" + ONE_DAY_RANGE_IT,
+      conf);
+    String rewrittenQuery = ctx.toHQL();
+    assertTrue(!rewrittenQuery.contains("time_range_in"));
+  }
+
+  @Test
   public void testCubeQueryWithMultipleRanges() throws Exception {
     String hqlQuery =
       rewrite("select SUM(msr2) from testCube" + " where " + TWO_DAYS_RANGE + " OR "

http://git-wip-us.apache.org/repos/asf/lens/blob/ff4f3726/lens-cube/src/test/resources/schema/cubes/base/testcube.xml
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/resources/schema/cubes/base/testcube.xml b/lens-cube/src/test/resources/schema/cubes/base/testcube.xml
index 9d2bb02..4cafd31 100644
--- a/lens-cube/src/test/resources/schema/cubes/base/testcube.xml
+++ b/lens-cube/src/test/resources/schema/cubes/base/testcube.xml
@@ -32,6 +32,8 @@
     <property name="cube.timedim.partition.test_time_dim2" value="ttd2"/>
   </properties>
   <measures>
+    <measure _type="FLOAT" default_aggr="SUM" unit="RS" name="msr23" display_string="Measure23"
+             description="twenty third measure"/>
     <measure _type="FLOAT" default_aggr="SUM" unit="RS" name="msr21" display_string="Measure22"
              description="second measure"/>
     <measure _type="BIGINT" default_aggr="COUNT" name="msr4" display_string="Measure4"
description="fourth measure"/>

http://git-wip-us.apache.org/repos/asf/lens/blob/ff4f3726/lens-cube/src/test/resources/schema/facts/summary5.xml
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/resources/schema/facts/summary5.xml b/lens-cube/src/test/resources/schema/facts/summary5.xml
new file mode 100644
index 0000000..a4bbf71
--- /dev/null
+++ b/lens-cube/src/test/resources/schema/facts/summary5.xml
@@ -0,0 +1,70 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements. See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership. The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License. You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied. See the License for the
+  specific language governing permissions and limitations
+  under the License.
+
+-->
+<x_fact_table name="summary5" cube_name="testCube" weight="15.0" xmlns="uri:lens:cube:0.1">
+  <columns>
+    <column name="msr23" _type="float" comment="twenty third measure"/>
+    <column name="dim1" _type="string" comment="dim1"/>
+    <column name="dim2" _type="string" comment="dim2"/>
+  </columns>
+  <properties>
+    <property name="cube.fact.relative.start.time" value="now.year - 90 days"/>
+    <property name="cube.table.summary5.weight" value="15.0"/>
+    <property name="cube.fact.summary5.storages" value="C98"/>
+    <property name="cube.fact.summary5.c98.updateperiods" value="HOURLY,DAILY"/>
+  </properties>
+  <storage_tables>
+    <storage_table>
+      <storage_name>C98</storage_name>
+      <update_periods>
+        <update_period_table_descriptor>
+          <update_period>HOURLY</update_period>
+          <table_desc external="false">
+            <part_cols>
+              <column name="pt" _type="string" comment="date partition"/>
+              <column name="it" _type="string" comment="date partition"/>
+            </part_cols>
+            <table_parameters>
+              <property name="cube.storagetable.start.times" value="now -4 days"/>
+            </table_parameters>
+            <time_part_cols>pt</time_part_cols>
+            <time_part_cols>it</time_part_cols>
+          </table_desc>
+        </update_period_table_descriptor>
+        <update_period_table_descriptor>
+          <update_period>DAILY</update_period>
+          <table_desc external="false">
+            <part_cols>
+              <column name="pt" _type="string" comment="date partition"/>
+              <column name="it" _type="string" comment="date partition"/>
+            </part_cols>
+            <table_parameters>
+              <property name="cube.storagetable.start.times" value="now.month -3 months"/>
+            </table_parameters>
+            <time_part_cols>pt</time_part_cols>
+            <time_part_cols>it</time_part_cols>
+          </table_desc>
+        </update_period_table_descriptor>
+      </update_periods>
+    </storage_table>
+  </storage_tables>
+</x_fact_table>
+

http://git-wip-us.apache.org/repos/asf/lens/blob/ff4f3726/lens-cube/src/test/resources/schema/storages/c98.xml
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/resources/schema/storages/c98.xml b/lens-cube/src/test/resources/schema/storages/c98.xml
new file mode 100644
index 0000000..550b421
--- /dev/null
+++ b/lens-cube/src/test/resources/schema/storages/c98.xml
@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements. See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership. The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License. You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied. See the License for the
+  specific language governing permissions and limitations
+  under the License.
+
+-->
+<x_storage name="c98" classname="org.apache.lens.cube.metadata.HDFSStorage" xmlns="uri:lens:cube:0.1">
+  <properties>
+    <property name="cube.table.c98.weight" value="0.0"/>
+  </properties>
+</x_storage>
\ No newline at end of file


Mime
View raw message