carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject [38/51] [abbrv] carbondata git commit: [CARBONDATA-1452] Issue with loading timestamp data beyond cutoff
Date Mon, 18 Sep 2017 10:01:57 GMT
[CARBONDATA-1452] Issue with loading timestamp data beyond cutoff

(1)Removed timeValue>=0 condition => this condition will restrict loading proper data
when the CARBON_CUTOFF_TIMESTAMP is set before 1970. In this case timeValue will always be
< 0
(2) Added test case for the same

This closes #1355


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/8791eabf
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/8791eabf
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/8791eabf

Branch: refs/heads/branch-1.2
Commit: 8791eabf0c6f2db385ed9c0886c0aa054a421b2f
Parents: 940f4d5
Author: dhatchayani <dhatcha.official@gmail.com>
Authored: Wed Sep 6 15:43:56 2017 +0530
Committer: Ravindra Pesala <ravi.pesala@gmail.com>
Committed: Fri Sep 15 13:54:21 2017 +0530

----------------------------------------------------------------------
 .../TimeStampDirectDictionaryGenerator.java     | 13 ++++------
 .../TimeStampDirectDictionaryGeneratorTest.java | 26 ++++++++++++++++++++
 2 files changed, 31 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/8791eabf/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
index c8b88d8..e0f5d41 100644
--- a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
+++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
@@ -210,15 +210,12 @@ public class TimeStampDirectDictionaryGenerator implements DirectDictionaryGener
   }
 
   private int generateKey(long timeValue) {
-    if (timeValue >= 0) {
-      long time = (timeValue - cutOffTimeStamp) / granularityFactor;
-      int keyValue = -1;
-      if (time <= (long) Integer.MAX_VALUE) {
-        keyValue = (int) time;
-      }
-      return keyValue < 0 ? 1 : keyValue + 2;
+    long time = (timeValue - cutOffTimeStamp) / granularityFactor;
+    int keyValue = -1;
+    if (time >= (long) Integer.MIN_VALUE && time <= (long) Integer.MAX_VALUE)
{
+      keyValue = (int) time;
     }
-    return 1;
+    return keyValue < 0 ? 1 : keyValue + 2;
   }
 
   public void initialize() {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/8791eabf/processing/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGeneratorTest.java
----------------------------------------------------------------------
diff --git a/processing/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGeneratorTest.java
b/processing/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGeneratorTest.java
index 7bb433a..bdc4ca5 100644
--- a/processing/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGeneratorTest.java
+++ b/processing/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGeneratorTest.java
@@ -23,6 +23,7 @@ import java.util.TimeZone;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
 import org.apache.carbondata.core.util.CarbonProperties;
 
+import mockit.Deencapsulation;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
@@ -77,6 +78,31 @@ public class TimeStampDirectDictionaryGeneratorTest {
    *
    * @throws Exception
    */
+  @Test public void getSurrogateWithCutoff() throws Exception {
+    SimpleDateFormat timeParser = new SimpleDateFormat(CarbonProperties.getInstance()
+        .getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+            CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT));
+    timeParser.setLenient(false);
+    TimeStampDirectDictionaryGenerator generator = new TimeStampDirectDictionaryGenerator();
+    long cutOffValue = timeParser.parse("1500-01-01 00:00:00").getTime();
+    //setting cutoff time to 1500-01-01 00:00:00 , so we can load data from this time
+    Deencapsulation.setField(generator, "cutOffTimeStamp", cutOffValue);
+    int surrogateFromValue = generator.generateDirectSurrogateKey("1500-01-01 00:00:01");
+    long valueFromSurrogate = (long) generator.getValueFromSurrogate(surrogateFromValue);
+    Date date = new Date(valueFromSurrogate / 1000);
+    Assert.assertEquals("1500-01-01 00:00:01", timeParser.format(date));
+    surrogateFromValue = generator.generateDirectSurrogateKey("1499-12-12 00:00:00");
+    //1499-12-12 00:00:00 is a value before cut off, so it is a bad record and surrogate
should be 1
+    Assert.assertEquals(1, surrogateFromValue);
+    //re setting the value to default
+    Deencapsulation.setField(generator, "cutOffTimeStamp", 0L);
+  }
+
+  /**
+   * The memberString should be retrieved from the actual surrogate key
+   *
+   * @throws Exception
+   */
   @Test public void lowerBoundaryValueTest() throws Exception {
     TimeStampDirectDictionaryGenerator generator = new TimeStampDirectDictionaryGenerator(
);
     long valueFromSurrogate = (long) generator.getValueFromSurrogate(2);


Mime
View raw message