carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kumarvisha...@apache.org
Subject carbondata git commit: [HOTFIX]Fixed int overflow and comparison gone wrong during blocklet min/max
Date Thu, 09 Aug 2018 09:31:39 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master 7158d5203 -> 8affab843


[HOTFIX]Fixed int overflow and comparison gone wrong during blocklet min/max

Problem: During calculating min/max for blocklet, it needs to calculate from all the pages.
During that comparison,
it is typecasting to int and overflows, so there is a chance the negative becomes positive
and positive become negative.
That's why min max of long comes wrong for bigger values.
Solution: Don't typecast directly, instead check first the negative or positive and then return.


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/8affab84
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/8affab84
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/8affab84

Branch: refs/heads/master
Commit: 8affab8433bb4dea70fbb4ea9d3abc7eaf9fd7b2
Parents: 7158d52
Author: ravipesala <ravi.pesala@gmail.com>
Authored: Tue Aug 7 21:19:36 2018 +0530
Committer: kumarvishal09 <kumarvishal1802@gmail.com>
Committed: Thu Aug 9 15:01:12 2018 +0530

----------------------------------------------------------------------
 .../core/util/CarbonMetadataUtil.java           | 16 +++++++-
 .../core/util/CarbonMetadataUtilTest.java       | 39 +++++++++++++-------
 2 files changed, 39 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/8affab84/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
index 8fc648b..70443d8 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
@@ -368,7 +368,13 @@ public class CarbonMetadataUtil {
       secondBuffer.put(second);
       firstBuffer.flip();
       secondBuffer.flip();
-      return (int) (firstBuffer.getDouble() - secondBuffer.getDouble());
+      double compare = firstBuffer.getDouble() - secondBuffer.getDouble();
+      if (compare > 0) {
+        compare = 1;
+      } else if (compare < 0) {
+        compare = -1;
+      }
+      return (int) compare;
     } else if (dataType == DataTypes.LONG || dataType == DataTypes.INT
         || dataType == DataTypes.SHORT) {
       firstBuffer = ByteBuffer.allocate(8);
@@ -377,7 +383,13 @@ public class CarbonMetadataUtil {
       secondBuffer.put(second);
       firstBuffer.flip();
       secondBuffer.flip();
-      return (int) (firstBuffer.getLong() - secondBuffer.getLong());
+      long compare = firstBuffer.getLong() - secondBuffer.getLong();
+      if (compare > 0) {
+        compare = 1;
+      } else if (compare < 0) {
+        compare = -1;
+      }
+      return (int) compare;
     } else if (DataTypes.isDecimal(dataType)) {
       return DataTypeUtil.byteToBigDecimal(first).compareTo(DataTypeUtil.byteToBigDecimal(second));
     } else {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/8affab84/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java
b/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java
index 2909dc4..14cd57a 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java
@@ -17,40 +17,28 @@
 
 package org.apache.carbondata.core.util;
 
+import java.lang.reflect.Method;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.List;
 
-import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.page.EncodedTablePage;
-import org.apache.carbondata.core.datastore.page.encoding.EncodedColumnPage;
-import org.apache.carbondata.core.datastore.page.key.TablePageKey;
-import org.apache.carbondata.core.datastore.page.statistics.PrimitivePageStatsCollector;
 import org.apache.carbondata.core.metadata.ValueEncoderMeta;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
 import org.apache.carbondata.core.metadata.index.BlockIndexInfo;
 import org.apache.carbondata.format.BlockIndex;
-import org.apache.carbondata.format.BlockletIndex;
 import org.apache.carbondata.format.BlockletInfo;
-import org.apache.carbondata.format.BlockletInfo3;
-import org.apache.carbondata.format.BlockletMinMaxIndex;
 import org.apache.carbondata.format.ColumnSchema;
 import org.apache.carbondata.format.DataChunk;
-import org.apache.carbondata.format.DataChunk2;
 import org.apache.carbondata.format.DataType;
 import org.apache.carbondata.format.Encoding;
-import org.apache.carbondata.format.FileFooter3;
 import org.apache.carbondata.format.IndexHeader;
 import org.apache.carbondata.format.SegmentInfo;
 
-import mockit.Mock;
-import mockit.MockUp;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
 import static junit.framework.TestCase.assertEquals;
-import static org.apache.carbondata.core.util.CarbonMetadataUtil.convertFileFooterVersion3;
 import static org.apache.carbondata.core.util.CarbonMetadataUtil.getBlockIndexInfo;
-import static org.apache.carbondata.core.util.CarbonMetadataUtil.getBlockletIndex;
 import static org.apache.carbondata.core.util.CarbonMetadataUtil.getIndexHeader;
 
 public class CarbonMetadataUtilTest {
@@ -205,4 +193,27 @@ public class CarbonMetadataUtilTest {
     assertEquals(result.get(0).file_name, expected);
   }
 
+  @Test public void testGetBlockletIndex() throws Exception {
+
+    long left = Long.MAX_VALUE;
+    long right = 100;
+    ByteBuffer buffer = ByteBuffer.allocate(8);
+    buffer.putLong(left);
+    buffer.flip();
+    byte[] l = buffer.array().clone();
+
+    buffer.rewind();
+    buffer.putLong(right);
+    buffer.flip();
+    byte[] r = buffer.array().clone();
+
+    Method method = CarbonMetadataUtil.class
+        .getDeclaredMethod("compareMeasureData", l.getClass(), r.getClass(),
+            org.apache.carbondata.core.metadata.datatype.DataType.class);
+    method.setAccessible(true);
+    int out = (int)method.invoke(method, l, r, DataTypes.LONG);
+    assertEquals(1, out);
+
+  }
+
 }


Mime
View raw message