carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [7/7] carbondata git commit: Resolve rebase conflicts when rebasing branch encoding_override onto master
Date Fri, 28 Jul 2017 06:10:07 GMT
Resolve rebase conflicts when rebasing branch encoding_override onto master


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/74226907
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/74226907
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/74226907

Branch: refs/heads/master
Commit: 74226907990cdee41a6ccbd69e2a813077792f89
Parents: a5af0ff
Author: Raghunandan S <carbondatacontributions@gmail.com>
Authored: Wed Jul 26 21:59:05 2017 +0800
Committer: Raghunandan S <carbondatacontributions@gmail.com>
Committed: Fri Jul 28 01:24:38 2017 +0800

----------------------------------------------------------------------
 .../carbondata/core/datastore/TableSpec.java    | 38 ++++++++++-------
 .../AbstractMeasureChunkReaderV2V3Format.java   |  1 -
 ...CompressedMeasureChunkFileBasedReaderV1.java |  2 +-
 ...CompressedMeasureChunkFileBasedReaderV2.java | 18 ++++++--
 ...CompressedMeasureChunkFileBasedReaderV3.java | 16 +++++--
 .../core/datastore/page/ColumnPage.java         |  5 +--
 .../page/encoding/AdaptiveIntegralCodec.java    |  3 +-
 .../page/encoding/DeltaIntegralCodec.java       |  3 +-
 .../page/encoding/EncodingStrategy.java         |  2 +-
 .../core/datastore/page/key/TablePageKey.java   |  4 +-
 .../statistics/PrimitivePageStatsCollector.java | 44 +++++++++++++++++---
 .../page/statistics/SimpleStatsResult.java      |  4 ++
 .../statistics/VarLengthPageStatsCollector.java |  8 ++++
 .../core/metadata/ColumnPageCodecMeta.java      | 20 +++++++--
 .../core/metadata/ValueEncoderMeta.java         | 20 +++++++++
 .../apache/carbondata/core/util/CarbonUtil.java |  1 +
 .../newflow/sort/SortStepRowUtil.java           |  4 --
 .../sortdata/SortTempFileChunkHolder.java       |  1 +
 .../carbondata/processing/store/TablePage.java  | 12 +++---
 .../store/writer/AbstractFactDataWriter.java    | 12 +++---
 20 files changed, 162 insertions(+), 56 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/74226907/core/src/main/java/org/apache/carbondata/core/datastore/TableSpec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/TableSpec.java b/core/src/main/java/org/apache/carbondata/core/datastore/TableSpec.java
index f1d3546..818f46e 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/TableSpec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/TableSpec.java
@@ -75,7 +75,8 @@ public class TableSpec {
   private void addMeasures(List<CarbonMeasure> measures) {
     for (int i = 0; i < measures.size(); i++) {
       CarbonMeasure measure = measures.get(i);
-      measureSpec[i] = new MeasureSpec(measure.getColName(), measure.getDataType());
+      measureSpec[i] = new MeasureSpec(measure.getColName(), measure.getDataType(), measure
+          .getScale(), measure.getPrecision());
     }
   }
 
@@ -95,20 +96,14 @@ public class TableSpec {
     return dimensionSpec.length;
   }
 
-    public int getScale(int index) {
-      assert (index >= 0 && index < precision.length);
-      return scale[index];
-    }
+  /**
+   * return number of measures
+   */
+  public int getNumMeasures() {
+    return measureSpec.length;
+  }
 
-    public int getPrecision(int index) {
-      assert (index >= 0 && index < precision.length);
-      return precision[index];
-    }/**
-     * return number of measures
-     */
-    public int getNumMeasures() {
-      return measureSpec.length;
-    }public class ColumnSpec {
+  public class ColumnSpec {
     // field name of this column
     private String fieldName;
 
@@ -162,8 +157,21 @@ public class TableSpec {
 
   public class MeasureSpec extends ColumnSpec {
 
-    MeasureSpec(String fieldName, DataType dataType) {
+    private int scale;
+    private int precision;
+
+    MeasureSpec(String fieldName, DataType dataType, int scale, int precision) {
       super(fieldName, dataType);
+      this.scale = scale;
+      this.precision = precision;
+    }
+
+    public int getScale() {
+      return scale;
+    }
+
+    public int getPrecision() {
+      return precision;
     }
   }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/carbondata/blob/74226907/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReaderV2V3Format.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReaderV2V3Format.java
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReaderV2V3Format.java
index dd61826..049aba9 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReaderV2V3Format.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReaderV2V3Format.java
@@ -17,7 +17,6 @@
 package org.apache.carbondata.core.datastore.chunk.reader.measure;
 
 import java.io.IOException;
-import java.nio.ByteBuffer;
 import java.util.BitSet;
 import java.util.List;
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/74226907/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
index 6bf65da..7df18db 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
@@ -98,7 +98,7 @@ public class CompressedMeasureChunkFileBasedReaderV1 extends AbstractMeasureChun
     DataChunk dataChunk = measureColumnChunks.get(blockIndex);
     ValueEncoderMeta meta = dataChunk.getValueEncoderMeta().get(0);
 
-    ColumnPageCodec codec = strategy.newCodec(meta);
+    ColumnPageCodec codec = strategy.newCodec(meta, -1, -1);
     ColumnPage page = codec.decode(measureRawColumnChunk.getRawData().array(),
         measureRawColumnChunk.getOffSet(), dataChunk.getDataPageLength());
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/74226907/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
index 7511b6e..f2679ae 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
@@ -18,6 +18,7 @@ package org.apache.carbondata.core.datastore.chunk.reader.measure.v2;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.util.List;
 
 import org.apache.carbondata.core.datastore.FileHolder;
 import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
@@ -130,12 +131,21 @@ public class CompressedMeasureChunkFileBasedReaderV2 extends AbstractMeasureChun
 
   protected ColumnPage decodeMeasure(MeasureRawColumnChunk measureRawColumnChunk,
       DataChunk2 measureColumnChunk, int copyPoint) throws MemoryException, IOException {
-    // for measure, it should have only one ValueEncoderMeta
-    assert (measureColumnChunk.getEncoder_meta().size() == 1);
-    byte[] encodedMeta = measureColumnChunk.getEncoder_meta().get(0).array();
+    assert (measureColumnChunk.getEncoder_meta().size() > 0);
+    List<ByteBuffer> encoder_meta = measureColumnChunk.getEncoder_meta();
+    byte[] encodedMeta = encoder_meta.get(0).array();
 
     ValueEncoderMeta meta = CarbonUtil.deserializeEncoderMetaV3(encodedMeta);
-    ColumnPageCodec codec = strategy.newCodec(meta);
+    int scale = -1;
+    int precision = -1;
+    if (encoder_meta.size() > 1) {
+      ByteBuffer decimalInfo = encoder_meta.get(1);
+      scale = decimalInfo.getInt();
+      precision = decimalInfo.getInt();
+    }
+
+
+    ColumnPageCodec codec = strategy.newCodec(meta, scale, precision);
     byte[] rawData = measureRawColumnChunk.getRawData().array();
     return codec.decode(rawData, copyPoint, measureColumnChunk.data_page_length);
   }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/74226907/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java
index 1881791..354aa38 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java
@@ -18,6 +18,7 @@ package org.apache.carbondata.core.datastore.chunk.reader.measure.v3;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.util.List;
 
 import org.apache.carbondata.core.datastore.FileHolder;
 import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
@@ -226,13 +227,22 @@ public class CompressedMeasureChunkFileBasedReaderV3 extends AbstractMeasureChun
 
   protected ColumnPage decodeMeasure(MeasureRawColumnChunk measureRawColumnChunk,
       DataChunk2 measureColumnChunk, int copyPoint) throws MemoryException {
+    List<ByteBuffer> encoder_meta = measureColumnChunk.getEncoder_meta();
     // for measure, it should have only one ValueEncoderMeta
-    assert (measureColumnChunk.getEncoder_meta().size() == 1);
-    byte[] encodedMeta = measureColumnChunk.getEncoder_meta().get(0).array();
+    assert (encoder_meta.size() > 0);
+    byte[] encodedMeta = encoder_meta.get(0).array();
+
+    int scale = -1;
+    int precision = -1;
+    if (encoder_meta.size() > 1) {
+      ByteBuffer decimalInfo = encoder_meta.get(1);
+      scale = decimalInfo.getInt();
+      precision = decimalInfo.getInt();
+    }
 
     ColumnPageCodecMeta meta = new ColumnPageCodecMeta();
     meta.deserialize(encodedMeta);
-    ColumnPageCodec codec = strategy.newCodec(meta);
+    ColumnPageCodec codec = strategy.newCodec(meta, scale, precision);
     byte[] rawData = measureRawColumnChunk.getRawData().array();
     return codec.decode(rawData, copyPoint, measureColumnChunk.data_page_length);
   }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/74226907/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
b/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
index 90300d9..245e95b 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
@@ -59,8 +59,6 @@ public abstract class ColumnPage {
     this.pageSize = pageSize;
     this.scale = scale;
     this.precision = precision;
-    this.stats = new ColumnPageStatsVO(dataType);
-    this.nullBitSet = new BitSet(pageSize);
     if (dataType == DECIMAL) {
       decimalConverter = DecimalConverterFactory.INSTANCE.getDecimalConverter(precision,
scale);
     }
@@ -184,7 +182,7 @@ public abstract class ColumnPage {
           instance = newDecimalPage(new byte[pageSize][], scale, precision);
           break;
         case BYTE_ARRAY:
-          instance = new SafeVarLengthColumnPage(dataType, pageSize);
+          instance = new SafeVarLengthColumnPage(dataType, pageSize, scale, precision);
           break;
         default:
           throw new RuntimeException("Unsupported data dataType: " + dataType);
@@ -328,6 +326,7 @@ public abstract class ColumnPage {
         break;
       case DECIMAL:
         putDecimal(rowId, (BigDecimal) value);
+        statsCollector.update(((BigDecimal) value).unscaledValue().longValue());
         break;
       case BYTE_ARRAY:
         putBytes(rowId, (byte[]) value);

http://git-wip-us.apache.org/repos/asf/carbondata/blob/74226907/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/AdaptiveIntegralCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/AdaptiveIntegralCodec.java
b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/AdaptiveIntegralCodec.java
index ed8d734..6480bcd 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/AdaptiveIntegralCodec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/AdaptiveIntegralCodec.java
@@ -65,7 +65,8 @@ class AdaptiveIntegralCodec extends AdaptiveCompressionCodec {
 
   @Override
   public ColumnPage decode(byte[] input, int offset, int length) throws MemoryException {
-    ColumnPage page = ColumnPage.decompress(compressor, targetDataType, input, offset, length);
+    ColumnPage page = ColumnPage.decompress(compressor, targetDataType, input, offset, length,
+        stats.getScale(), stats.getPrecision());
     return LazyColumnPage.newPage(page, codec);
   }
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/74226907/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DeltaIntegralCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DeltaIntegralCodec.java
b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DeltaIntegralCodec.java
index 53a8295..ba61cd3 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DeltaIntegralCodec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DeltaIntegralCodec.java
@@ -87,7 +87,8 @@ public class DeltaIntegralCodec extends AdaptiveCompressionCodec {
 
   @Override
   public ColumnPage decode(byte[] input, int offset, int length) throws MemoryException {
-    ColumnPage page = ColumnPage.decompress(compressor, targetDataType, input, offset, length);
+    ColumnPage page = ColumnPage.decompress(compressor, targetDataType, input, offset, length,
+        stats.getScale(), stats.getPrecision());
     return LazyColumnPage.newPage(page, codec);
   }
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/74226907/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/EncodingStrategy.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/EncodingStrategy.java
b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/EncodingStrategy.java
index 29219ea..53c565d 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/EncodingStrategy.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/EncodingStrategy.java
@@ -76,7 +76,7 @@ public abstract class EncodingStrategy {
           throw new RuntimeException("unsupported data type: " + stats.getDataType());
       }
     } else {
-      SimpleStatsResult stats = PrimitivePageStatsCollector.newInstance(meta, scale, precision);
+      SimpleStatsResult stats = PrimitivePageStatsCollector.newInstance(meta);
       switch (meta.getType()) {
         case BYTE:
         case SHORT:

http://git-wip-us.apache.org/repos/asf/carbondata/blob/74226907/core/src/main/java/org/apache/carbondata/core/datastore/page/key/TablePageKey.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/key/TablePageKey.java
b/core/src/main/java/org/apache/carbondata/core/datastore/page/key/TablePageKey.java
index ef8307e..f8a10ca 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/key/TablePageKey.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/key/TablePageKey.java
@@ -68,12 +68,12 @@ public class TablePageKey {
       currentNoDictionaryKey = WriteStepRowUtil.getNoDictAndComplexDimension(row);
     }
     if (rowId == 0) {
-      startKey = WriteStepRowUtil.getMdk(row, model.getMDKeyGenerator());
+      startKey = WriteStepRowUtil.getMdk(row, mdkGenerator);
       noDictStartKey = currentNoDictionaryKey;
     }
     noDictEndKey = currentNoDictionaryKey;
     if (rowId == pageSize - 1) {
-      endKey = WriteStepRowUtil.getMdk(row, model.getMDKeyGenerator());
+      endKey = WriteStepRowUtil.getMdk(row, mdkGenerator);
       finalizeKeys();
     }
   }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/74226907/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java
b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java
index 73ada4b..af40f03 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java
@@ -32,6 +32,7 @@ public class PrimitivePageStatsCollector implements ColumnPageStatsCollector,
Si
   private int minInt, maxInt;
   private long minLong, maxLong;
   private double minDouble, maxDouble;
+  private int scale, precision;
 
   // scale of the double value
   private int decimal;
@@ -40,17 +41,19 @@ public class PrimitivePageStatsCollector implements ColumnPageStatsCollector,
Si
   private BitSet nullBitSet;
 
   // this is for encode flow
-  public static PrimitivePageStatsCollector newInstance(DataType dataType, int pageSize)
{
+  public static PrimitivePageStatsCollector newInstance(DataType dataType, int pageSize,
int
+      scale, int precision) {
     switch (dataType) {
       default:
-        return new PrimitivePageStatsCollector(dataType, pageSize);
+        return new PrimitivePageStatsCollector(dataType, pageSize, scale, precision);
     }
   }
 
   // this is for decode flow, we do not need to create nullBits, so passing 0 as pageSize
   public static PrimitivePageStatsCollector newInstance(ColumnPageCodecMeta meta) {
     PrimitivePageStatsCollector instance =
-        new PrimitivePageStatsCollector(meta.getSrcDataType(), 0);
+        new PrimitivePageStatsCollector(meta.getSrcDataType(), 0, meta.getScale(),
+            meta.getPrecision());
     // set min max from meta
     switch (meta.getSrcDataType()) {
       case BYTE:
@@ -74,13 +77,20 @@ public class PrimitivePageStatsCollector implements ColumnPageStatsCollector,
Si
         instance.maxDouble = (double) meta.getMaxValue();
         instance.decimal = meta.getDecimal();
         break;
+      case DECIMAL:
+        instance.minLong = (long) meta.getMinValue();
+        instance.maxLong = (long) meta.getMaxValue();
+        instance.decimal = meta.getDecimal();
+        instance.scale = meta.getScale();
+        instance.precision = meta.getPrecision();
+        break;
     }
     return instance;
   }
 
   public static PrimitivePageStatsCollector newInstance(ValueEncoderMeta meta) {
     PrimitivePageStatsCollector instance =
-        new PrimitivePageStatsCollector(meta.getType(), 0);
+        new PrimitivePageStatsCollector(meta.getType(), 0, meta.getScale(), meta.getPrecision());
     // set min max from meta
     switch (meta.getType()) {
       case BYTE:
@@ -104,11 +114,18 @@ public class PrimitivePageStatsCollector implements ColumnPageStatsCollector,
Si
         instance.maxDouble = (double) meta.getMaxValue();
         instance.decimal = meta.getDecimal();
         break;
+      case DECIMAL:
+        instance.minDouble = (double) meta.getMinValue();
+        instance.maxDouble = (double) meta.getMaxValue();
+        instance.decimal = meta.getDecimal();
+        instance.scale = meta.getScale();
+        instance.precision = meta.getPrecision();
+        break;
     }
     return instance;
   }
 
-  private PrimitivePageStatsCollector(DataType dataType, int pageSize) {
+  private PrimitivePageStatsCollector(DataType dataType, int pageSize, int scale, int precision)
{
     this.dataType = dataType;
     this.nullBitSet = new BitSet(pageSize);
     switch (dataType) {
@@ -134,6 +151,11 @@ public class PrimitivePageStatsCollector implements ColumnPageStatsCollector,
Si
         decimal = 0;
         break;
       case DECIMAL:
+        minLong = Long.MAX_VALUE;
+        maxLong = Long.MIN_VALUE;
+        decimal = scale;
+        this.scale = scale;
+        this.precision = precision;
     }
   }
 
@@ -255,6 +277,8 @@ public class PrimitivePageStatsCollector implements ColumnPageStatsCollector,
Si
         return minLong;
       case DOUBLE:
         return minDouble;
+      case DECIMAL:
+        return minLong;
     }
     return null;
   }
@@ -272,6 +296,8 @@ public class PrimitivePageStatsCollector implements ColumnPageStatsCollector,
Si
         return maxLong;
       case DOUBLE:
         return maxDouble;
+      case DECIMAL:
+        return maxLong;
     }
     return null;
   }
@@ -291,4 +317,12 @@ public class PrimitivePageStatsCollector implements ColumnPageStatsCollector,
Si
     return dataType;
   }
 
+  @Override public int getScale() {
+    return scale;
+  }
+
+  @Override public int getPrecision() {
+    return precision;
+  }
+
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/carbondata/blob/74226907/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/SimpleStatsResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/SimpleStatsResult.java
b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/SimpleStatsResult.java
index 1db86ff..b40d023 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/SimpleStatsResult.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/SimpleStatsResult.java
@@ -32,4 +32,8 @@ public interface SimpleStatsResult {
   int getDecimalPoint();
 
   DataType getDataType();
+
+  int getScale();
+
+  int getPrecision();
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/74226907/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/VarLengthPageStatsCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/VarLengthPageStatsCollector.java
b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/VarLengthPageStatsCollector.java
index e985f90..0fe5960 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/VarLengthPageStatsCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/VarLengthPageStatsCollector.java
@@ -102,6 +102,14 @@ public class VarLengthPageStatsCollector implements ColumnPageStatsCollector
{
       @Override public DataType getDataType() {
         return null;
       }
+
+      @Override public int getScale() {
+        return 0;
+      }
+
+      @Override public int getPrecision() {
+        return 0;
+      }
     };
   }
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/74226907/core/src/main/java/org/apache/carbondata/core/metadata/ColumnPageCodecMeta.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/ColumnPageCodecMeta.java
b/core/src/main/java/org/apache/carbondata/core/metadata/ColumnPageCodecMeta.java
index 20a7568..6322670 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/ColumnPageCodecMeta.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/ColumnPageCodecMeta.java
@@ -57,6 +57,8 @@ public class ColumnPageCodecMeta extends ValueEncoderMeta implements Serializabl
     meta.setMaxValue(stats.getMax());
     meta.setMinValue(stats.getMin());
     meta.setDecimal(stats.getDecimalPoint());
+    meta.setScale(stats.getScale());
+    meta.setPrecision(stats.getPrecision());
     return meta;
   }
 
@@ -169,8 +171,17 @@ public class ColumnPageCodecMeta extends ValueEncoderMeta implements
Serializabl
         buffer.putDouble((Double) 0d); // unique value is obsoleted, maintain for compatibility
         break;
       case DECIMAL:
-        buffer = ByteBuffer.allocate(CarbonCommonConstants.INT_SIZE_IN_BYTE + 3);
+        buffer = ByteBuffer.allocate(
+            (CarbonCommonConstants.LONG_SIZE_IN_BYTE * 3) + (CarbonCommonConstants
+                .INT_SIZE_IN_BYTE * 3)
+                + 3);
         buffer.putChar(getSrcDataTypeInChar());
+        buffer.putLong((Long) getMaxValue());
+        buffer.putLong((Long) getMinValue());
+        buffer.putLong((Long) 0L); // unique value is obsoleted, maintain for compatibility
+        buffer.putInt(getScale());
+        buffer.putInt(getPrecision());
+
         break;
     }
     buffer.putInt(getDecimal());
@@ -190,8 +201,11 @@ public class ColumnPageCodecMeta extends ValueEncoderMeta implements
Serializabl
         buffer.getDouble(); // for non exist value which is obsoleted, it is backward compatibility;
         break;
       case BIG_DECIMAL_MEASURE:
-        this.setMaxValue(0.0);
-        this.setMinValue(0.0);
+        this.setMaxValue(buffer.getLong());
+        this.setMinValue(buffer.getLong());
+        buffer.getLong();
+        this.setScale(buffer.getInt());
+        this.setPrecision(buffer.getInt());
         break;
       case BYTE_VALUE_MEASURE:
         this.setMaxValue(buffer.get());

http://git-wip-us.apache.org/repos/asf/carbondata/blob/74226907/core/src/main/java/org/apache/carbondata/core/metadata/ValueEncoderMeta.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/ValueEncoderMeta.java
b/core/src/main/java/org/apache/carbondata/core/metadata/ValueEncoderMeta.java
index 971359d..f1ebbd3 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/ValueEncoderMeta.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/ValueEncoderMeta.java
@@ -49,6 +49,10 @@ public class ValueEncoderMeta implements Serializable {
 
   private byte dataTypeSelected;
 
+  private int scale;
+
+  private int precision;
+
   public Object getMaxValue() {
     return maxValue;
   }
@@ -109,4 +113,20 @@ public class ValueEncoderMeta implements Serializable {
   public void setDataTypeSelected(byte dataTypeSelected) {
     this.dataTypeSelected = dataTypeSelected;
   }
+
+  public int getScale() {
+    return scale;
+  }
+
+  public void setScale(int scale) {
+    this.scale = scale;
+  }
+
+  public int getPrecision() {
+    return precision;
+  }
+
+  public void setPrecision(int precision) {
+    this.precision = precision;
+  }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/carbondata/blob/74226907/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index eff8f0d..7628415 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -53,6 +53,7 @@ import org.apache.carbondata.core.datastore.columnar.ColumnGroupModel;
 import org.apache.carbondata.core.datastore.columnar.UnBlockIndexer;
 import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
 import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.indexstore.BlockletDetailInfo;
 import org.apache.carbondata.core.keygenerator.mdkey.NumberCompressor;
 import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
 import org.apache.carbondata.core.metadata.ColumnarFormatVersion;

http://git-wip-us.apache.org/repos/asf/carbondata/blob/74226907/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/SortStepRowUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/SortStepRowUtil.java
b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/SortStepRowUtil.java
index 1bcbe54..7857f4e 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/SortStepRowUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/SortStepRowUtil.java
@@ -17,10 +17,6 @@
 
 package org.apache.carbondata.processing.newflow.sort;
 
-import java.math.BigDecimal;
-
-import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.util.DataTypeUtil;
 import org.apache.carbondata.core.util.NonDictionaryUtil;
 import org.apache.carbondata.processing.sortandgroupby.sortdata.SortParameters;
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/74226907/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortTempFileChunkHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortTempFileChunkHolder.java
b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortTempFileChunkHolder.java
index b17c69a..9732e66 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortTempFileChunkHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortTempFileChunkHolder.java
@@ -35,6 +35,7 @@ import org.apache.carbondata.core.metadata.datatype.DataType;
 import org.apache.carbondata.core.util.ByteUtil.UnsafeComparer;
 import org.apache.carbondata.core.util.CarbonProperties;
 import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.DataTypeUtil;
 import org.apache.carbondata.core.util.NonDictionaryUtil;
 import org.apache.carbondata.processing.sortandgroupby.exception.CarbonSortKeyAndGroupByException;
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/74226907/processing/src/main/java/org/apache/carbondata/processing/store/TablePage.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/TablePage.java
b/processing/src/main/java/org/apache/carbondata/processing/store/TablePage.java
index 9881e8e..03f3e5e 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/TablePage.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/TablePage.java
@@ -47,7 +47,6 @@ import org.apache.carbondata.core.memory.MemoryException;
 import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
 import org.apache.carbondata.core.metadata.datatype.DataType;
 import org.apache.carbondata.core.util.CarbonProperties;
-import org.apache.carbondata.core.util.DataTypeUtil;
 
 import org.apache.spark.sql.types.Decimal;
 
@@ -82,13 +81,13 @@ public class TablePage {
     int numDictDimension = model.getMDKeyGenerator().getDimCount();
     dictDimensionPages = new ColumnPage[numDictDimension];
     for (int i = 0; i < dictDimensionPages.length; i++) {
-      ColumnPage page = ColumnPage.newPage(DataType.BYTE_ARRAY, pageSize);
+      ColumnPage page = ColumnPage.newPage(DataType.BYTE_ARRAY, pageSize, -1, -1);
       page.setStatsCollector(VarLengthPageStatsCollector.newInstance());
       dictDimensionPages[i] = page;
     }
     noDictDimensionPages = new ColumnPage[model.getNoDictionaryCount()];
     for (int i = 0; i < noDictDimensionPages.length; i++) {
-      ColumnPage page = ColumnPage.newPage(DataType.BYTE_ARRAY, pageSize);
+      ColumnPage page = ColumnPage.newPage(DataType.BYTE_ARRAY, pageSize, -1, -1);
       page.setStatsCollector(VarLengthPageStatsCollector.newInstance());
       noDictDimensionPages[i] = page;
     }
@@ -100,11 +99,12 @@ public class TablePage {
     }
     measurePage = new ColumnPage[model.getMeasureCount()];
     DataType[] dataTypes = model.getMeasureDataType();
-    TableSpec.MeasureSpec measureSpec = model.getTableSpec().getMeasureSpec();
     for (int i = 0; i < measurePage.length; i++) {
+      TableSpec.MeasureSpec measureSpec = model.getTableSpec().getMeasureSpec(i);
       ColumnPage page = ColumnPage
-          .newPage(dataTypes[i], pageSize, measureSpec.getScale(i), measureSpec.getPrecision(i));
-      page.setStatsCollector(PrimitivePageStatsCollector.newInstance(dataTypes[i], pageSize));
+          .newPage(dataTypes[i], pageSize, measureSpec.getScale(), measureSpec.getPrecision());
+      page.setStatsCollector(PrimitivePageStatsCollector.newInstance(dataTypes[i], pageSize,
+          measureSpec.getScale(), measureSpec.getPrecision()));
       measurePage[i] = page;
     }
     boolean hasNoDictionary = noDictDimensionPages.length > 0;

http://git-wip-us.apache.org/repos/asf/carbondata/blob/74226907/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
index b83a82a..a34ed01 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
@@ -24,7 +24,7 @@ import java.io.FileFilter;
 import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
-import java.nio.ByteBuffer;
+//import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -61,7 +61,6 @@ import org.apache.carbondata.core.util.CarbonMergerUtil;
 import org.apache.carbondata.core.util.CarbonMetadataUtil;
 import org.apache.carbondata.core.util.CarbonProperties;
 import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.NodeHolder;
 import org.apache.carbondata.core.util.path.CarbonTablePath;
 import org.apache.carbondata.core.writer.CarbonIndexFileWriter;
 import org.apache.carbondata.format.BlockIndex;
@@ -608,10 +607,11 @@ public abstract class AbstractFactDataWriter<T> implements CarbonFactDataWriter<
    * @return min max value without length
    */
   protected byte[] updateMinMaxForNoDictionary(byte[] valueWithLength) {
-    ByteBuffer buffer = ByteBuffer.wrap(valueWithLength);
-    byte[] actualValue = new byte[buffer.getShort()];
-    buffer.get(actualValue);
-    return actualValue;
+    return valueWithLength;
+//    ByteBuffer buffer = ByteBuffer.wrap(valueWithLength);
+//    byte[] actualValue = new byte[buffer.getShort()];
+//    buffer.get(actualValue);
+//    return actualValue;
   }
 
   /**


Mime
View raw message