carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [1/2] incubator-carbondata git commit: Adding Header And Making Footer Optional
Date Fri, 10 Mar 2017 12:53:38 GMT
Repository: incubator-carbondata
Updated Branches:
  refs/heads/master 5fb1ec088 -> 1f8e83f49


Adding Header And Making Footer Optional


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/b41e48f1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/b41e48f1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/b41e48f1

Branch: refs/heads/master
Commit: b41e48f1a41394eb0307ecf7308b3422029e7720
Parents: 5fb1ec0
Author: kumarvishal <kumarvishal.1802@gmail.com>
Authored: Tue Mar 7 20:54:13 2017 +0800
Committer: jackylk <jacky.likun@huawei.com>
Committed: Fri Mar 10 20:27:54 2017 +0800

----------------------------------------------------------------------
 .../carbondata/core/datastore/DataRefNode.java  |  6 ++
 ...mpressedDimensionChunkFileBasedReaderV3.java |  2 +
 ...CompressedMeasureChunkFileBasedReaderV3.java |  2 +
 .../datastore/impl/btree/BTreeNonLeafNode.java  | 10 +++
 .../impl/btree/BlockBTreeLeafNode.java          |  9 +++
 .../impl/btree/BlockletBTreeLeafNode.java       | 13 ++++
 .../core/metadata/blocklet/BlockletInfo.java    | 20 ++++++
 .../core/reader/CarbonFooterReaderV3.java       | 75 ++++++++++++++++++++
 .../core/reader/CarbonHeaderReader.java         | 68 ++++++++++++++++++
 .../core/util/CarbonMetadataUtil.java           | 41 +++++++----
 .../core/util/DataFileFooterConverterV3.java    | 29 ++++----
 format/src/main/thrift/carbondata.thrift        | 18 ++++-
 format/src/main/thrift/carbondata_index.thrift  |  2 +-
 pom.xml                                         |  2 +-
 .../writer/v3/CarbonFactDataWriterImplV3.java   | 40 ++++++-----
 15 files changed, 291 insertions(+), 46 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b41e48f1/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java b/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java
index 456710a..8914196 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java
@@ -122,4 +122,10 @@ public interface DataRefNode {
    * @return
    */
   BlockletLevelDeleteDeltaDataCache getDeleteDeltaDataCache();
+
+  /**
+   * number of pages in blocklet
+   * @return
+   */
+  int numberOfPages();
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b41e48f1/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimensionChunkFileBasedReaderV3.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimensionChunkFileBasedReaderV3.java
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimensionChunkFileBasedReaderV3.java
index acaa2fa..79b75e4 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimensionChunkFileBasedReaderV3.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimensionChunkFileBasedReaderV3.java
@@ -40,10 +40,12 @@ import org.apache.commons.lang.ArrayUtils;
  * V3 format data
  * data format
  * Data Format
+ * <FileHeader>
  * <Column1 Data ChunkV3><Column1<Page1><Page2><Page3><Page4>>
  * <Column2 Data ChunkV3><Column2<Page1><Page2><Page3><Page4>>
  * <Column3 Data ChunkV3><Column3<Page1><Page2><Page3><Page4>>
  * <Column4 Data ChunkV3><Column4<Page1><Page2><Page3><Page4>>
+ * <File Footer>
  */
 public class CompressedDimensionChunkFileBasedReaderV3 extends AbstractChunkReaderV2V3Format
{
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b41e48f1/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java
index 307af41..58b2937 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java
@@ -41,10 +41,12 @@ import org.apache.commons.lang.ArrayUtils;
  * V3 format data
  * data format
  * Data Format
+ * <FileHeader>
  * <Column1 Data ChunkV3><Column1<Page1><Page2><Page3><Page4>>
  * <Column2 Data ChunkV3><Column2<Page1><Page2><Page3><Page4>>
  * <Column3 Data ChunkV3><Column3<Page1><Page2><Page3><Page4>>
  * <Column4 Data ChunkV3><Column4<Page1><Page2><Page3><Page4>>
+ * <File Footer>
  */
 public class CompressedMeasureChunkFileBasedReaderV3 extends AbstractMeasureChunkReaderV2V3Format
{
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b41e48f1/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeNonLeafNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeNonLeafNode.java
b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeNonLeafNode.java
index 8e5976d..01c0177 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeNonLeafNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeNonLeafNode.java
@@ -244,4 +244,14 @@ public class BTreeNonLeafNode implements BTreeNode {
   public BlockletLevelDeleteDeltaDataCache getDeleteDeltaDataCache() {
     return deleteDeltaDataCache;
   }
+
+  /**
+   * number of pages in blocklet
+   * @return
+   */
+  @Override
+  public int numberOfPages() {
+    // TODO Auto-generated method stub
+    throw new UnsupportedOperationException("Unsupported operation");
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b41e48f1/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockBTreeLeafNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockBTreeLeafNode.java
b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockBTreeLeafNode.java
index 0a2a343..f248ce0 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockBTreeLeafNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockBTreeLeafNode.java
@@ -59,4 +59,13 @@ public class BlockBTreeLeafNode extends AbstractBTreeLeafNode {
     return blockInfo.getTableBlockInfo();
   }
 
+  /**
+   * number of pages in blocklet
+   * @return
+   */
+  @Override
+  public int numberOfPages() {
+    throw new UnsupportedOperationException("Unsupported operation");
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b41e48f1/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeLeafNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeLeafNode.java
b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeLeafNode.java
index 00a0e01..9a123d1 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeLeafNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeLeafNode.java
@@ -43,6 +43,11 @@ public class BlockletBTreeLeafNode extends AbstractBTreeLeafNode {
   private MeasureColumnChunkReader measureColumnChunkReader;
 
   /**
+   * number of pages in blocklet
+   */
+  private int numberOfPages;
+
+  /**
    * Create a leaf node
    *
    * @param builderInfos builder infos which have required metadata to create a leaf node
@@ -124,4 +129,12 @@ public class BlockletBTreeLeafNode extends AbstractBTreeLeafNode {
       throws IOException {
     return measureColumnChunkReader.readRawMeasureChunk(fileReader, blockIndex);
   }
+
+  /**
+   * @return the number of pages in blocklet
+   */
+  @Override
+  public int numberOfPages() {
+    return numberOfPages;
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b41e48f1/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/BlockletInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/BlockletInfo.java
b/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/BlockletInfo.java
index 97f3822..bfa9d7e 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/BlockletInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/BlockletInfo.java
@@ -61,11 +61,23 @@ public class BlockletInfo implements Serializable {
    */
   private BlockletIndex blockletIndex;
 
+  /**
+   * last dimension end offset
+   */
   private long dimensionOffset;
 
+  /**
+   * last measure end offsets
+   */
   private long measureOffsets;
 
   /**
+   * number of pages in blocklet
+   * default value is one for V1 and V2 version
+   */
+  private int numberOfPages = 1;
+
+  /**
    * @return the numberOfRows
    */
   public int getNumberOfRows() {
@@ -169,4 +181,12 @@ public class BlockletInfo implements Serializable {
     this.measureOffsets = measureOffsets;
   }
 
+  public int getNumberOfPages() {
+    return numberOfPages;
+  }
+
+  public void setNumberOfPages(int numberOfPages) {
+    this.numberOfPages = numberOfPages;
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b41e48f1/core/src/main/java/org/apache/carbondata/core/reader/CarbonFooterReaderV3.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/reader/CarbonFooterReaderV3.java
b/core/src/main/java/org/apache/carbondata/core/reader/CarbonFooterReaderV3.java
new file mode 100644
index 0000000..8171e12
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/reader/CarbonFooterReaderV3.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.reader;
+
+import java.io.IOException;
+
+import org.apache.carbondata.format.FileFooter3;
+
+import org.apache.thrift.TBase;
+
+/**
+ * Below class to read file footer of version3
+ * carbon data file
+ */
+public class CarbonFooterReaderV3 {
+
+  //Fact file path
+  private String filePath;
+
+  //start offset of the file footer
+  private long footerOffset;
+
+  public CarbonFooterReaderV3(String filePath, long offset) {
+    this.filePath = filePath;
+    this.footerOffset = offset;
+  }
+
+  /**
+   * It reads the metadata in FileFooter thrift object format.
+   *
+   * @return
+   * @throws IOException
+   */
+  public FileFooter3 readFooterVersion3() throws IOException {
+    ThriftReader thriftReader = openThriftReader(filePath);
+    thriftReader.open();
+    //Set the offset from where it should read
+    thriftReader.setReadOffset(footerOffset);
+    FileFooter3 footer = (FileFooter3) thriftReader.read();
+    thriftReader.close();
+    return footer;
+  }
+
+  /**
+   * Open the thrift reader
+   *
+   * @param filePath
+   * @return
+   * @throws IOException
+   */
+  private ThriftReader openThriftReader(String filePath) {
+
+    return new ThriftReader(filePath, new ThriftReader.TBaseCreator() {
+      @Override public TBase create() {
+        return new FileFooter3();
+      }
+    });
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b41e48f1/core/src/main/java/org/apache/carbondata/core/reader/CarbonHeaderReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/reader/CarbonHeaderReader.java
b/core/src/main/java/org/apache/carbondata/core/reader/CarbonHeaderReader.java
new file mode 100644
index 0000000..ab557cc
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/reader/CarbonHeaderReader.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.reader;
+
+import java.io.IOException;
+
+import org.apache.carbondata.format.FileHeader;
+
+import org.apache.thrift.TBase;
+
+/**
+ * Below class to read file header of version3
+ * carbon data file
+ */
+public class CarbonHeaderReader {
+
+  //Fact file path
+  private String filePath;
+
+  public CarbonHeaderReader(String filePath) {
+    this.filePath = filePath;
+  }
+
+  /**
+   * It reads the metadata in FileFooter thrift object format.
+   *
+   * @return
+   * @throws IOException
+   */
+  public FileHeader readHeader() throws IOException {
+    ThriftReader thriftReader = openThriftReader(filePath);
+    thriftReader.open();
+    FileHeader header = (FileHeader) thriftReader.read();
+    thriftReader.close();
+    return header;
+  }
+
+  /**
+   * Open the thrift reader
+   *
+   * @param filePath
+   * @return
+   * @throws IOException
+   */
+  private ThriftReader openThriftReader(String filePath) {
+
+    return new ThriftReader(filePath, new ThriftReader.TBaseCreator() {
+      @Override public TBase create() {
+        return new FileHeader();
+      }
+    });
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b41e48f1/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
index 371855b..5e227fb 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
@@ -54,6 +54,8 @@ import org.apache.carbondata.format.DataChunk2;
 import org.apache.carbondata.format.DataChunk3;
 import org.apache.carbondata.format.Encoding;
 import org.apache.carbondata.format.FileFooter;
+import org.apache.carbondata.format.FileFooter3;
+import org.apache.carbondata.format.FileHeader;
 import org.apache.carbondata.format.IndexHeader;
 import org.apache.carbondata.format.PresenceMeta;
 import org.apache.carbondata.format.SegmentInfo;
@@ -114,17 +116,18 @@ public class CarbonMetadataUtil {
   }
 
   /**
-   * It converts list of BlockletInfoColumnar to FileFooter thrift objects
+   * Below method prepares the file footer object for carbon data file version 3
    *
    * @param infoList
    * @param numCols
    * @param cardinalities
    * @return FileFooter
    */
-  public static FileFooter convertFileFooter3(List<BlockletInfo3> infoList,
-      List<BlockletIndex> blockletIndexs, int[] cardinalities, List<ColumnSchema>
columnSchemaList,
+  public static FileFooter3 convertFileFooterVersion3(List<BlockletInfo3> infoList,
+      List<BlockletIndex> blockletIndexs, int[] cardinalities, int numberOfColumns,
       SegmentProperties segmentProperties) throws IOException {
-    FileFooter footer = getFileFooter3(infoList, blockletIndexs, cardinalities, columnSchemaList);
+    FileFooter3 footer =
+        getFileFooter3(infoList, blockletIndexs, cardinalities, numberOfColumns);
     for (BlockletInfo3 info : infoList) {
       footer.addToBlocklet_info_list3(info);
     }
@@ -139,18 +142,14 @@ public class CarbonMetadataUtil {
    * @param columnSchemaList column schema list
    * @return file footer
    */
-  private static FileFooter getFileFooter3(List<BlockletInfo3> infoList,
-      List<BlockletIndex> blockletIndexs, int[] cardinalities,
-      List<ColumnSchema> columnSchemaList) {
+  private static FileFooter3 getFileFooter3(List<BlockletInfo3> infoList,
+      List<BlockletIndex> blockletIndexs, int[] cardinalities, int numberOfColumns)
{
     SegmentInfo segmentInfo = new SegmentInfo();
-    segmentInfo.setNum_cols(columnSchemaList.size());
+    segmentInfo.setNum_cols(numberOfColumns);
     segmentInfo.setColumn_cardinalities(CarbonUtil.convertToIntegerList(cardinalities));
-    ColumnarFormatVersion version = CarbonProperties.getInstance().getFormatVersion();
-    FileFooter footer = new FileFooter();
-    footer.setVersion(version.number());
+    FileFooter3 footer = new FileFooter3();
     footer.setNum_rows(getNumberOfRowForFooter(infoList));
     footer.setSegment_info(segmentInfo);
-    footer.setTable_columns(columnSchemaList);
     for (BlockletIndex info : blockletIndexs) {
       footer.addToBlocklet_index_list(info);
     }
@@ -851,7 +850,7 @@ public class CarbonMetadataUtil {
         b.flip();
         return b.array();
       case DECIMAL:
-        return DataTypeUtil.bigDecimalToByte((BigDecimal)data);
+        return DataTypeUtil.bigDecimalToByte((BigDecimal) data);
       default:
         throw new IllegalArgumentException("Invalid data type");
     }
@@ -887,4 +886,20 @@ public class CarbonMetadataUtil {
     }
   }
 
+  /**
+   * Below method will be used to prepare the file header object for carbondata file
+   *
+   * @param isFooterPresent  is footer present in carbon data file
+   * @param columnSchemaList list of column schema
+   * @return file header thrift object
+   */
+  public static FileHeader getFileHeader(boolean isFooterPresent,
+      List<ColumnSchema> columnSchemaList) {
+    FileHeader fileHeader = new FileHeader();
+    ColumnarFormatVersion version = CarbonProperties.getInstance().getFormatVersion();
+    fileHeader.setIs_footer_present(isFooterPresent);
+    fileHeader.setColumn_schema(columnSchemaList);
+    fileHeader.setVersion(version.number());
+    return fileHeader;
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b41e48f1/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverterV3.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverterV3.java
b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverterV3.java
index 1ab3133..3fcf427 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverterV3.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverterV3.java
@@ -26,18 +26,21 @@ import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
 import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
 import org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex;
 import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
-import org.apache.carbondata.core.reader.CarbonFooterReader;
-import org.apache.carbondata.format.FileFooter;
+import org.apache.carbondata.core.reader.CarbonFooterReaderV3;
+import org.apache.carbondata.core.reader.CarbonHeaderReader;
+import org.apache.carbondata.format.FileFooter3;
+import org.apache.carbondata.format.FileHeader;
 
 public class DataFileFooterConverterV3 extends AbstractDataFileFooterConverter {
 
   /**
    * Below method will be used to convert thrift file meta to wrapper file meta
    * This method will read the footer from footer offset present in the data file
-   * 1. It will set the stream offset
-   * 2. It will read the footer data from file
-   * 3. parse the footer to thrift object
-   * 4. convert to wrapper object
+   * 1. It will read the header from carbon data file, header starts from 0 offset
+   * 2. It will set the stream offset
+   * 3. It will read the footer data from file
+   * 4. parse the footer to thrift object
+   * 5. convert to wrapper object
    *
    * @param tableBlockInfo
    *        table block info
@@ -46,19 +49,20 @@ public class DataFileFooterConverterV3 extends AbstractDataFileFooterConverter
{
   @Override public DataFileFooter readDataFileFooter(TableBlockInfo tableBlockInfo)
       throws IOException {
     DataFileFooter dataFileFooter = new DataFileFooter();
-    CarbonFooterReader reader =
-        new CarbonFooterReader(tableBlockInfo.getFilePath(), tableBlockInfo.getBlockOffset());
-    FileFooter footer = reader.readFooter();
-    dataFileFooter.setVersionId(ColumnarFormatVersion.valueOf((short) footer.getVersion()));
+    CarbonHeaderReader carbonHeaderReader = new CarbonHeaderReader(tableBlockInfo.getFilePath());
+    FileHeader fileHeader = carbonHeaderReader.readHeader();
+    CarbonFooterReaderV3 reader =
+        new CarbonFooterReaderV3(tableBlockInfo.getFilePath(), tableBlockInfo.getBlockOffset());
+    FileFooter3 footer = reader.readFooterVersion3();
+    dataFileFooter.setVersionId(ColumnarFormatVersion.valueOf((short) fileHeader.getVersion()));
     dataFileFooter.setNumberOfRows(footer.getNum_rows());
     dataFileFooter.setSegmentInfo(getSegmentInfo(footer.getSegment_info()));
     List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();
-    List<org.apache.carbondata.format.ColumnSchema> table_columns = footer.getTable_columns();
+    List<org.apache.carbondata.format.ColumnSchema> table_columns = fileHeader.getColumn_schema();
     for (int i = 0; i < table_columns.size(); i++) {
       columnSchemaList.add(thriftColumnSchmeaToWrapperColumnSchema(table_columns.get(i)));
     }
     dataFileFooter.setColumnInTable(columnSchemaList);
-
     List<org.apache.carbondata.format.BlockletIndex> leaf_node_indices_Thrift =
         footer.getBlocklet_index_list();
     List<BlockletIndex> blockletIndexList = new ArrayList<BlockletIndex>();
@@ -107,6 +111,7 @@ public class DataFileFooterConverterV3 extends AbstractDataFileFooterConverter
{
     blockletInfo.setNumberOfRows(blockletInfoThrift.getNum_rows());
     blockletInfo.setDimensionOffset(blockletInfoThrift.getDimension_offsets());
     blockletInfo.setMeasureOffsets(blockletInfoThrift.getMeasure_offsets());
+    blockletInfo.setNumberOfPages(blockletInfoThrift.getNumber_number_of_pages());
     return blockletInfo;
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b41e48f1/format/src/main/thrift/carbondata.thrift
----------------------------------------------------------------------
diff --git a/format/src/main/thrift/carbondata.thrift b/format/src/main/thrift/carbondata.thrift
index 3114ee1..198afde 100644
--- a/format/src/main/thrift/carbondata.thrift
+++ b/format/src/main/thrift/carbondata.thrift
@@ -166,6 +166,7 @@ struct BlockletInfo3{
     3: required list<i32> column_data_chunks_length;	// Information about length all
column chunks in this blocklet
     4: required i64 dimension_offsets;
     5: required i64 measure_offsets;
+    6: required i32 number_number_of_pages; // this is rquired for alter table, in case of
alter table when filter is only selected on new added column this will help
   }
 /**
 * Footer for indexed carbon file
@@ -178,8 +179,18 @@ struct FileFooter{
     5: required list<BlockletIndex> blocklet_index_list;	// blocklet index of all blocklets
in this file
     6: optional list<BlockletInfo> blocklet_info_list;	// Information about blocklets
of all columns in this file
     7: optional list<BlockletInfo2> blocklet_info_list2;	// Information about blocklets
of all columns in this file
-    8: optional list<BlockletInfo3> blocklet_info_list3;	// Information about blocklets
of all columns in this file
-    9: optional dictionary.ColumnDictionaryChunk dictionary; // blocklet local dictionary
+    8: optional dictionary.ColumnDictionaryChunk dictionary; // blocklet local dictionary
+}
+
+/**
+* Footer for indexed carbon file for V3 format
+*/
+struct FileFooter3{
+    1: required i64 num_rows; // Total number of rows in this file
+    2: required SegmentInfo segment_info;	// Segment info (will be same/repeated for all
files in this segment)
+    3: required list<BlockletIndex> blocklet_index_list;	// blocklet index of all blocklets
in this file
+    4: optional list<BlockletInfo3> blocklet_info_list3;	// Information about blocklets
of all columns in this file
+    5: optional dictionary.ColumnDictionaryChunk dictionary; // blocklet local dictionary
 }
 
 /**
@@ -187,7 +198,8 @@ struct FileFooter{
  */
 struct FileHeader{
 	1: required i32 version; // version used for data compatibility
-	2: required list<schema.ColumnSchema> table_columns;  // Description of columns in
this file
+	2: required list<schema.ColumnSchema> column_schema;  // Description of columns in
this file
+	3: optional bool is_footer_present; //  to check whether footer is present or not      
 }
 
 /**

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b41e48f1/format/src/main/thrift/carbondata_index.thrift
----------------------------------------------------------------------
diff --git a/format/src/main/thrift/carbondata_index.thrift b/format/src/main/thrift/carbondata_index.thrift
index 364a7e5..ef6fb18 100644
--- a/format/src/main/thrift/carbondata_index.thrift
+++ b/format/src/main/thrift/carbondata_index.thrift
@@ -41,6 +41,6 @@ struct IndexHeader{
 struct BlockIndex{
   1: required i64 num_rows; // Total number of rows in this file
   2: required string file_name; // Block file name
-  3: required i64 offset; // Offset of block
+  3: required i64 offset; // Offset of the footer
   4: required carbondata.BlockletIndex block_index;	// Block index
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b41e48f1/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 95a3ff4..cf37b50 100644
--- a/pom.xml
+++ b/pom.xml
@@ -93,7 +93,7 @@
 
   <modules>
     <module>common</module>
-    <module>core</module>
+	<module>core</module>
     <module>processing</module>
     <module>hadoop</module>
     <module>integration/spark-common</module>

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b41e48f1/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
index fa7bf27..49d7ab2 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
@@ -26,7 +26,6 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants;
 import org.apache.carbondata.core.constants.CarbonV3DataFormatConstants;
 import org.apache.carbondata.core.datastore.columnar.IndexStorage;
 import org.apache.carbondata.core.datastore.compression.WriterCompressModel;
-import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
 import org.apache.carbondata.core.metadata.blocklet.index.BlockletBTreeIndex;
 import org.apache.carbondata.core.metadata.blocklet.index.BlockletMinMaxIndex;
 import org.apache.carbondata.core.metadata.index.BlockIndexInfo;
@@ -35,9 +34,8 @@ import org.apache.carbondata.core.util.CarbonMetadataUtil;
 import org.apache.carbondata.core.util.CarbonProperties;
 import org.apache.carbondata.core.util.CarbonUtil;
 import org.apache.carbondata.core.util.NodeHolder;
-import org.apache.carbondata.core.writer.CarbonFooterWriter;
 import org.apache.carbondata.format.BlockletInfo3;
-import org.apache.carbondata.format.FileFooter;
+import org.apache.carbondata.format.FileFooter3;
 import org.apache.carbondata.processing.store.colgroup.ColGroupBlockStorage;
 import org.apache.carbondata.processing.store.writer.AbstractFactDataWriter;
 import org.apache.carbondata.processing.store.writer.CarbonDataWriterVo;
@@ -45,6 +43,10 @@ import org.apache.carbondata.processing.store.writer.exception.CarbonDataWriterE
 
 /**
  * Below class will be used to write the data in V3 format
+ * <Column1 Data ChunkV3><Column1<Page1><Page2><Page3><Page4>>
+ * <Column2 Data ChunkV3><Column2<Page1><Page2><Page3><Page4>>
+ * <Column3 Data ChunkV3><Column3<Page1><Page2><Page3><Page4>>
+ * <Column4 Data ChunkV3><Column4<Page1><Page2><Page3><Page4>>
  */
 public class CarbonFactDataWriterImplV3 extends AbstractFactDataWriter<short[]> {
 
@@ -61,8 +63,8 @@ public class CarbonFactDataWriterImplV3 extends AbstractFactDataWriter<short[]>
   public CarbonFactDataWriterImplV3(CarbonDataWriterVo dataWriterVo) {
     super(dataWriterVo);
     this.numberOfChunksInBlocklet = Integer.parseInt(CarbonProperties.getInstance()
-          .getProperty(CarbonV3DataFormatConstants.NUMBER_OF_PAGE_IN_BLOCKLET_COLUMN,
-              CarbonV3DataFormatConstants.NUMBER_OF_PAGE_IN_BLOCKLET_COLUMN_DEFAULT_VALUE));
+        .getProperty(CarbonV3DataFormatConstants.NUMBER_OF_PAGE_IN_BLOCKLET_COLUMN,
+            CarbonV3DataFormatConstants.NUMBER_OF_PAGE_IN_BLOCKLET_COLUMN_DEFAULT_VALUE));
     dataWriterHolder = new DataWriterHolder();
   }
 
@@ -239,15 +241,20 @@ public class CarbonFactDataWriterImplV3 extends AbstractFactDataWriter<short[]>
     try {
       // get the current file position
       long currentPosition = channel.size();
-      CarbonFooterWriter writer = new CarbonFooterWriter(filePath);
       // get thrift file footer instance
-      FileFooter convertFileMeta = CarbonMetadataUtil
-          .convertFileFooter3(blockletMetadata, blockletIndex, localCardinality,
-              thriftColumnSchemaList, dataWriterVo.getSegmentProperties());
+      FileFooter3 convertFileMeta = CarbonMetadataUtil
+          .convertFileFooterVersion3(blockletMetadata, blockletIndex, localCardinality,
+              thriftColumnSchemaList.size(), dataWriterVo.getSegmentProperties());
       // fill the carbon index details
       fillBlockIndexInfoDetails(convertFileMeta.getNum_rows(), filePath, currentPosition);
       // write the footer
-      writer.writeFooter(convertFileMeta, currentPosition);
+      byte[] byteArray = CarbonUtil.getByteArray(convertFileMeta);
+      ByteBuffer buffer =
+          ByteBuffer.allocate(byteArray.length + CarbonCommonConstants.LONG_SIZE_IN_BYTE);
+      buffer.put(byteArray);
+      buffer.putLong(currentPosition);
+      buffer.flip();
+      channel.write(buffer);
     } catch (IOException e) {
       throw new CarbonDataWriterException("Problem while writing the carbon file: ", e);
     }
@@ -319,11 +326,12 @@ public class CarbonFactDataWriterImplV3 extends AbstractFactDataWriter<short[]>
     // write the header
     try {
       if (fileChannel.size() == 0) {
-        ColumnarFormatVersion version = CarbonProperties.getInstance().getFormatVersion();
-        byte[] header = (CarbonCommonConstants.CARBON_DATA_VERSION_HEADER + version).getBytes();
-        ByteBuffer buffer = ByteBuffer.allocate(header.length);
-        buffer.put(header);
-        buffer.rewind();
+        // below code is to write the file header
+        byte[] fileHeader =
+            CarbonUtil.getByteArray(CarbonMetadataUtil.getFileHeader(true, thriftColumnSchemaList));
+        ByteBuffer buffer = ByteBuffer.allocate(fileHeader.length);
+        buffer.put(fileHeader);
+        buffer.flip();
         fileChannel.write(buffer);
       }
       offset = channel.size();
@@ -411,7 +419,7 @@ public class CarbonFactDataWriterImplV3 extends AbstractFactDataWriter<short[]>
         .getBlockletIndex(nodeHolderList, dataWriterVo.getSegmentProperties().getMeasures()));
     BlockletInfo3 blockletInfo3 =
         new BlockletInfo3(numberOfRows, currentDataChunksOffset, currentDataChunksLength,
-            dimensionOffset, measureOffset);
+            dimensionOffset, measureOffset, dataWriterHolder.getNumberOfPagesAdded());
     blockletMetadata.add(blockletInfo3);
   }
 


Mime
View raw message