carbondata-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jackylk <...@git.apache.org>
Subject [GitHub] carbondata pull request #1248: [CARBONDATA-1371] Support creating decoder ba...
Date Mon, 21 Aug 2017 04:59:07 GMT
Github user jackylk commented on a diff in the pull request:

    https://github.com/apache/carbondata/pull/1248#discussion_r134145643
  
    --- Diff: core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimensionChunkFileBasedReaderV3.java
---
    @@ -194,72 +202,118 @@ public DimensionRawColumnChunk readRawDimensionChunk(FileHolder
fileReader,
       /**
        * Below method will be used to convert the compressed dimension chunk raw data to
actual data
        *
    -   * @param dimensionRawColumnChunk dimension raw chunk
    +   * @param rawColumnPage dimension raw chunk
        * @param pageNumber              number
        * @return DimensionColumnDataChunk
        */
       @Override public DimensionColumnDataChunk convertToDimensionChunk(
    -      DimensionRawColumnChunk dimensionRawColumnChunk, int pageNumber) throws IOException
{
    -    byte[] dataPage = null;
    -    int[] invertedIndexes = null;
    -    int[] invertedIndexesReverse = null;
    -    int[] rlePage = null;
    -    // data chunk of page
    -    DataChunk2 dimensionColumnChunk = null;
    +      DimensionRawColumnChunk rawColumnPage, int pageNumber) throws IOException, MemoryException
{
         // data chunk of blocklet column
    -    DataChunk3 dataChunk3 = dimensionRawColumnChunk.getDataChunkV3();
    +    DataChunk3 dataChunk3 = rawColumnPage.getDataChunkV3();
         // get the data buffer
    -    ByteBuffer rawData = dimensionRawColumnChunk.getRawData();
    -    dimensionColumnChunk = dataChunk3.getData_chunk_list().get(pageNumber);
    +    ByteBuffer rawData = rawColumnPage.getRawData();
    +    DataChunk2 pageMetadata = dataChunk3.getData_chunk_list().get(pageNumber);
         // calculating the start point of data
         // as buffer can contain multiple column data, start point will be datachunkoffset
+
         // data chunk length + page offset
    -    int copySourcePoint = dimensionRawColumnChunk.getOffSet() + dimensionChunksLength
    -        .get(dimensionRawColumnChunk.getBlockletId()) + dataChunk3.getPage_offset().get(pageNumber);
    +    int offset = rawColumnPage.getOffSet() + dimensionChunksLength
    +        .get(rawColumnPage.getBlockletId()) + dataChunk3.getPage_offset().get(pageNumber);
         // first read the data and uncompressed it
    -    dataPage = COMPRESSOR
    -        .unCompressByte(rawData.array(), copySourcePoint, dimensionColumnChunk.data_page_length);
    -    copySourcePoint += dimensionColumnChunk.data_page_length;
    +    return decodeDimension(rawColumnPage, rawData, pageMetadata, offset);
    +  }
    +
    +  private DimensionColumnDataChunk decodeDimensionByMeta(DataChunk2 pageMetadata,
    +      ByteBuffer pageData, int offset)
    +      throws IOException, MemoryException {
    +    List<Encoding> encodings = pageMetadata.getEncoders();
    +    List<ByteBuffer> encoderMetas = pageMetadata.getEncoder_meta();
    +    assert (encodings.size() == 1);
    +    assert (encoderMetas.size() == 1);
    +    Encoding encoding = encodings.get(0);
    +    ColumnPageEncoderMeta metadata = null;
    +    ByteArrayInputStream stream = new ByteArrayInputStream(encoderMetas.get(0).array());
    +    DataInputStream in = new DataInputStream(stream);
    +    switch (encoding) {
    +      case DIRECT_COMPRESS:
    +        DirectCompressorEncoderMeta meta = new DirectCompressorEncoderMeta();
    --- End diff --
    
    ok, fixed


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---

Mime
View raw message