carbondata-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jackylk <...@git.apache.org>
Subject [GitHub] incubator-carbondata pull request #584: [CARBONDATA-726] Handled query and s...
Date Thu, 23 Feb 2017 02:16:57 GMT
Github user jackylk commented on a diff in the pull request:

    https://github.com/apache/incubator-carbondata/pull/584#discussion_r102619669
  
    --- Diff: core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java
---
    @@ -56,51 +58,74 @@ public CompressedDimensionChunkFileBasedReaderV1(final BlockletInfo
blockletInfo
       }
     
       /**
    -   * Below method will be used to read the chunk based on block indexes
    +   * Below method will be used to read the raw chunk based on block indexes
        *
        * @param fileReader   file reader to read the blocks from file
        * @param blockIndexes blocks to be read
        * @return dimension column chunks
        */
    -  @Override public DimensionColumnDataChunk[] readDimensionChunks(FileHolder fileReader,
    +  @Override public DimensionRawColumnChunk[] readRawDimensionChunks(FileHolder fileReader,
           int[][] blockIndexes) throws IOException {
    -    // read the column chunk based on block index and add
    -    DimensionColumnDataChunk[] dataChunks =
    -        new DimensionColumnDataChunk[dimensionColumnChunk.size()];
    +    DimensionRawColumnChunk[] dataChunks = new DimensionRawColumnChunk[dimensionColumnChunk.size()];
         for (int i = 0; i < blockIndexes.length; i++) {
           for (int j = blockIndexes[i][0]; j <= blockIndexes[i][1]; j++) {
    -        dataChunks[j] = readDimensionChunk(fileReader, j);
    +        dataChunks[j] = readRawDimensionChunk(fileReader, j);
           }
         }
         return dataChunks;
       }
     
       /**
    -   * Below method will be used to read the chunk based on block index
    +   * Below method will be used to read the raw chunk based on block index
        *
        * @param fileReader file reader to read the blocks from file
        * @param blockIndex block to be read
        * @return dimension column chunk
        */
    -  @Override public DimensionColumnDataChunk readDimensionChunk(FileHolder fileReader,
    +  @Override public DimensionRawColumnChunk readRawDimensionChunk(FileHolder fileReader,
           int blockIndex) throws IOException {
    +    ByteBuffer buffer =
    +        ByteBuffer.allocateDirect(dimensionColumnChunk.get(blockIndex).getDataPageLength());
    +    synchronized (fileReader) {
    +      fileReader.readByteBuffer(filePath, buffer,
    +          dimensionColumnChunk.get(blockIndex).getDataPageOffset(),
    +          dimensionColumnChunk.get(blockIndex).getDataPageLength());
    +    }
    +    DimensionRawColumnChunk rawColumnChunk = new DimensionRawColumnChunk(blockIndex,
buffer, 0,
    +        dimensionColumnChunk.get(blockIndex).getDataPageLength(), this);
    +    rawColumnChunk.setFileHolder(fileReader);
    +    rawColumnChunk.setPagesCount(1);
    +    rawColumnChunk.setRowCount(new int[] { numberOfRows });
    +    return rawColumnChunk;
    +  }
    +
    +  @Override public DimensionColumnDataChunk convertToDimensionChunk(
    +      DimensionRawColumnChunk dimensionRawColumnChunk, int pageNumber) throws IOException
{
    +    int blockIndex = dimensionRawColumnChunk.getBlockId();
         byte[] dataPage = null;
         int[] invertedIndexes = null;
         int[] invertedIndexesReverse = null;
         int[] rlePage = null;
    +    FileHolder fileReader = dimensionRawColumnChunk.getFileReader();
    +
    +    ByteBuffer rawData = dimensionRawColumnChunk.getRawData();
    +    rawData.position(dimensionRawColumnChunk.getOffSet());
    +    byte[] data = new byte[dimensionRawColumnChunk.getLength()];
    +    rawData.get(data);
    +    dataPage = COMPRESSOR.unCompressByte(data);
     
    -    // first read the data and uncompressed it
    -    dataPage = COMPRESSOR.unCompressByte(fileReader
    -        .readByteArray(filePath, dimensionColumnChunk.get(blockIndex).getDataPageOffset(),
    -            dimensionColumnChunk.get(blockIndex).getDataPageLength()));
         // if row id block is present then read the row id chunk and uncompress it
         if (CarbonUtil.hasEncoding(dimensionColumnChunk.get(blockIndex).getEncodingList(),
    --- End diff --
    
     In this function, multiple place invoke `dimensionColumnChunk.get(blockIndex)`, can you
get it once and use it


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---

Mime
View raw message