carbondata-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jackylk <...@git.apache.org>
Subject [GitHub] carbondata pull request #1265: [CARBONDATA-1128] Add encoding for non-dictio...
Date Fri, 08 Sep 2017 14:50:16 GMT
Github user jackylk commented on a diff in the pull request:

    https://github.com/apache/carbondata/pull/1265#discussion_r137809952
  
    --- Diff: core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/ColumnPageWrapper.java
---
    @@ -17,46 +17,183 @@
     
     package org.apache.carbondata.core.datastore.chunk.store;
     
    +import java.nio.ByteBuffer;
    +import java.util.BitSet;
    +
    +import org.apache.carbondata.core.constants.CarbonCommonConstants;
    +import org.apache.carbondata.core.datastore.ColumnType;
    +import org.apache.carbondata.core.datastore.TableSpec;
     import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
     import org.apache.carbondata.core.datastore.page.ColumnPage;
    +import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
    +import org.apache.carbondata.core.metadata.datatype.DataType;
     import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
    +import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
     import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
    +import org.apache.carbondata.core.util.ByteUtil;
     
    +/**
    + * ColumnPage wrapper for dimension column reader
    + */
     public class ColumnPageWrapper implements DimensionColumnDataChunk {
     
    +  private TableSpec.ColumnSpec columnSpec;
       private ColumnPage columnPage;
    +  private int columnValueSize;
     
    -  public ColumnPageWrapper(ColumnPage columnPage) {
    +  public ColumnPageWrapper(ColumnPage columnPage, int columnValueSize) {
         this.columnPage = columnPage;
    +    this.columnValueSize = columnValueSize;
    +    this.columnSpec = columnPage.getColumnSpec();
       }
     
       @Override
       public int fillChunkData(byte[] data, int offset, int columnIndex,
           KeyStructureInfo restructuringInfo) {
    -    throw new UnsupportedOperationException("internal error");
    +    int surrogate = columnPage.getInt(columnIndex);
    +    ByteBuffer buffer = ByteBuffer.wrap(data);
    +    buffer.putInt(offset, surrogate);
    +    return columnValueSize;
       }
     
       @Override
       public int fillConvertedChunkData(int rowId, int columnIndex, int[] row,
           KeyStructureInfo restructuringInfo) {
    -    throw new UnsupportedOperationException("internal error");
    +    row[columnIndex] = columnPage.getInt(rowId);
    +    return columnIndex + 1;
       }
     
       @Override
       public int fillConvertedChunkData(ColumnVectorInfo[] vectorInfo, int column,
           KeyStructureInfo restructuringInfo) {
    -    throw new UnsupportedOperationException("internal error");
    +    // fill the vector with data in column page
    +    ColumnVectorInfo columnVectorInfo = vectorInfo[column];
    +    CarbonColumnVector vector = columnVectorInfo.vector;
    +    fillData(null, columnVectorInfo, vector);
    +    return column + 1;
       }
     
       @Override
       public int fillConvertedChunkData(int[] rowMapping, ColumnVectorInfo[] vectorInfo,
int column,
           KeyStructureInfo restructuringInfo) {
    -    throw new UnsupportedOperationException("internal error");
    +    ColumnVectorInfo columnVectorInfo = vectorInfo[column];
    +    CarbonColumnVector vector = columnVectorInfo.vector;
    +    fillData(rowMapping, columnVectorInfo, vector);
    +    return column + 1;
    +  }
    +
    +  /**
    +   * Copy columnar data from internal column page to `vector`, row information described
by
    +   * `columnVectorInfo`
    +   */
    +  private void fillData(int[] rowMapping, ColumnVectorInfo columnVectorInfo,
    --- End diff --
    
    For dictionary column, it still using `IndexStorageCodec` to do encoding and using `decodeDimensionLegacy`
to decode, so `FixedLengthDimensionDataChunk` is still required. `VariableLengthDimensionDataChunk`
is also required for backward compatiblity to read old data file


---

Mime
View raw message