drill-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "ASF GitHub Bot (JIRA)" <j...@apache.org>
Subject [jira] [Commented] (DRILL-5356) Refactor Parquet Record Reader
Date Thu, 25 May 2017 21:42:04 GMT

    [ https://issues.apache.org/jira/browse/DRILL-5356?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16025436#comment-16025436
] 

ASF GitHub Bot commented on DRILL-5356:
---------------------------------------

Github user paul-rogers commented on a diff in the pull request:

    https://github.com/apache/drill/pull/789#discussion_r118590602
  
    --- Diff: exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetColumnMetadata.java
---
    @@ -0,0 +1,151 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements.  See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership.  The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License.  You may obtain a copy of the License at
    + *
    + * http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +package org.apache.drill.exec.store.parquet.columnreaders;
    +
    +import java.util.Map;
    +
    +import org.apache.drill.common.exceptions.ExecutionSetupException;
    +import org.apache.drill.common.expression.SchemaPath;
    +import org.apache.drill.common.types.TypeProtos;
    +import org.apache.drill.common.types.TypeProtos.DataMode;
    +import org.apache.drill.common.types.TypeProtos.MajorType;
    +import org.apache.drill.exec.exception.SchemaChangeException;
    +import org.apache.drill.exec.expr.TypeHelper;
    +import org.apache.drill.exec.physical.impl.OutputMutator;
    +import org.apache.drill.exec.record.MaterializedField;
    +import org.apache.drill.exec.server.options.OptionManager;
    +import org.apache.drill.exec.vector.ValueVector;
    +import org.apache.drill.exec.vector.complex.RepeatedValueVector;
    +import org.apache.parquet.column.ColumnDescriptor;
    +import org.apache.parquet.format.SchemaElement;
    +import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData;
    +import org.apache.parquet.schema.PrimitiveType;
    +import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
    +
    +/**
    + * Represents a single column read from the Parquet file by the record reader.
    + */
    +
    +public class ParquetColumnMetadata {
    +
    +  ColumnDescriptor column;
    +  private SchemaElement se;
    +  MaterializedField field;
    +  int length;
    +  private MajorType type;
    +  ColumnChunkMetaData columnChunkMetaData;
    +  private ValueVector vector;
    +
    +  public ParquetColumnMetadata(ColumnDescriptor column) {
    +    this.column = column;
    +  }
    +
    +  public void resolveDrillType(Map<String, SchemaElement> schemaElements, OptionManager
options) {
    +    se = schemaElements.get(column.getPath()[0]);
    +    type = ParquetToDrillTypeConverter.toMajorType(column.getType(), se.getType_length(),
    +        getDataMode(column), se, options);
    +    field = MaterializedField.create(toFieldName(column.getPath()), type);
    +    length = getDataTypeLength();
    +  }
    +
    +  private String toFieldName(String[] paths) {
    +    return SchemaPath.getCompoundPath(paths).getAsUnescapedPath();
    +  }
    +
    +  private TypeProtos.DataMode getDataMode(ColumnDescriptor column) {
    +    if (isRepeated()) {
    +      return DataMode.REPEATED;
    +    } else if (column.getMaxDefinitionLevel() == 0) {
    +      return TypeProtos.DataMode.REQUIRED;
    +    } else {
    +      return TypeProtos.DataMode.OPTIONAL;
    +    }
    +  }
    +
    +  /**
    +   * @param type
    +   * @param type a fixed length type from the parquet library enum
    +   * @return the length in pageDataByteArray of the type
    +   */
    +  public static int getTypeLengthInBits(PrimitiveTypeName type) {
    +    switch (type) {
    +      case INT64:   return 64;
    +      case INT32:   return 32;
    +      case BOOLEAN: return 1;
    +      case FLOAT:   return 32;
    +      case DOUBLE:  return 64;
    +      case INT96:   return 96;
    +      // binary and fixed length byte array
    +      default:
    +        throw new IllegalStateException("Length cannot be determined for type " + type);
    +    }
    +  }
    +
    +  /**
    +   * Returns data type length for a given {@see ColumnDescriptor} and it's corresponding
    +   * {@see SchemaElement}. Neither is enough information alone as the max
    +   * repetition level (indicating if it is an array type) is in the ColumnDescriptor
and
    +   * the length of a fixed width field is stored at the schema level.
    +   *
    +   * @return the length if fixed width, else -1
    +   */
    +  private int getDataTypeLength() {
    +    if (! isFixedLength()) {
    +      return -1;
    +    } else if (isRepeated()) {
    +      return -1;
    +    } else if (column.getType() == PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY)
{
    +      return se.getType_length() * 8;
    +    } else {
    +      return getTypeLengthInBits(column.getType());
    +    }
    +  }
    +
    +  public boolean isFixedLength( ) {
    +    return column.getType() != PrimitiveType.PrimitiveTypeName.BINARY;
    +  }
    +
    +  public boolean isRepeated() {
    +    return column.getMaxRepetitionLevel() > 0;
    +  }
    +
    +  ValueVector buildVector(OutputMutator output) throws SchemaChangeException {
    +    Class<? extends ValueVector> vectorClass = TypeHelper.getValueVectorClass(type.getMinorType(),
type.getMode());
    +    vector = output.addField(field, vectorClass);
    +    return vector;
    +  }
    +
    +  ColumnReader<?> makeFixedWidthReader(ParquetRecordReader reader, int recordsPerBatch)
throws Exception {
    +    return ColumnReaderFactory.createFixedColumnReader(reader, true,
    +        column, columnChunkMetaData, recordsPerBatch, vector, se);
    +  }
    +
    +  @SuppressWarnings("resource") FixedWidthRepeatedReader makeRepeatedFixedWidthReader(ParquetRecordReader
reader, int recordsPerBatch) throws Exception {
    +    final RepeatedValueVector repeatedVector = RepeatedValueVector.class.cast(vector);
    --- End diff --
    
    Fixed.


> Refactor Parquet Record Reader
> ------------------------------
>
>                 Key: DRILL-5356
>                 URL: https://issues.apache.org/jira/browse/DRILL-5356
>             Project: Apache Drill
>          Issue Type: Improvement
>    Affects Versions: 1.10.0, 1.11.0
>            Reporter: Paul Rogers
>            Assignee: Paul Rogers
>            Priority: Minor
>              Labels: ready-to-commit
>             Fix For: 1.11.0
>
>
> The Parquet record reader class is a key part of Drill that has evolved over time to
become somewhat hard to follow.
> A number of us are working on Parquet-related tasks and find we have to spend an uncomfortable
amount of time trying to understand the code. In particular, this writer needs to figure out
how to convince the reader to provide higher-density record batches.
> Rather than continue to decypher the complex code multiple times, this ticket requests
to refactor the code to make it functionally identical, but structurally cleaner. The result
will be faster time to value when working with this code.
> This is a lower-priority change and will be coordinated with others working on this code
base. This ticket is only for the record reader class itself; it does not include the various
readers and writers that Parquet uses since another project is actively modifying those classes.



--
This message was sent by Atlassian JIRA
(v6.3.15#6346)

Mime
View raw message