drill-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "ASF GitHub Bot (JIRA)" <j...@apache.org>
Subject [jira] [Commented] (DRILL-4982) Hive Queries degrade when queries switch between different formats
Date Thu, 08 Dec 2016 05:41:59 GMT

    [ https://issues.apache.org/jira/browse/DRILL-4982?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15731188#comment-15731188
] 

ASF GitHub Bot commented on DRILL-4982:
---------------------------------------

Github user paul-rogers commented on a diff in the pull request:

    https://github.com/apache/drill/pull/638#discussion_r91447651
  
    --- Diff: contrib/storage-hive/core/src/main/codegen/templates/HiveRecordReaders.java
---
    @@ -0,0 +1,300 @@
    +/**
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements.  See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership.  The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License.  You may obtain a copy of the License at
    + *
    + * http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +
    +/**
    + * This template is used to generate different Hive record reader classes for different
data formats
    + * to avoid JIT profile pullusion. These readers are derived from HiveAbstractReader
which implements
    + * codes for init and setup stage, but the repeated - and performance critical part -
next() method is
    + * separately implemented in the classes generated from this template. The internal SkipRecordReeader
    + * class is also separated as well due to the same reason.
    + *
    + * As to the performance gain with this change, please refer to:
    + * https://issues.apache.org/jira/browse/DRILL-4982
    + *
    + */
    +<@pp.dropOutputFile />
    +<#list hiveFormat.map as entry>
    +<@pp.changeOutputFile name="/org/apache/drill/exec/store/hive/Hive${entry.hiveReader}Reader.java"
/>
    +<#include "/@includes/license.ftl" />
    +
    +package org.apache.drill.exec.store.hive;
    +
    +import java.io.IOException;
    +import java.util.List;
    +import java.util.Properties;
    +import org.apache.drill.common.exceptions.DrillRuntimeException;
    +import org.apache.drill.common.exceptions.ExecutionSetupException;
    +import org.apache.drill.common.expression.SchemaPath;
    +import org.apache.drill.exec.ops.FragmentContext;
    +import org.apache.drill.exec.vector.AllocationHelper;
    +import org.apache.drill.exec.vector.ValueVector;
    +import org.apache.hadoop.hive.metastore.api.Partition;
    +import org.apache.hadoop.hive.metastore.api.Table;
    +import org.apache.hadoop.io.Writable;
    +import org.apache.hadoop.mapred.InputSplit;
    +import org.apache.hadoop.security.UserGroupInformation;
    +import org.apache.hadoop.hive.conf.HiveConf;
    +
    +import org.apache.hadoop.hive.serde2.SerDeException;
    +
    +import org.apache.hadoop.mapred.RecordReader;
    +<#if entry.hasHeaderFooter == true>
    +import org.apache.hadoop.hive.serde2.SerDe;
    +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
    +import com.google.common.collect.Lists;
    +import java.util.ArrayList;
    +import java.util.Arrays;
    +import java.util.HashSet;
    +import java.util.Queue;
    +import java.util.Set;
    +import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
    +import org.apache.hadoop.hive.serde.serdeConstants;
    +</#if>
    +
    +public class Hive${entry.hiveReader}Reader extends HiveAbstractReader {
    +
    +  Object key;
    +<#if entry.hasHeaderFooter == true>
    +  SkipRecordsInspector skipRecordsInspector;
    +<#else>
    +  Object value;
    +</#if>
    +
    +  public Hive${entry.hiveReader}Reader(Table table, Partition partition, InputSplit inputSplit,
List<SchemaPath> projectedColumns,
    +                       FragmentContext context, final HiveConf hiveConf,
    +                       UserGroupInformation proxyUgi) throws ExecutionSetupException
{
    +    super(table, partition, inputSplit, projectedColumns, context, hiveConf, proxyUgi);
    +  }
    +
    +  public  void internalInit(Properties tableProperties, RecordReader<Object, Object>
reader) {
    +
    +    key = reader.createKey();
    +<#if entry.hasHeaderFooter == true>
    +    skipRecordsInspector = new SkipRecordsInspector(tableProperties, reader);
    +<#else>
    +    value = reader.createValue();
    +</#if>
    +
    +  }
    +  private void readHiveRecordAndInsertIntoRecordBatch(Object deSerializedValue, int outputRecordIndex)
{
    +    for (int i = 0; i < selectedStructFieldRefs.size(); i++) {
    +      Object hiveValue = finalOI.getStructFieldData(deSerializedValue, selectedStructFieldRefs.get(i));
    +      if (hiveValue != null) {
    +        selectedColumnFieldConverters.get(i).setSafeValue(selectedColumnObjInspectors.get(i),
hiveValue,
    +          vectors.get(i), outputRecordIndex);
    +      }
    +    }
    +  }
    +
    +<#if entry.hasHeaderFooter == true>
    +  @Override
    +  public int next() {
    +    for (ValueVector vv : vectors) {
    +      AllocationHelper.allocateNew(vv, TARGET_RECORD_COUNT);
    +    }
    +    if (empty) {
    +      setValueCountAndPopulatePartitionVectors(0);
    +      return 0;
    +    }
    +
    +    try {
    +      skipRecordsInspector.reset();
    +      Object value;
    +
    +      int recordCount = 0;
    +
    +      while (recordCount < TARGET_RECORD_COUNT && reader.next(key, value =
skipRecordsInspector.getNextValue())) {
    +        if (skipRecordsInspector.doSkipHeader(recordCount++)) {
    +          continue;
    +        }
    +        Object bufferedValue = skipRecordsInspector.bufferAdd(value);
    +        if (bufferedValue != null) {
    +          Object deSerializedValue = partitionSerDe.deserialize((Writable) bufferedValue);
    +          if (partTblObjectInspectorConverter != null) {
    +            deSerializedValue = partTblObjectInspectorConverter.convert(deSerializedValue);
    +          }
    +          readHiveRecordAndInsertIntoRecordBatch(deSerializedValue, skipRecordsInspector.getActualCount());
    +          skipRecordsInspector.incrementActualCount();
    +        }
    +        skipRecordsInspector.incrementTempCount();
    +      }
    +
    +      setValueCountAndPopulatePartitionVectors(skipRecordsInspector.getActualCount());
    +      skipRecordsInspector.updateContinuance();
    +      return skipRecordsInspector.getActualCount();
    +    } catch (IOException | SerDeException e) {
    +      throw new DrillRuntimeException(e);
    +    }
    +  }
    +
    +/**
    + * SkipRecordsInspector encapsulates logic to skip header and footer from file.
    + * Logic is applicable only for predefined in constructor file formats.
    + */
    +protected class SkipRecordsInspector {
    --- End diff --
    
    Small point, but nested classes are usually indented one level to help highlight that
they are, in fact, nested.


> Hive Queries degrade when queries switch between different formats
> ------------------------------------------------------------------
>
>                 Key: DRILL-4982
>                 URL: https://issues.apache.org/jira/browse/DRILL-4982
>             Project: Apache Drill
>          Issue Type: Bug
>            Reporter: Chunhui Shi
>            Assignee: Karthikeyan Manivannan
>            Priority: Critical
>             Fix For: 1.10.0
>
>
> We have seen degraded performance by doing these steps:
> 1) generate the repro data:
> python script repro.py as below:
> import string
> import random
>  
> for i in range(30000000):
>     x1 = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(random.randrange(19,
27)))
>     x2 = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(random.randrange(19,
27)))
>     x3 = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(random.randrange(19,
27)))
>     x4 = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(random.randrange(19,
27)))
>     x5 = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(random.randrange(19,
27)))
>     x6 = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(random.randrange(19,
27)))
>     print "{0}".format(x1),"{0}".format(x2),"{0}".format(x3),"{0}".format(x4),"{0}".format(x5),"{0}".format(x6)
> python repro.py > repro.csv
> 2) put these files in a dfs directory e.g. '/tmp/hiveworkspace/plain'. Under hive prompt,
use the following sql command to create an external table:
> CREATE EXTERNAL TABLE `hiveworkspace`.`plain` (`id1` string, `id2` string, `id3` string,
`id4` string, `id5` string, `id6` string) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
STORED AS TEXTFILE LOCATION '/tmp/hiveworkspace/plain'
> 3) create Hive's table of ORC|PARQUET format:
> CREATE TABLE `hiveworkspace`.`plainorc` STORED AS ORC AS SELECT id1,id2,id3,id4,id5,id6
from `hiveworkspace`.`plain`;
> CREATE TABLE `hiveworkspace`.`plainparquet` STORED AS PARQUET AS SELECT id1,id2,id3,id4,id5,id6
from `hiveworkspace`.`plain`;
> 4) Query switch between these two tables, then the query time on the same table significantly
lengthened. On my setup, for ORC, it was 15sec -> 26secs. Queries on table of other formats,
after injecting a query to other formats, all have significant slow down.



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

Mime
View raw message