hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ser...@apache.org
Subject [28/51] [abbrv] hive git commit: HIVE-12000 : LLAP: Merge master into branch (Sergey Shelukhin)
Date Tue, 13 Oct 2015 00:12:53 GMT
HIVE-12000 : LLAP: Merge master into branch (Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f272ccb2
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f272ccb2
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f272ccb2

Branch: refs/heads/master
Commit: f272ccb25bc495d600a713058f6c2082ebd6d966
Parents: 7273a4c 064e37c
Author: Sergey Shelukhin <sershe@apache.org>
Authored: Wed Sep 30 12:22:32 2015 -0700
Committer: Sergey Shelukhin <sershe@apache.org>
Committed: Wed Sep 30 12:22:32 2015 -0700

----------------------------------------------------------------------
 .../common/metrics/common/MetricsConstant.java  |   1 +
 .../hive/hcatalog/templeton/AppConfig.java      |  21 +++
 .../apache/hive/hcatalog/templeton/Server.java  |  12 +-
 itests/qtest/pom.xml                            |   2 +-
 .../test/resources/testconfiguration.properties |  27 ++-
 .../hadoop/hive/ql/io/orc/MetadataReader.java   |   2 +-
 .../hive/ql/io/orc/MetadataReaderImpl.java      |  20 ++-
 .../hadoop/hive/ql/io/orc/OrcInputFormat.java   |  49 ++++++
 .../hadoop/hive/ql/io/orc/ReaderImpl.java       |  12 +-
 .../hadoop/hive/ql/io/orc/RecordReaderImpl.java |  35 ++++
 .../hive/ql/io/orc/RecordReaderUtils.java       |   5 +-
 .../hive/ql/optimizer/StatsOptimizer.java       |  46 ++++-
 .../ql/optimizer/calcite/RelOptHiveTable.java   |  15 ++
 .../ql/optimizer/physical/Vectorizer.java.rej   |  86 ----------
 .../hadoop/hive/ql/session/SessionState.java    |   9 +-
 .../test/queries/clientpositive/add_jar_pfile.q |   8 +
 .../clientpositive/metadata_only_queries.q      |  15 ++
 .../results/clientpositive/add_jar_pfile.q.out  |  12 ++
 .../clientpositive/metadata_only_queries.q.out  | 158 +++++++++++++++++
 .../spark/metadata_only_queries.q.out           | 170 +++++++++++++++++++
 .../tez/metadata_only_queries.q.out             | 170 +++++++++++++++++++
 .../tez/vector_groupby_reduce.q.out             |  70 +++++---
 .../clientpositive/vector_groupby_reduce.q.out  |  69 +++++---
 .../hive/service/cli/operation/Operation.java   |  11 ++
 .../service/cli/operation/OperationManager.java |  11 ++
 .../service/cli/session/HiveSessionProxy.java   |   6 +
 .../service/cli/session/SessionManager.java     |   6 +-
 .../thrift/EmbeddedThriftBinaryCLIService.java  |   2 +-
 .../thrift/ThreadPoolExecutorWithOomHook.java   |  55 ++++++
 .../cli/thrift/ThriftBinaryCLIService.java      |  12 +-
 .../service/cli/thrift/ThriftCLIService.java    |   3 +
 .../cli/thrift/ThriftHttpCLIService.java        |  10 +-
 .../apache/hive/service/server/HiveServer2.java |  12 +-
 .../hive/service/auth/TestPlainSaslHelper.java  |   2 +-
 .../session/TestPluggableHiveSessionImpl.java   |   2 +-
 .../cli/session/TestSessionGlobalInitFile.java  |   2 +-
 36 files changed, 957 insertions(+), 191 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/f272ccb2/itests/qtest/pom.xml
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/f272ccb2/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/f272ccb2/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReader.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReader.java
index 35cc05f,43d2933..cea324c
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReader.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReader.java
@@@ -18,17 -18,102 +18,17 @@@
  package org.apache.hadoop.hive.ql.io.orc;
  
  import java.io.IOException;
 -import java.nio.ByteBuffer;
 -import java.util.List;
  
 -import org.apache.hadoop.fs.FSDataInputStream;
 -import org.apache.hadoop.fs.FileSystem;
 -import org.apache.hadoop.fs.Path;
 -import org.apache.hadoop.hive.common.DiskRange;
 -import org.apache.hadoop.hive.ql.io.orc.RecordReaderImpl.BufferChunk;
 +import org.apache.hadoop.hive.ql.io.orc.OrcProto.BloomFilterIndex;
 +import org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndex;
 +import org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeFooter;
  
 -import com.google.common.collect.Lists;
 +public interface MetadataReader {
 +  RecordReaderImpl.Index readRowIndex(StripeInformation stripe, StripeFooter footer,
 +      boolean[] included, RowIndex[] indexes, boolean[] sargColumns,
 +      BloomFilterIndex[] bloomFilterIndices) throws IOException;
  
 -public class MetadataReader {
 -  private final FSDataInputStream file;
 -  private final CompressionCodec codec;
 -  private final int bufferSize;
 -  private final int typeCount;
 +  StripeFooter readStripeFooter(StripeInformation stripe) throws IOException;
  
 -  public MetadataReader(FileSystem fileSystem, Path path,
 -      CompressionCodec codec, int bufferSize, int typeCount) throws IOException {
 -    this(fileSystem.open(path), codec, bufferSize, typeCount);
 -  }
 -
 -  public MetadataReader(FSDataInputStream file,
 -      CompressionCodec codec, int bufferSize, int typeCount) {
 -    this.file = file;
 -    this.codec = codec;
 -    this.bufferSize = bufferSize;
 -    this.typeCount = typeCount;
 -  }
 -
 -  public RecordReaderImpl.Index readRowIndex(StripeInformation stripe, OrcProto.StripeFooter
footer,
 -      boolean[] included, OrcProto.RowIndex[] indexes, boolean[] sargColumns,
 -      OrcProto.BloomFilterIndex[] bloomFilterIndices) throws IOException {
 -    if (footer == null) {
 -      footer = readStripeFooter(stripe);
 -    }
 -    if (indexes == null) {
 -      indexes = new OrcProto.RowIndex[typeCount];
 -    }
 -    if (bloomFilterIndices == null) {
 -      bloomFilterIndices = new OrcProto.BloomFilterIndex[typeCount];
 -    }
 -    long offset = stripe.getOffset();
 -    List<OrcProto.Stream> streams = footer.getStreamsList();
 -    for (int i = 0; i < streams.size(); i++) {
 -      OrcProto.Stream stream = streams.get(i);
 -      OrcProto.Stream nextStream = null;
 -      if (i < streams.size() - 1) {
 -        nextStream = streams.get(i+1);
 -      }
 -      int col = stream.getColumn();
 -      int len = (int) stream.getLength();
 -      // row index stream and bloom filter are interlaced, check if the sarg column contains
bloom
 -      // filter and combine the io to read row index and bloom filters for that column together
 -      if (stream.hasKind() && (stream.getKind() == OrcProto.Stream.Kind.ROW_INDEX))
{
 -        boolean readBloomFilter = false;
 -        if (sargColumns != null && sargColumns[col] &&
 -            nextStream.getKind() == OrcProto.Stream.Kind.BLOOM_FILTER) {
 -          len += nextStream.getLength();
 -          i += 1;
 -          readBloomFilter = true;
 -        }
 -        if ((included == null || included[col]) && indexes[col] == null) {
 -          byte[] buffer = new byte[len];
 -          file.readFully(offset, buffer, 0, buffer.length);
 -          ByteBuffer[] bb = new ByteBuffer[] {ByteBuffer.wrap(buffer)};
 -          indexes[col] = OrcProto.RowIndex.parseFrom(InStream.create("index",
 -              bb, new long[]{0}, stream.getLength(), codec, bufferSize));
 -          if (readBloomFilter) {
 -            bb[0].position((int) stream.getLength());
 -            bloomFilterIndices[col] = OrcProto.BloomFilterIndex.parseFrom(
 -                InStream.create("bloom_filter", bb, new long[]{0}, nextStream.getLength(),
 -                    codec, bufferSize));
 -          }
 -        }
 -      }
 -      offset += len;
 -    }
 -
 -    RecordReaderImpl.Index index = new RecordReaderImpl.Index(indexes, bloomFilterIndices);
 -    return index;
 -  }
 -
 -  public OrcProto.StripeFooter readStripeFooter(StripeInformation stripe) throws IOException
{
 -    long offset = stripe.getOffset() + stripe.getIndexLength() + stripe.getDataLength();
 -    int tailLength = (int) stripe.getFooterLength();
 -
 -    // read the footer
 -    ByteBuffer tailBuf = ByteBuffer.allocate(tailLength);
 -    file.readFully(offset, tailBuf.array(), tailBuf.arrayOffset(), tailLength);
 -    return OrcProto.StripeFooter.parseFrom(InStream.create("footer",
 -        Lists.<DiskRange>newArrayList(new BufferChunk(tailBuf, 0)),
 -        tailLength, codec, bufferSize));
 -  }
 -
 -  public void close() throws IOException {
 -    file.close();
 -  }
 +  void close() throws IOException;
- }
+ }

http://git-wip-us.apache.org/repos/asf/hive/blob/f272ccb2/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReaderImpl.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReaderImpl.java
index 83594f7,0000000..1456df3
mode 100644,000000..100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReaderImpl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReaderImpl.java
@@@ -1,119 -1,0 +1,123 @@@
 +/**
 + * Licensed to the Apache Software Foundation (ASF) under one
 + * or more contributor license agreements.  See the NOTICE file
 + * distributed with this work for additional information
 + * regarding copyright ownership.  The ASF licenses this file
 + * to you under the Apache License, Version 2.0 (the
 + * "License"); you may not use this file except in compliance
 + * with the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.hadoop.hive.ql.io.orc;
 +
 +import java.io.IOException;
 +import java.nio.ByteBuffer;
 +import java.util.List;
 +
 +import org.apache.hadoop.fs.FSDataInputStream;
 +import org.apache.hadoop.fs.FileSystem;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.hadoop.hive.common.io.DiskRange;
 +import org.apache.hadoop.hive.ql.io.orc.RecordReaderImpl.BufferChunk;
 +
 +import com.google.common.collect.Lists;
 +
 +public class MetadataReaderImpl implements MetadataReader {
 +  private final FSDataInputStream file;
 +  private final CompressionCodec codec;
 +  private final int bufferSize;
 +  private final int typeCount;
 +
-   public MetadataReaderImpl(FileSystem fileSystem, Path path, CompressionCodec codec,
-       int bufferSize, int typeCount) throws IOException {
-     this.file = fileSystem.open(path);
++  public MetadataReaderImpl(FileSystem fileSystem, Path path,
++      CompressionCodec codec, int bufferSize, int typeCount) throws IOException {
++    this(fileSystem.open(path), codec, bufferSize, typeCount);
++  }
++
++  public MetadataReaderImpl(FSDataInputStream file,
++      CompressionCodec codec, int bufferSize, int typeCount) {
++    this.file = file;
 +    this.codec = codec;
 +    this.bufferSize = bufferSize;
 +    this.typeCount = typeCount;
 +  }
 +
 +  @Override
 +  public RecordReaderImpl.Index readRowIndex(StripeInformation stripe,
 +      OrcProto.StripeFooter footer, boolean[] included, OrcProto.RowIndex[] indexes,
 +      boolean[] sargColumns, OrcProto.BloomFilterIndex[] bloomFilterIndices) throws IOException
{
 +    if (footer == null) {
 +      footer = readStripeFooter(stripe);
 +    }
 +    if (indexes == null) {
 +      indexes = new OrcProto.RowIndex[typeCount];
 +    }
 +    if (bloomFilterIndices == null) {
 +      bloomFilterIndices = new OrcProto.BloomFilterIndex[typeCount];
 +    }
 +    long offset = stripe.getOffset();
 +    List<OrcProto.Stream> streams = footer.getStreamsList();
 +    for (int i = 0; i < streams.size(); i++) {
 +      OrcProto.Stream stream = streams.get(i);
 +      OrcProto.Stream nextStream = null;
 +      if (i < streams.size() - 1) {
 +        nextStream = streams.get(i+1);
 +      }
 +      int col = stream.getColumn();
 +      int len = (int) stream.getLength();
 +      // row index stream and bloom filter are interlaced, check if the sarg column contains
bloom
 +      // filter and combine the io to read row index and bloom filters for that column together
 +      if (stream.hasKind() && (stream.getKind() == OrcProto.Stream.Kind.ROW_INDEX))
{
 +        boolean readBloomFilter = false;
 +        if (sargColumns != null && sargColumns[col] &&
 +            nextStream.getKind() == OrcProto.Stream.Kind.BLOOM_FILTER) {
 +          len += nextStream.getLength();
 +          i += 1;
 +          readBloomFilter = true;
 +        }
 +        if ((included == null || included[col]) && indexes[col] == null) {
 +          byte[] buffer = new byte[len];
++          file.readFully(offset, buffer, 0, buffer.length);
 +          ByteBuffer bb = ByteBuffer.wrap(buffer);
-           file.seek(offset);
-           file.readFully(buffer);
 +          indexes[col] = OrcProto.RowIndex.parseFrom(InStream.create(null, "index",
 +              Lists.<DiskRange>newArrayList(new BufferChunk(bb, 0)), stream.getLength(),
-               codec, bufferSize));
++               codec, bufferSize));
 +          if (readBloomFilter) {
 +            bb.position((int) stream.getLength());
 +            bloomFilterIndices[col] = OrcProto.BloomFilterIndex.parseFrom(InStream.create(
 +                null, "bloom_filter", Lists.<DiskRange>newArrayList(new BufferChunk(bb,
0)),
 +                nextStream.getLength(), codec, bufferSize));
 +          }
 +        }
 +      }
 +      offset += len;
 +    }
 +
 +    RecordReaderImpl.Index index = new RecordReaderImpl.Index(indexes, bloomFilterIndices);
 +    return index;
 +  }
 +
 +  @Override
 +  public OrcProto.StripeFooter readStripeFooter(StripeInformation stripe) throws IOException
{
 +    long offset = stripe.getOffset() + stripe.getIndexLength() + stripe.getDataLength();
 +    int tailLength = (int) stripe.getFooterLength();
++
 +    // read the footer
 +    ByteBuffer tailBuf = ByteBuffer.allocate(tailLength);
-     file.seek(offset);
-     file.readFully(tailBuf.array(), tailBuf.arrayOffset(), tailLength);
++    file.readFully(offset, tailBuf.array(), tailBuf.arrayOffset(), tailLength);
 +    return OrcProto.StripeFooter.parseFrom(InStream.create(null, "footer",
 +        Lists.<DiskRange>newArrayList(new BufferChunk(tailBuf, 0)),
 +        tailLength, codec, bufferSize));
 +  }
 +
 +  @Override
 +  public void close() throws IOException {
 +    file.close();
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/hive/blob/f272ccb2/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/f272ccb2/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/f272ccb2/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/f272ccb2/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderUtils.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/f272ccb2/ql/src/test/results/clientpositive/tez/vector_groupby_reduce.q.out
----------------------------------------------------------------------


Mime
View raw message