Return-Path: X-Original-To: apmail-hive-commits-archive@www.apache.org Delivered-To: apmail-hive-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 560FF18985 for ; Tue, 13 Oct 2015 00:12:32 +0000 (UTC) Received: (qmail 42986 invoked by uid 500); 13 Oct 2015 00:12:28 -0000 Delivered-To: apmail-hive-commits-archive@hive.apache.org Received: (qmail 42925 invoked by uid 500); 13 Oct 2015 00:12:28 -0000 Mailing-List: contact commits-help@hive.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hive-dev@hive.apache.org Delivered-To: mailing list commits@hive.apache.org Received: (qmail 40933 invoked by uid 99); 13 Oct 2015 00:12:27 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 13 Oct 2015 00:12:27 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id A3320E07EE; Tue, 13 Oct 2015 00:12:26 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: sershe@apache.org To: commits@hive.apache.org Date: Tue, 13 Oct 2015 00:12:53 -0000 Message-Id: <526d5eccbfab41ac8a1336278e91a544@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [28/51] [abbrv] hive git commit: HIVE-12000 : LLAP: Merge master into branch (Sergey Shelukhin) HIVE-12000 : LLAP: Merge master into branch (Sergey Shelukhin) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f272ccb2 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f272ccb2 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f272ccb2 Branch: refs/heads/master Commit: f272ccb25bc495d600a713058f6c2082ebd6d966 Parents: 7273a4c 064e37c Author: Sergey Shelukhin Authored: Wed Sep 30 12:22:32 2015 -0700 Committer: Sergey Shelukhin Committed: Wed Sep 30 12:22:32 2015 -0700 ---------------------------------------------------------------------- .../common/metrics/common/MetricsConstant.java | 1 + .../hive/hcatalog/templeton/AppConfig.java | 21 +++ .../apache/hive/hcatalog/templeton/Server.java | 12 +- itests/qtest/pom.xml | 2 +- .../test/resources/testconfiguration.properties | 27 ++- .../hadoop/hive/ql/io/orc/MetadataReader.java | 2 +- .../hive/ql/io/orc/MetadataReaderImpl.java | 20 ++- .../hadoop/hive/ql/io/orc/OrcInputFormat.java | 49 ++++++ .../hadoop/hive/ql/io/orc/ReaderImpl.java | 12 +- .../hadoop/hive/ql/io/orc/RecordReaderImpl.java | 35 ++++ .../hive/ql/io/orc/RecordReaderUtils.java | 5 +- .../hive/ql/optimizer/StatsOptimizer.java | 46 ++++- .../ql/optimizer/calcite/RelOptHiveTable.java | 15 ++ .../ql/optimizer/physical/Vectorizer.java.rej | 86 ---------- .../hadoop/hive/ql/session/SessionState.java | 9 +- .../test/queries/clientpositive/add_jar_pfile.q | 8 + .../clientpositive/metadata_only_queries.q | 15 ++ .../results/clientpositive/add_jar_pfile.q.out | 12 ++ .../clientpositive/metadata_only_queries.q.out | 158 +++++++++++++++++ .../spark/metadata_only_queries.q.out | 170 +++++++++++++++++++ .../tez/metadata_only_queries.q.out | 170 +++++++++++++++++++ .../tez/vector_groupby_reduce.q.out | 70 +++++--- .../clientpositive/vector_groupby_reduce.q.out | 69 +++++--- .../hive/service/cli/operation/Operation.java | 11 ++ .../service/cli/operation/OperationManager.java | 11 ++ .../service/cli/session/HiveSessionProxy.java | 6 + .../service/cli/session/SessionManager.java | 6 +- .../thrift/EmbeddedThriftBinaryCLIService.java | 2 +- .../thrift/ThreadPoolExecutorWithOomHook.java | 55 ++++++ .../cli/thrift/ThriftBinaryCLIService.java | 12 +- .../service/cli/thrift/ThriftCLIService.java | 3 + .../cli/thrift/ThriftHttpCLIService.java | 10 +- .../apache/hive/service/server/HiveServer2.java | 12 +- .../hive/service/auth/TestPlainSaslHelper.java | 2 +- .../session/TestPluggableHiveSessionImpl.java | 2 +- .../cli/session/TestSessionGlobalInitFile.java | 2 +- 36 files changed, 957 insertions(+), 191 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/f272ccb2/itests/qtest/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/f272ccb2/itests/src/test/resources/testconfiguration.properties ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/f272ccb2/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReader.java ---------------------------------------------------------------------- diff --cc ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReader.java index 35cc05f,43d2933..cea324c --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReader.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReader.java @@@ -18,17 -18,102 +18,17 @@@ package org.apache.hadoop.hive.ql.io.orc; import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.List; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.common.DiskRange; -import org.apache.hadoop.hive.ql.io.orc.RecordReaderImpl.BufferChunk; +import org.apache.hadoop.hive.ql.io.orc.OrcProto.BloomFilterIndex; +import org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndex; +import org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeFooter; -import com.google.common.collect.Lists; +public interface MetadataReader { + RecordReaderImpl.Index readRowIndex(StripeInformation stripe, StripeFooter footer, + boolean[] included, RowIndex[] indexes, boolean[] sargColumns, + BloomFilterIndex[] bloomFilterIndices) throws IOException; -public class MetadataReader { - private final FSDataInputStream file; - private final CompressionCodec codec; - private final int bufferSize; - private final int typeCount; + StripeFooter readStripeFooter(StripeInformation stripe) throws IOException; - public MetadataReader(FileSystem fileSystem, Path path, - CompressionCodec codec, int bufferSize, int typeCount) throws IOException { - this(fileSystem.open(path), codec, bufferSize, typeCount); - } - - public MetadataReader(FSDataInputStream file, - CompressionCodec codec, int bufferSize, int typeCount) { - this.file = file; - this.codec = codec; - this.bufferSize = bufferSize; - this.typeCount = typeCount; - } - - public RecordReaderImpl.Index readRowIndex(StripeInformation stripe, OrcProto.StripeFooter footer, - boolean[] included, OrcProto.RowIndex[] indexes, boolean[] sargColumns, - OrcProto.BloomFilterIndex[] bloomFilterIndices) throws IOException { - if (footer == null) { - footer = readStripeFooter(stripe); - } - if (indexes == null) { - indexes = new OrcProto.RowIndex[typeCount]; - } - if (bloomFilterIndices == null) { - bloomFilterIndices = new OrcProto.BloomFilterIndex[typeCount]; - } - long offset = stripe.getOffset(); - List streams = footer.getStreamsList(); - for (int i = 0; i < streams.size(); i++) { - OrcProto.Stream stream = streams.get(i); - OrcProto.Stream nextStream = null; - if (i < streams.size() - 1) { - nextStream = streams.get(i+1); - } - int col = stream.getColumn(); - int len = (int) stream.getLength(); - // row index stream and bloom filter are interlaced, check if the sarg column contains bloom - // filter and combine the io to read row index and bloom filters for that column together - if (stream.hasKind() && (stream.getKind() == OrcProto.Stream.Kind.ROW_INDEX)) { - boolean readBloomFilter = false; - if (sargColumns != null && sargColumns[col] && - nextStream.getKind() == OrcProto.Stream.Kind.BLOOM_FILTER) { - len += nextStream.getLength(); - i += 1; - readBloomFilter = true; - } - if ((included == null || included[col]) && indexes[col] == null) { - byte[] buffer = new byte[len]; - file.readFully(offset, buffer, 0, buffer.length); - ByteBuffer[] bb = new ByteBuffer[] {ByteBuffer.wrap(buffer)}; - indexes[col] = OrcProto.RowIndex.parseFrom(InStream.create("index", - bb, new long[]{0}, stream.getLength(), codec, bufferSize)); - if (readBloomFilter) { - bb[0].position((int) stream.getLength()); - bloomFilterIndices[col] = OrcProto.BloomFilterIndex.parseFrom( - InStream.create("bloom_filter", bb, new long[]{0}, nextStream.getLength(), - codec, bufferSize)); - } - } - } - offset += len; - } - - RecordReaderImpl.Index index = new RecordReaderImpl.Index(indexes, bloomFilterIndices); - return index; - } - - public OrcProto.StripeFooter readStripeFooter(StripeInformation stripe) throws IOException { - long offset = stripe.getOffset() + stripe.getIndexLength() + stripe.getDataLength(); - int tailLength = (int) stripe.getFooterLength(); - - // read the footer - ByteBuffer tailBuf = ByteBuffer.allocate(tailLength); - file.readFully(offset, tailBuf.array(), tailBuf.arrayOffset(), tailLength); - return OrcProto.StripeFooter.parseFrom(InStream.create("footer", - Lists.newArrayList(new BufferChunk(tailBuf, 0)), - tailLength, codec, bufferSize)); - } - - public void close() throws IOException { - file.close(); - } + void close() throws IOException; - } + } http://git-wip-us.apache.org/repos/asf/hive/blob/f272ccb2/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReaderImpl.java ---------------------------------------------------------------------- diff --cc ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReaderImpl.java index 83594f7,0000000..1456df3 mode 100644,000000..100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReaderImpl.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReaderImpl.java @@@ -1,119 -1,0 +1,123 @@@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.io.orc; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.List; + +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.common.io.DiskRange; +import org.apache.hadoop.hive.ql.io.orc.RecordReaderImpl.BufferChunk; + +import com.google.common.collect.Lists; + +public class MetadataReaderImpl implements MetadataReader { + private final FSDataInputStream file; + private final CompressionCodec codec; + private final int bufferSize; + private final int typeCount; + - public MetadataReaderImpl(FileSystem fileSystem, Path path, CompressionCodec codec, - int bufferSize, int typeCount) throws IOException { - this.file = fileSystem.open(path); ++ public MetadataReaderImpl(FileSystem fileSystem, Path path, ++ CompressionCodec codec, int bufferSize, int typeCount) throws IOException { ++ this(fileSystem.open(path), codec, bufferSize, typeCount); ++ } ++ ++ public MetadataReaderImpl(FSDataInputStream file, ++ CompressionCodec codec, int bufferSize, int typeCount) { ++ this.file = file; + this.codec = codec; + this.bufferSize = bufferSize; + this.typeCount = typeCount; + } + + @Override + public RecordReaderImpl.Index readRowIndex(StripeInformation stripe, + OrcProto.StripeFooter footer, boolean[] included, OrcProto.RowIndex[] indexes, + boolean[] sargColumns, OrcProto.BloomFilterIndex[] bloomFilterIndices) throws IOException { + if (footer == null) { + footer = readStripeFooter(stripe); + } + if (indexes == null) { + indexes = new OrcProto.RowIndex[typeCount]; + } + if (bloomFilterIndices == null) { + bloomFilterIndices = new OrcProto.BloomFilterIndex[typeCount]; + } + long offset = stripe.getOffset(); + List streams = footer.getStreamsList(); + for (int i = 0; i < streams.size(); i++) { + OrcProto.Stream stream = streams.get(i); + OrcProto.Stream nextStream = null; + if (i < streams.size() - 1) { + nextStream = streams.get(i+1); + } + int col = stream.getColumn(); + int len = (int) stream.getLength(); + // row index stream and bloom filter are interlaced, check if the sarg column contains bloom + // filter and combine the io to read row index and bloom filters for that column together + if (stream.hasKind() && (stream.getKind() == OrcProto.Stream.Kind.ROW_INDEX)) { + boolean readBloomFilter = false; + if (sargColumns != null && sargColumns[col] && + nextStream.getKind() == OrcProto.Stream.Kind.BLOOM_FILTER) { + len += nextStream.getLength(); + i += 1; + readBloomFilter = true; + } + if ((included == null || included[col]) && indexes[col] == null) { + byte[] buffer = new byte[len]; ++ file.readFully(offset, buffer, 0, buffer.length); + ByteBuffer bb = ByteBuffer.wrap(buffer); - file.seek(offset); - file.readFully(buffer); + indexes[col] = OrcProto.RowIndex.parseFrom(InStream.create(null, "index", + Lists.newArrayList(new BufferChunk(bb, 0)), stream.getLength(), - codec, bufferSize)); ++ codec, bufferSize)); + if (readBloomFilter) { + bb.position((int) stream.getLength()); + bloomFilterIndices[col] = OrcProto.BloomFilterIndex.parseFrom(InStream.create( + null, "bloom_filter", Lists.newArrayList(new BufferChunk(bb, 0)), + nextStream.getLength(), codec, bufferSize)); + } + } + } + offset += len; + } + + RecordReaderImpl.Index index = new RecordReaderImpl.Index(indexes, bloomFilterIndices); + return index; + } + + @Override + public OrcProto.StripeFooter readStripeFooter(StripeInformation stripe) throws IOException { + long offset = stripe.getOffset() + stripe.getIndexLength() + stripe.getDataLength(); + int tailLength = (int) stripe.getFooterLength(); ++ + // read the footer + ByteBuffer tailBuf = ByteBuffer.allocate(tailLength); - file.seek(offset); - file.readFully(tailBuf.array(), tailBuf.arrayOffset(), tailLength); ++ file.readFully(offset, tailBuf.array(), tailBuf.arrayOffset(), tailLength); + return OrcProto.StripeFooter.parseFrom(InStream.create(null, "footer", + Lists.newArrayList(new BufferChunk(tailBuf, 0)), + tailLength, codec, bufferSize)); + } + + @Override + public void close() throws IOException { + file.close(); + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/f272ccb2/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/f272ccb2/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/f272ccb2/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/f272ccb2/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderUtils.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/f272ccb2/ql/src/test/results/clientpositive/tez/vector_groupby_reduce.q.out ----------------------------------------------------------------------