Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 6CA52200B4C for ; Fri, 22 Jul 2016 19:57:25 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 6B371160A5A; Fri, 22 Jul 2016 17:57:25 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 8E3EC160A6D for ; Fri, 22 Jul 2016 19:57:24 +0200 (CEST) Received: (qmail 55902 invoked by uid 500); 22 Jul 2016 17:57:23 -0000 Mailing-List: contact commits-help@hive.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hive-dev@hive.apache.org Delivered-To: mailing list commits@hive.apache.org Received: (qmail 55890 invoked by uid 99); 22 Jul 2016 17:57:23 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 22 Jul 2016 17:57:23 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 7B093E058E; Fri, 22 Jul 2016 17:57:23 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: sseth@apache.org To: commits@hive.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: hive git commit: Revert "HIVE-14167 : Use work directories provided by Tez instead of directly using YARN local dirs (Wei Zheng, reviewed by Siddharth Seth)" Date: Fri, 22 Jul 2016 17:57:23 +0000 (UTC) archived-at: Fri, 22 Jul 2016 17:57:25 -0000 Repository: hive Updated Branches: refs/heads/branch-2.1 0efafc0df -> d9bea1400 Revert "HIVE-14167 : Use work directories provided by Tez instead of directly using YARN local dirs (Wei Zheng, reviewed by Siddharth Seth)" This reverts commit e96994b1078ee8b79d59beee481d96b922147650. Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d9bea140 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d9bea140 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d9bea140 Branch: refs/heads/branch-2.1 Commit: d9bea1400409e369e0031f94b90c04ba0806aae4 Parents: 0efafc0 Author: Siddharth Seth Authored: Fri Jul 22 10:57:13 2016 -0700 Committer: Siddharth Seth Committed: Fri Jul 22 10:57:13 2016 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/hive/llap/LlapUtil.java | 6 +++++ .../hive/llap/daemon/impl/LlapDaemon.java | 3 +-- .../persistence/HybridHashTableContainer.java | 3 +-- .../hive/ql/exec/persistence/RowContainer.java | 10 +++++++-- .../hadoop/hive/ql/metadata/HiveUtils.java | 23 -------------------- 5 files changed, 16 insertions(+), 29 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/d9bea140/llap-common/src/java/org/apache/hadoop/hive/llap/LlapUtil.java ---------------------------------------------------------------------- diff --git a/llap-common/src/java/org/apache/hadoop/hive/llap/LlapUtil.java b/llap-common/src/java/org/apache/hadoop/hive/llap/LlapUtil.java index 06323f5..505ddb1 100644 --- a/llap-common/src/java/org/apache/hadoop/hive/llap/LlapUtil.java +++ b/llap-common/src/java/org/apache/hadoop/hive/llap/LlapUtil.java @@ -27,6 +27,12 @@ import org.slf4j.LoggerFactory; public class LlapUtil { private static final Logger LOG = LoggerFactory.getLogger(LlapUtil.class); + public static String getDaemonLocalDirList(Configuration conf) { + String localDirList = HiveConf.getVar(conf, ConfVars.LLAP_DAEMON_WORK_DIRS); + if (localDirList != null && !localDirList.isEmpty()) return localDirList; + return conf.get("yarn.nodemanager.local-dirs"); + } + public static UserGroupInformation loginWithKerberos( String principal, String keytabFile) throws IOException { if (!UserGroupInformation.isSecurityEnabled()) return null; http://git-wip-us.apache.org/repos/asf/hive/blob/d9bea140/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java ---------------------------------------------------------------------- diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java index d6cf01d..2faedcd 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java @@ -57,7 +57,6 @@ import org.apache.hadoop.hive.llap.registry.impl.LlapRegistryService; import org.apache.hadoop.hive.llap.shufflehandler.ShuffleHandler; import org.apache.hadoop.hive.ql.exec.SerializationUtilities; import org.apache.hadoop.hive.ql.exec.UDF; -import org.apache.hadoop.hive.ql.metadata.HiveUtils; import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge.UdfWhitelistChecker; @@ -421,7 +420,7 @@ public class LlapDaemon extends CompositeService implements ContainerRunner, Lla int numExecutors = HiveConf.getIntVar(daemonConf, ConfVars.LLAP_DAEMON_NUM_EXECUTORS); - String localDirList = HiveUtils.getLocalDirList(daemonConf); + String localDirList = LlapUtil.getDaemonLocalDirList(daemonConf); String[] localDirs = (localDirList == null || localDirList.isEmpty()) ? new String[0] : StringUtils.getTrimmedStrings(localDirList); int rpcPort = HiveConf.getIntVar(daemonConf, ConfVars.LLAP_DAEMON_RPC_PORT); http://git-wip-us.apache.org/repos/asf/hive/blob/d9bea140/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java index e4a2b35..233f66b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java @@ -45,7 +45,6 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriter; import org.apache.hadoop.hive.ql.exec.vector.mapjoin.VectorMapJoinRowBytesContainer; import org.apache.hadoop.hive.ql.io.HiveKey; import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.metadata.HiveUtils; import org.apache.hadoop.hive.serde2.ByteStream.Output; import org.apache.hadoop.hive.serde2.SerDe; import org.apache.hadoop.hive.serde2.SerDeException; @@ -277,7 +276,7 @@ public class HybridHashTableContainer HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVEHYBRIDGRACEHASHJOINMINNUMPARTITIONS), HiveConf.getFloatVar(hconf, HiveConf.ConfVars.HIVEMAPJOINOPTIMIZEDTABLEPROBEPERCENT), estimatedTableSize, keyCount, memoryAvailable, nwayConf, - HiveUtils.getLocalDirList(hconf)); + RowContainer.getLocalDirsForSpillFiles(hconf)); } private HybridHashTableContainer(float keyCountAdj, int threshold, float loadFactor, http://git-wip-us.apache.org/repos/asf/hive/blob/d9bea140/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java index e928719..893d265 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java @@ -32,12 +32,13 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.FileUtils; +import org.apache.hadoop.hive.llap.LlapUtil; +import org.apache.hadoop.hive.llap.io.api.LlapProxy; import org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils; import org.apache.hadoop.hive.ql.io.HiveOutputFormat; import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.metadata.HiveUtils; import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.serde2.SerDe; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; @@ -134,7 +135,7 @@ public class RowContainer> this.size = 0; this.itrCursor = 0; this.addCursor = 0; - this.spillFileDirs = HiveUtils.getLocalDirList(jc); + this.spillFileDirs = getLocalDirsForSpillFiles(jc); this.numFlushedBlocks = 0; this.tmpFile = null; this.currentWriteBlock = (ROW[]) new ArrayList[blockSize]; @@ -150,6 +151,11 @@ public class RowContainer> } } + public static String getLocalDirsForSpillFiles(Configuration conf) { + return LlapProxy.isDaemon() + ? LlapUtil.getDaemonLocalDirList(conf) : conf.get("yarn.nodemanager.local-dirs"); + } + private JobConf getLocalFSJobConfClone(Configuration jc) { if (this.jobCloneUsingLocalFs == null) { this.jobCloneUsingLocalFs = new JobConf(jc); http://git-wip-us.apache.org/repos/asf/hive/blob/d9bea140/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveUtils.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveUtils.java index 47a93fb..feb471a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveUtils.java @@ -26,10 +26,8 @@ import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.JavaUtils; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.llap.io.api.LlapProxy; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.exec.Utilities; -import org.apache.hadoop.hive.ql.exec.tez.TezContext; import org.apache.hadoop.hive.ql.index.HiveIndexHandler; import org.apache.hadoop.hive.ql.security.HadoopDefaultAuthenticator; import org.apache.hadoop.hive.ql.security.HiveAuthenticationProvider; @@ -40,7 +38,6 @@ import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthorizerFac import org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory; import org.apache.hadoop.io.Text; import org.apache.hadoop.util.ReflectionUtils; -import org.apache.hadoop.util.StringUtils; /** * General collection of helper functions. @@ -454,24 +451,4 @@ public final class HiveUtils { } return sb.toString(); } - - public static String getLocalDirList(Configuration conf) { - String localDirList; - - if (LlapProxy.isDaemon()) { - localDirList = HiveConf.getVar(conf, HiveConf.ConfVars.LLAP_DAEMON_WORK_DIRS); - if (localDirList != null && !localDirList.isEmpty()) { - return localDirList; - } // otherwise, fall back to use tez work dirs - } - - if (HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("tez")) { - TezContext tezContext = (TezContext) TezContext.get(); - if (tezContext != null && tezContext.getTezProcessorContext() != null) { - return StringUtils.arrayToString(tezContext.getTezProcessorContext().getWorkDirs()); - } // otherwise fall back to return null, i.e. to use local tmp dir only - } - - return null; - } }