Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id AFC70200D35 for ; Tue, 24 Oct 2017 00:20:56 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id AE64A160BF0; Mon, 23 Oct 2017 22:20:56 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id CDC971609E0 for ; Tue, 24 Oct 2017 00:20:55 +0200 (CEST) Received: (qmail 86228 invoked by uid 500); 23 Oct 2017 22:20:55 -0000 Mailing-List: contact commits-help@hive.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hive-dev@hive.apache.org Delivered-To: mailing list commits@hive.apache.org Received: (qmail 86217 invoked by uid 99); 23 Oct 2017 22:20:54 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 23 Oct 2017 22:20:54 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id D7922DFBC7; Mon, 23 Oct 2017 22:20:54 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: mithun@apache.org To: commits@hive.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: hive git commit: HIVE-17781: Map MR settings to Tez settings via DeprecatedKeys (Chris Drome, Thiruvel Thirumoolan, reviewed by Mithun Radhakrishnan) Date: Mon, 23 Oct 2017 22:20:54 +0000 (UTC) archived-at: Mon, 23 Oct 2017 22:20:56 -0000 Repository: hive Updated Branches: refs/heads/master 3c7f09ae1 -> 4fd12205a HIVE-17781: Map MR settings to Tez settings via DeprecatedKeys (Chris Drome, Thiruvel Thirumoolan, reviewed by Mithun Radhakrishnan) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4fd12205 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4fd12205 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4fd12205 Branch: refs/heads/master Commit: 4fd12205a4ab367e912f1ad9ecf79a5e7f8e5c33 Parents: 3c7f09a Author: Mithun RK Authored: Wed Oct 11 17:15:38 2017 -0700 Committer: Mithun RK Committed: Mon Oct 23 15:04:29 2017 -0700 ---------------------------------------------------------------------- .../hive/ql/exec/tez/TezSessionState.java | 77 +++++++++++++++++++- 1 file changed, 76 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/4fd12205/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java index aab4ac4..1448168 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java @@ -62,11 +62,13 @@ import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.session.SessionState.LogHelper; import org.apache.hadoop.hive.ql.wm.TriggerContext; import org.apache.hadoop.hive.shims.Utils; +import org.apache.hadoop.mapred.JobContext; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.tez.client.TezClient; import org.apache.tez.common.TezUtils; import org.apache.tez.dag.api.PreWarmVertex; @@ -74,7 +76,9 @@ import org.apache.tez.dag.api.SessionNotRunning; import org.apache.tez.dag.api.TezConfiguration; import org.apache.tez.dag.api.TezException; import org.apache.tez.dag.api.UserPayload; +import org.apache.tez.mapreduce.hadoop.DeprecatedKeys; import org.apache.tez.mapreduce.hadoop.MRHelpers; +import org.apache.tez.mapreduce.hadoop.MRJobConfig; import org.apache.tez.serviceplugins.api.ContainerLauncherDescriptor; import org.apache.tez.serviceplugins.api.ServicePluginsDescriptor; import org.apache.tez.serviceplugins.api.TaskCommunicatorDescriptor; @@ -275,7 +279,10 @@ public class TezSessionState { // and finally we're ready to create and start the session // generate basic tez config - final TezConfiguration tezConfig = new TezConfiguration(conf); + final TezConfiguration tezConfig = new TezConfiguration(true); + tezConfig.addResource(conf); + + setupTezParamsBasedOnMR(tezConfig); // set up the staging directory to use tezConfig.set(TezConfiguration.TEZ_AM_STAGING_DIR, tezScratchDir.toUri().toString()); @@ -440,6 +447,74 @@ public class TezSessionState { } } + /** + * This takes settings from MR and applies them to the appropriate Tez configuration. This is + * similar to what Pig on Tez does (refer MRToTezHelper.java). + * + * @param conf configuration with MR settings + */ + private void setupTezParamsBasedOnMR(TezConfiguration conf) { + + String env = conf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV); + if (conf.get(MRJobConfig.MR_AM_ENV) != null) { + env = (env == null) ? conf.get(MRJobConfig.MR_AM_ENV) : env + "," + conf.get(MRJobConfig.MR_AM_ENV); + } + if (env != null) { + conf.setIfUnset(TezConfiguration.TEZ_AM_LAUNCH_ENV, env); + } + + conf.setIfUnset(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS, + org.apache.tez.mapreduce.hadoop.MRHelpers.getJavaOptsForMRAM(conf)); + + String queueName = conf.get(JobContext.QUEUE_NAME, YarnConfiguration.DEFAULT_QUEUE_NAME); + conf.setIfUnset(TezConfiguration.TEZ_QUEUE_NAME, queueName); + + int amMemMB = conf.getInt(MRJobConfig.MR_AM_VMEM_MB, MRJobConfig.DEFAULT_MR_AM_VMEM_MB); + conf.setIfUnset(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, "" + amMemMB); + + int amCores = conf.getInt(MRJobConfig.MR_AM_CPU_VCORES, MRJobConfig.DEFAULT_MR_AM_CPU_VCORES); + conf.setIfUnset(TezConfiguration.TEZ_AM_RESOURCE_CPU_VCORES, "" + amCores); + + conf.setIfUnset(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS, "" + + conf.getInt(MRJobConfig.MR_AM_MAX_ATTEMPTS, MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS)); + + conf.setIfUnset(TezConfiguration.TEZ_AM_VIEW_ACLS, + conf.get(MRJobConfig.JOB_ACL_VIEW_JOB, MRJobConfig.DEFAULT_JOB_ACL_VIEW_JOB)); + + conf.setIfUnset(TezConfiguration.TEZ_AM_MODIFY_ACLS, + conf.get(MRJobConfig.JOB_ACL_MODIFY_JOB, MRJobConfig.DEFAULT_JOB_ACL_MODIFY_JOB)); + + + // Refer to org.apache.tez.mapreduce.hadoop.MRHelpers.processDirectConversion. + ArrayList> maps = new ArrayList>(2); + maps.add(DeprecatedKeys.getMRToTezRuntimeParamMap()); + maps.add(DeprecatedKeys.getMRToDAGParamMap()); + + boolean preferTez = true; // Can make this configurable. + + for (Map map : maps) { + for (Map.Entry dep : map.entrySet()) { + if (conf.get(dep.getKey()) != null) { + // TODO Deprecation reason does not seem to reflect in the config ? + // The ordering is important in case of keys which are also deprecated. + // Unset will unset the deprecated keys and all its variants. + final String mrValue = conf.get(dep.getKey()); + final String tezValue = conf.get(dep.getValue()); + conf.unset(dep.getKey()); + if (tezValue == null) { + conf.set(dep.getValue(), mrValue, "TRANSLATED_TO_TEZ"); + } else if (!preferTez) { + conf.set(dep.getValue(), mrValue, "TRANSLATED_TO_TEZ_AND_MR_OVERRIDE"); + } + LOG.info("Config: mr(unset):" + dep.getKey() + ", mr initial value=" + + mrValue + + ", tez(original):" + dep.getValue() + "=" + tezValue + + ", tez(final):" + dep.getValue() + "=" + conf.get(dep.getValue())); + } + } + } + } + private void setupSessionAcls(Configuration tezConf, HiveConf hiveConf) throws IOException {