hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ser...@apache.org
Subject [05/17] hive git commit: HIVE-14418 : Hive config validation prevents unsetting the settings (Sergey Shelukhin, reviewed by Ashutosh Chauhan)
Date Tue, 30 Aug 2016 22:21:12 GMT
HIVE-14418 : Hive config validation prevents unsetting the settings (Sergey Shelukhin, reviewed
by Ashutosh Chauhan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0705323d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0705323d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0705323d

Branch: refs/heads/hive-14535
Commit: 0705323db28edf13777d29d3a0add48f19936db0
Parents: b80bcd0
Author: Sergey Shelukhin <sershe@apache.org>
Authored: Mon Aug 29 15:36:39 2016 -0700
Committer: Sergey Shelukhin <sershe@apache.org>
Committed: Mon Aug 29 15:36:39 2016 -0700

----------------------------------------------------------------------
 .../hive/ql/processors/ResetProcessor.java      | 109 +++++++++++++++++--
 .../hadoop/hive/ql/processors/SetProcessor.java |   3 +-
 ql/src/test/queries/clientpositive/reset_conf.q |  18 +++
 .../test/queries/clientpositive/set_metaconf.q  |   4 +
 .../results/clientpositive/reset_conf.q.out     |  32 ++++++
 .../results/clientpositive/set_metaconf.q.out   |   2 +
 6 files changed, 158 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/0705323d/ql/src/java/org/apache/hadoop/hive/ql/processors/ResetProcessor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/processors/ResetProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/processors/ResetProcessor.java
index e67422b..bbd4501 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/processors/ResetProcessor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/processors/ResetProcessor.java
@@ -18,41 +18,134 @@
 
 package org.apache.hadoop.hive.ql.processors;
 
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
 
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveVariableSource;
+import org.apache.hadoop.hive.conf.SystemVariables;
+import org.apache.hadoop.hive.conf.VariableSubstitution;
 import org.apache.hadoop.hive.ql.CommandNeedRetryException;
+import org.apache.hadoop.hive.ql.metadata.Hive;
 import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveOperationType;
 import org.apache.hadoop.hive.ql.session.SessionState;
 
+import com.google.common.collect.Lists;
+
 public class ResetProcessor implements CommandProcessor {
 
   @Override
   public void init() {
   }
 
+  private final static String DEFAULT_ARG = "-d";
+
   @Override
   public CommandProcessorResponse run(String command) throws CommandNeedRetryException {
     SessionState ss = SessionState.get();
 
     CommandProcessorResponse authErrResp =
         CommandUtil.authorizeCommand(ss, HiveOperationType.RESET, Arrays.asList(command));
-    if(authErrResp != null){
+    if (authErrResp != null) {
       // there was an authorization issue
       return authErrResp;
     }
-
-    if (ss.getOverriddenConfigurations().isEmpty()) {
+    command = command.trim();
+    if (StringUtils.isBlank(command)) {
+      resetOverridesOnly(ss);
       return new CommandProcessorResponse(0);
     }
+    String[] parts = command.split("\\s+");
+    boolean isDefault = false;
+    List<String> varnames = new ArrayList<>(parts.length);
+    for (String part : parts) {
+      if (part.isEmpty()) continue;
+      if (DEFAULT_ARG.equals(part)) {
+        isDefault = true;
+      } else {
+        varnames.add(part);
+      }
+    }
+    if (varnames.isEmpty()) {
+      return new CommandProcessorResponse(1, "No variable names specified", "42000");
+    }
+    String message = "";
+    for (String varname : varnames) {
+      if (isDefault) {
+        if (!message.isEmpty()) {
+          message += ", ";
+        }
+        message += varname;
+        resetToDefault(ss, varname);
+      } else {
+        resetOverrideOnly(ss, varname);
+      }
+    }
+    return new CommandProcessorResponse(0, isDefault
+        ? Lists.newArrayList("Resetting " + message + " to default values") : null);
+  }
+
+  private void resetOverridesOnly(SessionState ss) {
+    if (ss.getOverriddenConfigurations().isEmpty()) return;
     HiveConf conf = new HiveConf();
     for (String key : ss.getOverriddenConfigurations().keySet()) {
-      String value = conf.get(key);
-      if (value != null) {
-        ss.getConf().set(key, value);
-      }
+      setSessionVariableFromConf(ss, key, conf);
     }
     ss.getOverriddenConfigurations().clear();
-    return new CommandProcessorResponse(0);
+  }
+
+  private void resetOverrideOnly(SessionState ss, String varname) {
+    if (!ss.getOverriddenConfigurations().containsKey(varname)) return;
+    setSessionVariableFromConf(ss, varname, new HiveConf());
+    ss.getOverriddenConfigurations().remove(varname);
+  }
+
+  private void setSessionVariableFromConf(SessionState ss, String varname,
+      HiveConf conf) {
+    String value = conf.get(varname);
+    if (value != null) {
+      ss.getConf().set(varname, value);
+    }
+  }
+
+  private CommandProcessorResponse resetToDefault(SessionState ss, String varname) {
+    varname = varname.trim();
+    try {
+      String nonErrorMessage = null;
+      if (varname.startsWith(SystemVariables.HIVECONF_PREFIX)){
+        String propName = varname.substring(SystemVariables.HIVECONF_PREFIX.length());
+        nonErrorMessage = SetProcessor.setConf(
+            varname, propName, getConfVar(propName).getDefaultValue(), false);
+      } else if (varname.startsWith(SystemVariables.METACONF_PREFIX)) {
+        String propName = varname.substring(SystemVariables.METACONF_PREFIX.length());
+        HiveConf.ConfVars confVars = getConfVar(propName);
+        Hive.get(ss.getConf()).setMetaConf(propName, new VariableSubstitution(new HiveVariableSource()
{
+          @Override
+          public Map<String, String> getHiveVariable() {
+            return SessionState.get().getHiveVariables();
+          }
+        }).substitute(ss.getConf(), confVars.getDefaultValue()));
+      } else {
+        String defaultVal = getConfVar(varname).getDefaultValue();
+        nonErrorMessage = SetProcessor.setConf(varname, varname, defaultVal, true);
+        if (varname.equals(HiveConf.ConfVars.HIVE_SESSION_HISTORY_ENABLED.toString())) {
+          SessionState.get().updateHistory(Boolean.parseBoolean(defaultVal), ss);
+        }
+      }
+      return nonErrorMessage == null ? new CommandProcessorResponse(0)
+        : new CommandProcessorResponse(0, Lists.newArrayList(nonErrorMessage));
+    } catch (Exception e) {
+      return new CommandProcessorResponse(1, e.getMessage(), "42000",
+          e instanceof IllegalArgumentException ? null : e);
+    }
+  }
+
+  private static HiveConf.ConfVars getConfVar(String propName) {
+    HiveConf.ConfVars confVars = HiveConf.getConfVars(propName);
+    if (confVars == null) throw new IllegalArgumentException(propName + " not found");
+    return confVars;
   }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/0705323d/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java
index 2e13dab..c9d06ba 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java
@@ -24,7 +24,6 @@ import static org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe.defaultN
 
 import static org.apache.hadoop.hive.conf.SystemVariables.*;
 
-import java.util.HashSet;
 import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
@@ -179,7 +178,7 @@ public class SetProcessor implements CommandProcessor {
   /**
    * @return A console message that is not strong enough to fail the command (e.g. deprecation).
    */
-  private static String setConf(String varname, String key, String varvalue, boolean register)
+  static String setConf(String varname, String key, String varvalue, boolean register)
         throws IllegalArgumentException {
     String result = null;
     HiveConf conf = SessionState.get().getConf();

http://git-wip-us.apache.org/repos/asf/hive/blob/0705323d/ql/src/test/queries/clientpositive/reset_conf.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/reset_conf.q b/ql/src/test/queries/clientpositive/reset_conf.q
index 8ddde23..8420d02 100644
--- a/ql/src/test/queries/clientpositive/reset_conf.q
+++ b/ql/src/test/queries/clientpositive/reset_conf.q
@@ -9,3 +9,21 @@ reset;
 
 set hive.skewjoin.key;
 set hive.skewjoin.mapjoin.min.split;
+
+set hive.skewjoin.key=300000;
+set hive.skewjoin.mapjoin.min.split=256000000;
+select 'After setting hive.skewjoin.key and hive.skewjoin.mapjoin.min.split';
+set hive.skewjoin.key;
+
+reset -d hive.skewjoin.key;
+select 'After resetting hive.skewjoin.key to default';
+set hive.skewjoin.key;
+set hive.skewjoin.mapjoin.min.split;
+
+set hive.skewjoin.key=300000;
+
+reset -d hive.skewjoin.key hive.skewjoin.mapjoin.min.split;
+select 'After resetting both to default';
+set hive.skewjoin.key;
+set hive.skewjoin.mapjoin.min.split;
+

http://git-wip-us.apache.org/repos/asf/hive/blob/0705323d/ql/src/test/queries/clientpositive/set_metaconf.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/set_metaconf.q b/ql/src/test/queries/clientpositive/set_metaconf.q
index a679489..6186122 100644
--- a/ql/src/test/queries/clientpositive/set_metaconf.q
+++ b/ql/src/test/queries/clientpositive/set_metaconf.q
@@ -4,3 +4,7 @@ set metaconf:hive.metastore.try.direct.sql;
 set metaconf:hive.metastore.try.direct.sql=false;
 set metaconf:hive.metastore.try.direct.sql;
 set hive.metastore.try.direct.sql;
+
+reset -d metaconf:hive.metastore.try.direct.sql;
+set metaconf:hive.metastore.try.direct.sql;
+set hive.metastore.try.direct.sql;

http://git-wip-us.apache.org/repos/asf/hive/blob/0705323d/ql/src/test/results/clientpositive/reset_conf.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/reset_conf.q.out b/ql/src/test/results/clientpositive/reset_conf.q.out
index e4e15be..12f2555 100644
--- a/ql/src/test/results/clientpositive/reset_conf.q.out
+++ b/ql/src/test/results/clientpositive/reset_conf.q.out
@@ -4,3 +4,35 @@ hive.skewjoin.key=300000
 hive.skewjoin.mapjoin.min.split=256000000
 hive.skewjoin.key=100000
 hive.skewjoin.mapjoin.min.split=33554432
+PREHOOK: query: select 'After setting hive.skewjoin.key and hive.skewjoin.mapjoin.min.split'
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: select 'After setting hive.skewjoin.key and hive.skewjoin.mapjoin.min.split'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+After setting hive.skewjoin.key and hive.skewjoin.mapjoin.min.split
+hive.skewjoin.key=300000
+PREHOOK: query: select 'After resetting hive.skewjoin.key to default'
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: select 'After resetting hive.skewjoin.key to default'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+After resetting hive.skewjoin.key to default
+hive.skewjoin.key=100000
+hive.skewjoin.mapjoin.min.split=256000000
+PREHOOK: query: select 'After resetting both to default'
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: select 'After resetting both to default'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+After resetting both to default
+hive.skewjoin.key=100000
+hive.skewjoin.mapjoin.min.split=33554432

http://git-wip-us.apache.org/repos/asf/hive/blob/0705323d/ql/src/test/results/clientpositive/set_metaconf.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/set_metaconf.q.out b/ql/src/test/results/clientpositive/set_metaconf.q.out
index ec33e59..41b8957 100644
--- a/ql/src/test/results/clientpositive/set_metaconf.q.out
+++ b/ql/src/test/results/clientpositive/set_metaconf.q.out
@@ -1,3 +1,5 @@
 metaconf:hive.metastore.try.direct.sql=true
 metaconf:hive.metastore.try.direct.sql=false
 hive.metastore.try.direct.sql=true
+metaconf:hive.metastore.try.direct.sql=true
+hive.metastore.try.direct.sql=true


Mime
View raw message