hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From br...@apache.org
Subject svn commit: r1538648 - in /hive/trunk: common/src/java/org/apache/hadoop/hive/conf/ ql/src/java/org/apache/hadoop/hive/ql/processors/ ql/src/test/queries/clientnegative/ ql/src/test/queries/clientpositive/ ql/src/test/results/clientnegative/
Date Mon, 04 Nov 2013 15:26:31 GMT
Author: brock
Date: Mon Nov  4 15:26:31 2013
New Revision: 1538648

URL: http://svn.apache.org/r1538648
Log:
HIVE-5707: Validate values for ConfVar (Navis via Brock Noland)

Added:
    hive/trunk/ql/src/test/queries/clientnegative/set_hiveconf_validation2.q
    hive/trunk/ql/src/test/results/clientnegative/set_hiveconf_validation2.q.out
Modified:
    hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java
    hive/trunk/ql/src/test/queries/clientpositive/orc_create.q

Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1538648&r1=1538647&r2=1538648&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Mon Nov  4 15:26:31
2013
@@ -27,10 +27,12 @@ import java.net.URL;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Iterator;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Properties;
+import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -440,8 +442,10 @@ public class HiveConf extends Configurat
 
     // Default file format for CREATE TABLE statement
     // Options: TextFile, SequenceFile
-    HIVEDEFAULTFILEFORMAT("hive.default.fileformat", "TextFile"),
-    HIVEQUERYRESULTFILEFORMAT("hive.query.result.fileformat", "TextFile"),
+    HIVEDEFAULTFILEFORMAT("hive.default.fileformat", "TextFile",
+        new StringsValidator("TextFile", "SequenceFile", "RCfile", "ORC")),
+    HIVEQUERYRESULTFILEFORMAT("hive.query.result.fileformat", "TextFile",
+        new StringsValidator("TextFile", "SequenceFile", "RCfile")),
     HIVECHECKFILEFORMAT("hive.fileformat.check", true),
 
     // default serde for rcfile
@@ -654,7 +658,8 @@ public class HiveConf extends Configurat
     HIVEOUTERJOINSUPPORTSFILTERS("hive.outerjoin.supports.filters", true),
 
     // 'minimal', 'more' (and 'all' later)
-    HIVEFETCHTASKCONVERSION("hive.fetch.task.conversion", "minimal"),
+    HIVEFETCHTASKCONVERSION("hive.fetch.task.conversion", "minimal",
+        new StringsValidator("minimal", "more")),
 
     HIVEFETCHTASKAGGR("hive.fetch.task.aggr", false),
 
@@ -732,7 +737,8 @@ public class HiveConf extends Configurat
     HIVE_ENTITY_SEPARATOR("hive.entity.separator", "@"),
 
     // binary or http
-    HIVE_SERVER2_TRANSPORT_MODE("hive.server2.transport.mode", "binary"),
+    HIVE_SERVER2_TRANSPORT_MODE("hive.server2.transport.mode", "binary",
+        new StringsValidator("binary", "http")),
 
     // http (over thrift) transport settings
     HIVE_SERVER2_THRIFT_HTTP_PORT("hive.server2.thrift.http.port", 10001),
@@ -743,7 +749,8 @@ public class HiveConf extends Configurat
     // binary transport settings
     HIVE_SERVER2_THRIFT_PORT("hive.server2.thrift.port", 10000),
     HIVE_SERVER2_THRIFT_BIND_HOST("hive.server2.thrift.bind.host", ""),
-    HIVE_SERVER2_THRIFT_SASL_QOP("hive.server2.thrift.sasl.qop", "auth"),
+    HIVE_SERVER2_THRIFT_SASL_QOP("hive.server2.thrift.sasl.qop", "auth",
+        new StringsValidator("auth", "auth-int", "auth-conf")),
     HIVE_SERVER2_THRIFT_MIN_WORKER_THREADS("hive.server2.thrift.min.worker.threads", 5),
     HIVE_SERVER2_THRIFT_MAX_WORKER_THREADS("hive.server2.thrift.max.worker.threads", 500),
 
@@ -755,7 +762,8 @@ public class HiveConf extends Configurat
 
 
     // HiveServer2 auth configuration
-    HIVE_SERVER2_AUTHENTICATION("hive.server2.authentication", "NONE"),
+    HIVE_SERVER2_AUTHENTICATION("hive.server2.authentication", "NONE",
+        new StringsValidator("NOSASL", "NONE", "LDAP", "KERBEROS", "CUSTOM")),
     HIVE_SERVER2_KERBEROS_KEYTAB("hive.server2.authentication.kerberos.keytab", ""),
     HIVE_SERVER2_KERBEROS_PRINCIPAL("hive.server2.authentication.kerberos.principal", ""),
     HIVE_SERVER2_PLAIN_LDAP_URL("hive.server2.authentication.ldap.url", null),
@@ -763,7 +771,8 @@ public class HiveConf extends Configurat
     HIVE_SERVER2_PLAIN_LDAP_DOMAIN("hive.server2.authentication.ldap.Domain", null),
     HIVE_SERVER2_CUSTOM_AUTHENTICATION_CLASS("hive.server2.custom.authentication.class",
null),
     HIVE_SERVER2_ENABLE_DOAS("hive.server2.enable.doAs", true),
-    HIVE_SERVER2_TABLE_TYPE_MAPPING("hive.server2.table.type.mapping", "CLASSIC"),
+    HIVE_SERVER2_TABLE_TYPE_MAPPING("hive.server2.table.type.mapping", "CLASSIC",
+        new StringsValidator("CLASSIC", "HIVE")),
     HIVE_SERVER2_SESSION_HOOK("hive.server2.session.hook", ""),
 
     HIVE_SECURITY_COMMAND_WHITELIST("hive.security.command.whitelist", "set,reset,dfs,add,delete,compile"),
@@ -824,7 +833,13 @@ public class HiveConf extends Configurat
 
     private final VarType type;
 
+    private final Validator validator;
+
     ConfVars(String varname, String defaultVal) {
+      this(varname, defaultVal, null);
+    }
+
+    ConfVars(String varname, String defaultVal, Validator validator) {
       this.varname = varname;
       this.valClass = String.class;
       this.defaultVal = defaultVal;
@@ -833,9 +848,14 @@ public class HiveConf extends Configurat
       this.defaultFloatVal = -1;
       this.defaultBoolVal = false;
       this.type = VarType.STRING;
+      this.validator = validator;
     }
 
-    ConfVars(String varname, int defaultIntVal) {
+    ConfVars(String varname, int defaultVal) {
+      this(varname, defaultVal, null);
+    }
+
+    ConfVars(String varname, int defaultIntVal, Validator validator) {
       this.varname = varname;
       this.valClass = Integer.class;
       this.defaultVal = Integer.toString(defaultIntVal);
@@ -844,9 +864,14 @@ public class HiveConf extends Configurat
       this.defaultFloatVal = -1;
       this.defaultBoolVal = false;
       this.type = VarType.INT;
+      this.validator = validator;
+    }
+
+    ConfVars(String varname, long defaultVal) {
+      this(varname, defaultVal, null);
     }
 
-    ConfVars(String varname, long defaultLongVal) {
+    ConfVars(String varname, long defaultLongVal, Validator validator) {
       this.varname = varname;
       this.valClass = Long.class;
       this.defaultVal = Long.toString(defaultLongVal);
@@ -855,9 +880,14 @@ public class HiveConf extends Configurat
       this.defaultFloatVal = -1;
       this.defaultBoolVal = false;
       this.type = VarType.LONG;
+      this.validator = validator;
     }
 
-    ConfVars(String varname, float defaultFloatVal) {
+    ConfVars(String varname, float defaultVal) {
+      this(varname, defaultVal, null);
+    }
+
+    ConfVars(String varname, float defaultFloatVal, Validator validator) {
       this.varname = varname;
       this.valClass = Float.class;
       this.defaultVal = Float.toString(defaultFloatVal);
@@ -866,6 +896,7 @@ public class HiveConf extends Configurat
       this.defaultFloatVal = defaultFloatVal;
       this.defaultBoolVal = false;
       this.type = VarType.FLOAT;
+      this.validator = validator;
     }
 
     ConfVars(String varname, boolean defaultBoolVal) {
@@ -877,12 +908,17 @@ public class HiveConf extends Configurat
       this.defaultFloatVal = -1;
       this.defaultBoolVal = defaultBoolVal;
       this.type = VarType.BOOLEAN;
+      this.validator = null;
     }
 
     public boolean isType(String value) {
       return type.isType(value);
     }
 
+    public String validate(String value) {
+      return validator == null ? null : validator.validate(value);
+    }
+
     public String typeString() {
       return type.typeString();
     }
@@ -1282,4 +1318,41 @@ public class HiveConf extends Configurat
     }
   }
 
+  /**
+   * validate value for a ConfVar, return non-null string for fail message
+   */
+  public static interface Validator {
+    String validate(String value);
+  }
+
+  public static class StringsValidator implements Validator {
+    private final Set<String> expected = new LinkedHashSet<String>();
+    private StringsValidator(String... values) {
+      for (String value : values) {
+        expected.add(value.toLowerCase());
+      }
+    }
+    @Override
+    public String validate(String value) {
+      if (value == null || !expected.contains(value.toLowerCase())) {
+        return "Invalid value.. expects one of " + expected;
+      }
+      return null;
+    }
+  }
+
+  public static class RatioValidator implements Validator {
+    @Override
+    public String validate(String value) {
+      try {
+        float fvalue = Float.valueOf(value);
+        if (fvalue <= 0 || fvalue >= 1) {
+          return "Invalid ratio " + value + ", which should be in between 0 to 1";
+        }
+      } catch (NumberFormatException e) {
+        return e.toString();
+      }
+      return null;
+    }
+  }
 }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java?rev=1538648&r1=1538647&r2=1538648&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java Mon Nov
 4 15:26:31 2013
@@ -143,12 +143,21 @@ public class SetProcessor implements Com
     String value = new VariableSubstitution().substitute(conf, varvalue);
     if (conf.getBoolVar(HiveConf.ConfVars.HIVECONFVALIDATION)) {
       HiveConf.ConfVars confVars = HiveConf.getConfVars(key);
-      if (confVars != null && !confVars.isType(value)) {
-        StringBuilder message = new StringBuilder();
-        message.append("'SET ").append(varname).append('=').append(varvalue);
-        message.append("' FAILED because "); message.append(key).append(" expects an ");
-        message.append(confVars.typeString()).append(" value.");
-        throw new IllegalArgumentException(message.toString());
+      if (confVars != null) {
+        if (!confVars.isType(value)) {
+          StringBuilder message = new StringBuilder();
+          message.append("'SET ").append(varname).append('=').append(varvalue);
+          message.append("' FAILED because ").append(key).append(" expects an ");
+          message.append(confVars.typeString()).append(" value.");
+          throw new IllegalArgumentException(message.toString());
+        }
+        String fail = confVars.validate(value);
+        if (fail != null) {
+          StringBuilder message = new StringBuilder();
+          message.append("'SET ").append(varname).append('=').append(varvalue);
+          message.append("' FAILED in validation : ").append(fail).append('.');
+          throw new IllegalArgumentException(message.toString());
+        }
       }
     }
     conf.verifyAndSet(key, value);

Added: hive/trunk/ql/src/test/queries/clientnegative/set_hiveconf_validation2.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientnegative/set_hiveconf_validation2.q?rev=1538648&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientnegative/set_hiveconf_validation2.q (added)
+++ hive/trunk/ql/src/test/queries/clientnegative/set_hiveconf_validation2.q Mon Nov  4 15:26:31
2013
@@ -0,0 +1,5 @@
+-- should fail: hive.fetch.task.conversion accepts minimal or more
+desc src;
+
+set hive.conf.validation=true;
+set hive.fetch.task.conversion=true;

Modified: hive/trunk/ql/src/test/queries/clientpositive/orc_create.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/orc_create.q?rev=1538648&r1=1538647&r2=1538648&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/orc_create.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/orc_create.q Mon Nov  4 15:26:31 2013
@@ -40,7 +40,7 @@ set hive.default.fileformat=orc;
 CREATE TABLE orc_create (key INT, value STRING)
    PARTITIONED BY (ds string);
 
-set hive.default.fileformat=text;
+set hive.default.fileformat=TextFile;
 
 DESCRIBE FORMATTED orc_create;
 

Added: hive/trunk/ql/src/test/results/clientnegative/set_hiveconf_validation2.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/set_hiveconf_validation2.q.out?rev=1538648&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientnegative/set_hiveconf_validation2.q.out (added)
+++ hive/trunk/ql/src/test/results/clientnegative/set_hiveconf_validation2.q.out Mon Nov 
4 15:26:31 2013
@@ -0,0 +1,9 @@
+PREHOOK: query: -- should fail: hive.fetch.task.conversion accepts minimal or more
+desc src
+PREHOOK: type: DESCTABLE
+POSTHOOK: query: -- should fail: hive.fetch.task.conversion accepts minimal or more
+desc src
+POSTHOOK: type: DESCTABLE
+key                 	string              	default             
+value               	string              	default             
+Query returned non-zero code: 1, cause: 'SET hive.fetch.task.conversion=true' FAILED in validation
: Invalid value.. expects one of [minimal, more].



Mime
View raw message