hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1674132 [5/5] - in /hive/branches/cbo: ./ common/src/java/org/apache/hadoop/hive/conf/ common/src/java/org/apache/hive/common/util/ dev-support/ hcatalog/src/test/e2e/templeton/deployers/ hcatalog/src/test/e2e/templeton/deployers/config/hi...
Date Thu, 16 Apr 2015 18:35:41 GMT
Modified: hive/branches/cbo/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java?rev=1674132&r1=1674131&r2=1674132&view=diff
==============================================================================
--- hive/branches/cbo/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java
(original)
+++ hive/branches/cbo/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java
Thu Apr 16 18:35:39 2015
@@ -19,18 +19,26 @@
 package org.apache.hive.service.cli.thrift;
 
 import java.io.IOException;
+import java.io.UnsupportedEncodingException;
 import java.security.PrivilegedExceptionAction;
+import java.util.Arrays;
 import java.util.Map;
+import java.util.Random;
 import java.util.Set;
+import java.util.concurrent.TimeUnit;
 
 import javax.servlet.ServletException;
+import javax.servlet.http.Cookie;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
+import javax.ws.rs.core.NewCookie;
 
 import org.apache.commons.codec.binary.Base64;
 import org.apache.commons.codec.binary.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.shims.HadoopShims.KerberosNameShim;
 import org.apache.hadoop.hive.shims.ShimLoader;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -41,6 +49,7 @@ import org.apache.hive.service.auth.Http
 import org.apache.hive.service.auth.HttpAuthenticationException;
 import org.apache.hive.service.auth.PasswdAuthenticationProvider;
 import org.apache.hive.service.cli.session.SessionManager;
+import org.apache.hive.service.CookieSigner;
 import org.apache.thrift.TProcessor;
 import org.apache.thrift.protocol.TProtocolFactory;
 import org.apache.thrift.server.TServlet;
@@ -63,6 +72,18 @@ public class ThriftHttpServlet extends T
   private final String authType;
   private final UserGroupInformation serviceUGI;
   private final UserGroupInformation httpUGI;
+  private HiveConf hiveConf = new HiveConf();
+
+  // Class members for cookie based authentication.
+  private CookieSigner signer;
+  public static final String AUTH_COOKIE = "hive.server2.auth";
+  private static final Random RAN = new Random();
+  private boolean isCookieAuthEnabled;
+  private String cookieDomain;
+  private String cookiePath;
+  private int cookieMaxAge;
+  private boolean isCookieSecure;
+  private boolean isHttpOnlyCookie;
 
   public ThriftHttpServlet(TProcessor processor, TProtocolFactory protocolFactory,
       String authType, UserGroupInformation serviceUGI, UserGroupInformation httpUGI) {
@@ -70,34 +91,80 @@ public class ThriftHttpServlet extends T
     this.authType = authType;
     this.serviceUGI = serviceUGI;
     this.httpUGI = httpUGI;
+    this.isCookieAuthEnabled = hiveConf.getBoolVar(
+      ConfVars.HIVE_SERVER2_THRIFT_HTTP_COOKIE_AUTH_ENABLED);
+    // Initialize the cookie based authentication related variables.
+    if (isCookieAuthEnabled) {
+      // Generate the signer with secret.
+      String secret = Long.toString(RAN.nextLong());
+      LOG.debug("Using the random number as the secret for cookie generation " + secret);
+      this.signer = new CookieSigner(secret.getBytes());
+      this.cookieMaxAge = (int) hiveConf.getTimeVar(
+        ConfVars.HIVE_SERVER2_THRIFT_HTTP_COOKIE_MAX_AGE, TimeUnit.SECONDS);
+      this.cookieDomain = hiveConf.getVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_COOKIE_DOMAIN);
+      this.cookiePath = hiveConf.getVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_COOKIE_PATH);
+      this.isCookieSecure = hiveConf.getBoolVar(
+        ConfVars.HIVE_SERVER2_THRIFT_HTTP_COOKIE_IS_SECURE);
+      this.isHttpOnlyCookie = hiveConf.getBoolVar(
+        ConfVars.HIVE_SERVER2_THRIFT_HTTP_COOKIE_IS_HTTPONLY);
+    }
   }
 
   @Override
   protected void doPost(HttpServletRequest request, HttpServletResponse response)
       throws ServletException, IOException {
-    String clientUserName;
+    String clientUserName = null;
     String clientIpAddress;
+    boolean requireNewCookie = false;
+
     try {
-      // For a kerberos setup
-      if(isKerberosAuthMode(authType)) {
-        clientUserName = doKerberosAuth(request);
-        String doAsQueryParam = getDoAsQueryParam(request.getQueryString());
-        if (doAsQueryParam != null) {
-          SessionManager.setProxyUserName(doAsQueryParam);
+      // If the cookie based authentication is already enabled, parse the
+      // request and validate the request cookies.
+      if (isCookieAuthEnabled) {
+        clientUserName = validateCookie(request);
+        requireNewCookie = (clientUserName == null);
+        if (requireNewCookie) {
+          LOG.info("Could not validate cookie sent, will try to generate a new cookie");
         }
       }
-      else {
-        clientUserName = doPasswdAuth(request, authType);
+      // If the cookie based authentication is not enabled or the request does
+      // not have a valid cookie, use the kerberos or password based authentication
+      // depending on the server setup.
+      if (clientUserName == null) {
+        // For a kerberos setup
+        if (isKerberosAuthMode(authType)) {
+          clientUserName = doKerberosAuth(request);
+          String doAsQueryParam = getDoAsQueryParam(request.getQueryString());
+
+          if (doAsQueryParam != null) {
+            SessionManager.setProxyUserName(doAsQueryParam);
+          }
+        }
+        // For password based authentication
+        else {
+          clientUserName = doPasswdAuth(request, authType);
+        }
       }
       LOG.debug("Client username: " + clientUserName);
       // Set the thread local username to be used for doAs if true
       SessionManager.setUserName(clientUserName);
-
       clientIpAddress = request.getRemoteAddr();
       LOG.debug("Client IP Address: " + clientIpAddress);
       // Set the thread local ip address
       SessionManager.setIpAddress(clientIpAddress);
+      // Generate new cookie and add it to the response
+      if (requireNewCookie &&
+          !authType.equalsIgnoreCase(HiveAuthFactory.AuthTypes.NOSASL.toString())) {
+        String cookieToken = HttpAuthUtils.createCookieToken(clientUserName);
+        Cookie hs2Cookie = createCookie(signer.signCookie(cookieToken));
 
+        if (isHttpOnlyCookie) {
+          response.setHeader("SET-COOKIE", getHttpOnlyCookieHeader(hs2Cookie));
+        } else {
+          response.addCookie(hs2Cookie);
+        }
+        LOG.info("Cookie added for clientUserName " + clientUserName);
+      }
       super.doPost(request, response);
     }
     catch (HttpAuthenticationException e) {
@@ -118,6 +185,127 @@ public class ThriftHttpServlet extends T
   }
 
   /**
+   * Retrieves the client name from cookieString. If the cookie does not
+   * correspond to a valid client, the function returns null.
+   * @param cookies HTTP Request cookies.
+   * @return Client Username if cookieString has a HS2 Generated cookie that is currently
valid.
+   * Else, returns null.
+   */
+  private String getClientNameFromCookie(Cookie[] cookies) {
+    // Current Cookie Name, Current Cookie Value
+    String currName, currValue;
+
+    // Following is the main loop which iterates through all the cookies send by the client.
+    // The HS2 generated cookies are of the format hive.server2.auth=<value>
+    // A cookie which is identified as a hiveserver2 generated cookie is validated
+    // by calling signer.verifyAndExtract(). If the validation passes, send the
+    // username for which the cookie is validated to the caller. If no client side
+    // cookie passes the validation, return null to the caller.
+    for (Cookie currCookie : cookies) {
+      // Get the cookie name
+      currName = currCookie.getName();
+      if (!currName.equals(AUTH_COOKIE)) {
+        // Not a HS2 generated cookie, continue.
+        continue;
+      }
+      // If we reached here, we have match for HS2 generated cookie
+      currValue = currCookie.getValue();
+      // Validate the value.
+      currValue = signer.verifyAndExtract(currValue);
+      // Retrieve the user name, do the final validation step.
+      if (currValue != null) {
+        String userName = HttpAuthUtils.getUserNameFromCookieToken(currValue);
+
+        if (userName == null) {
+          LOG.warn("Invalid cookie token " + currValue);
+          continue;
+        }
+        //We have found a valid cookie in the client request.
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Validated the cookie for user " + userName);
+        }
+        return userName;
+	  }
+    }
+    // No valid HS2 generated cookies found, return null
+    return null;
+  }
+
+  /**
+   * Convert cookie array to human readable cookie string
+   * @param cookies Cookie Array
+   * @return String containing all the cookies separated by a newline character.
+   * Each cookie is of the format [key]=[value]
+   */
+  private String toCookieStr(Cookie[] cookies) {
+	String cookieStr = "";
+
+	for (Cookie c : cookies) {
+     cookieStr += c.getName() + "=" + c.getValue() + " ;\n";
+    }
+    return cookieStr;
+  }
+
+  /**
+   * Validate the request cookie. This function iterates over the request cookie headers
+   * and finds a cookie that represents a valid client/server session. If it finds one, it
+   * returns the client name associated with the session. Else, it returns null.
+   * @param request The HTTP Servlet Request send by the client
+   * @return Client Username if the request has valid HS2 cookie, else returns null
+   * @throws UnsupportedEncodingException
+   */
+  private String validateCookie(HttpServletRequest request) throws UnsupportedEncodingException
{
+    // Find all the valid cookies associated with the request.
+    Cookie[] cookies = request.getCookies();
+
+    if (cookies == null) {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("No valid cookies associated with the request " + request);
+      }
+      return null;
+    }
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Received cookies: " + toCookieStr(cookies));
+    }
+    return getClientNameFromCookie(cookies);
+  }
+
+  /**
+   * Generate a server side cookie given the cookie value as the input.
+   * @param str Input string token.
+   * @return The generated cookie.
+   * @throws UnsupportedEncodingException
+   */
+  private Cookie createCookie(String str) throws UnsupportedEncodingException {
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Cookie name = " + AUTH_COOKIE + " value = " + str);
+    }
+    Cookie cookie = new Cookie(AUTH_COOKIE, str);
+
+    cookie.setMaxAge(cookieMaxAge);
+    if (cookieDomain != null) {
+      cookie.setDomain(cookieDomain);
+    }
+    if (cookiePath != null) {
+      cookie.setPath(cookiePath);
+    }
+    cookie.setSecure(isCookieSecure);
+    return cookie;
+  }
+
+  /**
+   * Generate httponly cookie from HS2 cookie
+   * @param cookie HS2 generated cookie
+   * @return The httponly cookie
+   */
+  private static String getHttpOnlyCookieHeader(Cookie cookie) {
+    NewCookie newCookie = new NewCookie(cookie.getName(), cookie.getValue(),
+      cookie.getPath(), cookie.getDomain(), cookie.getVersion(),
+      cookie.getComment(), cookie.getMaxAge(), cookie.getSecure());
+    return newCookie + "; HttpOnly";
+  }
+
+  /**
    * Do the LDAP/PAM authentication
    * @param request
    * @param authType

Modified: hive/branches/cbo/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java?rev=1674132&r1=1674131&r2=1674132&view=diff
==============================================================================
--- hive/branches/cbo/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
(original)
+++ hive/branches/cbo/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
Thu Apr 16 18:35:39 2015
@@ -156,6 +156,11 @@ public class Hadoop20SShims extends Hado
   }
 
   @Override
+  public void startPauseMonitor(Configuration conf) {
+    /* no supported */
+  }
+
+  @Override
   public boolean isLocalMode(Configuration conf) {
     return "local".equals(getJobLauncherRpcAddress(conf));
   }

Modified: hive/branches/cbo/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java?rev=1674132&r1=1674131&r2=1674132&view=diff
==============================================================================
--- hive/branches/cbo/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
(original)
+++ hive/branches/cbo/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
Thu Apr 16 18:35:39 2015
@@ -210,6 +210,19 @@ public class Hadoop23Shims extends Hadoo
   }
 
   @Override
+  public void startPauseMonitor(Configuration conf) {
+    try {
+      Class.forName("org.apache.hadoop.util.JvmPauseMonitor");
+      org.apache.hadoop.util.JvmPauseMonitor pauseMonitor = new org.apache.hadoop.util
+          .JvmPauseMonitor(conf);
+      pauseMonitor.start();
+    } catch (Throwable t) {
+      LOG.warn("Could not initiate the JvmPauseMonitor thread." + " GCs and Pauses may not
be " +
+          "warned upon.", t);
+    }
+  }
+
+  @Override
   public boolean isLocalMode(Configuration conf) {
     return "local".equals(conf.get("mapreduce.framework.name"));
   }

Modified: hive/branches/cbo/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java?rev=1674132&r1=1674131&r2=1674132&view=diff
==============================================================================
--- hive/branches/cbo/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
(original)
+++ hive/branches/cbo/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
Thu Apr 16 18:35:39 2015
@@ -144,6 +144,8 @@ public interface HadoopShims {
 
   public JobContext newJobContext(Job job);
 
+  public void startPauseMonitor(Configuration conf);
+
   /**
    * Check wether MR is configured to run in local-mode
    * @param conf

Modified: hive/branches/cbo/testutils/metastore/execute-test-on-lxc.sh
URL: http://svn.apache.org/viewvc/hive/branches/cbo/testutils/metastore/execute-test-on-lxc.sh?rev=1674132&r1=1674131&r2=1674132&view=diff
==============================================================================
--- hive/branches/cbo/testutils/metastore/execute-test-on-lxc.sh (original)
+++ hive/branches/cbo/testutils/metastore/execute-test-on-lxc.sh Thu Apr 16 18:35:39 2015
@@ -70,9 +70,11 @@ lxc_exists() {
 }
 
 lxc_create() {
-	lxc-create -n $1 -t download -- --dist "ubuntu" --release "trusty" --arch "amd64"
+	lxc-create -n $1 -t download -- --dist "ubuntu" --release "trusty" --arch "amd64" || return
1
+	lxc_start $1 || return 1
 	lxc-attach -n $1 -- apt-get update
-	lxc-attach -n $1 -- apt-get install -y openssh-server subversion
+	lxc-attach -n $1 -- apt-get install -y openssh-server subversion patch
+
 	printf "root\nroot" | sudo lxc-attach -n $1 -- passwd
 	lxc-attach -n $1 -- sed -i /etc/ssh/sshd_config 's/^PermitRootLogin without-password/PermitRootLogin
yes/'
 	lxc-attach -n $1 -- service ssh restart
@@ -83,20 +85,29 @@ lxc_running() {
 }
 
 lxc_start() {
-	lxc-start -n $1 --daemon
-	lxc-wait -n mysql -s RUNNING
+	lxc-start -n $1 --daemon || return 1
+	lxc-wait -n $1 -s RUNNING || return 1
 	sleep 10 # wait a little longer
 }
 
+lxc_stop() {
+	lxc-stop -n $1
+}
+
 lxc_prepare() {
 	echo "Downloading hive source code from SVN, branch='$BRANCH' ..."
-	lxc-attach -n $1 -- rm -rf /tmp/hive
-	lxc-attach -n $1 -- mkdir /tmp/hive
 
-	lxc-attach -n $1 -- svn co http://svn.apache.org/repos/asf/hive/$BRANCH /tmp/hive >/dev/null
+	tmpfile=$(mktemp)
+	cat>$tmpfile<<EOF
+rm -rf hive
+mkdir hive
+svn co http://svn.apache.org/repos/asf/hive/$BRANCH hive >/dev/null
+cd hive
+wget $PATCH_URL -O hms.patch
+bash -x testutils/ptest2/src/main/resources/smart-apply-patch.sh hms.patch
+EOF
 
-	lxc-attach -n $1 -- wget $PATCH_URL -O /tmp/hive/hms.patch
-	lxc-attach -n $1 -- patch -s -N -d /tmp/hive -p1 -i /tmp/hive/hms.patch
+	lxc-attach -n $1 -- bash -x -e < $tmpfile
 }
 
 lxc_print_metastore_log() {
@@ -104,7 +115,7 @@ lxc_print_metastore_log() {
 }
 
 run_tests() {
-	lxc-attach -n $1 -- bash /tmp/hive/testutils/metastore/metastore-upgrade-test.sh --db $1
+	lxc-attach -n $1 -- bash hive/testutils/metastore/metastore-upgrade-test.sh --db $1
 }
 
 # Install LXC packages if needed
@@ -142,4 +153,6 @@ do
 	echo "Running metastore upgrade tests for $name..."
 	run_tests $name
 	log "$(lxc_print_metastore_log $name)"
+
+	lxc_stop $name
 done



Mime
View raw message