hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ga...@apache.org
Subject svn commit: r1673437 [2/8] - in /hive/branches/hbase-metastore: ./ beeline/src/java/org/apache/hive/beeline/ bin/ cli/src/java/org/apache/hadoop/hive/cli/ common/src/java/org/apache/hadoop/hive/common/type/ common/src/java/org/apache/hadoop/hive/conf/ ...
Date Tue, 14 Apr 2015 14:47:33 GMT
Modified: hive/branches/hbase-metastore/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java
URL: http://svn.apache.org/viewvc/hive/branches/hbase-metastore/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java?rev=1673437&r1=1673436&r2=1673437&view=diff
==============================================================================
--- hive/branches/hbase-metastore/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java (original)
+++ hive/branches/hbase-metastore/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java Tue Apr 14 14:47:30 2015
@@ -24,6 +24,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import com.google.common.collect.Lists;
 import junit.framework.TestCase;
 
 import org.apache.hadoop.hive.cli.CliSessionState;
@@ -296,7 +297,8 @@ public class TestMetaStoreEventListener
     AddPartitionEvent partEvent = (AddPartitionEvent)(notifyList.get(listSize-1));
     assert partEvent.getStatus();
     Partition part = msc.getPartition("hive2038", "tmptbl", "b=2011");
-    validateAddPartition(part, partEvent.getPartitions().get(0));
+    Partition partAdded = partEvent.getPartitionIterator().next();
+    validateAddPartition(part, partAdded);
     validateTableInAddPartition(tbl, partEvent.getTable());
     validateAddPartition(part, prePartEvent.getPartitions().get(0));
 
@@ -313,11 +315,12 @@ public class TestMetaStoreEventListener
     hmsClient.add_partitions(Arrays.asList(partition1, partition2, partition3));
     ++listSize;
     AddPartitionEvent multiplePartitionEvent = (AddPartitionEvent)(notifyList.get(listSize-1));
-    assertEquals("Unexpected number of partitions in event!", 3, multiplePartitionEvent.getPartitions().size());
     assertEquals("Unexpected table value.", table, multiplePartitionEvent.getTable());
-    assertEquals("Unexpected partition value.", partition1.getValues(), multiplePartitionEvent.getPartitions().get(0).getValues());
-    assertEquals("Unexpected partition value.", partition2.getValues(), multiplePartitionEvent.getPartitions().get(1).getValues());
-    assertEquals("Unexpected partition value.", partition3.getValues(), multiplePartitionEvent.getPartitions().get(2).getValues());
+    List<Partition> multiParts = Lists.newArrayList(multiplePartitionEvent.getPartitionIterator());
+    assertEquals("Unexpected number of partitions in event!", 3, multiParts.size());
+    assertEquals("Unexpected partition value.", partition1.getValues(), multiParts.get(0).getValues());
+    assertEquals("Unexpected partition value.", partition2.getValues(), multiParts.get(1).getValues());
+    assertEquals("Unexpected partition value.", partition3.getValues(), multiParts.get(2).getValues());
 
     driver.run(String.format("alter table %s touch partition (%s)", tblName, "b='2011'"));
     listSize++;
@@ -352,7 +355,8 @@ public class TestMetaStoreEventListener
 
     AddPartitionEvent appendPartEvent =
         (AddPartitionEvent)(notifyList.get(listSize-1));
-    validateAddPartition(newPart, appendPartEvent.getPartitions().get(0));
+    Partition partAppended = appendPartEvent.getPartitionIterator().next();
+    validateAddPartition(newPart, partAppended);
 
     PreAddPartitionEvent preAppendPartEvent =
         (PreAddPartitionEvent)(preNotifyList.get(preNotifyList.size() - 1));

Modified: hive/branches/hbase-metastore/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java
URL: http://svn.apache.org/viewvc/hive/branches/hbase-metastore/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java?rev=1673437&r1=1673436&r2=1673437&view=diff
==============================================================================
--- hive/branches/hbase-metastore/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java (original)
+++ hive/branches/hbase-metastore/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java Tue Apr 14 14:47:30 2015
@@ -97,7 +97,7 @@ public class TestHiveAuthorizerCheckInvo
         + " (i int, j int, k string) partitioned by (city string, `date` string) ");
     runCmd("create database " + dbName);
     // Need a separate table for ACID testing since it has to be bucketed and it has to be Acid
-    runCmd("create table " + acidTableName + " (i int, j int) clustered by (i) into 2 buckets " +
+    runCmd("create table " + acidTableName + " (i int, j int, k int) clustered by (k) into 2 buckets " +
         "stored as orc TBLPROPERTIES ('transactional'='true')");
   }
 
@@ -273,7 +273,7 @@ public class TestHiveAuthorizerCheckInvo
     List<HivePrivilegeObject> inputs = io.getLeft();
     assertEquals(1, inputs.size());
     tableObj = inputs.get(0);
-    assertEquals(1, tableObj.getColumns().size());
+    assertEquals(2, tableObj.getColumns().size());
     assertEquals("j", tableObj.getColumns().get(0));
   }
 

Modified: hive/branches/hbase-metastore/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
URL: http://svn.apache.org/viewvc/hive/branches/hbase-metastore/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java?rev=1673437&r1=1673436&r2=1673437&view=diff
==============================================================================
--- hive/branches/hbase-metastore/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java (original)
+++ hive/branches/hbase-metastore/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java Tue Apr 14 14:47:30 2015
@@ -371,7 +371,7 @@ public class TestCompactor {
     executeStatementOnDriver("insert into " + tblName + " partition (ds) values (1, 'fred', " +
         "'today'), (2, 'wilma', 'yesterday')", driver);
 
-    executeStatementOnDriver("update " + tblName + " set a = 3", driver);
+    executeStatementOnDriver("update " + tblName + " set b = 'fred' where a = 1", driver);
 
     executeStatementOnDriver("delete from " + tblName + " where b = 'fred'", driver);
 

Modified: hive/branches/hbase-metastore/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java
URL: http://svn.apache.org/viewvc/hive/branches/hbase-metastore/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java?rev=1673437&r1=1673436&r2=1673437&view=diff
==============================================================================
--- hive/branches/hbase-metastore/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java (original)
+++ hive/branches/hbase-metastore/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java Tue Apr 14 14:47:30 2015
@@ -108,7 +108,7 @@ public class TestJdbcDriver2 {
   public static void setUpBeforeClass() throws SQLException, ClassNotFoundException{
     Class.forName(driverName);
     Connection con1 = getConnection("default");
-    System.setProperty(ConfVars.HIVE_SERVER2_LOGGING_OPERATION_VERBOSE.varname, "" + true);
+    System.setProperty(ConfVars.HIVE_SERVER2_LOGGING_OPERATION_LEVEL.varname, "verbose");
 
     Statement stmt1 = con1.createStatement();
     assertNotNull("Statement is null", stmt1);

Modified: hive/branches/hbase-metastore/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java
URL: http://svn.apache.org/viewvc/hive/branches/hbase-metastore/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java?rev=1673437&r1=1673436&r2=1673437&view=diff
==============================================================================
--- hive/branches/hbase-metastore/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java (original)
+++ hive/branches/hbase-metastore/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java Tue Apr 14 14:47:30 2015
@@ -266,7 +266,7 @@ public class TestJdbcWithMiniHS2 {
 
     // Set some conf parameters
     String hiveConf = "hive.cli.print.header=true;hive.server2.async.exec.shutdown.timeout=20;"
-        + "hive.server2.async.exec.threads=30;hive.server2.thrift.http.max.worker.threads=15";
+        + "hive.server2.async.exec.threads=30;hive.server2.thrift.max.worker.threads=15";
     // Set some conf vars
     String hiveVar = "stab=salesTable;icol=customerID";
     String jdbcUri = miniHS2.getJdbcURL() + "?" + hiveConf + "#" + hiveVar;
@@ -284,7 +284,7 @@ public class TestJdbcWithMiniHS2 {
     verifyConfProperty(stmt, "hive.cli.print.header", "true");
     verifyConfProperty(stmt, "hive.server2.async.exec.shutdown.timeout", "20");
     verifyConfProperty(stmt, "hive.server2.async.exec.threads", "30");
-    verifyConfProperty(stmt, "hive.server2.thrift.http.max.worker.threads",
+    verifyConfProperty(stmt, "hive.server2.thrift.max.worker.threads",
         "15");
     verifyConfProperty(stmt, "stab", "salesTable");
     verifyConfProperty(stmt, "icol", "customerID");

Modified: hive/branches/hbase-metastore/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestSSL.java
URL: http://svn.apache.org/viewvc/hive/branches/hbase-metastore/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestSSL.java?rev=1673437&r1=1673436&r2=1673437&view=diff
==============================================================================
--- hive/branches/hbase-metastore/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestSSL.java (original)
+++ hive/branches/hbase-metastore/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestSSL.java Tue Apr 14 14:47:30 2015
@@ -155,7 +155,7 @@ public class TestSSL {
         cause = cause.getCause();
       }
       Assert.assertEquals("org.apache.http.NoHttpResponseException", cause.getClass().getName());
-      Assert.assertEquals("The target server failed to respond", cause.getMessage());
+      Assert.assertTrue(cause.getMessage().contains("failed to respond"));
     }
     miniHS2.stop();
   }

Modified: hive/branches/hbase-metastore/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/TestThriftHttpCLIService.java
URL: http://svn.apache.org/viewvc/hive/branches/hbase-metastore/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/TestThriftHttpCLIService.java?rev=1673437&r1=1673436&r2=1673437&view=diff
==============================================================================
--- hive/branches/hbase-metastore/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/TestThriftHttpCLIService.java (original)
+++ hive/branches/hbase-metastore/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/TestThriftHttpCLIService.java Tue Apr 14 14:47:30 2015
@@ -160,7 +160,7 @@ public class TestThriftHttpCLIService ex
     String httpUrl = transportMode + "://" + host + ":" + port +
         "/" + thriftHttpPath + "/";
     httpClient.addRequestInterceptor(
-        new HttpBasicAuthInterceptor(USERNAME, PASSWORD));
+        new HttpBasicAuthInterceptor(USERNAME, PASSWORD, null, null));
     return new THttpClient(httpUrl, httpClient);
   }
 

Modified: hive/branches/hbase-metastore/itests/src/test/resources/testconfiguration.properties
URL: http://svn.apache.org/viewvc/hive/branches/hbase-metastore/itests/src/test/resources/testconfiguration.properties?rev=1673437&r1=1673436&r2=1673437&view=diff
==============================================================================
--- hive/branches/hbase-metastore/itests/src/test/resources/testconfiguration.properties (original)
+++ hive/branches/hbase-metastore/itests/src/test/resources/testconfiguration.properties Tue Apr 14 14:47:30 2015
@@ -181,9 +181,11 @@ minitez.query.files.shared=alter_merge_2
   update_where_non_partitioned.q,\
   update_where_partitioned.q,\
   update_two_cols.q,\
+  vector_aggregate_9.q,\
   vector_between_in.q,\
   vector_bucket.q,\
   vector_cast_constant.q,\
+  vector_char_2.q,\
   vector_char_4.q,\
   vector_char_mapjoin1.q,\
   vector_char_simple.q,\
@@ -191,6 +193,7 @@ minitez.query.files.shared=alter_merge_2
   vector_coalesce_2.q,\
   vector_count_distinct.q,\
   vector_data_types.q,\
+  vector_date_1.q,\
   vector_decimal_1.q,\
   vector_decimal_10_0.q,\
   vector_decimal_2.q,\
@@ -204,6 +207,8 @@ minitez.query.files.shared=alter_merge_2
   vector_decimal_mapjoin.q,\
   vector_decimal_math_funcs.q,\
   vector_decimal_precision.q,\
+  vector_decimal_round.q,\
+  vector_decimal_round_2.q,\
   vector_decimal_trailing.q,\
   vector_decimal_udf.q,\
   vector_decimal_udf2.q,\
@@ -211,8 +216,12 @@ minitez.query.files.shared=alter_merge_2
   vector_elt.q,\
   vector_groupby_3.q,\
   vector_groupby_reduce.q,\
+  vector_if_expr.q,\
+  vector_interval_1.q,\
+  vector_interval_2.q,\
   vector_left_outer_join.q,\
   vector_mapjoin_reduce.q,\
+  vector_multi_insert.q,\
   vector_non_string_partition.q,\
   vector_orderby_5.q,\
   vector_partition_diff_num_cols.q,\

Modified: hive/branches/hbase-metastore/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
URL: http://svn.apache.org/viewvc/hive/branches/hbase-metastore/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java?rev=1673437&r1=1673436&r2=1673437&view=diff
==============================================================================
--- hive/branches/hbase-metastore/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java (original)
+++ hive/branches/hbase-metastore/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java Tue Apr 14 14:47:30 2015
@@ -50,10 +50,11 @@ import java.util.concurrent.TimeUnit;
 import javax.security.sasl.Sasl;
 import javax.security.sasl.SaslException;
 
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.protocol.HttpContext;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.shims.ShimLoader;
 import org.apache.hive.jdbc.Utils.JdbcConnectionParams;
 import org.apache.hive.service.auth.HiveAuthFactory;
 import org.apache.hive.service.auth.KerberosSaslHelper;
@@ -73,9 +74,17 @@ import org.apache.hive.service.cli.thrif
 import org.apache.hive.service.cli.thrift.TRenewDelegationTokenResp;
 import org.apache.hive.service.cli.thrift.TSessionHandle;
 import org.apache.http.HttpRequestInterceptor;
-import org.apache.http.conn.scheme.Scheme;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.CookieStore;
+import org.apache.http.client.ServiceUnavailableRetryStrategy;
+import org.apache.http.config.Registry;
+import org.apache.http.config.RegistryBuilder;
+import org.apache.http.conn.socket.ConnectionSocketFactory;
 import org.apache.http.conn.ssl.SSLSocketFactory;
-import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.http.impl.client.BasicCookieStore;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.impl.conn.BasicHttpClientConnectionManager;
 import org.apache.thrift.TException;
 import org.apache.thrift.protocol.TBinaryProtocol;
 import org.apache.thrift.transport.THttpClient;
@@ -236,7 +245,7 @@ public class HiveConnection implements j
   }
 
   private TTransport createHttpTransport() throws SQLException, TTransportException {
-    DefaultHttpClient httpClient;
+    CloseableHttpClient httpClient;
     boolean useSsl = isSslConnection();
     // Create an http client from the configs
     httpClient = getHttpClient(useSsl);
@@ -260,35 +269,76 @@ public class HiveConnection implements j
     return transport;
   }
 
-  private DefaultHttpClient getHttpClient(Boolean useSsl) throws SQLException {
-    DefaultHttpClient httpClient = new DefaultHttpClient();
+  private CloseableHttpClient getHttpClient(Boolean useSsl) throws SQLException {
+    boolean isCookieEnabled = sessConfMap.get(JdbcConnectionParams.COOKIE_AUTH) == null ||
+      (!JdbcConnectionParams.COOKIE_AUTH_FALSE.equalsIgnoreCase(
+      sessConfMap.get(JdbcConnectionParams.COOKIE_AUTH)));
+    String cookieName = sessConfMap.get(JdbcConnectionParams.COOKIE_NAME) == null ?
+      JdbcConnectionParams.DEFAULT_COOKIE_NAMES_HS2 :
+      sessConfMap.get(JdbcConnectionParams.COOKIE_NAME);
+    CookieStore cookieStore = isCookieEnabled ? new BasicCookieStore() : null;
+    HttpClientBuilder httpClientBuilder;
     // Request interceptor for any request pre-processing logic
     HttpRequestInterceptor requestInterceptor;
-    // If Kerberos
+
+    // Configure http client for kerberos/password based authentication
     if (isKerberosAuthMode()) {
       /**
        * Add an interceptor which sets the appropriate header in the request.
        * It does the kerberos authentication and get the final service ticket,
        * for sending to the server before every request.
        * In https mode, the entire information is encrypted
-       * TODO: Optimize this with a mix of kerberos + using cookie.
        */
       requestInterceptor =
           new HttpKerberosRequestInterceptor(sessConfMap.get(JdbcConnectionParams.AUTH_PRINCIPAL),
-              host, getServerHttpUrl(useSsl), assumeSubject);
+              host, getServerHttpUrl(useSsl), assumeSubject, cookieStore, cookieName);
     }
     else {
       /**
        * Add an interceptor to pass username/password in the header.
        * In https mode, the entire information is encrypted
        */
-      requestInterceptor = new HttpBasicAuthInterceptor(getUserName(), getPassword());
+      requestInterceptor = new HttpBasicAuthInterceptor(getUserName(), getPassword(),
+                                                        cookieStore, cookieName);
+    }
+    // Configure http client for cookie based authentication
+    if (isCookieEnabled) {
+      // Create a http client with a retry mechanism when the server returns a status code of 401.
+      httpClientBuilder =
+      HttpClients.custom().setServiceUnavailableRetryStrategy(
+        new  ServiceUnavailableRetryStrategy() {
+
+      @Override
+      public boolean retryRequest(
+        final HttpResponse response,
+        final int executionCount,
+        final HttpContext context) {
+        int statusCode = response.getStatusLine().getStatusCode();
+        boolean ret = statusCode == 401 && executionCount <= 1;
+
+        // Set the context attribute to true which will be interpreted by the request interceptor
+        if (ret) {
+          context.setAttribute(Utils.HIVE_SERVER2_RETRY_KEY, Utils.HIVE_SERVER2_RETRY_TRUE);
+        }
+        return ret;
+      }
+
+      @Override
+      public long getRetryInterval() {
+        // Immediate retry
+        return 0;
+      }
+    });
+    } else {
+      httpClientBuilder = HttpClientBuilder.create();
     }
-    // Configure httpClient for SSL
+    // Add the request interceptor to the client builder
+    httpClientBuilder.addInterceptorFirst(requestInterceptor);
+    // Configure http client for SSL
     if (useSsl) {
       String sslTrustStorePath = sessConfMap.get(JdbcConnectionParams.SSL_TRUST_STORE);
       String sslTrustStorePassword = sessConfMap.get(
-          JdbcConnectionParams.SSL_TRUST_STORE_PASSWORD);
+        JdbcConnectionParams.SSL_TRUST_STORE_PASSWORD);
       KeyStore sslTrustStore;
       SSLSocketFactory socketFactory;
       /**
@@ -312,21 +362,25 @@ public class HiveConnection implements j
           // Pick trust store config from the given path
           sslTrustStore = KeyStore.getInstance(JdbcConnectionParams.SSL_TRUST_STORE_TYPE);
           sslTrustStore.load(new FileInputStream(sslTrustStorePath),
-              sslTrustStorePassword.toCharArray());
+            sslTrustStorePassword.toCharArray());
           socketFactory = new SSLSocketFactory(sslTrustStore);
         }
         socketFactory.setHostnameVerifier(SSLSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
-        Scheme sslScheme = new Scheme("https", 443, socketFactory);
-        httpClient.getConnectionManager().getSchemeRegistry().register(sslScheme);
+
+        final Registry<ConnectionSocketFactory> registry =
+          RegistryBuilder.<ConnectionSocketFactory>create()
+          .register("https", socketFactory)
+          .build();
+
+        httpClientBuilder.setConnectionManager(new BasicHttpClientConnectionManager(registry));
       }
       catch (Exception e) {
         String msg =  "Could not create an https connection to " +
-            jdbcUriString + ". " + e.getMessage();
+          jdbcUriString + ". " + e.getMessage();
         throw new SQLException(msg, " 08S01", e);
       }
     }
-    httpClient.addRequestInterceptor(requestInterceptor);
-    return httpClient;
+    return httpClientBuilder.build();
   }
 
   /**

Modified: hive/branches/hbase-metastore/jdbc/src/java/org/apache/hive/jdbc/HttpBasicAuthInterceptor.java
URL: http://svn.apache.org/viewvc/hive/branches/hbase-metastore/jdbc/src/java/org/apache/hive/jdbc/HttpBasicAuthInterceptor.java?rev=1673437&r1=1673436&r2=1673437&view=diff
==============================================================================
--- hive/branches/hbase-metastore/jdbc/src/java/org/apache/hive/jdbc/HttpBasicAuthInterceptor.java (original)
+++ hive/branches/hbase-metastore/jdbc/src/java/org/apache/hive/jdbc/HttpBasicAuthInterceptor.java Tue Apr 14 14:47:30 2015
@@ -25,6 +25,8 @@ import org.apache.http.HttpException;
 import org.apache.http.HttpRequest;
 import org.apache.http.HttpRequestInterceptor;
 import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.CookieStore;
+import org.apache.http.client.protocol.ClientContext;
 import org.apache.http.impl.auth.AuthSchemeBase;
 import org.apache.http.impl.auth.BasicScheme;
 import org.apache.http.protocol.HttpContext;
@@ -37,20 +39,42 @@ import org.apache.http.protocol.HttpCont
 public class HttpBasicAuthInterceptor implements HttpRequestInterceptor {
   UsernamePasswordCredentials credentials;
   AuthSchemeBase authScheme;
+  CookieStore cookieStore;
+  boolean isCookieEnabled;
+  String cookieName;
 
-  public HttpBasicAuthInterceptor(String username, String password) {
+  public HttpBasicAuthInterceptor(String username, String password, CookieStore cookieStore,
+                           String cn) {
     if(username != null){
       credentials = new UsernamePasswordCredentials(username, password);
     }
     authScheme = new BasicScheme();
+    this.cookieStore = cookieStore;
+    isCookieEnabled = (cookieStore != null);
+    cookieName = cn;
   }
 
   @Override
   public void process(HttpRequest httpRequest, HttpContext httpContext)
       throws HttpException, IOException {
-    Header basicAuthHeader = authScheme.authenticate(
-        credentials, httpRequest, httpContext);
-    httpRequest.addHeader(basicAuthHeader);
+    if (isCookieEnabled) {
+      httpContext.setAttribute(ClientContext.COOKIE_STORE, cookieStore);
+    }
+    // Add the authentication details under the following scenarios:
+    // 1. Cookie Authentication is disabled OR
+    // 2. The first time when the request is sent OR
+    // 3. The server returns a 401, which sometimes means the cookie has expired
+    if (!isCookieEnabled || ((httpContext.getAttribute(Utils.HIVE_SERVER2_RETRY_KEY) == null &&
+        (cookieStore == null || (cookieStore != null &&
+        Utils.needToSendCredentials(cookieStore, cookieName)))) ||
+        (httpContext.getAttribute(Utils.HIVE_SERVER2_RETRY_KEY) != null &&
+         httpContext.getAttribute(Utils.HIVE_SERVER2_RETRY_KEY).
+         equals(Utils.HIVE_SERVER2_RETRY_TRUE)))) {
+      Header basicAuthHeader = authScheme.authenticate(credentials, httpRequest, httpContext);
+      httpRequest.addHeader(basicAuthHeader);
+    }
+    if (isCookieEnabled) {
+      httpContext.setAttribute(Utils.HIVE_SERVER2_RETRY_KEY, Utils.HIVE_SERVER2_RETRY_FALSE);
+    }
   }
-
 }

Modified: hive/branches/hbase-metastore/jdbc/src/java/org/apache/hive/jdbc/HttpKerberosRequestInterceptor.java
URL: http://svn.apache.org/viewvc/hive/branches/hbase-metastore/jdbc/src/java/org/apache/hive/jdbc/HttpKerberosRequestInterceptor.java?rev=1673437&r1=1673436&r2=1673437&view=diff
==============================================================================
--- hive/branches/hbase-metastore/jdbc/src/java/org/apache/hive/jdbc/HttpKerberosRequestInterceptor.java (original)
+++ hive/branches/hbase-metastore/jdbc/src/java/org/apache/hive/jdbc/HttpKerberosRequestInterceptor.java Tue Apr 14 14:47:30 2015
@@ -25,6 +25,8 @@ import org.apache.hive.service.auth.Http
 import org.apache.http.HttpException;
 import org.apache.http.HttpRequest;
 import org.apache.http.HttpRequestInterceptor;
+import org.apache.http.client.CookieStore;
+import org.apache.http.client.protocol.ClientContext;
 import org.apache.http.protocol.HttpContext;
 
 /**
@@ -40,31 +42,59 @@ public class HttpKerberosRequestIntercep
   String host;
   String serverHttpUrl;
   boolean assumeSubject;
+  CookieStore cookieStore;
+  boolean isCookieEnabled;
+  String cookieName;
 
   // A fair reentrant lock
   private static ReentrantLock kerberosLock = new ReentrantLock(true);
 
   public HttpKerberosRequestInterceptor(String principal, String host,
-      String serverHttpUrl, boolean assumeSubject) {
+      String serverHttpUrl, boolean assumeSubject, CookieStore cs, String cn) {
     this.principal = principal;
     this.host = host;
     this.serverHttpUrl = serverHttpUrl;
     this.assumeSubject = assumeSubject;
+    this.cookieStore = cs;
+    isCookieEnabled = (cs != null);
+    cookieName = cn;
   }
 
   @Override
   public void process(HttpRequest httpRequest, HttpContext httpContext)
       throws HttpException, IOException {
     String kerberosAuthHeader;
+
     try {
       // Generate the service ticket for sending to the server.
       // Locking ensures the tokens are unique in case of concurrent requests
       kerberosLock.lock();
-      kerberosAuthHeader = HttpAuthUtils.getKerberosServiceTicket(
-          principal, host, serverHttpUrl, assumeSubject);
-      // Set the session key token (Base64 encoded) in the headers
-      httpRequest.addHeader(HttpAuthUtils.AUTHORIZATION + ": " +
-          HttpAuthUtils.NEGOTIATE + " ", kerberosAuthHeader);
+      // If cookie based authentication is allowed, generate ticket only when necessary.
+      // The necessary condition is either when there are no server side cookies in the
+      // cookiestore which can be send back or when the server returns a 401 error code
+      // indicating that the previous cookie has expired.
+      if (isCookieEnabled) {
+        httpContext.setAttribute(ClientContext.COOKIE_STORE, cookieStore);
+      }
+      // Generate the kerberos ticket under the following scenarios:
+      // 1. Cookie Authentication is disabled OR
+      // 2. The first time when the request is sent OR
+      // 3. The server returns a 401, which sometimes means the cookie has expired
+      if (!isCookieEnabled || ((httpContext.getAttribute(Utils.HIVE_SERVER2_RETRY_KEY) == null &&
+          (cookieStore == null || (cookieStore != null &&
+          Utils.needToSendCredentials(cookieStore, cookieName)))) ||
+          (httpContext.getAttribute(Utils.HIVE_SERVER2_RETRY_KEY) != null &&
+          httpContext.getAttribute(Utils.HIVE_SERVER2_RETRY_KEY).
+          equals(Utils.HIVE_SERVER2_RETRY_TRUE)))) {
+        kerberosAuthHeader = HttpAuthUtils.getKerberosServiceTicket(
+            principal, host, serverHttpUrl, assumeSubject);
+        // Set the session key token (Base64 encoded) in the headers
+        httpRequest.addHeader(HttpAuthUtils.AUTHORIZATION + ": " +
+            HttpAuthUtils.NEGOTIATE + " ", kerberosAuthHeader);
+      }
+      if (isCookieEnabled) {
+        httpContext.setAttribute(Utils.HIVE_SERVER2_RETRY_KEY, Utils.HIVE_SERVER2_RETRY_FALSE);
+      }
     } catch (Exception e) {
       throw new HttpException(e.getMessage(), e);
     }

Modified: hive/branches/hbase-metastore/jdbc/src/java/org/apache/hive/jdbc/Utils.java
URL: http://svn.apache.org/viewvc/hive/branches/hbase-metastore/jdbc/src/java/org/apache/hive/jdbc/Utils.java?rev=1673437&r1=1673436&r2=1673437&view=diff
==============================================================================
--- hive/branches/hbase-metastore/jdbc/src/java/org/apache/hive/jdbc/Utils.java (original)
+++ hive/branches/hbase-metastore/jdbc/src/java/org/apache/hive/jdbc/Utils.java Tue Apr 14 14:47:30 2015
@@ -34,6 +34,8 @@ import org.apache.commons.logging.LogFac
 import org.apache.hive.service.cli.HiveSQLException;
 import org.apache.hive.service.cli.thrift.TStatus;
 import org.apache.hive.service.cli.thrift.TStatusCode;
+import org.apache.http.client.CookieStore;
+import org.apache.http.cookie.Cookie;
 
 public class Utils {
   public static final Log LOG = LogFactory.getLog(Utils.class.getName());
@@ -56,6 +58,11 @@ public class Utils {
 
   private static final String URI_HIVE_PREFIX = "hive2:";
 
+  // This value is set to true by the setServiceUnavailableRetryStrategy() when the server returns 401
+  static final String HIVE_SERVER2_RETRY_KEY = "hive.server2.retryserver";
+  static final String HIVE_SERVER2_RETRY_TRUE = "true";
+  static final String HIVE_SERVER2_RETRY_FALSE = "false";
+
   public static class JdbcConnectionParams {
     // Note on client side parameter naming convention:
     // Prefer using a shorter camelCase param name instead of using the same name as the
@@ -98,6 +105,11 @@ public class Utils {
     // Default namespace value on ZooKeeper.
     // This value is used if the param "zooKeeperNamespace" is not specified in the JDBC Uri.
     static final String ZOOKEEPER_DEFAULT_NAMESPACE = "hiveserver2";
+    static final String COOKIE_AUTH = "cookieAuth";
+    static final String COOKIE_AUTH_FALSE = "false";
+    static final String COOKIE_NAME = "cookieName";
+    // The default value of the cookie name when CookieAuth=true
+    static final String DEFAULT_COOKIE_NAMES_HS2 = "hive.server2.auth";
 
     // Non-configurable params:
     // Currently supports JKS keystore format
@@ -560,4 +572,28 @@ public class Utils {
     }
     return version;
   }
+
+  /**
+   * The function iterates through the list of cookies in the cookiestore and tries to
+   * match them with the cookieName. If there is a match, the cookieStore already
+   * has a valid cookie and the client need not send Credentials for validation purpose.
+   * @param cookieStore The cookie Store
+   * @param cookieName Name of the cookie which needs to be validated
+   * @return true or false based on whether the client needs to send the credentials or
+   * not to the server.
+   */
+  static boolean needToSendCredentials(CookieStore cookieStore, String cookieName) {
+    if (cookieName == null || cookieStore == null) {
+      return true;
+    }
+
+    List<Cookie> cookies = cookieStore.getCookies();
+
+    for (Cookie c : cookies) {
+      if (c.getName().equals(cookieName)) {
+        return false;
+      }
+    }
+    return true;
+  }
 }

Modified: hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
URL: http://svn.apache.org/viewvc/hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java?rev=1673437&r1=1673436&r2=1673437&view=diff
==============================================================================
--- hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (original)
+++ hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java Tue Apr 14 14:47:30 2015
@@ -5393,7 +5393,7 @@ public class HiveMetaStore extends Thrif
         ex = e;
         throw newMetaException(e);
       } finally {
-        endFunction("get_database", func != null, ex);
+        endFunction("get_function", func != null, ex);
       }
 
       return func;

Modified: hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/IExtrapolatePartStatus.java
URL: http://svn.apache.org/viewvc/hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/IExtrapolatePartStatus.java?rev=1673437&r1=1673436&r2=1673437&view=diff
==============================================================================
--- hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/IExtrapolatePartStatus.java (original)
+++ hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/IExtrapolatePartStatus.java Tue Apr 14 14:47:30 2015
@@ -25,23 +25,33 @@ public interface IExtrapolatePartStatus
   /**
    * The sequence of colStatNames.
    */
-  static String[] colStatNames = new String[] { "LONG_LOW_VALUE",
-      "LONG_HIGH_VALUE", "DOUBLE_LOW_VALUE", "DOUBLE_HIGH_VALUE",
-      "BIG_DECIMAL_LOW_VALUE", "BIG_DECIMAL_HIGH_VALUE", "NUM_NULLS",
-      "NUM_DISTINCTS", "AVG_COL_LEN", "MAX_COL_LEN", "NUM_TRUES", "NUM_FALSES" };
-  
+  static String[] colStatNames = new String[] { "LONG_LOW_VALUE", "LONG_HIGH_VALUE",
+      "DOUBLE_LOW_VALUE", "DOUBLE_HIGH_VALUE", "BIG_DECIMAL_LOW_VALUE", "BIG_DECIMAL_HIGH_VALUE",
+      "NUM_NULLS", "NUM_DISTINCTS", "AVG_COL_LEN", "MAX_COL_LEN", "NUM_TRUES", "NUM_FALSES",
+      "AVG_NDV_LONG", "AVG_NDV_DOUBLE", "AVG_NDV_DECIMAL", "SUM_NUM_DISTINCTS" };
+
   /**
    * The indexes for colstats.
    */
-  static HashMap<String, Integer[]> indexMaps = new HashMap<String, Integer []>(){{
-    put("long", new Integer [] {0,1,6,7});
-    put("double", new Integer [] {2,3,6,7});
-    put("string", new Integer [] {8,9,6,7});
-    put("boolean", new Integer [] {10,11,6});
-    put("binary", new Integer [] {8,9,6});
-    put("decimal", new Integer [] {4,5,6,7});
-    put("default", new Integer [] {0,1,2,3,4,5,6,7,8,9,10,11});
-}};
+  static HashMap<String, Integer[]> indexMaps = new HashMap<String, Integer[]>() {
+    {
+      put("bigint", new Integer[] { 0, 1, 6, 7, 12, 15 });
+      put("int", new Integer[] { 0, 1, 6, 7, 12, 15 });
+      put("smallint", new Integer[] { 0, 1, 6, 7, 12, 15 });
+      put("tinyint", new Integer[] { 0, 1, 6, 7, 12, 15 });
+      put("timestamp", new Integer[] { 0, 1, 6, 7, 12, 15 });
+      put("long", new Integer[] { 0, 1, 6, 7, 12, 15 });
+      put("double", new Integer[] { 2, 3, 6, 7, 13, 15 });
+      put("float", new Integer[] { 2, 3, 6, 7, 13, 15 });
+      put("varchar", new Integer[] { 8, 9, 6, 7, 15 });
+      put("char", new Integer[] { 8, 9, 6, 7, 15 });
+      put("string", new Integer[] { 8, 9, 6, 7, 15 });
+      put("boolean", new Integer[] { 10, 11, 6, 15 });
+      put("binary", new Integer[] { 8, 9, 6, 15 });
+      put("decimal", new Integer[] { 4, 5, 6, 7, 14, 15 });
+      put("default", new Integer[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 15 });
+    }
+  };
 
   /**
    * The sequence of colStatTypes.
@@ -50,23 +60,24 @@ public interface IExtrapolatePartStatus
     Long, Double, Decimal
   }
 
-  static ColStatType[] colStatTypes = new ColStatType[] { ColStatType.Long,
-      ColStatType.Long, ColStatType.Double, ColStatType.Double,
-      ColStatType.Decimal, ColStatType.Decimal, ColStatType.Long,
-      ColStatType.Long, ColStatType.Double, ColStatType.Long, ColStatType.Long,
+  static ColStatType[] colStatTypes = new ColStatType[] { ColStatType.Long, ColStatType.Long,
+      ColStatType.Double, ColStatType.Double, ColStatType.Decimal, ColStatType.Decimal,
+      ColStatType.Long, ColStatType.Long, ColStatType.Double, ColStatType.Long, ColStatType.Long,
+      ColStatType.Long, ColStatType.Double, ColStatType.Double, ColStatType.Double,
       ColStatType.Long };
 
   /**
    * The sequence of aggregation function on colStats.
    */
   static enum AggrType {
-    Min, Max, Sum
+    Min, Max, Sum, Avg
   }
 
-  static AggrType[] aggrTypes = new AggrType[] { AggrType.Min, AggrType.Max,
-      AggrType.Min, AggrType.Max, AggrType.Min, AggrType.Max, AggrType.Sum,
-      AggrType.Max, AggrType.Max, AggrType.Max, AggrType.Sum, AggrType.Sum };
-  
+  static AggrType[] aggrTypes = new AggrType[] { AggrType.Min, AggrType.Max, AggrType.Min,
+      AggrType.Max, AggrType.Min, AggrType.Max, AggrType.Sum, AggrType.Max, AggrType.Max,
+      AggrType.Max, AggrType.Sum, AggrType.Sum, AggrType.Avg, AggrType.Avg, AggrType.Avg,
+      AggrType.Sum };
+
   public Object extrapolate(Object[] min, Object[] max, int colStatIndex,
       Map<String, Integer> indexMap);
 

Modified: hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/LinearExtrapolatePartStatus.java
URL: http://svn.apache.org/viewvc/hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/LinearExtrapolatePartStatus.java?rev=1673437&r1=1673436&r2=1673437&view=diff
==============================================================================
--- hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/LinearExtrapolatePartStatus.java (original)
+++ hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/LinearExtrapolatePartStatus.java Tue Apr 14 14:47:30 2015
@@ -19,11 +19,8 @@
 package org.apache.hadoop.hive.metastore;
 
 import java.math.BigDecimal;
-import java.nio.ByteBuffer;
 import java.util.Map;
 
-import org.apache.hadoop.hive.metastore.api.Decimal;
-
 public class LinearExtrapolatePartStatus implements IExtrapolatePartStatus {
 
   @Override
@@ -35,6 +32,15 @@ public class LinearExtrapolatePartStatus
     if (minInd == maxInd) {
       return min[0];
     }
+    //note that recent metastore stores decimal in string.
+    double decimalmin= 0;
+    double decimalmax = 0;
+    if (colStatTypes[colStatIndex] == ColStatType.Decimal) {
+      BigDecimal bdmin = new BigDecimal(min[0].toString());
+      decimalmin = bdmin.doubleValue();
+      BigDecimal bdmax = new BigDecimal(max[0].toString());
+      decimalmax = bdmax.doubleValue();
+    }
     if (aggrTypes[colStatIndex] == AggrType.Max) {
       if (minInd < maxInd) {
         // right border is the max
@@ -45,15 +51,9 @@ public class LinearExtrapolatePartStatus
           return (Double) ((Double) min[0] + (((Double) max[0] - (Double) min[0])
               * (rightBorderInd - minInd) / (maxInd - minInd)));
         } else {
-          Decimal dmax = (Decimal) max[0];
-          BigDecimal bdmax = new BigDecimal(dmax.toString());
-          double doublemax = bdmax.doubleValue();
-          Decimal dmin = (Decimal) min[0];
-          BigDecimal bdmin = new BigDecimal(dmin.toString());
-          double doublemin = bdmin.doubleValue();
-          double ret = doublemin + (doublemax - doublemin)
+          double ret = decimalmin + (decimalmax - decimalmin)
               * (rightBorderInd - minInd) / (maxInd - minInd);
-          return createThriftDecimal(String.valueOf(ret));
+          return String.valueOf(ret);
         }
       } else {
         // left border is the max
@@ -62,17 +62,11 @@ public class LinearExtrapolatePartStatus
               * minInd / (minInd - maxInd));
         } else if (colStatTypes[colStatIndex] == ColStatType.Double) {
           return (Double) ((Double) min[0] + ((Double) max[0] - (Double) min[0])
-              * minInd / (maxInd - minInd));
+              * minInd / (minInd - maxInd));
         } else {
-          Decimal dmax = (Decimal) max[0];
-          BigDecimal bdmax = new BigDecimal(dmax.toString());
-          double doublemax = bdmax.doubleValue();
-          Decimal dmin = (Decimal) min[0];
-          BigDecimal bdmin = new BigDecimal(dmin.toString());
-          double doublemin = bdmin.doubleValue();
-          double ret = doublemin + (doublemax - doublemin) * minInd
-              / (maxInd - minInd);
-          return createThriftDecimal(String.valueOf(ret));
+          double ret = decimalmin + (decimalmax - decimalmin) * minInd
+              / (minInd - maxInd);
+          return String.valueOf(ret);
         }
       }
     } else {
@@ -87,16 +81,9 @@ public class LinearExtrapolatePartStatus
               * maxInd / (maxInd - minInd);
           return ret;
         } else {
-          Decimal dmax = (Decimal) max[0];
-          BigDecimal bdmax = new BigDecimal(dmax.toString());
-          double doublemax = bdmax.doubleValue();
-          Decimal dmin = (Decimal) min[0];
-          BigDecimal bdmin = new BigDecimal(dmin.toString());
-          double doublemin = bdmin.doubleValue();
-          double ret = doublemax - (doublemax - doublemin) * maxInd
+          double ret = decimalmax - (decimalmax - decimalmin) * maxInd
               / (maxInd - minInd);
-          return createThriftDecimal(String.valueOf(ret));
-
+          return String.valueOf(ret);
         }
       } else {
         // right border is the min
@@ -109,24 +96,11 @@ public class LinearExtrapolatePartStatus
               * (rightBorderInd - maxInd) / (minInd - maxInd);
           return ret;
         } else {
-          Decimal dmax = (Decimal) max[0];
-          BigDecimal bdmax = new BigDecimal(dmax.toString());
-          double doublemax = bdmax.doubleValue();
-          Decimal dmin = (Decimal) min[0];
-          BigDecimal bdmin = new BigDecimal(dmin.toString());
-          double doublemin = bdmin.doubleValue();
-          double ret = doublemax - (doublemax - doublemin)
+          double ret = decimalmax - (decimalmax - decimalmin)
               * (rightBorderInd - maxInd) / (minInd - maxInd);
-          return createThriftDecimal(String.valueOf(ret));
+          return String.valueOf(ret);
         }
       }
     }
   }
-
-  private static Decimal createThriftDecimal(String s) {
-    BigDecimal d = new BigDecimal(s);
-    return new Decimal(ByteBuffer.wrap(d.unscaledValue().toByteArray()),
-        (short) d.scale());
-  }
-
 }

Modified: hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
URL: http://svn.apache.org/viewvc/hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java?rev=1673437&r1=1673436&r2=1673437&view=diff
==============================================================================
--- hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java (original)
+++ hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java Tue Apr 14 14:47:30 2015
@@ -803,6 +803,15 @@ class MetaStoreDirectSql {
     if (value == null) return null;
     return value.toString();
   }
+  
+  static Double extractSqlDouble(Object obj) throws MetaException {
+    if (obj == null)
+      return null;
+    if (!(obj instanceof Number)) {
+      throw new MetaException("Expected numeric type but got " + obj.getClass().getName());
+    }
+    return ((Number) obj).doubleValue();
+  }
 
   private static String trimCommaList(StringBuilder sb) {
     if (sb.length() > 0) {
@@ -1082,10 +1091,13 @@ class MetaStoreDirectSql {
   }
 
   public AggrStats aggrColStatsForPartitions(String dbName, String tableName,
-      List<String> partNames, List<String> colNames) throws MetaException {
+      List<String> partNames, List<String> colNames, boolean useDensityFunctionForNDVEstimation) throws MetaException {
     long partsFound = partsFoundForPartitions(dbName, tableName, partNames, colNames);
     List<ColumnStatisticsObj> stats = columnStatisticsObjForPartitions(dbName,
-        tableName, partNames, colNames, partsFound);
+        tableName, partNames, colNames, partsFound, useDensityFunctionForNDVEstimation);
+    LOG.info("useDensityFunctionForNDVEstimation = " + useDensityFunctionForNDVEstimation
+        + "\npartsFound = " + partsFound + "\nColumnStatisticsObj = "
+        + Arrays.toString(stats.toArray()));
     return new AggrStats(stats, partsFound);
   }
 
@@ -1114,15 +1126,33 @@ class MetaStoreDirectSql {
     return partsFound;
   }
 
-  private List<ColumnStatisticsObj> columnStatisticsObjForPartitions(
-      String dbName, String tableName, List<String> partNames,
-      List<String> colNames, long partsFound) throws MetaException {
+  private List<ColumnStatisticsObj> columnStatisticsObjForPartitions(String dbName,
+      String tableName, List<String> partNames, List<String> colNames, long partsFound, boolean useDensityFunctionForNDVEstimation)
+      throws MetaException {
     // TODO: all the extrapolation logic should be moved out of this class,
-    //       only mechanical data retrieval should remain here.
+    // only mechanical data retrieval should remain here.
     String commonPrefix = "select \"COLUMN_NAME\", \"COLUMN_TYPE\", "
         + "min(\"LONG_LOW_VALUE\"), max(\"LONG_HIGH_VALUE\"), min(\"DOUBLE_LOW_VALUE\"), max(\"DOUBLE_HIGH_VALUE\"), "
-        + "min(\"BIG_DECIMAL_LOW_VALUE\"), max(\"BIG_DECIMAL_HIGH_VALUE\"), sum(\"NUM_NULLS\"), max(\"NUM_DISTINCTS\"), "
-        + "max(\"AVG_COL_LEN\"), max(\"MAX_COL_LEN\"), sum(\"NUM_TRUES\"), sum(\"NUM_FALSES\") from \"PART_COL_STATS\""
+        + "min(cast(\"BIG_DECIMAL_LOW_VALUE\" as decimal)), max(cast(\"BIG_DECIMAL_HIGH_VALUE\" as decimal)), "
+        + "sum(\"NUM_NULLS\"), max(\"NUM_DISTINCTS\"), "
+        + "max(\"AVG_COL_LEN\"), max(\"MAX_COL_LEN\"), sum(\"NUM_TRUES\"), sum(\"NUM_FALSES\"), "
+        // The following data is used to compute a partitioned table's NDV based
+        // on partitions' NDV when useDensityFunctionForNDVEstimation = true. Global NDVs cannot be
+        // accurately derived from partition NDVs, because the domain of column value two partitions
+        // can overlap. If there is no overlap then global NDV is just the sum
+        // of partition NDVs (UpperBound). But if there is some overlay then
+        // global NDV can be anywhere between sum of partition NDVs (no overlap)
+        // and same as one of the partition NDV (domain of column value in all other
+        // partitions is subset of the domain value in one of the partition)
+        // (LowerBound).But under uniform distribution, we can roughly estimate the global
+        // NDV by leveraging the min/max values.
+        // And, we also guarantee that the estimation makes sense by comparing it to the
+        // UpperBound (calculated by "sum(\"NUM_DISTINCTS\")")
+        // and LowerBound (calculated by "max(\"NUM_DISTINCTS\")")
+        + "avg((\"LONG_HIGH_VALUE\"-\"LONG_LOW_VALUE\")/cast(\"NUM_DISTINCTS\" as decimal)),"
+        + "avg((\"DOUBLE_HIGH_VALUE\"-\"DOUBLE_LOW_VALUE\")/\"NUM_DISTINCTS\"),"
+        + "avg((cast(\"BIG_DECIMAL_HIGH_VALUE\" as decimal)-cast(\"BIG_DECIMAL_LOW_VALUE\" as decimal))/\"NUM_DISTINCTS\"),"
+        + "sum(\"NUM_DISTINCTS\")" + " from \"PART_COL_STATS\""
         + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ? ";
     String queryText = null;
     long start = 0;
@@ -1134,14 +1164,13 @@ class MetaStoreDirectSql {
     // Check if the status of all the columns of all the partitions exists
     // Extrapolation is not needed.
     if (partsFound == partNames.size()) {
-      queryText = commonPrefix
-          + " and \"COLUMN_NAME\" in (" + makeParams(colNames.size()) + ")"
+      queryText = commonPrefix + " and \"COLUMN_NAME\" in (" + makeParams(colNames.size()) + ")"
           + " and \"PARTITION_NAME\" in (" + makeParams(partNames.size()) + ")"
           + " group by \"COLUMN_NAME\", \"COLUMN_TYPE\"";
       start = doTrace ? System.nanoTime() : 0;
       query = pm.newQuery("javax.jdo.query.SQL", queryText);
-      qResult = executeWithArray(query, prepareParams(
-          dbName, tableName, partNames, colNames), queryText);
+      qResult = executeWithArray(query, prepareParams(dbName, tableName, partNames, colNames),
+          queryText);
       if (qResult == null) {
         query.closeAll();
         return Lists.newArrayList();
@@ -1149,10 +1178,9 @@ class MetaStoreDirectSql {
       end = doTrace ? System.nanoTime() : 0;
       timingTrace(doTrace, queryText, start, end);
       List<Object[]> list = ensureList(qResult);
-      List<ColumnStatisticsObj> colStats = new ArrayList<ColumnStatisticsObj>(
-          list.size());
+      List<ColumnStatisticsObj> colStats = new ArrayList<ColumnStatisticsObj>(list.size());
       for (Object[] row : list) {
-        colStats.add(prepareCSObj(row, 0));
+        colStats.add(prepareCSObjWithAdjustedNDV(row, 0, useDensityFunctionForNDVEstimation));
         Deadline.checkTimeout();
       }
       query.closeAll();
@@ -1161,18 +1189,16 @@ class MetaStoreDirectSql {
       // Extrapolation is needed for some columns.
       // In this case, at least a column status for a partition is missing.
       // We need to extrapolate this partition based on the other partitions
-      List<ColumnStatisticsObj> colStats = new ArrayList<ColumnStatisticsObj>(
-          colNames.size());
+      List<ColumnStatisticsObj> colStats = new ArrayList<ColumnStatisticsObj>(colNames.size());
       queryText = "select \"COLUMN_NAME\", \"COLUMN_TYPE\", count(\"PARTITION_NAME\") "
-          + " from \"PART_COL_STATS\""
-          + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ? "
+          + " from \"PART_COL_STATS\"" + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ? "
           + " and \"COLUMN_NAME\" in (" + makeParams(colNames.size()) + ")"
           + " and \"PARTITION_NAME\" in (" + makeParams(partNames.size()) + ")"
           + " group by \"COLUMN_NAME\", \"COLUMN_TYPE\"";
       start = doTrace ? System.nanoTime() : 0;
       query = pm.newQuery("javax.jdo.query.SQL", queryText);
-      qResult = executeWithArray(query, prepareParams(
-          dbName, tableName, partNames, colNames), queryText);
+      qResult = executeWithArray(query, prepareParams(dbName, tableName, partNames, colNames),
+          queryText);
       end = doTrace ? System.nanoTime() : 0;
       timingTrace(doTrace, queryText, start, end);
       if (qResult == null) {
@@ -1200,21 +1226,20 @@ class MetaStoreDirectSql {
       query.closeAll();
       // Extrapolation is not needed for columns noExtraColumnNames
       if (noExtraColumnNames.size() != 0) {
-        queryText = commonPrefix
-            + " and \"COLUMN_NAME\" in ("+ makeParams(noExtraColumnNames.size()) + ")"
-            + " and \"PARTITION_NAME\" in ("+ makeParams(partNames.size()) +")"
-            + " group by \"COLUMN_NAME\", \"COLUMN_TYPE\"";
+        queryText = commonPrefix + " and \"COLUMN_NAME\" in ("
+            + makeParams(noExtraColumnNames.size()) + ")" + " and \"PARTITION_NAME\" in ("
+            + makeParams(partNames.size()) + ")" + " group by \"COLUMN_NAME\", \"COLUMN_TYPE\"";
         start = doTrace ? System.nanoTime() : 0;
         query = pm.newQuery("javax.jdo.query.SQL", queryText);
-        qResult = executeWithArray(query, prepareParams(
-            dbName, tableName, partNames, noExtraColumnNames), queryText);
+        qResult = executeWithArray(query,
+            prepareParams(dbName, tableName, partNames, noExtraColumnNames), queryText);
         if (qResult == null) {
           query.closeAll();
           return Lists.newArrayList();
         }
         list = ensureList(qResult);
         for (Object[] row : list) {
-          colStats.add(prepareCSObj(row, 0));
+          colStats.add(prepareCSObjWithAdjustedNDV(row, 0, useDensityFunctionForNDVEstimation));
           Deadline.checkTimeout();
         }
         end = doTrace ? System.nanoTime() : 0;
@@ -1230,38 +1255,42 @@ class MetaStoreDirectSql {
         }
         // get sum for all columns to reduce the number of queries
         Map<String, Map<Integer, Object>> sumMap = new HashMap<String, Map<Integer, Object>>();
-        queryText = "select \"COLUMN_NAME\", sum(\"NUM_NULLS\"), sum(\"NUM_TRUES\"), sum(\"NUM_FALSES\")"
+        queryText = "select \"COLUMN_NAME\", sum(\"NUM_NULLS\"), sum(\"NUM_TRUES\"), sum(\"NUM_FALSES\"), sum(\"NUM_DISTINCTS\")"
             + " from \"PART_COL_STATS\""
             + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ? "
-            + " and \"COLUMN_NAME\" in (" +makeParams(extraColumnNameTypeParts.size())+ ")"
-            + " and \"PARTITION_NAME\" in (" + makeParams(partNames.size()) + ")"
+            + " and \"COLUMN_NAME\" in ("
+            + makeParams(extraColumnNameTypeParts.size())
+            + ")"
+            + " and \"PARTITION_NAME\" in ("
+            + makeParams(partNames.size())
+            + ")"
             + " group by \"COLUMN_NAME\"";
         start = doTrace ? System.nanoTime() : 0;
         query = pm.newQuery("javax.jdo.query.SQL", queryText);
         List<String> extraColumnNames = new ArrayList<String>();
         extraColumnNames.addAll(extraColumnNameTypeParts.keySet());
-        qResult = executeWithArray(query, prepareParams(
-            dbName, tableName, partNames, extraColumnNames), queryText);
+        qResult = executeWithArray(query,
+            prepareParams(dbName, tableName, partNames, extraColumnNames), queryText);
         if (qResult == null) {
           query.closeAll();
           return Lists.newArrayList();
         }
         list = ensureList(qResult);
         // see the indexes for colstats in IExtrapolatePartStatus
-        Integer[] sumIndex = new Integer[] { 6, 10, 11 };
+        Integer[] sumIndex = new Integer[] { 6, 10, 11, 15 };
         for (Object[] row : list) {
           Map<Integer, Object> indexToObject = new HashMap<Integer, Object>();
           for (int ind = 1; ind < row.length; ind++) {
             indexToObject.put(sumIndex[ind - 1], row[ind]);
           }
+          // row[0] is the column name
           sumMap.put((String) row[0], indexToObject);
           Deadline.checkTimeout();
         }
         end = doTrace ? System.nanoTime() : 0;
         timingTrace(doTrace, queryText, start, end);
         query.closeAll();
-        for (Map.Entry<String, String[]> entry : extraColumnNameTypeParts
-            .entrySet()) {
+        for (Map.Entry<String, String[]> entry : extraColumnNameTypeParts.entrySet()) {
           Object[] row = new Object[IExtrapolatePartStatus.colStatNames.length + 2];
           String colName = entry.getKey();
           String colType = entry.getValue()[0];
@@ -1270,12 +1299,20 @@ class MetaStoreDirectSql {
           row[0] = colName;
           // fill in coltype
           row[1] = colType;
-          // use linear extrapolation. more complicated one can be added in the future.
+          // use linear extrapolation. more complicated one can be added in the
+          // future.
           IExtrapolatePartStatus extrapolateMethod = new LinearExtrapolatePartStatus();
           // fill in colstatus
-          Integer[] index = IExtrapolatePartStatus.indexMaps.get(colType
-              .toLowerCase());
-          //if the colType is not the known type, long, double, etc, then get all index.
+          Integer[] index = null;
+          boolean decimal = false;
+          if (colType.toLowerCase().startsWith("decimal")) {
+            index = IExtrapolatePartStatus.indexMaps.get("decimal");
+            decimal = true;
+          } else {
+            index = IExtrapolatePartStatus.indexMaps.get(colType.toLowerCase());
+          }
+          // if the colType is not the known type, long, double, etc, then get
+          // all index.
           if (index == null) {
             index = IExtrapolatePartStatus.indexMaps.get("default");
           }
@@ -1290,20 +1327,27 @@ class MetaStoreDirectSql {
                 Long val = extractSqlLong(o);
                 row[2 + colStatIndex] = (Long) (val / sumVal * (partNames.size()));
               }
-            } else {
+            } else if (IExtrapolatePartStatus.aggrTypes[colStatIndex] == IExtrapolatePartStatus.AggrType.Min
+                || IExtrapolatePartStatus.aggrTypes[colStatIndex] == IExtrapolatePartStatus.AggrType.Max) {
               // if the aggregation type is min/max, we extrapolate from the
               // left/right borders
-              queryText = "select \""
-                  + colStatName
-                  + "\",\"PARTITION_NAME\" from \"PART_COL_STATS\""
-                  + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ?"
-                  + " and \"COLUMN_NAME\" = ?"
-                  + " and \"PARTITION_NAME\" in (" + makeParams(partNames.size()) + ")"
-                  + " order by \'" + colStatName + "\'";
+              if (!decimal) {
+                queryText = "select \"" + colStatName
+                    + "\",\"PARTITION_NAME\" from \"PART_COL_STATS\""
+                    + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ?" + " and \"COLUMN_NAME\" = ?"
+                    + " and \"PARTITION_NAME\" in (" + makeParams(partNames.size()) + ")"
+                    + " order by \"" + colStatName + "\"";
+              } else {
+                queryText = "select \"" + colStatName
+                    + "\",\"PARTITION_NAME\" from \"PART_COL_STATS\""
+                    + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ?" + " and \"COLUMN_NAME\" = ?"
+                    + " and \"PARTITION_NAME\" in (" + makeParams(partNames.size()) + ")"
+                    + " order by cast(\"" + colStatName + "\" as decimal)";
+              }
               start = doTrace ? System.nanoTime() : 0;
               query = pm.newQuery("javax.jdo.query.SQL", queryText);
-              qResult = executeWithArray(query, prepareParams(
-                  dbName, tableName, partNames, Arrays.asList(colName)), queryText);
+              qResult = executeWithArray(query,
+                  prepareParams(dbName, tableName, partNames, Arrays.asList(colName)), queryText);
               if (qResult == null) {
                 query.closeAll();
                 return Lists.newArrayList();
@@ -1317,12 +1361,38 @@ class MetaStoreDirectSql {
               if (min[0] == null || max[0] == null) {
                 row[2 + colStatIndex] = null;
               } else {
-                row[2 + colStatIndex] = extrapolateMethod.extrapolate(min, max,
-                    colStatIndex, indexMap);
+                row[2 + colStatIndex] = extrapolateMethod.extrapolate(min, max, colStatIndex,
+                    indexMap);
               }
+            } else {
+              // if the aggregation type is avg, we use the average on the
+              // existing ones.
+              queryText = "select "
+                  + "avg((\"LONG_HIGH_VALUE\"-\"LONG_LOW_VALUE\")/cast(\"NUM_DISTINCTS\" as decimal)),"
+                  + "avg((\"DOUBLE_HIGH_VALUE\"-\"DOUBLE_LOW_VALUE\")/\"NUM_DISTINCTS\"),"
+                  + "avg((cast(\"BIG_DECIMAL_HIGH_VALUE\" as decimal)-cast(\"BIG_DECIMAL_LOW_VALUE\" as decimal))/\"NUM_DISTINCTS\")"
+                  + " from \"PART_COL_STATS\"" + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ?"
+                  + " and \"COLUMN_NAME\" = ?" + " and \"PARTITION_NAME\" in ("
+                  + makeParams(partNames.size()) + ")" + " group by \"COLUMN_NAME\"";
+              start = doTrace ? System.nanoTime() : 0;
+              query = pm.newQuery("javax.jdo.query.SQL", queryText);
+              qResult = executeWithArray(query,
+                  prepareParams(dbName, tableName, partNames, Arrays.asList(colName)), queryText);
+              if (qResult == null) {
+                query.closeAll();
+                return Lists.newArrayList();
+              }
+              fqr = (ForwardQueryResult) qResult;
+              Object[] avg = (Object[]) (fqr.get(0));
+              // colStatIndex=12,13,14 respond to "AVG_LONG", "AVG_DOUBLE",
+              // "AVG_DECIMAL"
+              row[2 + colStatIndex] = avg[colStatIndex - 12];
+              end = doTrace ? System.nanoTime() : 0;
+              timingTrace(doTrace, queryText, start, end);
+              query.closeAll();
             }
           }
-          colStats.add(prepareCSObj(row, 0));
+          colStats.add(prepareCSObjWithAdjustedNDV(row, 0, useDensityFunctionForNDVEstimation));
           Deadline.checkTimeout();
         }
       }
@@ -1341,6 +1411,17 @@ class MetaStoreDirectSql {
     return cso;
   }
 
+  private ColumnStatisticsObj prepareCSObjWithAdjustedNDV(Object[] row, int i,
+      boolean useDensityFunctionForNDVEstimation) throws MetaException {
+    ColumnStatisticsData data = new ColumnStatisticsData();
+    ColumnStatisticsObj cso = new ColumnStatisticsObj((String) row[i++], (String) row[i++], data);
+    Object llow = row[i++], lhigh = row[i++], dlow = row[i++], dhigh = row[i++], declow = row[i++], dechigh = row[i++], nulls = row[i++], dist = row[i++], avglen = row[i++], maxlen = row[i++], trues = row[i++], falses = row[i++], avgLong = row[i++], avgDouble = row[i++], avgDecimal = row[i++], sumDist = row[i++];
+    StatObjectConverter.fillColumnStatisticsData(cso.getColType(), data, llow, lhigh, dlow, dhigh,
+        declow, dechigh, nulls, dist, avglen, maxlen, trues, falses, avgLong, avgDouble,
+        avgDecimal, sumDist, useDensityFunctionForNDVEstimation);
+    return cso;
+  }
+  
   private Object[] prepareParams(String dbName, String tableName, List<String> partNames,
     List<String> colNames) throws MetaException {
 

Modified: hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
URL: http://svn.apache.org/viewvc/hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java?rev=1673437&r1=1673436&r2=1673437&view=diff
==============================================================================
--- hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (original)
+++ hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java Tue Apr 14 14:47:30 2015
@@ -6228,12 +6228,13 @@ public class ObjectStore implements RawS
   @Override
   public AggrStats get_aggr_stats_for(String dbName, String tblName,
       final List<String> partNames, final List<String> colNames) throws MetaException, NoSuchObjectException {
+    final boolean  useDensityFunctionForNDVEstimation = HiveConf.getBoolVar(getConf(), HiveConf.ConfVars.HIVE_METASTORE_STATS_NDV_DENSITY_FUNCTION);
     return new GetHelper<AggrStats>(dbName, tblName, true, false) {
       @Override
       protected AggrStats getSqlResult(GetHelper<AggrStats> ctx)
           throws MetaException {
         return directSql.aggrColStatsForPartitions(dbName, tblName, partNames,
-            colNames);
+            colNames, useDensityFunctionForNDVEstimation);
       }
       @Override
       protected AggrStats getJdoResult(GetHelper<AggrStats> ctx)

Modified: hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/StatObjectConverter.java
URL: http://svn.apache.org/viewvc/hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/StatObjectConverter.java?rev=1673437&r1=1673436&r2=1673437&view=diff
==============================================================================
--- hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/StatObjectConverter.java (original)
+++ hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/StatObjectConverter.java Tue Apr 14 14:47:30 2015
@@ -476,6 +476,133 @@ public class StatObjectConverter {
     }
   }
 
+  public static void fillColumnStatisticsData(String colType, ColumnStatisticsData data,
+      Object llow, Object lhigh, Object dlow, Object dhigh, Object declow, Object dechigh,
+      Object nulls, Object dist, Object avglen, Object maxlen, Object trues, Object falses,
+      Object avgLong, Object avgDouble, Object avgDecimal, Object sumDist,
+      boolean useDensityFunctionForNDVEstimation) throws MetaException {
+    colType = colType.toLowerCase();
+    if (colType.equals("boolean")) {
+      BooleanColumnStatsData boolStats = new BooleanColumnStatsData();
+      boolStats.setNumFalses(MetaStoreDirectSql.extractSqlLong(falses));
+      boolStats.setNumTrues(MetaStoreDirectSql.extractSqlLong(trues));
+      boolStats.setNumNulls(MetaStoreDirectSql.extractSqlLong(nulls));
+      data.setBooleanStats(boolStats);
+    } else if (colType.equals("string") || colType.startsWith("varchar")
+        || colType.startsWith("char")) {
+      StringColumnStatsData stringStats = new StringColumnStatsData();
+      stringStats.setNumNulls(MetaStoreDirectSql.extractSqlLong(nulls));
+      stringStats.setAvgColLen((Double) avglen);
+      stringStats.setMaxColLen(MetaStoreDirectSql.extractSqlLong(maxlen));
+      stringStats.setNumDVs(MetaStoreDirectSql.extractSqlLong(dist));
+      data.setStringStats(stringStats);
+    } else if (colType.equals("binary")) {
+      BinaryColumnStatsData binaryStats = new BinaryColumnStatsData();
+      binaryStats.setNumNulls(MetaStoreDirectSql.extractSqlLong(nulls));
+      binaryStats.setAvgColLen((Double) avglen);
+      binaryStats.setMaxColLen(MetaStoreDirectSql.extractSqlLong(maxlen));
+      data.setBinaryStats(binaryStats);
+    } else if (colType.equals("bigint") || colType.equals("int") || colType.equals("smallint")
+        || colType.equals("tinyint") || colType.equals("timestamp")) {
+      LongColumnStatsData longStats = new LongColumnStatsData();
+      longStats.setNumNulls(MetaStoreDirectSql.extractSqlLong(nulls));
+      if (lhigh != null) {
+        longStats.setHighValue(MetaStoreDirectSql.extractSqlLong(lhigh));
+      }
+      if (llow != null) {
+        longStats.setLowValue(MetaStoreDirectSql.extractSqlLong(llow));
+      }
+      long lowerBound = MetaStoreDirectSql.extractSqlLong(dist);
+      long higherBound = MetaStoreDirectSql.extractSqlLong(sumDist);
+      if (useDensityFunctionForNDVEstimation && lhigh != null && llow != null && avgLong != null
+          && MetaStoreDirectSql.extractSqlDouble(avgLong) != 0.0) {
+        // We have estimation, lowerbound and higherbound. We use estimation if
+        // it is between lowerbound and higherbound.
+        long estimation = MetaStoreDirectSql
+            .extractSqlLong((MetaStoreDirectSql.extractSqlLong(lhigh) - MetaStoreDirectSql
+                .extractSqlLong(llow)) / MetaStoreDirectSql.extractSqlDouble(avgLong));
+        if (estimation < lowerBound) {
+          longStats.setNumDVs(lowerBound);
+        } else if (estimation > higherBound) {
+          longStats.setNumDVs(higherBound);
+        } else {
+          longStats.setNumDVs(estimation);
+        }
+      } else {
+        longStats.setNumDVs(lowerBound);
+      }
+      data.setLongStats(longStats);
+    } else if (colType.equals("double") || colType.equals("float")) {
+      DoubleColumnStatsData doubleStats = new DoubleColumnStatsData();
+      doubleStats.setNumNulls(MetaStoreDirectSql.extractSqlLong(nulls));
+      if (dhigh != null) {
+        doubleStats.setHighValue((Double) dhigh);
+      }
+      if (dlow != null) {
+        doubleStats.setLowValue((Double) dlow);
+      }
+      long lowerBound = MetaStoreDirectSql.extractSqlLong(dist);
+      long higherBound = MetaStoreDirectSql.extractSqlLong(sumDist);
+      if (useDensityFunctionForNDVEstimation && dhigh != null && dlow != null && avgDouble != null
+          && MetaStoreDirectSql.extractSqlDouble(avgDouble) != 0.0) {
+        long estimation = MetaStoreDirectSql
+            .extractSqlLong((MetaStoreDirectSql.extractSqlLong(dhigh) - MetaStoreDirectSql
+                .extractSqlLong(dlow)) / MetaStoreDirectSql.extractSqlDouble(avgDouble));
+        if (estimation < lowerBound) {
+          doubleStats.setNumDVs(lowerBound);
+        } else if (estimation > higherBound) {
+          doubleStats.setNumDVs(higherBound);
+        } else {
+          doubleStats.setNumDVs(estimation);
+        }
+      } else {
+        doubleStats.setNumDVs(lowerBound);
+      }
+      data.setDoubleStats(doubleStats);
+    } else if (colType.startsWith("decimal")) {
+      DecimalColumnStatsData decimalStats = new DecimalColumnStatsData();
+      decimalStats.setNumNulls(MetaStoreDirectSql.extractSqlLong(nulls));
+      Decimal low = null;
+      Decimal high = null;
+      BigDecimal blow = null;
+      BigDecimal bhigh = null;
+      if (dechigh instanceof BigDecimal) {
+        bhigh = (BigDecimal) dechigh;
+        high = new Decimal(ByteBuffer.wrap(bhigh.unscaledValue().toByteArray()),
+            (short) bhigh.scale());
+      } else if (dechigh instanceof String) {
+        bhigh = new BigDecimal((String) dechigh);
+        high = createThriftDecimal((String) dechigh);
+      }
+      decimalStats.setHighValue(high);
+      if (declow instanceof BigDecimal) {
+        blow = (BigDecimal) declow;
+        low = new Decimal(ByteBuffer.wrap(blow.unscaledValue().toByteArray()), (short) blow.scale());
+      } else if (dechigh instanceof String) {
+        blow = new BigDecimal((String) declow);
+        low = createThriftDecimal((String) declow);
+      }
+      decimalStats.setLowValue(low);
+      long lowerBound = MetaStoreDirectSql.extractSqlLong(dist);
+      long higherBound = MetaStoreDirectSql.extractSqlLong(sumDist);
+      if (useDensityFunctionForNDVEstimation && dechigh != null && declow != null && avgDecimal != null
+          && MetaStoreDirectSql.extractSqlDouble(avgDecimal) != 0.0) {
+        long estimation = MetaStoreDirectSql.extractSqlLong(MetaStoreDirectSql.extractSqlLong(bhigh
+            .subtract(blow).floatValue() / MetaStoreDirectSql.extractSqlDouble(avgDecimal)));
+        if (estimation < lowerBound) {
+          decimalStats.setNumDVs(lowerBound);
+        } else if (estimation > higherBound) {
+          decimalStats.setNumDVs(higherBound);
+        } else {
+          decimalStats.setNumDVs(estimation);
+        }
+      } else {
+        decimalStats.setNumDVs(lowerBound);
+      }
+      data.setDecimalStats(decimalStats);
+    }
+  }
+
   private static Decimal createThriftDecimal(String s) {
     BigDecimal d = new BigDecimal(s);
     return new Decimal(ByteBuffer.wrap(d.unscaledValue().toByteArray()), (short)d.scale());
@@ -484,4 +611,5 @@ public class StatObjectConverter {
   private static String createJdoDecimalString(Decimal d) {
     return new BigDecimal(new BigInteger(d.getUnscaled()), d.getScale()).toString();
   }
+
 }

Modified: hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/events/AddPartitionEvent.java
URL: http://svn.apache.org/viewvc/hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/events/AddPartitionEvent.java?rev=1673437&r1=1673436&r2=1673437&view=diff
==============================================================================
--- hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/events/AddPartitionEvent.java (original)
+++ hive/branches/hbase-metastore/metastore/src/java/org/apache/hadoop/hive/metastore/events/AddPartitionEvent.java Tue Apr 14 14:47:30 2015
@@ -61,18 +61,18 @@ public class AddPartitionEvent extends L
     return table;
   }
 
-  /**
-   * @return List of partitions.
-   */
-  public List<Partition> getPartitions() {
-    return partitions;
-  }
+
+  // Note : List<Partition> getPartitions() removed with HIVE-9609 because it will result in OOM errors with large add_partitions.
 
   /**
    * @return Iterator for partitions.
    */
   public Iterator<Partition> getPartitionIterator() {
-    return partitionSpecProxy == null ? null : partitionSpecProxy.getPartitionIterator();
+    if (partitions != null){
+      return partitions.iterator();
+    } else {
+      return partitionSpecProxy == null ? null : partitionSpecProxy.getPartitionIterator();
+    }
   }
 
 }

Modified: hive/branches/hbase-metastore/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/hbase-metastore/pom.xml?rev=1673437&r1=1673436&r2=1673437&view=diff
==============================================================================
--- hive/branches/hbase-metastore/pom.xml (original)
+++ hive/branches/hbase-metastore/pom.xml Tue Apr 14 14:47:30 2015
@@ -124,8 +124,8 @@
     <hbase.hadoop1.version>0.98.9-hadoop1</hbase.hadoop1.version>
     <hbase.hadoop2.version>0.98.9-hadoop2</hbase.hadoop2.version>
     <!-- httpcomponents are not always in version sync -->
-    <httpcomponents.client.version>4.2.5</httpcomponents.client.version>
-    <httpcomponents.core.version>4.2.5</httpcomponents.core.version>
+    <httpcomponents.client.version>4.4</httpcomponents.client.version>
+    <httpcomponents.core.version>4.4</httpcomponents.core.version>
     <ivy.version>2.4.0</ivy.version>
     <jackson.version>1.9.2</jackson.version>
     <javaewah.version>0.3.2</javaewah.version>
@@ -1084,6 +1084,16 @@
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-common</artifactId>
             <version>${hadoop-23.version}</version>
+            <exclusions>
+              <exclusion>
+                <groupId>org.apache.httpcomponents</groupId>
+                <artifactId>httpcore</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>org.apache.httpcomponents</groupId>
+                <artifactId>httpclient</artifactId>
+              </exclusion>
+            </exclusions>
           </dependency>
           <dependency>
             <groupId>org.apache.hadoop</groupId>

Modified: hive/branches/hbase-metastore/ql/src/gen/vectorization/ExpressionTemplates/ColumnArithmeticColumn.txt
URL: http://svn.apache.org/viewvc/hive/branches/hbase-metastore/ql/src/gen/vectorization/ExpressionTemplates/ColumnArithmeticColumn.txt?rev=1673437&r1=1673436&r2=1673437&view=diff
==============================================================================
--- hive/branches/hbase-metastore/ql/src/gen/vectorization/ExpressionTemplates/ColumnArithmeticColumn.txt (original)
+++ hive/branches/hbase-metastore/ql/src/gen/vectorization/ExpressionTemplates/ColumnArithmeticColumn.txt Tue Apr 14 14:47:30 2015
@@ -83,25 +83,27 @@ public class <ClassName> extends VectorE
     if (inputColVector1.isRepeating && inputColVector2.isRepeating) { 
       outputVector[0] = vector1[0] <OperatorSymbol> vector2[0];
     } else if (inputColVector1.isRepeating) {
+      final <OperandType1> vector1Value = vector1[0];
       if (batch.selectedInUse) {
         for(int j = 0; j != n; j++) {
           int i = sel[j];
-          outputVector[i] = vector1[0] <OperatorSymbol> vector2[i];
+          outputVector[i] = vector1Value <OperatorSymbol> vector2[i];
         }
       } else {
         for(int i = 0; i != n; i++) {
-          outputVector[i] = vector1[0] <OperatorSymbol> vector2[i];
+          outputVector[i] = vector1Value <OperatorSymbol> vector2[i];
         }
       }
     } else if (inputColVector2.isRepeating) {
+      final <OperandType2> vector2Value = vector2[0];
       if (batch.selectedInUse) {
         for(int j = 0; j != n; j++) {
           int i = sel[j];
-          outputVector[i] = vector1[i] <OperatorSymbol> vector2[0];
+          outputVector[i] = vector1[i] <OperatorSymbol> vector2Value;
         }
       } else {
         for(int i = 0; i != n; i++) {
-          outputVector[i] = vector1[i] <OperatorSymbol> vector2[0];
+          outputVector[i] = vector1[i] <OperatorSymbol> vector2Value;
         }
       }
     } else {

Modified: hive/branches/hbase-metastore/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
URL: http://svn.apache.org/viewvc/hive/branches/hbase-metastore/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java?rev=1673437&r1=1673436&r2=1673437&view=diff
==============================================================================
--- hive/branches/hbase-metastore/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java (original)
+++ hive/branches/hbase-metastore/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java Tue Apr 14 14:47:30 2015
@@ -429,6 +429,7 @@ public enum ErrorMsg {
       "Alter table partition type {0} does not support cascade", true),
 
   DROP_NATIVE_FUNCTION(10301, "Cannot drop native function"),
+  UPDATE_CANNOT_UPDATE_BUCKET_VALUE(10302, "Updating values of bucketing columns is not supported.  Column {0}.", true),
 
   //========================== 20000 range starts here ========================//
   SCRIPT_INIT_ERROR(20000, "Unable to initialize custom script."),
@@ -443,6 +444,9 @@ public enum ErrorMsg {
       "is controlled by hive.exec.max.dynamic.partitions and hive.exec.max.dynamic.partitions.pernode. "),
   PARTITION_SCAN_LIMIT_EXCEEDED(20005, "Number of partitions scanned (={0}) on table {1} exceeds limit" +
       " (={2}). This is controlled by hive.limit.query.max.table.partition.", true),
+  OP_NOT_ALLOWED_IN_AUTOCOMMIT(20006, "Operation {0} is not allowed when autoCommit=true.", true),//todo: better SQLState?
+  OP_NOT_ALLOWED_IN_TXN(20007, "Operation {0} is not allowed in a transaction.  TransactionID={1}.", true),
+  OP_NOT_ALLOWED_WITHOUT_TXN(2008, "Operation {0} is not allowed since autoCommit=false and there is no active transaction", true),
 
   //========================== 30000 range starts here ========================//
   STATSPUBLISHER_NOT_OBTAINED(30000, "StatsPublisher cannot be obtained. " +
@@ -508,7 +512,7 @@ public enum ErrorMsg {
   static {
     for (ErrorMsg errorMsg : values()) {
       if (errorMsg.format != null) {
-        String pattern = errorMsg.mesg.replaceAll("\\{.*\\}", ".*");
+        String pattern = errorMsg.mesg.replaceAll("\\{[0-9]+\\}", ".*");
         formatToErrorMsgMap.put(Pattern.compile("^" + pattern + "$"), errorMsg);
       } else {
         mesgToErrorMsgMap.put(errorMsg.getMsg().trim(), errorMsg);

Modified: hive/branches/hbase-metastore/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/hbase-metastore/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java?rev=1673437&r1=1673436&r2=1673437&view=diff
==============================================================================
--- hive/branches/hbase-metastore/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java (original)
+++ hive/branches/hbase-metastore/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java Tue Apr 14 14:47:30 2015
@@ -691,11 +691,11 @@ public abstract class CommonJoinOperator
         Byte alias = order[i];
         AbstractRowContainer<List<Object>> alw = storage[alias];
 
-        if (alw.rowCount() != 1) {
+        if (!alw.isSingleRow()) {
           allOne = false;
         }
 
-        if (alw.rowCount() == 0) {
+        if (!alw.hasRows()) {
           alw.addRow(dummyObj[i]);
           hasNulls = true;
         } else if (condn[i].getPreserved()) {
@@ -721,16 +721,16 @@ public abstract class CommonJoinOperator
         AbstractRowContainer<List<Object>> alw = storage[alias];
 
         if (noOuterJoin) {
-          if (alw.rowCount() == 0) {
+          if (!alw.hasRows()) {
             return;
-          } else if (alw.rowCount() > 1) {
+          } else if (!alw.isSingleRow()) {
             mayHasMoreThanOne = true;
           }
         } else {
-          if (alw.rowCount() == 0) {
+          if (!alw.hasRows()) {
             hasEmpty = true;
             alw.addRow(dummyObj[i]);
-          } else if (!hasEmpty && alw.rowCount() == 1) {
+          } else if (!hasEmpty && alw.isSingleRow()) {
             if (hasAnyFiltered(alias, alw.rowIter().first())) {
               hasEmpty = true;
             }

Modified: hive/branches/hbase-metastore/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/hbase-metastore/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java?rev=1673437&r1=1673436&r2=1673437&view=diff
==============================================================================
--- hive/branches/hbase-metastore/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java (original)
+++ hive/branches/hbase-metastore/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java Tue Apr 14 14:47:30 2015
@@ -391,7 +391,7 @@ public class CommonMergeJoinOperator ext
         if (candidateStorage[pos] == null) {
           continue;
         }
-        if (this.candidateStorage[pos].rowCount() > 0) {
+        if (this.candidateStorage[pos].hasRows()) {
           dataInCache = true;
           break;
         }

Modified: hive/branches/hbase-metastore/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
URL: http://svn.apache.org/viewvc/hive/branches/hbase-metastore/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java?rev=1673437&r1=1673436&r2=1673437&view=diff
==============================================================================
--- hive/branches/hbase-metastore/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (original)
+++ hive/branches/hbase-metastore/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java Tue Apr 14 14:47:30 2015
@@ -3404,13 +3404,13 @@ public class DDLTask extends Task<DDLWor
 
   private int alterTableOrSinglePartition(AlterTableDesc alterTbl, Table tbl, Partition part)
       throws HiveException {
-    List<FieldSchema> oldCols = (part == null ? tbl.getCols() : part.getCols());
-    StorageDescriptor sd = (part == null ? tbl.getTTable().getSd() : part.getTPartition().getSd());
 
     if (alterTbl.getOp() == AlterTableDesc.AlterTableTypes.RENAME) {
       tbl.setDbName(Utilities.getDatabaseName(alterTbl.getNewName()));
       tbl.setTableName(Utilities.getTableName(alterTbl.getNewName()));
     } else if (alterTbl.getOp() == AlterTableDesc.AlterTableTypes.ADDCOLS) {
+      List<FieldSchema> oldCols = (part == null ? tbl.getCols() : part.getCols());
+      StorageDescriptor sd = (part == null ? tbl.getTTable().getSd() : part.getTPartition().getSd());
       List<FieldSchema> newCols = alterTbl.getNewCols();
       String serializationLib = sd.getSerdeInfo().getSerializationLib();
       if (serializationLib.equals(
@@ -3437,6 +3437,8 @@ public class DDLTask extends Task<DDLWor
         sd.setCols(oldCols);
       }
     } else if (alterTbl.getOp() == AlterTableDesc.AlterTableTypes.RENAMECOLUMN) {
+      List<FieldSchema> oldCols = (part == null ? tbl.getCols() : part.getCols());
+      StorageDescriptor sd = (part == null ? tbl.getTTable().getSd() : part.getTPartition().getSd());
       List<FieldSchema> newCols = new ArrayList<FieldSchema>();
       Iterator<FieldSchema> iterOldCols = oldCols.iterator();
       String oldName = alterTbl.getOldColName();
@@ -3499,6 +3501,7 @@ public class DDLTask extends Task<DDLWor
 
       sd.setCols(newCols);
     } else if (alterTbl.getOp() == AlterTableDesc.AlterTableTypes.REPLACECOLS) {
+      StorageDescriptor sd = (part == null ? tbl.getTTable().getSd() : part.getTPartition().getSd());
       // change SerDe to LazySimpleSerDe if it is columnsetSerDe
       String serializationLib = sd.getSerdeInfo().getSerializationLib();
       if (serializationLib.equals(
@@ -3523,8 +3526,10 @@ public class DDLTask extends Task<DDLWor
         tbl.getTTable().getParameters().remove(keyItr.next());
       }
     } else if (alterTbl.getOp() == AlterTableDesc.AlterTableTypes.ADDSERDEPROPS) {
+      StorageDescriptor sd = (part == null ? tbl.getTTable().getSd() : part.getTPartition().getSd());
       sd.getSerdeInfo().getParameters().putAll(alterTbl.getProps());
     } else if (alterTbl.getOp() == AlterTableDesc.AlterTableTypes.ADDSERDE) {
+      StorageDescriptor sd = (part == null ? tbl.getTTable().getSd() : part.getTPartition().getSd());
       String serdeName = alterTbl.getSerdeName();
       sd.getSerdeInfo().setSerializationLib(serdeName);
       if ((alterTbl.getProps() != null) && (alterTbl.getProps().size() > 0)) {
@@ -3539,6 +3544,7 @@ public class DDLTask extends Task<DDLWor
         }
       }
     } else if (alterTbl.getOp() == AlterTableDesc.AlterTableTypes.ADDFILEFORMAT) {
+      StorageDescriptor sd = (part == null ? tbl.getTTable().getSd() : part.getTPartition().getSd());
       sd.setInputFormat(alterTbl.getInputFormat());
       sd.setOutputFormat(alterTbl.getOutputFormat());
       if (alterTbl.getSerdeName() != null) {
@@ -3559,6 +3565,7 @@ public class DDLTask extends Task<DDLWor
         tbl.setProtectMode(mode);
       }
     } else if (alterTbl.getOp() == AlterTableDesc.AlterTableTypes.ADDCLUSTERSORTCOLUMN) {
+      StorageDescriptor sd = (part == null ? tbl.getTTable().getSd() : part.getTPartition().getSd());
       // validate sort columns and bucket columns
       List<String> columns = Utilities.getColumnNamesFromFieldSchema(tbl
           .getCols());
@@ -3583,6 +3590,7 @@ public class DDLTask extends Task<DDLWor
         sd.setSortCols(alterTbl.getSortColumns());
       }
     } else if (alterTbl.getOp() == AlterTableDesc.AlterTableTypes.ALTERLOCATION) {
+      StorageDescriptor sd = (part == null ? tbl.getTTable().getSd() : part.getTPartition().getSd());
       String newLocation = alterTbl.getNewLocation();
       try {
         URI locUri = new URI(newLocation);

Modified: hive/branches/hbase-metastore/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/hbase-metastore/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java?rev=1673437&r1=1673436&r2=1673437&view=diff
==============================================================================
--- hive/branches/hbase-metastore/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java (original)
+++ hive/branches/hbase-metastore/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java Tue Apr 14 14:47:30 2015
@@ -342,7 +342,7 @@ public class SMBMapJoinOperator extends
       joinOneGroup();
       dataInCache = false;
       for (byte pos = 0; pos < order.length; pos++) {
-        if (this.candidateStorage[pos].rowCount() > 0) {
+        if (this.candidateStorage[pos].hasRows()) {
           dataInCache = true;
           break;
         }

Modified: hive/branches/hbase-metastore/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL: http://svn.apache.org/viewvc/hive/branches/hbase-metastore/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1673437&r1=1673436&r2=1673437&view=diff
==============================================================================
--- hive/branches/hbase-metastore/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (original)
+++ hive/branches/hbase-metastore/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Tue Apr 14 14:47:30 2015
@@ -46,6 +46,7 @@ import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.PreparedStatement;
 import java.sql.SQLException;
+import java.sql.SQLFeatureNotSupportedException;
 import java.sql.SQLTransientException;
 import java.sql.Timestamp;
 import java.text.SimpleDateFormat;
@@ -464,9 +465,9 @@ public final class Utilities {
     }
   }
 
-  public static Map<String, Map<Integer, String>> getMapWorkAllScratchColumnVectorTypeMaps(Configuration hiveConf) {
+  public static Map<Integer, String> getMapWorkVectorScratchColumnTypeMap(Configuration hiveConf) {
     MapWork mapWork = getMapWork(hiveConf);
-    return mapWork.getAllScratchColumnVectorTypeMaps();
+    return mapWork.getVectorScratchColumnTypeMap();
   }
 
   public static void setWorkflowAdjacencies(Configuration conf, QueryPlan plan) {
@@ -3089,6 +3090,24 @@ public final class Utilities {
     }
   }
 
+  public static void setQueryTimeout(java.sql.Statement stmt, int timeout) throws SQLException {
+    if (timeout < 0) {
+      LOG.info("Invalid query timeout " + timeout);
+      return;
+    }
+    try {
+      stmt.setQueryTimeout(timeout);
+    } catch (SQLException e) {
+      String message = e.getMessage() == null ? null : e.getMessage().toLowerCase();
+      if (e instanceof SQLFeatureNotSupportedException ||
+         (message != null && (message.contains("implemented") || message.contains("supported")))) {
+        LOG.info("setQueryTimeout is not supported");
+        return;
+      }
+      throw e;
+    }
+  }
+
   /**
    * Introducing a random factor to the wait time before another retry.
    * The wait time is dependent on # of failures and a random factor.

Modified: hive/branches/hbase-metastore/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/AbstractRowContainer.java
URL: http://svn.apache.org/viewvc/hive/branches/hbase-metastore/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/AbstractRowContainer.java?rev=1673437&r1=1673436&r2=1673437&view=diff
==============================================================================
--- hive/branches/hbase-metastore/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/AbstractRowContainer.java (original)
+++ hive/branches/hbase-metastore/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/AbstractRowContainer.java Tue Apr 14 14:47:30 2015
@@ -37,6 +37,17 @@ public interface AbstractRowContainer<RO
   public void addRow(ROW t) throws HiveException;
 
   /**
+   * @return whether the row container has at least 1 row.
+   * NOTE: Originally we named this isEmpty, but that name conflicted with another interface.
+   */
+  public boolean hasRows() throws HiveException;
+
+  /**
+   * @return whether the row container has 1 row.
+   */
+  public boolean isSingleRow() throws HiveException;
+
+  /**
    * @return number of elements in the RowContainer
    */
   public int rowCount() throws HiveException;



Mime
View raw message