hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jd...@apache.org
Subject [10/10] hive git commit: HIVE-10319: Hive CLI startup takes a long time with a large number of databases (Nezih Yigitbasi, reviewed by Jason Dere)
Date Tue, 04 Aug 2015 00:52:53 GMT
HIVE-10319: Hive CLI startup takes a long time with a large number of databases (Nezih Yigitbasi,
reviewed by Jason Dere)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/cb648903
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/cb648903
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/cb648903

Branch: refs/heads/master
Commit: cb648903c7afa9a17420248f7956d85c653f6987
Parents: 607f23d
Author: Jason Dere <jdere@hortonworks.com>
Authored: Mon Aug 3 17:51:35 2015 -0700
Committer: Jason Dere <jdere@hortonworks.com>
Committed: Mon Aug 3 17:51:35 2015 -0700

----------------------------------------------------------------------
 .../hive/metastore/TestHiveMetaStore.java       |   30 +-
 metastore/if/hive_metastore.thrift              |    5 +
 .../gen/thrift/gen-cpp/ThriftHiveMetastore.cpp  | 2046 +++++++-----
 .../gen/thrift/gen-cpp/ThriftHiveMetastore.h    |  135 +
 .../ThriftHiveMetastore_server.skeleton.cpp     |    5 +
 .../gen/thrift/gen-cpp/hive_metastore_types.cpp |  304 +-
 .../gen/thrift/gen-cpp/hive_metastore_types.h   |   47 +
 .../hive/metastore/api/AbortTxnRequest.java     |    2 +-
 .../metastore/api/AddDynamicPartitions.java     |    2 +-
 .../metastore/api/AddPartitionsRequest.java     |    2 +-
 .../hive/metastore/api/AddPartitionsResult.java |    2 +-
 .../hadoop/hive/metastore/api/AggrStats.java    |    2 +-
 .../metastore/api/AlreadyExistsException.java   |    2 +-
 .../metastore/api/BinaryColumnStatsData.java    |    2 +-
 .../metastore/api/BooleanColumnStatsData.java   |    2 +-
 .../hive/metastore/api/CheckLockRequest.java    |    2 +-
 .../hive/metastore/api/ColumnStatistics.java    |    2 +-
 .../metastore/api/ColumnStatisticsDesc.java     |    2 +-
 .../hive/metastore/api/ColumnStatisticsObj.java |    2 +-
 .../hive/metastore/api/CommitTxnRequest.java    |    2 +-
 .../hive/metastore/api/CompactionRequest.java   |    2 +-
 .../api/ConfigValSecurityException.java         |    2 +-
 .../api/CurrentNotificationEventId.java         |    2 +-
 .../hadoop/hive/metastore/api/Database.java     |    2 +-
 .../apache/hadoop/hive/metastore/api/Date.java  |    2 +-
 .../hive/metastore/api/DateColumnStatsData.java |    2 +-
 .../hadoop/hive/metastore/api/Decimal.java      |    2 +-
 .../metastore/api/DecimalColumnStatsData.java   |    2 +-
 .../metastore/api/DoubleColumnStatsData.java    |    2 +-
 .../hive/metastore/api/DropPartitionsExpr.java  |    2 +-
 .../metastore/api/DropPartitionsRequest.java    |    2 +-
 .../metastore/api/DropPartitionsResult.java     |    2 +-
 .../hive/metastore/api/EnvironmentContext.java  |    2 +-
 .../hadoop/hive/metastore/api/FieldSchema.java  |    2 +-
 .../hive/metastore/api/FireEventRequest.java    |    2 +-
 .../hive/metastore/api/FireEventResponse.java   |    2 +-
 .../hadoop/hive/metastore/api/Function.java     |    2 +-
 .../metastore/api/GetAllFunctionsResponse.java  |  447 +++
 .../metastore/api/GetOpenTxnsInfoResponse.java  |    2 +-
 .../hive/metastore/api/GetOpenTxnsResponse.java |    2 +-
 .../api/GetPrincipalsInRoleRequest.java         |    2 +-
 .../api/GetPrincipalsInRoleResponse.java        |    2 +-
 .../api/GetRoleGrantsForPrincipalRequest.java   |    2 +-
 .../api/GetRoleGrantsForPrincipalResponse.java  |    2 +-
 .../api/GrantRevokePrivilegeRequest.java        |    2 +-
 .../api/GrantRevokePrivilegeResponse.java       |    2 +-
 .../metastore/api/GrantRevokeRoleRequest.java   |    2 +-
 .../metastore/api/GrantRevokeRoleResponse.java  |    2 +-
 .../hive/metastore/api/HeartbeatRequest.java    |    2 +-
 .../metastore/api/HeartbeatTxnRangeRequest.java |    2 +-
 .../api/HeartbeatTxnRangeResponse.java          |    2 +-
 .../hive/metastore/api/HiveObjectPrivilege.java |    2 +-
 .../hive/metastore/api/HiveObjectRef.java       |    2 +-
 .../apache/hadoop/hive/metastore/api/Index.java |    2 +-
 .../api/IndexAlreadyExistsException.java        |    2 +-
 .../metastore/api/InsertEventRequestData.java   |    2 +-
 .../metastore/api/InvalidInputException.java    |    2 +-
 .../metastore/api/InvalidObjectException.java   |    2 +-
 .../api/InvalidOperationException.java          |    2 +-
 .../api/InvalidPartitionException.java          |    2 +-
 .../hive/metastore/api/LockComponent.java       |    2 +-
 .../hadoop/hive/metastore/api/LockRequest.java  |    2 +-
 .../hadoop/hive/metastore/api/LockResponse.java |    2 +-
 .../hive/metastore/api/LongColumnStatsData.java |    2 +-
 .../hive/metastore/api/MetaException.java       |    2 +-
 .../hive/metastore/api/NoSuchLockException.java |    2 +-
 .../metastore/api/NoSuchObjectException.java    |    2 +-
 .../hive/metastore/api/NoSuchTxnException.java  |    2 +-
 .../hive/metastore/api/NotificationEvent.java   |    2 +-
 .../metastore/api/NotificationEventRequest.java |    2 +-
 .../api/NotificationEventResponse.java          |    2 +-
 .../hive/metastore/api/OpenTxnRequest.java      |    2 +-
 .../hive/metastore/api/OpenTxnsResponse.java    |    2 +-
 .../apache/hadoop/hive/metastore/api/Order.java |    2 +-
 .../hadoop/hive/metastore/api/Partition.java    |    2 +-
 .../api/PartitionListComposingSpec.java         |    2 +-
 .../hive/metastore/api/PartitionSpec.java       |    2 +-
 .../api/PartitionSpecWithSharedSD.java          |    2 +-
 .../hive/metastore/api/PartitionWithoutSD.java  |    2 +-
 .../metastore/api/PartitionsByExprRequest.java  |    2 +-
 .../metastore/api/PartitionsByExprResult.java   |    2 +-
 .../metastore/api/PartitionsStatsRequest.java   |    2 +-
 .../metastore/api/PartitionsStatsResult.java    |    2 +-
 .../metastore/api/PrincipalPrivilegeSet.java    |    2 +-
 .../hadoop/hive/metastore/api/PrivilegeBag.java |    2 +-
 .../hive/metastore/api/PrivilegeGrantInfo.java  |    2 +-
 .../hadoop/hive/metastore/api/ResourceUri.java  |    2 +-
 .../apache/hadoop/hive/metastore/api/Role.java  |    2 +-
 .../hive/metastore/api/RolePrincipalGrant.java  |    2 +-
 .../hadoop/hive/metastore/api/Schema.java       |    2 +-
 .../hadoop/hive/metastore/api/SerDeInfo.java    |    2 +-
 .../api/SetPartitionsStatsRequest.java          |    2 +-
 .../hive/metastore/api/ShowCompactRequest.java  |    2 +-
 .../hive/metastore/api/ShowCompactResponse.java |    2 +-
 .../api/ShowCompactResponseElement.java         |    2 +-
 .../hive/metastore/api/ShowLocksRequest.java    |    2 +-
 .../hive/metastore/api/ShowLocksResponse.java   |    2 +-
 .../metastore/api/ShowLocksResponseElement.java |    2 +-
 .../hadoop/hive/metastore/api/SkewedInfo.java   |    2 +-
 .../hive/metastore/api/StorageDescriptor.java   |    2 +-
 .../metastore/api/StringColumnStatsData.java    |    2 +-
 .../apache/hadoop/hive/metastore/api/Table.java |    2 +-
 .../hive/metastore/api/TableStatsRequest.java   |    2 +-
 .../hive/metastore/api/TableStatsResult.java    |    2 +-
 .../hive/metastore/api/ThriftHiveMetastore.java | 3140 +++++++++++-------
 .../hive/metastore/api/TxnAbortedException.java |    2 +-
 .../hadoop/hive/metastore/api/TxnInfo.java      |    2 +-
 .../hive/metastore/api/TxnOpenException.java    |    2 +-
 .../apache/hadoop/hive/metastore/api/Type.java  |    2 +-
 .../hive/metastore/api/UnknownDBException.java  |    2 +-
 .../api/UnknownPartitionException.java          |    2 +-
 .../metastore/api/UnknownTableException.java    |    2 +-
 .../hive/metastore/api/UnlockRequest.java       |    2 +-
 .../hadoop/hive/metastore/api/Version.java      |    2 +-
 .../gen-php/metastore/ThriftHiveMetastore.php   | 1265 ++++---
 .../src/gen/thrift/gen-php/metastore/Types.php  |  103 +
 .../hive_metastore/ThriftHiveMetastore-remote   |   21 +-
 .../hive_metastore/ThriftHiveMetastore.py       |  879 +++--
 .../gen/thrift/gen-py/hive_metastore/ttypes.py  |   74 +
 .../gen/thrift/gen-rb/hive_metastore_types.rb   |   16 +
 .../gen/thrift/gen-rb/thrift_hive_metastore.rb  |   60 +
 .../hadoop/hive/metastore/HiveMetaStore.java    |   21 +
 .../hive/metastore/HiveMetaStoreClient.java     |    7 +
 .../hadoop/hive/metastore/IMetaStoreClient.java |    4 +
 .../hadoop/hive/metastore/ObjectStore.java      |   28 +
 .../apache/hadoop/hive/metastore/RawStore.java  |    7 +
 .../DummyRawStoreControlledCommit.java          |    7 +
 .../DummyRawStoreForJdoConnection.java          |    6 +
 .../hadoop/hive/ql/plan/api/Adjacency.java      |    2 +-
 .../apache/hadoop/hive/ql/plan/api/Graph.java   |    2 +-
 .../hadoop/hive/ql/plan/api/Operator.java       |    2 +-
 .../apache/hadoop/hive/ql/plan/api/Query.java   |    2 +-
 .../hadoop/hive/ql/plan/api/QueryPlan.java      |    2 +-
 .../apache/hadoop/hive/ql/plan/api/Stage.java   |    2 +-
 .../apache/hadoop/hive/ql/plan/api/Task.java    |    2 +-
 .../apache/hadoop/hive/ql/metadata/Hive.java    |   30 +-
 .../hadoop/hive/serde/test/InnerStruct.java     |    2 +-
 .../hadoop/hive/serde/test/ThriftTestObj.java   |    2 +-
 .../hadoop/hive/serde2/thrift/test/Complex.java |    2 +-
 .../hive/serde2/thrift/test/IntString.java      |    2 +-
 .../hive/serde2/thrift/test/MegaStruct.java     |    2 +-
 .../hive/serde2/thrift/test/MiniStruct.java     |    2 +-
 .../hive/serde2/thrift/test/SetIntString.java   |    2 +-
 .../hadoop/hive/service/HiveClusterStatus.java  |    2 +-
 .../hive/service/HiveServerException.java       |    2 +-
 .../apache/hadoop/hive/service/ThriftHive.java  |    2 +-
 .../service/cli/thrift/TArrayTypeEntry.java     |    2 +-
 .../hive/service/cli/thrift/TBinaryColumn.java  |    2 +-
 .../hive/service/cli/thrift/TBoolColumn.java    |    2 +-
 .../hive/service/cli/thrift/TBoolValue.java     |    2 +-
 .../hive/service/cli/thrift/TByteColumn.java    |    2 +-
 .../hive/service/cli/thrift/TByteValue.java     |    2 +-
 .../hive/service/cli/thrift/TCLIService.java    |    2 +-
 .../cli/thrift/TCancelDelegationTokenReq.java   |    2 +-
 .../cli/thrift/TCancelDelegationTokenResp.java  |    2 +-
 .../service/cli/thrift/TCancelOperationReq.java |    2 +-
 .../cli/thrift/TCancelOperationResp.java        |    2 +-
 .../service/cli/thrift/TCloseOperationReq.java  |    2 +-
 .../service/cli/thrift/TCloseOperationResp.java |    2 +-
 .../service/cli/thrift/TCloseSessionReq.java    |    2 +-
 .../service/cli/thrift/TCloseSessionResp.java   |    2 +-
 .../hive/service/cli/thrift/TColumnDesc.java    |    2 +-
 .../hive/service/cli/thrift/TDoubleColumn.java  |    2 +-
 .../hive/service/cli/thrift/TDoubleValue.java   |    2 +-
 .../cli/thrift/TExecuteStatementReq.java        |    2 +-
 .../cli/thrift/TExecuteStatementResp.java       |    2 +-
 .../service/cli/thrift/TFetchResultsReq.java    |    2 +-
 .../service/cli/thrift/TFetchResultsResp.java   |    2 +-
 .../service/cli/thrift/TGetCatalogsReq.java     |    2 +-
 .../service/cli/thrift/TGetCatalogsResp.java    |    2 +-
 .../hive/service/cli/thrift/TGetColumnsReq.java |    2 +-
 .../service/cli/thrift/TGetColumnsResp.java     |    2 +-
 .../cli/thrift/TGetDelegationTokenReq.java      |    2 +-
 .../cli/thrift/TGetDelegationTokenResp.java     |    2 +-
 .../service/cli/thrift/TGetFunctionsReq.java    |    2 +-
 .../service/cli/thrift/TGetFunctionsResp.java   |    2 +-
 .../hive/service/cli/thrift/TGetInfoReq.java    |    2 +-
 .../hive/service/cli/thrift/TGetInfoResp.java   |    2 +-
 .../cli/thrift/TGetOperationStatusReq.java      |    2 +-
 .../cli/thrift/TGetOperationStatusResp.java     |    2 +-
 .../cli/thrift/TGetResultSetMetadataReq.java    |    2 +-
 .../cli/thrift/TGetResultSetMetadataResp.java   |    2 +-
 .../hive/service/cli/thrift/TGetSchemasReq.java |    2 +-
 .../service/cli/thrift/TGetSchemasResp.java     |    2 +-
 .../service/cli/thrift/TGetTableTypesReq.java   |    2 +-
 .../service/cli/thrift/TGetTableTypesResp.java  |    2 +-
 .../hive/service/cli/thrift/TGetTablesReq.java  |    2 +-
 .../hive/service/cli/thrift/TGetTablesResp.java |    2 +-
 .../service/cli/thrift/TGetTypeInfoReq.java     |    2 +-
 .../service/cli/thrift/TGetTypeInfoResp.java    |    2 +-
 .../service/cli/thrift/THandleIdentifier.java   |    2 +-
 .../hive/service/cli/thrift/TI16Column.java     |    2 +-
 .../hive/service/cli/thrift/TI16Value.java      |    2 +-
 .../hive/service/cli/thrift/TI32Column.java     |    2 +-
 .../hive/service/cli/thrift/TI32Value.java      |    2 +-
 .../hive/service/cli/thrift/TI64Column.java     |    2 +-
 .../hive/service/cli/thrift/TI64Value.java      |    2 +-
 .../hive/service/cli/thrift/TMapTypeEntry.java  |    2 +-
 .../service/cli/thrift/TOpenSessionReq.java     |    2 +-
 .../service/cli/thrift/TOpenSessionResp.java    |    2 +-
 .../service/cli/thrift/TOperationHandle.java    |    2 +-
 .../service/cli/thrift/TPrimitiveTypeEntry.java |    2 +-
 .../cli/thrift/TRenewDelegationTokenReq.java    |    2 +-
 .../cli/thrift/TRenewDelegationTokenResp.java   |    2 +-
 .../apache/hive/service/cli/thrift/TRow.java    |    2 +-
 .../apache/hive/service/cli/thrift/TRowSet.java |    2 +-
 .../hive/service/cli/thrift/TSessionHandle.java |    2 +-
 .../apache/hive/service/cli/thrift/TStatus.java |    2 +-
 .../hive/service/cli/thrift/TStringColumn.java  |    2 +-
 .../hive/service/cli/thrift/TStringValue.java   |    2 +-
 .../service/cli/thrift/TStructTypeEntry.java    |    2 +-
 .../hive/service/cli/thrift/TTableSchema.java   |    2 +-
 .../hive/service/cli/thrift/TTypeDesc.java      |    2 +-
 .../service/cli/thrift/TTypeQualifiers.java     |    2 +-
 .../service/cli/thrift/TUnionTypeEntry.java     |    2 +-
 .../cli/thrift/TUserDefinedTypeEntry.java       |    2 +-
 .../gen-py/hive_service/ThriftHive-remote       |   21 +-
 217 files changed, 5877 insertions(+), 3215 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/cb648903/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
index 06c6b76..5a344bb 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
@@ -51,6 +51,7 @@ import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.metastore.api.Function;
 import org.apache.hadoop.hive.metastore.api.FunctionType;
+import org.apache.hadoop.hive.metastore.api.GetAllFunctionsResponse;
 import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
 import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
 import org.apache.hadoop.hive.metastore.api.MetaException;
@@ -2533,6 +2534,7 @@ public abstract class TestHiveMetaStore extends TestCase {
     String funcName = "test_func";
     String className = "org.apache.hadoop.hive.ql.udf.generic.GenericUDFUpper";
     String owner = "test_owner";
+    final int N_FUNCTIONS = 5;
     PrincipalType ownerType = PrincipalType.USER;
     int createTime = (int) (System.currentTimeMillis() / 1000);
     FunctionType funcType = FunctionType.JAVA;
@@ -2542,14 +2544,16 @@ public abstract class TestHiveMetaStore extends TestCase {
 
       createDb(dbName);
 
-      createFunction(dbName, funcName, className, owner, ownerType, createTime, funcType,
null);
+      for (int i = 0; i < N_FUNCTIONS; i++) {
+        createFunction(dbName, funcName + "_" + i, className, owner, ownerType, createTime,
funcType, null);
+      }
 
       // Try the different getters
 
       // getFunction()
-      Function func = client.getFunction(dbName, funcName);
+      Function func = client.getFunction(dbName, funcName + "_0");
       assertEquals("function db name", dbName, func.getDbName());
-      assertEquals("function name", funcName, func.getFunctionName());
+      assertEquals("function name", funcName + "_0", func.getFunctionName());
       assertEquals("function class name", className, func.getClassName());
       assertEquals("function owner name", owner, func.getOwnerName());
       assertEquals("function owner type", PrincipalType.USER, func.getOwnerType());
@@ -2566,21 +2570,31 @@ public abstract class TestHiveMetaStore extends TestCase {
       }
       assertEquals(true, gotException);
 
+      // getAllFunctions()
+      GetAllFunctionsResponse response = client.getAllFunctions();
+      List<Function> allFunctions = response.getFunctions();
+      assertEquals(N_FUNCTIONS, allFunctions.size());
+      assertEquals(funcName + "_3", allFunctions.get(3).getFunctionName());
+
       // getFunctions()
-      List<String> funcs = client.getFunctions(dbName, "*_func");
-      assertEquals(1, funcs.size());
-      assertEquals(funcName, funcs.get(0));
+      List<String> funcs = client.getFunctions(dbName, "*_func_*");
+      assertEquals(N_FUNCTIONS, funcs.size());
+      assertEquals(funcName + "_0", funcs.get(0));
 
       funcs = client.getFunctions(dbName, "nonexistent_func");
       assertEquals(0, funcs.size());
 
       // dropFunction()
-      client.dropFunction(dbName, funcName);
+      for (int i = 0; i < N_FUNCTIONS; i++) {
+        client.dropFunction(dbName, funcName + "_" + i);
+      }
 
       // Confirm that the function is now gone
       funcs = client.getFunctions(dbName, funcName);
       assertEquals(0, funcs.size());
-
+      response = client.getAllFunctions();
+      allFunctions = response.getFunctions();
+      assertEquals(0, allFunctions.size());
     } catch (Exception e) {
       System.err.println(StringUtils.stringifyException(e));
       System.err.println("testConcurrentMetastores() failed.");

http://git-wip-us.apache.org/repos/asf/hive/blob/cb648903/metastore/if/hive_metastore.thrift
----------------------------------------------------------------------
diff --git a/metastore/if/hive_metastore.thrift b/metastore/if/hive_metastore.thrift
index adf4830..d651195 100755
--- a/metastore/if/hive_metastore.thrift
+++ b/metastore/if/hive_metastore.thrift
@@ -715,6 +715,9 @@ struct FireEventResponse {
 }
     
 
+struct GetAllFunctionsResponse {
+  1: optional list<Function> functions
+}
 
 exception MetaException {
   1: string message
@@ -1114,6 +1117,8 @@ service ThriftHiveMetastore extends fb303.FacebookService
   Function get_function(1:string dbName, 2:string funcName)
       throws (1:MetaException o1, 2:NoSuchObjectException o2)
 
+  GetAllFunctionsResponse get_all_functions() throws (1:MetaException o1)
+
   //authorization privileges
 
   bool create_role(1:Role role) throws(1:MetaException o1)


Mime
View raw message