hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From xu...@apache.org
Subject [48/54] [abbrv] hive git commit: HIVE-9152 - Dynamic Partition Pruning [Spark Branch] (Chao Sun, reviewed by Xuefu Zhang and Chengxiang Li)
Date Fri, 31 Jul 2015 22:57:49 GMT
HIVE-9152 - Dynamic Partition Pruning [Spark Branch] (Chao Sun, reviewed by Xuefu Zhang and
Chengxiang Li)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/42216997
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/42216997
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/42216997

Branch: refs/heads/master
Commit: 42216997f7fcff1853524b03e3961ec5c21f3fd7
Parents: dddffe1
Author: Chao Sun <sunchao@apache.org>
Authored: Fri Jul 17 12:51:27 2015 -0700
Committer: Chao Sun <sunchao@apache.org>
Committed: Fri Jul 17 12:51:27 2015 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/hive/conf/HiveConf.java   |     7 +
 itests/qtest/pom.xml                            |     2 +-
 .../test/resources/testconfiguration.properties |     6 +
 .../gen/thrift/gen-cpp/ThriftHiveMetastore.cpp  |  4708 ++++-
 .../gen/thrift/gen-cpp/ThriftHiveMetastore.h    |  7807 +++----
 .../thrift/gen-cpp/hive_metastore_constants.cpp |     2 +-
 .../thrift/gen-cpp/hive_metastore_constants.h   |     2 +-
 .../gen/thrift/gen-cpp/hive_metastore_types.cpp |  6044 +++++-
 .../gen/thrift/gen-cpp/hive_metastore_types.h   |  2619 ++-
 .../hive/metastore/api/AbortTxnRequest.java     |    24 +-
 .../metastore/api/AddDynamicPartitions.java     |    69 +-
 .../metastore/api/AddPartitionsRequest.java     |    80 +-
 .../hive/metastore/api/AddPartitionsResult.java |    48 +-
 .../hadoop/hive/metastore/api/AggrStats.java    |    54 +-
 .../metastore/api/AlreadyExistsException.java   |    24 +-
 .../metastore/api/BinaryColumnStatsData.java    |    40 +-
 .../metastore/api/BooleanColumnStatsData.java   |    40 +-
 .../hive/metastore/api/CheckLockRequest.java    |    24 +-
 .../hive/metastore/api/ColumnStatistics.java    |    54 +-
 .../metastore/api/ColumnStatisticsData.java     |    20 +-
 .../metastore/api/ColumnStatisticsDesc.java     |    58 +-
 .../hive/metastore/api/ColumnStatisticsObj.java |    40 +-
 .../hive/metastore/api/CommitTxnRequest.java    |    24 +-
 .../hive/metastore/api/CompactionRequest.java   |    62 +-
 .../hive/metastore/api/CompactionType.java      |     2 +-
 .../api/ConfigValSecurityException.java         |    24 +-
 .../api/CurrentNotificationEventId.java         |    24 +-
 .../hadoop/hive/metastore/api/Database.java     |   115 +-
 .../apache/hadoop/hive/metastore/api/Date.java  |    24 +-
 .../hive/metastore/api/DateColumnStatsData.java |    50 +-
 .../hadoop/hive/metastore/api/Decimal.java      |    41 +-
 .../metastore/api/DecimalColumnStatsData.java   |    50 +-
 .../metastore/api/DoubleColumnStatsData.java    |    50 +-
 .../hive/metastore/api/DropPartitionsExpr.java  |    43 +-
 .../metastore/api/DropPartitionsRequest.java    |    82 +-
 .../metastore/api/DropPartitionsResult.java     |    48 +-
 .../hive/metastore/api/EnvironmentContext.java  |    61 +-
 .../hive/metastore/api/EventRequestType.java    |     2 +-
 .../hadoop/hive/metastore/api/FieldSchema.java  |    58 +-
 .../hive/metastore/api/FireEventRequest.java    |    79 +-
 .../metastore/api/FireEventRequestData.java     |    20 +-
 .../hive/metastore/api/FireEventResponse.java   |    16 +-
 .../hadoop/hive/metastore/api/Function.java     |   110 +-
 .../hadoop/hive/metastore/api/FunctionType.java |     2 +-
 .../metastore/api/GetOpenTxnsInfoResponse.java  |    54 +-
 .../hive/metastore/api/GetOpenTxnsResponse.java |    53 +-
 .../api/GetPrincipalsInRoleRequest.java         |    24 +-
 .../api/GetPrincipalsInRoleResponse.java        |    46 +-
 .../api/GetRoleGrantsForPrincipalRequest.java   |    36 +-
 .../api/GetRoleGrantsForPrincipalResponse.java  |    46 +-
 .../api/GrantRevokePrivilegeRequest.java        |    46 +-
 .../api/GrantRevokePrivilegeResponse.java       |    26 +-
 .../metastore/api/GrantRevokeRoleRequest.java   |    86 +-
 .../metastore/api/GrantRevokeRoleResponse.java  |    26 +-
 .../hive/metastore/api/GrantRevokeType.java     |     2 +-
 .../hive/metastore/api/HeartbeatRequest.java    |    34 +-
 .../metastore/api/HeartbeatTxnRangeRequest.java |    32 +-
 .../api/HeartbeatTxnRangeResponse.java          |    74 +-
 .../hive/metastore/api/HiveObjectPrivilege.java |    52 +-
 .../hive/metastore/api/HiveObjectRef.java       |    81 +-
 .../hive/metastore/api/HiveObjectType.java      |     2 +-
 .../apache/hadoop/hive/metastore/api/Index.java |   133 +-
 .../api/IndexAlreadyExistsException.java        |    24 +-
 .../metastore/api/InsertEventRequestData.java   |    45 +-
 .../metastore/api/InvalidInputException.java    |    24 +-
 .../metastore/api/InvalidObjectException.java   |    24 +-
 .../api/InvalidOperationException.java          |    24 +-
 .../api/InvalidPartitionException.java          |    24 +-
 .../hive/metastore/api/LockComponent.java       |    66 +-
 .../hadoop/hive/metastore/api/LockLevel.java    |     2 +-
 .../hadoop/hive/metastore/api/LockRequest.java  |    72 +-
 .../hadoop/hive/metastore/api/LockResponse.java |    36 +-
 .../hadoop/hive/metastore/api/LockState.java    |     2 +-
 .../hadoop/hive/metastore/api/LockType.java     |     2 +-
 .../hive/metastore/api/LongColumnStatsData.java |    50 +-
 .../hive/metastore/api/MetaException.java       |    24 +-
 .../hive/metastore/api/NoSuchLockException.java |    24 +-
 .../metastore/api/NoSuchObjectException.java    |    24 +-
 .../hive/metastore/api/NoSuchTxnException.java  |    24 +-
 .../hive/metastore/api/NotificationEvent.java   |    66 +-
 .../metastore/api/NotificationEventRequest.java |    34 +-
 .../api/NotificationEventResponse.java          |    46 +-
 .../hive/metastore/api/OpenTxnRequest.java      |    40 +-
 .../hive/metastore/api/OpenTxnsResponse.java    |    45 +-
 .../apache/hadoop/hive/metastore/api/Order.java |    32 +-
 .../hadoop/hive/metastore/api/Partition.java    |   156 +-
 .../hive/metastore/api/PartitionEventType.java  |     2 +-
 .../api/PartitionListComposingSpec.java         |    46 +-
 .../hive/metastore/api/PartitionSpec.java       |    58 +-
 .../api/PartitionSpecWithSharedSD.java          |    54 +-
 .../hive/metastore/api/PartitionWithoutSD.java  |   124 +-
 .../metastore/api/PartitionsByExprRequest.java  |    67 +-
 .../metastore/api/PartitionsByExprResult.java   |    54 +-
 .../metastore/api/PartitionsStatsRequest.java   |    90 +-
 .../metastore/api/PartitionsStatsResult.java    |    72 +-
 .../metastore/api/PrincipalPrivilegeSet.java    |   184 +-
 .../hive/metastore/api/PrincipalType.java       |     2 +-
 .../hadoop/hive/metastore/api/PrivilegeBag.java |    46 +-
 .../hive/metastore/api/PrivilegeGrantInfo.java  |    60 +-
 .../hive/metastore/api/RequestPartsSpec.java    |    56 +-
 .../hadoop/hive/metastore/api/ResourceType.java |     2 +-
 .../hadoop/hive/metastore/api/ResourceUri.java  |    36 +-
 .../apache/hadoop/hive/metastore/api/Role.java  |    40 +-
 .../hive/metastore/api/RolePrincipalGrant.java  |    80 +-
 .../hadoop/hive/metastore/api/Schema.java       |    91 +-
 .../hadoop/hive/metastore/api/SerDeInfo.java    |    93 +-
 .../api/SetPartitionsStatsRequest.java          |    46 +-
 .../hive/metastore/api/ShowCompactRequest.java  |    16 +-
 .../hive/metastore/api/ShowCompactResponse.java |    46 +-
 .../api/ShowCompactResponseElement.java         |    86 +-
 .../hive/metastore/api/ShowLocksRequest.java    |    16 +-
 .../hive/metastore/api/ShowLocksResponse.java   |    46 +-
 .../metastore/api/ShowLocksResponseElement.java |   114 +-
 .../hadoop/hive/metastore/api/SkewedInfo.java   |   147 +-
 .../hive/metastore/api/StorageDescriptor.java   |   242 +-
 .../metastore/api/StringColumnStatsData.java    |    48 +-
 .../apache/hadoop/hive/metastore/api/Table.java |   189 +-
 .../hive/metastore/api/TableStatsRequest.java   |    61 +-
 .../hive/metastore/api/TableStatsResult.java    |    46 +-
 .../hive/metastore/api/ThriftHiveMetastore.java | 18497 ++++++++++++-----
 .../hive/metastore/api/TxnAbortedException.java |    24 +-
 .../hadoop/hive/metastore/api/TxnInfo.java      |    52 +-
 .../hive/metastore/api/TxnOpenException.java    |    24 +-
 .../hadoop/hive/metastore/api/TxnState.java     |     2 +-
 .../apache/hadoop/hive/metastore/api/Type.java  |    72 +-
 .../hive/metastore/api/UnknownDBException.java  |    24 +-
 .../api/UnknownPartitionException.java          |    24 +-
 .../metastore/api/UnknownTableException.java    |    24 +-
 .../hive/metastore/api/UnlockRequest.java       |    24 +-
 .../hadoop/hive/metastore/api/Version.java      |    32 +-
 .../metastore/api/hive_metastoreConstants.java  |     7 +-
 .../gen-php/metastore/ThriftHiveMetastore.php   |  3328 ++-
 .../src/gen/thrift/gen-php/metastore/Types.php  |  1081 +-
 .../hive_metastore/ThriftHiveMetastore-remote   |   609 +-
 .../hive_metastore/ThriftHiveMetastore.py       |  3726 +++-
 .../thrift/gen-py/hive_metastore/constants.py   |     2 +-
 .../gen/thrift/gen-py/hive_metastore/ttypes.py  |   782 +-
 .../thrift/gen-rb/hive_metastore_constants.rb   |     2 +-
 .../gen/thrift/gen-rb/hive_metastore_types.rb   |     2 +-
 .../gen/thrift/gen-rb/thrift_hive_metastore.rb  |     2 +-
 ql/if/queryplan.thrift                          |     1 +
 .../gen/thrift/gen-cpp/queryplan_constants.cpp  |     2 +-
 ql/src/gen/thrift/gen-cpp/queryplan_constants.h |     2 +-
 ql/src/gen/thrift/gen-cpp/queryplan_types.cpp   |   796 +-
 ql/src/gen/thrift/gen-cpp/queryplan_types.h     |   294 +-
 .../hadoop/hive/ql/plan/api/Adjacency.java      |    65 +-
 .../hadoop/hive/ql/plan/api/AdjacencyType.java  |     2 +-
 .../apache/hadoop/hive/ql/plan/api/Graph.java   |    87 +-
 .../hadoop/hive/ql/plan/api/NodeType.java       |     2 +-
 .../hadoop/hive/ql/plan/api/Operator.java       |   142 +-
 .../hadoop/hive/ql/plan/api/OperatorType.java   |     7 +-
 .../apache/hadoop/hive/ql/plan/api/Query.java   |   176 +-
 .../hadoop/hive/ql/plan/api/QueryPlan.java      |    62 +-
 .../apache/hadoop/hive/ql/plan/api/Stage.java   |   172 +-
 .../hadoop/hive/ql/plan/api/StageType.java      |     2 +-
 .../apache/hadoop/hive/ql/plan/api/Task.java    |   182 +-
 .../hadoop/hive/ql/plan/api/TaskType.java       |     2 +-
 ql/src/gen/thrift/gen-php/Types.php             |   119 +-
 ql/src/gen/thrift/gen-py/queryplan/constants.py |     2 +-
 ql/src/gen/thrift/gen-py/queryplan/ttypes.py    |    87 +-
 ql/src/gen/thrift/gen-rb/queryplan_constants.rb |     2 +-
 ql/src/gen/thrift/gen-rb/queryplan_types.rb     |     7 +-
 .../hadoop/hive/ql/exec/OperatorFactory.java    |     8 +
 .../ql/exec/spark/HiveSparkClientFactory.java   |     5 +-
 .../exec/spark/SparkDynamicPartitionPruner.java |   268 +
 .../hive/ql/exec/spark/SparkUtilities.java      |    56 +
 ...VectorSparkPartitionPruningSinkOperator.java |    99 +
 .../hive/ql/io/CombineHiveInputFormat.java      |    59 +-
 .../hadoop/hive/ql/io/HiveInputFormat.java      |    46 +-
 .../DynamicPartitionPruningOptimization.java    |    44 +-
 .../hadoop/hive/ql/optimizer/Optimizer.java     |     2 +-
 .../SparkRemoveDynamicPruningBySize.java        |    73 +
 .../hive/ql/optimizer/physical/Vectorizer.java  |     2 +
 .../spark/SparkPartitionPruningSinkDesc.java    |   100 +
 .../ql/parse/spark/GenSparkProcContext.java     |    14 +-
 .../hive/ql/parse/spark/GenSparkUtils.java      |   111 +-
 .../parse/spark/OptimizeSparkProcContext.java   |    16 +-
 .../hive/ql/parse/spark/SparkCompiler.java      |   177 +-
 .../SparkPartitionPruningSinkOperator.java      |   142 +
 .../hive/ql/parse/spark/SplitOpTreeForDPP.java  |   151 +
 .../org/apache/hadoop/hive/ql/plan/MapWork.java |    10 +
 .../hadoop/hive/ql/plan/TableScanDesc.java      |     6 +-
 .../hive/ql/ppd/SyntheticJoinPredicate.java     |    14 +-
 .../spark_dynamic_partition_pruning.q           |   180 +
 .../spark_dynamic_partition_pruning_2.q         |   118 +
 ...spark_vectorized_dynamic_partition_pruning.q |   192 +
 .../results/clientpositive/spark/bucket2.q.out  |     3 -
 .../results/clientpositive/spark/bucket3.q.out  |     3 -
 .../results/clientpositive/spark/bucket4.q.out  |     3 -
 .../spark/column_access_stats.q.out             |     4 -
 .../spark/limit_partition_metadataonly.q.out    |     2 -
 .../spark/list_bucket_dml_2.q.java1.7.out       |     3 -
 .../spark/optimize_nullscan.q.out               |     3 -
 .../test/results/clientpositive/spark/pcr.q.out |     6 -
 .../results/clientpositive/spark/sample3.q.out  |     3 -
 .../results/clientpositive/spark/sample9.q.out  |     3 -
 .../clientpositive/spark/smb_mapjoin_11.q.out   |     6 -
 .../spark/spark_dynamic_partition_pruning.q.out |  5573 +++++
 .../spark_dynamic_partition_pruning_2.q.out     |  1015 +
 ...k_vectorized_dynamic_partition_pruning.q.out |  5822 ++++++
 .../clientpositive/spark/udf_example_add.q.out  |     3 -
 .../clientpositive/spark/udf_in_file.q.out      |     3 -
 .../clientpositive/spark/union_view.q.out       |     9 -
 .../clientpositive/spark/vector_elt.q.out       |     7 -
 .../spark/vector_string_concat.q.out            |     3 -
 .../spark/vectorization_decimal_date.q.out      |     4 -
 .../spark/vectorization_div0.q.out              |     3 -
 .../clientpositive/spark/vectorized_case.q.out  |     3 -
 .../spark/vectorized_math_funcs.q.out           |     3 -
 .../spark/vectorized_string_funcs.q.out         |     3 -
 .../gen/thrift/gen-cpp/complex_constants.cpp    |     2 +-
 .../src/gen/thrift/gen-cpp/complex_constants.h  |     2 +-
 serde/src/gen/thrift/gen-cpp/complex_types.cpp  |   442 +-
 serde/src/gen/thrift/gen-cpp/complex_types.h    |   174 +-
 .../gen/thrift/gen-cpp/megastruct_constants.cpp |     2 +-
 .../gen/thrift/gen-cpp/megastruct_constants.h   |     2 +-
 .../src/gen/thrift/gen-cpp/megastruct_types.cpp |   585 +-
 serde/src/gen/thrift/gen-cpp/megastruct_types.h |   175 +-
 .../src/gen/thrift/gen-cpp/serde_constants.cpp  |     2 +-
 serde/src/gen/thrift/gen-cpp/serde_constants.h  |     2 +-
 serde/src/gen/thrift/gen-cpp/serde_types.cpp    |     5 +-
 serde/src/gen/thrift/gen-cpp/serde_types.h      |     5 +-
 .../gen/thrift/gen-cpp/testthrift_constants.cpp |     2 +-
 .../gen/thrift/gen-cpp/testthrift_constants.h   |     2 +-
 .../src/gen/thrift/gen-cpp/testthrift_types.cpp |    95 +-
 serde/src/gen/thrift/gen-cpp/testthrift_types.h |    45 +-
 .../hadoop/hive/serde/serdeConstants.java       |     7 +-
 .../hadoop/hive/serde/test/InnerStruct.java     |    24 +-
 .../hadoop/hive/serde/test/ThriftTestObj.java   |    62 +-
 .../hadoop/hive/serde2/thrift/test/Complex.java |   279 +-
 .../hive/serde2/thrift/test/IntString.java      |    40 +-
 .../hive/serde2/thrift/test/MegaStruct.java     |   521 +-
 .../hive/serde2/thrift/test/MiniStruct.java     |    38 +-
 .../hadoop/hive/serde2/thrift/test/MyEnum.java  |     2 +-
 .../hive/serde2/thrift/test/PropValueUnion.java |    60 +-
 .../hive/serde2/thrift/test/SetIntString.java   |    54 +-
 serde/src/gen/thrift/gen-php/Types.php          |    15 +-
 .../org/apache/hadoop/hive/serde/Types.php      |   373 +-
 .../src/gen/thrift/gen-py/complex/constants.py  |     2 +-
 serde/src/gen/thrift/gen-py/complex/ttypes.py   |    50 +-
 .../gen/thrift/gen-py/megastruct/constants.py   |     2 +-
 .../src/gen/thrift/gen-py/megastruct/ttypes.py  |    44 +-
 .../org_apache_hadoop_hive_serde/constants.py   |     2 +-
 .../org_apache_hadoop_hive_serde/ttypes.py      |     2 +-
 .../gen/thrift/gen-py/testthrift/constants.py   |     2 +-
 .../src/gen/thrift/gen-py/testthrift/ttypes.py  |    14 +-
 .../src/gen/thrift/gen-rb/complex_constants.rb  |     2 +-
 serde/src/gen/thrift/gen-rb/complex_types.rb    |     2 +-
 .../gen/thrift/gen-rb/megastruct_constants.rb   |     2 +-
 serde/src/gen/thrift/gen-rb/megastruct_types.rb |     2 +-
 serde/src/gen/thrift/gen-rb/serde_constants.rb  |     2 +-
 serde/src/gen/thrift/gen-rb/serde_types.rb      |     2 +-
 .../gen/thrift/gen-rb/testthrift_constants.rb   |     2 +-
 serde/src/gen/thrift/gen-rb/testthrift_types.rb |     2 +-
 service/src/gen/thrift/gen-cpp/TCLIService.cpp  |   458 +-
 service/src/gen/thrift/gen-cpp/TCLIService.h    |   821 +-
 .../thrift/gen-cpp/TCLIService_constants.cpp    |     2 +-
 .../gen/thrift/gen-cpp/TCLIService_constants.h  |     2 +-
 .../gen/thrift/gen-cpp/TCLIService_types.cpp    |  3250 ++-
 .../src/gen/thrift/gen-cpp/TCLIService_types.h  |  1482 +-
 service/src/gen/thrift/gen-cpp/ThriftHive.cpp   |   286 +-
 service/src/gen/thrift/gen-cpp/ThriftHive.h     |   389 +-
 .../thrift/gen-cpp/hive_service_constants.cpp   |     2 +-
 .../gen/thrift/gen-cpp/hive_service_constants.h |     2 +-
 .../gen/thrift/gen-cpp/hive_service_types.cpp   |   110 +-
 .../src/gen/thrift/gen-cpp/hive_service_types.h |    75 +-
 .../hadoop/hive/service/HiveClusterStatus.java  |    68 +-
 .../hive/service/HiveServerException.java       |    40 +-
 .../hadoop/hive/service/JobTrackerState.java    |     2 +-
 .../apache/hadoop/hive/service/ThriftHive.java  |   914 +-
 .../service/cli/thrift/TArrayTypeEntry.java     |    24 +-
 .../hive/service/cli/thrift/TBinaryColumn.java  |    64 +-
 .../hive/service/cli/thrift/TBoolColumn.java    |    62 +-
 .../hive/service/cli/thrift/TBoolValue.java     |    26 +-
 .../hive/service/cli/thrift/TByteColumn.java    |    62 +-
 .../hive/service/cli/thrift/TByteValue.java     |    26 +-
 .../hive/service/cli/thrift/TCLIService.java    |  1734 +-
 .../cli/thrift/TCLIServiceConstants.java        |     7 +-
 .../cli/thrift/TCancelDelegationTokenReq.java   |    32 +-
 .../cli/thrift/TCancelDelegationTokenResp.java  |    24 +-
 .../service/cli/thrift/TCancelOperationReq.java |    24 +-
 .../cli/thrift/TCancelOperationResp.java        |    24 +-
 .../service/cli/thrift/TCloseOperationReq.java  |    24 +-
 .../service/cli/thrift/TCloseOperationResp.java |    24 +-
 .../service/cli/thrift/TCloseSessionReq.java    |    24 +-
 .../service/cli/thrift/TCloseSessionResp.java   |    24 +-
 .../apache/hive/service/cli/thrift/TColumn.java |    20 +-
 .../hive/service/cli/thrift/TColumnDesc.java    |    50 +-
 .../hive/service/cli/thrift/TColumnValue.java   |    20 +-
 .../hive/service/cli/thrift/TDoubleColumn.java  |    62 +-
 .../hive/service/cli/thrift/TDoubleValue.java   |    26 +-
 .../cli/thrift/TExecuteStatementReq.java        |    87 +-
 .../cli/thrift/TExecuteStatementResp.java       |    34 +-
 .../service/cli/thrift/TFetchOrientation.java   |     2 +-
 .../service/cli/thrift/TFetchResultsReq.java    |    54 +-
 .../service/cli/thrift/TFetchResultsResp.java   |    42 +-
 .../service/cli/thrift/TGetCatalogsReq.java     |    24 +-
 .../service/cli/thrift/TGetCatalogsResp.java    |    34 +-
 .../hive/service/cli/thrift/TGetColumnsReq.java |    58 +-
 .../service/cli/thrift/TGetColumnsResp.java     |    34 +-
 .../cli/thrift/TGetDelegationTokenReq.java      |    40 +-
 .../cli/thrift/TGetDelegationTokenResp.java     |    34 +-
 .../service/cli/thrift/TGetFunctionsReq.java    |    50 +-
 .../service/cli/thrift/TGetFunctionsResp.java   |    34 +-
 .../hive/service/cli/thrift/TGetInfoReq.java    |    36 +-
 .../hive/service/cli/thrift/TGetInfoResp.java   |    32 +-
 .../hive/service/cli/thrift/TGetInfoType.java   |     2 +-
 .../hive/service/cli/thrift/TGetInfoValue.java  |    20 +-
 .../cli/thrift/TGetOperationStatusReq.java      |    24 +-
 .../cli/thrift/TGetOperationStatusResp.java     |    62 +-
 .../cli/thrift/TGetResultSetMetadataReq.java    |    24 +-
 .../cli/thrift/TGetResultSetMetadataResp.java   |    34 +-
 .../hive/service/cli/thrift/TGetSchemasReq.java |    42 +-
 .../service/cli/thrift/TGetSchemasResp.java     |    34 +-
 .../service/cli/thrift/TGetTableTypesReq.java   |    24 +-
 .../service/cli/thrift/TGetTableTypesResp.java  |    34 +-
 .../hive/service/cli/thrift/TGetTablesReq.java  |    79 +-
 .../hive/service/cli/thrift/TGetTablesResp.java |    34 +-
 .../service/cli/thrift/TGetTypeInfoReq.java     |    24 +-
 .../service/cli/thrift/TGetTypeInfoResp.java    |    34 +-
 .../service/cli/thrift/THandleIdentifier.java   |    50 +-
 .../hive/service/cli/thrift/TI16Column.java     |    62 +-
 .../hive/service/cli/thrift/TI16Value.java      |    26 +-
 .../hive/service/cli/thrift/TI32Column.java     |    62 +-
 .../hive/service/cli/thrift/TI32Value.java      |    26 +-
 .../hive/service/cli/thrift/TI64Column.java     |    62 +-
 .../hive/service/cli/thrift/TI64Value.java      |    26 +-
 .../hive/service/cli/thrift/TMapTypeEntry.java  |    32 +-
 .../service/cli/thrift/TOpenSessionReq.java     |    91 +-
 .../service/cli/thrift/TOpenSessionResp.java    |    91 +-
 .../service/cli/thrift/TOperationHandle.java    |    54 +-
 .../service/cli/thrift/TOperationState.java     |     2 +-
 .../hive/service/cli/thrift/TOperationType.java |     2 +-
 .../service/cli/thrift/TPrimitiveTypeEntry.java |    38 +-
 .../service/cli/thrift/TProtocolVersion.java    |     2 +-
 .../cli/thrift/TRenewDelegationTokenReq.java    |    32 +-
 .../cli/thrift/TRenewDelegationTokenResp.java   |    24 +-
 .../apache/hive/service/cli/thrift/TRow.java    |    46 +-
 .../apache/hive/service/cli/thrift/TRowSet.java |    86 +-
 .../hive/service/cli/thrift/TSessionHandle.java |    24 +-
 .../apache/hive/service/cli/thrift/TStatus.java |    83 +-
 .../hive/service/cli/thrift/TStatusCode.java    |     2 +-
 .../hive/service/cli/thrift/TStringColumn.java  |    62 +-
 .../hive/service/cli/thrift/TStringValue.java   |    26 +-
 .../service/cli/thrift/TStructTypeEntry.java    |    50 +-
 .../hive/service/cli/thrift/TTableSchema.java   |    46 +-
 .../hive/service/cli/thrift/TTypeDesc.java      |    46 +-
 .../hive/service/cli/thrift/TTypeEntry.java     |    20 +-
 .../apache/hive/service/cli/thrift/TTypeId.java |     2 +-
 .../service/cli/thrift/TTypeQualifierValue.java |    20 +-
 .../service/cli/thrift/TTypeQualifiers.java     |    54 +-
 .../service/cli/thrift/TUnionTypeEntry.java     |    50 +-
 .../cli/thrift/TUserDefinedTypeEntry.java       |    24 +-
 service/src/gen/thrift/gen-php/TCLIService.php  |   269 +-
 service/src/gen/thrift/gen-php/ThriftHive.php   |   125 +-
 service/src/gen/thrift/gen-php/Types.php        |    30 +-
 .../gen-py/TCLIService/TCLIService-remote       |   102 +-
 .../thrift/gen-py/TCLIService/TCLIService.py    |   439 +-
 .../gen/thrift/gen-py/TCLIService/constants.py  |     2 +-
 .../src/gen/thrift/gen-py/TCLIService/ttypes.py |   478 +-
 .../gen-py/hive_service/ThriftHive-remote       |  1014 +-
 .../thrift/gen-py/hive_service/ThriftHive.py    |   266 +-
 .../gen/thrift/gen-py/hive_service/constants.py |     2 +-
 .../gen/thrift/gen-py/hive_service/ttypes.py    |    19 +-
 .../gen/thrift/gen-rb/hive_service_constants.rb |     2 +-
 .../src/gen/thrift/gen-rb/hive_service_types.rb |     2 +-
 .../src/gen/thrift/gen-rb/t_c_l_i_service.rb    |     2 +-
 .../thrift/gen-rb/t_c_l_i_service_constants.rb  |     2 +-
 .../gen/thrift/gen-rb/t_c_l_i_service_types.rb  |     2 +-
 service/src/gen/thrift/gen-rb/thrift_hive.rb    |     2 +-
 370 files changed, 67652 insertions(+), 22750 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/42216997/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 27f68df..736a22d 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2187,6 +2187,13 @@ public class HiveConf extends Configuration {
       "Channel logging level for remote Spark driver.  One of {DEBUG, ERROR, INFO, TRACE,
WARN}."),
     SPARK_RPC_SASL_MECHANISM("hive.spark.client.rpc.sasl.mechanisms", "DIGEST-MD5",
       "Name of the SASL mechanism to use for authentication."),
+    SPARK_DYNAMIC_PARTITION_PRUNING(
+        "hive.spark.dynamic.partition.pruning", false,
+        "When dynamic pruning is enabled, joins on partition keys will be processed by writing\n"
+
+            "to a temporary HDFS file, and read later for removing unnecessary partitions."),
+    SPARK_DYNAMIC_PARTITION_PRUNING_MAX_DATA_SIZE(
+        "hive.spark.dynamic.partition.pruning.max.data.size", 100*1024*1024L,
+        "Maximum total data size in dynamic pruning."),
     NWAYJOINREORDER("hive.reorder.nway.joins", true,
       "Runs reordering of tables within single n-way join (i.e.: picks streamtable)"),
     HIVE_LOG_N_RECORDS("hive.log.every.n.records", 0L, new RangeValidator(0L, null),

http://git-wip-us.apache.org/repos/asf/hive/blob/42216997/itests/qtest/pom.xml
----------------------------------------------------------------------
diff --git a/itests/qtest/pom.xml b/itests/qtest/pom.xml
index e195bee..44d30da 100644
--- a/itests/qtest/pom.xml
+++ b/itests/qtest/pom.xml
@@ -429,7 +429,7 @@
                   templatePath="${basedir}/${hive.path.to.root}/ql/src/test/templates/" template="TestCliDriver.vm"
                   queryDirectory="${basedir}/${hive.path.to.root}/ql/src/test/queries/clientpositive/"
                   queryFile="${qfile}"
-                  excludeQueryFile="${minimr.query.files},${minitez.query.files},${encrypted.query.files}"
+                  excludeQueryFile="${minimr.query.files},${minitez.query.files},${encrypted.query.files},${spark.only.query.files}"
                   queryFileRegex="${qfile_regex}"
                   clusterMode="${clustermode}"
                   runDisabled="${run_disabled}"

http://git-wip-us.apache.org/repos/asf/hive/blob/42216997/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index f4ae684..2cf4920 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -1122,6 +1122,12 @@ spark.query.files=add_part_multiple.q, \
   vectorized_timestamp_funcs.q, \
   windowing.q
 
+# Unlike "spark.query.files" above, these tests only run
+# under Spark engine.
+spark.only.query.files=spark_dynamic_partition_pruning.q,\
+  spark_dynamic_partition_pruning_2.q,\
+  spark_vectorized_dynamic_partition_pruning.q
+
 miniSparkOnYarn.query.files=auto_sortmerge_join_16.q,\
   bucket4.q,\
   bucket5.q,\


Mime
View raw message