hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ser...@apache.org
Subject [50/50] [abbrv] hive git commit: HIVE-14671 : merge master into hive-14535 (Sergey Shelukhin)
Date Fri, 24 Feb 2017 01:09:22 GMT
HIVE-14671 : merge master into hive-14535 (Sergey Shelukhin)

Conflicts:
	metastore/scripts/upgrade/mssql/upgrade-2.1.0-to-2.2.0.mssql.sql
	metastore/scripts/upgrade/mysql/upgrade-2.1.0-to-2.2.0.mysql.sql
	metastore/scripts/upgrade/oracle/upgrade-2.1.0-to-2.2.0.oracle.sql
	metastore/scripts/upgrade/postgres/upgrade-2.1.0-to-2.2.0.postgres.sql
	ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2014ece9
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2014ece9
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2014ece9

Branch: refs/heads/hive-14535
Commit: 2014ece97960f8d2f690f55c131d9a61c421c2f9
Parents: 1f0a5ef b8d7192
Author: Sergey Shelukhin <sershe@apache.org>
Authored: Thu Feb 23 17:08:03 2017 -0800
Committer: Sergey Shelukhin <sershe@apache.org>
Committed: Thu Feb 23 17:08:03 2017 -0800

----------------------------------------------------------------------
 ant/pom.xml                                     |   69 -
 .../hive/ant/DistinctElementsClassPath.java     |   90 -
 .../apache/hadoop/hive/ant/GenVectorCode.java   | 3309 --------------
 .../hadoop/hive/ant/GenVectorTestCode.java      |  261 --
 .../apache/hadoop/hive/ant/GetVersionPref.java  |   94 -
 ant/src/org/apache/hadoop/hive/ant/antlib.xml   |   24 -
 .../java/org/apache/hive/beeline/BeeLine.java   |   42 +-
 .../hive/beeline/BeeLineSignalHandler.java      |    1 -
 .../java/org/apache/hive/beeline/Commands.java  |    2 +-
 .../apache/hive/beeline/SunSignalHandler.java   |   13 +-
 beeline/src/main/resources/BeeLine.properties   |    1 +
 .../apache/hive/beeline/TestHiveSchemaTool.java |   73 +
 .../apache/hive/beeline/TestShutdownHook.java   |   46 +
 bin/beeline                                     |    5 -
 bin/beeline.cmd                                 |   66 -
 bin/derbyserver.cmd                             |   60 -
 bin/ext/cleardanglingscratchdir.cmd             |   34 -
 bin/ext/cli.cmd                                 |   58 -
 bin/ext/cli.sh                                  |    8 -
 bin/ext/debug.cmd                               |  110 -
 bin/ext/hbaseimport.cmd                         |   35 -
 bin/ext/help.cmd                                |   30 -
 bin/ext/hiveserver2.cmd                         |  139 -
 bin/ext/jar.cmd                                 |   43 -
 bin/ext/lineage.cmd                             |   30 -
 bin/ext/metastore.cmd                           |   47 -
 bin/ext/orcfiledump.cmd                         |   35 -
 bin/ext/rcfilecat.cmd                           |   34 -
 bin/ext/schemaTool.cmd                          |   33 -
 bin/ext/util/execHiveCmd.cmd                    |   24 -
 bin/hive                                        |    7 +
 bin/hive-config.cmd                             |   61 -
 bin/hive.cmd                                    |  383 --
 bin/hplsql.cmd                                  |   58 -
 .../hadoop/hive/cli/TestCliDriverMethods.java   |   28 -
 .../apache/hadoop/hive/common/FileUtils.java    |    9 -
 .../hadoop/hive/common/StatsSetupConst.java     |  223 +-
 .../hadoop/hive/common/log/InPlaceUpdate.java   |    1 +
 .../org/apache/hadoop/hive/conf/HiveConf.java   |   61 +-
 .../common/util/ACLConfigurationParser.java     |  167 +
 .../java/org/apache/hive/http/HttpServer.java   |    2 +-
 .../hadoop/hive/common/TestStatsSetupConst.java |   54 +
 .../apache/hadoop/hive/conf/TestHiveConf.java   |    5 -
 .../common/util/TestACLConfigurationParser.java |   99 +
 .../hadoop/hive/contrib/mr/TestGenericMR.java   |   13 +-
 data/conf/hive-site.xml                         |    2 +-
 druid-handler/pom.xml                           |   33 +-
 .../hadoop/hive/druid/DruidStorageHandler.java  |    7 +-
 .../hive/druid/DruidStorageHandlerUtils.java    |   64 +-
 .../druid/io/DruidQueryBasedInputFormat.java    |  198 +-
 .../hadoop/hive/druid/io/HiveDruidSplit.java    |   30 +-
 .../druid/serde/DruidQueryRecordReader.java     |    3 +-
 .../hadoop/hive/druid/serde/DruidSerDe.java     |   18 +-
 .../TestHiveDruidQueryBasedInputFormat.java     |  159 +
 .../hive/ql/io/DruidRecordWriterTest.java       |    2 +-
 hcatalog/bin/templeton.cmd                      |   90 -
 .../hive/hcatalog/mapreduce/HCatBaseTest.java   |    4 -
 .../mapreduce/TestHCatPartitionPublish.java     |   11 +-
 .../pig/TestHCatLoaderComplexSchema.java        |    5 -
 .../hcatalog/pig/TestHCatLoaderEncryption.java  |   11 +-
 .../listener/DbNotificationListener.java        |    1 +
 .../e2e/templeton/drivers/TestDriverCurl.pm     | 1984 ++++----
 .../hive/hcatalog/api/TestHCatClient.java       |    7 -
 .../hcatalog/templeton/ExecServiceImpl.java     |   48 +-
 .../hive/hcatalog/templeton/HiveDelegator.java  |    9 +-
 .../hive/hcatalog/templeton/JarDelegator.java   |    9 +-
 .../hive/hcatalog/templeton/PigDelegator.java   |    5 +-
 .../hive/hcatalog/templeton/SqoopDelegator.java |    7 +-
 .../hcatalog/templeton/StreamingDelegator.java  |    4 +-
 .../hcatalog/templeton/tool/LaunchMapper.java   |    9 -
 .../hcatalog/templeton/tool/TempletonUtils.java |   72 +-
 .../main/java/org/apache/hive/hplsql/Exec.java  |    8 +-
 .../listener/TestDbNotificationListener.java    |   82 +
 itests/hive-blobstore/pom.xml                   |    6 -
 .../org/apache/hive/jdbc/miniHS2/MiniHS2.java   |    5 +-
 .../hive/ql/TestReplicationScenarios.java       |  123 +-
 .../security/StorageBasedMetastoreTestBase.java |    4 -
 .../ql/session/TestClearDanglingScratchDir.java |    4 -
 .../hive/beeline/TestBeeLineWithArgs.java       |  169 +-
 .../TestOperationLoggingAPIWithTez.java         |    2 -
 .../server/TestHS2ClearDanglingScratchDir.java  |    4 -
 itests/qtest-accumulo/pom.xml                   |    6 -
 itests/qtest-spark/pom.xml                      |    6 -
 itests/qtest/pom.xml                            |    6 -
 .../test/resources/testconfiguration.properties |    7 +-
 .../org/apache/hadoop/hive/ql/QTestUtil.java    |   55 +-
 jdbc/pom.xml                                    |   23 +-
 .../org/apache/hive/jdbc/HiveBaseResultSet.java |  297 +-
 .../apache/hive/jdbc/HiveCallableStatement.java |  421 +-
 .../org/apache/hive/jdbc/HiveConnection.java    |   64 +-
 .../org/apache/hive/jdbc/HiveDataSource.java    |   12 +-
 .../apache/hive/jdbc/HiveDatabaseMetaData.java  |  215 +-
 .../apache/hive/jdbc/HivePreparedStatement.java |   73 +-
 .../apache/hive/jdbc/HiveQueryResultSet.java    |    7 +-
 .../apache/hive/jdbc/HiveResultSetMetaData.java |   19 +-
 .../org/apache/hive/jdbc/HiveStatement.java     |   32 +-
 .../org/apache/hive/jdbc/HiveStatementTest.java |   14 +-
 .../llap/registry/impl/LlapRegistryService.java |   13 +-
 .../impl/LlapZookeeperRegistryImpl.java         |   11 +-
 .../hive/llap/security/SecretManager.java       |   19 +-
 .../org/apache/hadoop/hive/llap/LlapDump.java   |   30 +-
 .../hive/llap/cache/SerDeLowLevelCacheImpl.java |   13 +-
 .../hive/llap/cli/LlapOptionsProcessor.java     |   28 +-
 .../hadoop/hive/llap/cli/LlapServiceDriver.java |  290 +-
 .../hadoop/hive/llap/cli/LlapSliderUtils.java   |  188 +
 .../hive/llap/cli/LlapStatusServiceDriver.java  |   15 +-
 .../hive/llap/daemon/impl/AMReporter.java       |    5 +-
 .../llap/daemon/impl/ContainerRunnerImpl.java   |   16 +-
 .../hive/llap/daemon/impl/LlapDaemon.java       |   57 +-
 .../hadoop/hive/llap/daemon/impl/QueryInfo.java |   29 +
 .../hive/llap/daemon/impl/QueryTracker.java     |    4 +-
 .../llap/daemon/impl/TaskRunnerCallable.java    |   31 +-
 .../hive/llap/io/api/impl/LlapRecordReader.java |   11 +-
 .../llap/io/decode/OrcEncodedDataConsumer.java  |   22 +-
 .../llap/io/encoded/OrcEncodedDataReader.java   |    6 +-
 .../llap/io/encoded/SerDeEncodedDataReader.java |  882 ++--
 .../io/encoded/VectorDeserializeOrcWriter.java  |  458 ++
 .../io/encoded/VertorDeserializeOrcWriter.java  |  261 --
 .../llap/shufflehandler/ShuffleHandler.java     |    5 +-
 .../resources/llap-daemon-log4j2.properties     |   14 +-
 llap-server/src/main/resources/package.py       |   29 +-
 llap-server/src/main/resources/templates.py     |    3 +-
 .../hive/llap/daemon/MiniLlapCluster.java       |   38 +-
 .../daemon/impl/TaskExecutorTestHelpers.java    |    4 +-
 .../llap/tezplugins/LlapTaskCommunicator.java   |   81 +-
 .../tezplugins/LlapTaskSchedulerService.java    |   16 +
 .../tezplugins/helpers/SourceStateTracker.java  |   44 +-
 metastore/if/hive_metastore.thrift              |    1 +
 .../upgrade/derby/038-HIVE-10562.derby.sql      |   11 +
 .../upgrade/derby/hive-schema-2.2.0.derby.sql   |    2 +-
 .../derby/upgrade-2.1.0-to-2.2.0.derby.sql      |    1 +
 .../upgrade/mssql/023-HIVE-10562.mssql.sql      |    1 +
 .../upgrade/mssql/hive-schema-2.2.0.mssql.sql   |    1 +
 .../mssql/upgrade-2.1.0-to-2.2.0.mssql.sql      |    1 +
 .../upgrade/mysql/038-HIVE-10562.mysql.sql      |    6 +
 .../upgrade/mysql/hive-schema-2.2.0.mysql.sql   |    3 +-
 .../mysql/upgrade-2.1.0-to-2.2.0.mysql.sql      |    1 +
 .../upgrade/oracle/038-HIVE-10562.oracle.sql    |    2 +
 .../upgrade/oracle/hive-schema-2.2.0.oracle.sql |    3 +-
 .../oracle/upgrade-2.1.0-to-2.2.0.oracle.sql    |    1 +
 .../postgres/037-HIVE-10562.postgres.sql        |    1 +
 .../postgres/hive-schema-2.2.0.postgres.sql     |    1 +
 .../upgrade-2.1.0-to-2.2.0.postgres.sql         |    1 +
 .../gen/thrift/gen-cpp/hive_metastore_types.cpp |   22 +
 .../gen/thrift/gen-cpp/hive_metastore_types.h   |   12 +-
 .../hive/metastore/api/NotificationEvent.java   |  114 +-
 .../src/gen/thrift/gen-php/metastore/Types.php  |   23 +
 .../gen/thrift/gen-py/hive_metastore/ttypes.py  |   15 +-
 .../gen/thrift/gen-rb/hive_metastore_types.rb   |    4 +-
 .../hadoop/hive/metastore/HiveMetaStore.java    |   38 +-
 .../hive/metastore/HiveMetaStoreClient.java     |    9 +-
 .../hadoop/hive/metastore/ObjectStore.java      |    2 +
 .../hive/metastore/RetryingMetaStoreClient.java |    4 +-
 .../hive/metastore/messaging/EventUtils.java    |   28 +-
 .../metastore/messaging/MessageFactory.java     |    5 -
 .../messaging/json/JSONMessageFactory.java      |    7 +-
 .../hive/metastore/model/MNotificationLog.java  |    9 +
 metastore/src/model/package.jdo                 |    3 +
 pom.xml                                         |   50 +-
 ql/pom.xml                                      |    4 +-
 .../FilterColumnBetweenDynamicValue.txt         |   11 +-
 .../java/org/apache/hadoop/hive/ql/Context.java |  121 +-
 .../java/org/apache/hadoop/hive/ql/Driver.java  |    9 +-
 .../org/apache/hadoop/hive/ql/ErrorMsg.java     |    3 +
 .../apache/hadoop/hive/ql/exec/ExplainTask.java |   10 +-
 .../hadoop/hive/ql/exec/OperatorUtils.java      |   34 +
 .../hadoop/hive/ql/exec/ScriptOperator.java     |   17 -
 .../apache/hadoop/hive/ql/exec/Utilities.java   |   38 +-
 .../hive/ql/exec/mr/HadoopJobExecHelper.java    |    6 +-
 .../persistence/BytesBytesMultiHashMap.java     |   10 +-
 .../hive/ql/exec/spark/GroupByShuffler.java     |   11 +-
 .../hive/ql/exec/spark/HiveReduceFunction.java  |   10 +-
 .../spark/HiveReduceFunctionResultList.java     |   18 +-
 .../ql/exec/spark/HiveSparkClientFactory.java   |    5 +
 .../hadoop/hive/ql/exec/spark/ReduceTran.java   |    8 +-
 .../hive/ql/exec/spark/RepartitionShuffler.java |   42 +
 .../hive/ql/exec/spark/SortByShuffler.java      |    2 +-
 .../hive/ql/exec/spark/SparkPlanGenerator.java  |    6 +-
 .../ql/exec/spark/SparkReduceRecordHandler.java |   56 +-
 .../hive/ql/exec/spark/SparkShuffler.java       |    4 +-
 .../spark/status/RemoteSparkJobMonitor.java     |    7 +-
 .../ql/exec/spark/status/SparkJobMonitor.java   |   12 +
 .../spark/status/impl/RemoteSparkJobStatus.java |    8 +
 .../hive/ql/exec/tez/TezJobExecHelper.java      |   29 +-
 .../hive/ql/exec/tez/TezSessionPoolManager.java |    2 +-
 .../hive/ql/exec/tez/TezSessionState.java       |   27 +
 .../apache/hadoop/hive/ql/exec/tez/TezTask.java |   45 +-
 .../apache/hadoop/hive/ql/exec/tez/Utils.java   |    6 +-
 .../hive/ql/exec/tez/monitoring/DAGSummary.java |   12 +-
 .../ql/exec/tez/monitoring/RenderStrategy.java  |  154 +
 .../ql/exec/tez/monitoring/TezJobMonitor.java   |  115 +-
 .../ql/exec/vector/VectorizationContext.java    |  114 +-
 .../fast/VectorMapJoinFastBytesHashMap.java     |    4 +-
 .../VectorMapJoinFastBytesHashMultiSet.java     |    4 +-
 .../fast/VectorMapJoinFastBytesHashSet.java     |    4 +-
 .../fast/VectorMapJoinFastBytesHashTable.java   |    8 +-
 .../mapjoin/fast/VectorMapJoinFastHashMap.java  |    4 +-
 .../fast/VectorMapJoinFastHashMultiSet.java     |    4 +-
 .../mapjoin/fast/VectorMapJoinFastHashSet.java  |    4 +-
 .../fast/VectorMapJoinFastHashTable.java        |   20 +-
 .../fast/VectorMapJoinFastLongHashMap.java      |    4 +-
 .../fast/VectorMapJoinFastLongHashMultiSet.java |    4 +-
 .../fast/VectorMapJoinFastLongHashSet.java      |    4 +-
 .../fast/VectorMapJoinFastLongHashTable.java    |    8 +-
 .../fast/VectorMapJoinFastMultiKeyHashMap.java  |    4 +-
 .../VectorMapJoinFastMultiKeyHashMultiSet.java  |    4 +-
 .../fast/VectorMapJoinFastMultiKeyHashSet.java  |    4 +-
 .../fast/VectorMapJoinFastStringHashMap.java    |    4 +-
 .../VectorMapJoinFastStringHashMultiSet.java    |    4 +-
 .../fast/VectorMapJoinFastStringHashSet.java    |    4 +-
 .../fast/VectorMapJoinFastTableContainer.java   |   27 +-
 .../apache/hadoop/hive/ql/hooks/ATSHook.java    |   17 +-
 .../hadoop/hive/ql/io/orc/OrcInputFormat.java   |   39 +
 .../hive/ql/io/orc/encoded/CacheChunk.java      |    3 +-
 .../orc/encoded/EncodedTreeReaderFactory.java   |  497 ++-
 .../hadoop/hive/ql/io/orc/encoded/Reader.java   |   69 +-
 .../vector/VectorizedParquetRecordReader.java   |    8 +-
 .../hadoop/hive/ql/lockmgr/DbLockManager.java   |   28 +-
 .../hadoop/hive/ql/lockmgr/DbTxnManager.java    |  144 +-
 .../metadata/HiveMaterializedViewsRegistry.java |    3 +-
 .../hive/ql/metadata/HiveMetaStoreChecker.java  |   45 +-
 .../AnnotateReduceSinkOutputOperator.java       |   73 +
 .../hive/ql/optimizer/ConvertJoinMapJoin.java   |  112 +-
 .../DynamicPartitionPruningOptimization.java    |   36 +-
 .../hadoop/hive/ql/optimizer/Optimizer.java     |    4 +
 .../SparkRemoveDynamicPruningBySize.java        |    4 +-
 .../calcite/CalciteViewSemanticException.java   |   52 +
 .../calcite/rules/HiveRelDecorrelator.java      |  724 +--
 .../calcite/translator/ASTBuilder.java          |   38 +-
 .../translator/SqlFunctionConverter.java        |   33 +-
 .../index/RewriteParseContextGenerator.java     |    4 +-
 .../spark/SetSparkReducerParallelism.java       |   79 +-
 .../optimizer/spark/SparkMapJoinOptimizer.java  |   34 +-
 .../stats/annotation/StatsRulesProcFactory.java |   44 +-
 .../hadoop/hive/ql/parse/CalcitePlanner.java    |  147 +-
 .../ql/parse/ColumnStatsAutoGatherContext.java  |    4 +-
 .../ql/parse/ColumnStatsSemanticAnalyzer.java   |    4 +-
 .../apache/hadoop/hive/ql/parse/HiveParser.g    |   11 +-
 .../hadoop/hive/ql/parse/IdentifiersParser.g    |  162 +-
 .../apache/hadoop/hive/ql/parse/ParseUtils.java |  190 +-
 .../org/apache/hadoop/hive/ql/parse/QB.java     |    4 +
 .../ql/parse/ReplicationSemanticAnalyzer.java   |    9 +-
 .../hadoop/hive/ql/parse/SemanticAnalyzer.java  |   47 +-
 .../ql/parse/UpdateDeleteSemanticAnalyzer.java  |   62 +-
 .../hive/ql/parse/spark/GenSparkUtils.java      |   60 +-
 .../hive/ql/parse/spark/SparkCompiler.java      |   26 +-
 .../hadoop/hive/ql/plan/ReduceSinkDesc.java     |   14 +
 .../hadoop/hive/ql/session/SessionState.java    |   36 +-
 .../apache/hadoop/hive/ql/stats/StatsUtils.java |    6 +-
 .../hadoop/hive/ql/tools/LineageInfo.java       |    5 +-
 .../hive/ql/txn/compactor/CompactorMR.java      |    2 +-
 .../ql/udf/generic/GenericUDAFBloomFilter.java  |   28 +-
 .../generic/GenericUDAFPercentileApprox.java    |    2 +
 .../hive/ql/udf/generic/GenericUDFBasePad.java  |    2 +-
 .../hive/ql/udf/generic/GenericUDFLpad.java     |    9 +-
 .../hive/ql/udf/generic/GenericUDFRpad.java     |    8 +-
 .../ql/udf/generic/GenericUDFSQCountCheck.java  |   11 +-
 .../apache/hadoop/hive/ql/util/DosToUnix.java   |  107 -
 .../hadoop/hive/ql/util/ResourceDownloader.java |   12 +-
 .../apache/hadoop/hive/ql/TestTxnCommands.java  |   22 +
 .../apache/hadoop/hive/ql/WindowsPathUtil.java  |   57 -
 .../hadoop/hive/ql/exec/TestExecDriver.java     |    8 +-
 .../hadoop/hive/ql/exec/tez/TestTezTask.java    |    6 +
 .../exec/vector/TestVectorizationContext.java   |   37 +
 .../fast/TestVectorMapJoinFastBytesHashMap.java |   56 +-
 .../TestVectorMapJoinFastBytesHashMultiSet.java |   14 +-
 .../fast/TestVectorMapJoinFastBytesHashSet.java |   14 +-
 .../fast/TestVectorMapJoinFastLongHashMap.java  |   50 +-
 .../TestVectorMapJoinFastLongHashMultiSet.java  |   14 +-
 .../fast/TestVectorMapJoinFastLongHashSet.java  |   14 +-
 .../fast/TestVectorMapJoinFastRowHashMap.java   |   48 +-
 .../io/parquet/TestVectorizedColumnReader.java  |   21 +
 .../parquet/VectorizedColumnReaderTestBase.java |    2 +-
 .../ql/metadata/TestHiveMetaStoreChecker.java   |    4 -
 .../ql/parse/TestMacroSemanticAnalyzer.java     |    4 +-
 .../hive/ql/parse/TestParseDriverIntervals.java |    2 +-
 .../hadoop/hive/ql/parse/TestQBSubQuery.java    |    2 +-
 .../parse/TestUpdateDeleteSemanticAnalyzer.java |    4 +-
 .../authorization/AuthorizationTestUtil.java    |    2 +-
 .../hadoop/hive/ql/session/TestAddResource.java |    8 +-
 .../hive/ql/txn/compactor/CompactorTest.java    |   11 +-
 .../hive/ql/txn/compactor/TestCleaner.java      |    6 +
 .../hive/ql/txn/compactor/TestInitiator.java    |    5 +
 .../hive/ql/txn/compactor/TestWorker.java       |    6 +
 .../hive/ql/udf/generic/TestGenericUDFLpad.java |    9 +-
 .../hive/ql/udf/generic/TestGenericUDFRpad.java |   11 +-
 .../hadoop/hive/ql/util/TestDosToUnix.java      |   77 -
 .../clientnegative/masking_acid_delete.q        |   10 +
 .../queries/clientnegative/masking_acid_merge.q |   15 +
 .../clientnegative/masking_acid_update.q        |   10 +
 .../clientnegative/subquery_corr_in_agg.q       |    8 +
 ql/src/test/queries/clientnegative/udf_in_2.q   |    2 +
 .../queries/clientpositive/avro_timestamp.q     |    2 -
 .../queries/clientpositive/avro_timestamp_win.q |   28 -
 .../queries/clientpositive/cbo_rp_auto_join1.q  |    2 +-
 ql/src/test/queries/clientpositive/combine2.q   |    3 -
 .../queries/clientpositive/combine2_hadoop20.q  |    3 -
 .../test/queries/clientpositive/combine2_win.q  |   41 -
 .../dynamic_semijoin_reduction_2.q              |   41 +
 ql/src/test/queries/clientpositive/escape1.q    |    3 -
 ql/src/test/queries/clientpositive/escape2.q    |    3 -
 .../clientpositive/explain_formatted_oid.q      |   18 +
 .../test/queries/clientpositive/input_part10.q  |    3 -
 .../queries/clientpositive/input_part10_win.q   |   23 -
 .../test/queries/clientpositive/interval_alt.q  |   19 +-
 .../clientpositive/interval_arithmetic.q        |    3 +
 ql/src/test/queries/clientpositive/join31.q     |    2 +-
 .../queries/clientpositive/join_max_hashtable.q |   37 +
 .../clientpositive/lateral_view_explode2.q      |    4 +-
 .../test/queries/clientpositive/llap_reader.q   |   40 +
 .../queries/clientpositive/load_dyn_part14.q    |    3 -
 .../clientpositive/load_dyn_part14_win.q        |   38 -
 ql/src/test/queries/clientpositive/masking_10.q |   25 +
 .../clientpositive/msck_repair_batchsize.q      |   10 +
 .../test/queries/clientpositive/multiMapJoin2.q |    2 +-
 ql/src/test/queries/clientpositive/parenthese.q |   80 +
 .../clientpositive/partition_timestamp.q        |    2 -
 .../clientpositive/partition_timestamp2.q       |    2 -
 .../clientpositive/partition_timestamp2_win.q   |   58 -
 .../clientpositive/partition_timestamp_win.q    |   59 -
 .../clientpositive/position_alias_test_1.q      |   18 +
 .../test/queries/clientpositive/scriptfile1.q   |    2 -
 .../queries/clientpositive/scriptfile1_win.q    |   16 -
 .../spark_use_file_size_for_mapjoin.q           |   30 +
 .../queries/clientpositive/spark_use_op_stats.q |   41 +
 ql/src/test/queries/clientpositive/sqlmerge.q   |    6 +
 .../queries/clientpositive/subquery_multi.q     |    8 +-
 .../queries/clientpositive/transform_acid.q     |    2 -
 .../clientpositive/udaf_percentile_approx_23.q  |    3 +
 ql/src/test/queries/clientpositive/udf_in.q     |    4 +-
 ql/src/test/queries/clientpositive/udf_notop.q  |    2 +-
 .../queries/clientpositive/union_pos_alias.q    |   30 +
 .../vector_partitioned_date_time.q              |    3 -
 .../vector_partitioned_date_time_win.q          |  129 -
 .../vectorized_dynamic_semijoin_reduction.q     |    6 +
 .../vectorized_dynamic_semijoin_reduction2.q    |   65 +
 ql/src/test/queries/clientpositive/view_cbo.q   |   72 +
 .../clientnegative/char_pad_convert_fail2.q.out |    2 +-
 .../create_or_replace_view4.q.out               |    2 +-
 .../clientnegative/create_view_failure3.q.out   |    2 +-
 .../clientnegative/create_view_failure6.q.out   |    2 +-
 .../clientnegative/create_view_failure7.q.out   |    2 +-
 .../clientnegative/create_view_failure8.q.out   |    2 +-
 .../clientnegative/create_view_failure9.q.out   |    2 +-
 .../invalid_select_expression.q.out             |    2 +-
 .../clientnegative/masking_acid_delete.q.out    |   13 +
 .../clientnegative/masking_acid_merge.q.out     |   21 +
 .../clientnegative/masking_acid_update.q.out    |   13 +
 .../ptf_negative_DistributeByOrderBy.q.out      |    2 +-
 .../ptf_negative_PartitionBySortBy.q.out        |    2 +-
 .../clientnegative/ptf_window_boundaries.q.out  |    2 +-
 .../clientnegative/ptf_window_boundaries2.q.out |    2 +-
 .../selectDistinctStarNeg_1.q.out               |    2 +-
 .../clientnegative/select_charliteral.q.out     |    3 +-
 .../clientnegative/subquery_corr_in_agg.q.out   |   39 +
 .../clientnegative/subquery_missing_from.q.out  |    2 +-
 .../subquery_subquery_chain.q.out               |    2 +-
 .../clientnegative/subquery_with_or_cond.q.out  |    2 +-
 ql/src/test/results/clientnegative/udf_in.q.out |    2 +-
 .../test/results/clientnegative/udf_in_2.q.out  |    1 +
 .../clientpositive/acid_globallimit.q.out       |   52 -
 .../avro_timestamp_win.q.java1.7.out            |  134 -
 .../avro_timestamp_win.q.java1.8.out            |  134 -
 .../results/clientpositive/combine2_win.q.out   |  767 ----
 .../clientpositive/constant_prop_1.q.out        |    4 +-
 .../clientpositive/constprog_partitioner.q.out  |  117 +-
 .../results/clientpositive/create_view.q.out    |   10 +-
 .../clientpositive/create_view_translate.q.out  |    4 +-
 ql/src/test/results/clientpositive/cte_2.q.out  |    4 +-
 ql/src/test/results/clientpositive/cte_4.q.out  |    4 +-
 ...on_join_with_different_encryption_keys.q.out |   14 +-
 .../results/clientpositive/explain_ddl.q.out    |    4 +-
 .../clientpositive/explain_dependency.q.out     |    2 +-
 .../clientpositive/explain_formatted_oid.q.out  |   38 +
 .../clientpositive/explain_logical.q.out        |    2 +-
 ql/src/test/results/clientpositive/input4.q.out |    2 +-
 .../clientpositive/input_part10_win.q.out       |  131 -
 .../results/clientpositive/interval_alt.q.out   |   79 +-
 .../clientpositive/interval_arithmetic.q.out    |   35 +
 ql/src/test/results/clientpositive/join0.q.out  |    2 +-
 .../results/clientpositive/llap/cbo_views.q.out |    2 +-
 .../results/clientpositive/llap/cte_2.q.out     |    4 +-
 .../results/clientpositive/llap/cte_4.q.out     |    4 +-
 .../llap/dynamic_semijoin_reduction_2.q.out     |  301 ++
 .../clientpositive/llap/explainuser_1.q.out     |  688 +--
 .../llap/join_max_hashtable.q.out               |  490 ++
 .../results/clientpositive/llap/lineage3.q.out  |    2 +-
 .../clientpositive/llap/llap_reader.q.out       |  167 +
 .../results/clientpositive/llap/mergejoin.q.out |    8 +-
 .../llap/metadata_only_queries.q.out            |    4 +-
 ...chema_evol_text_vec_part_all_primitive.q.out |   34 +-
 .../llap/selectDistinctStar.q.out               |    8 +-
 .../results/clientpositive/llap/sqlmerge.q.out  |  101 +
 .../clientpositive/llap/subquery_exists.q.out   |  169 +-
 .../clientpositive/llap/subquery_in.q.out       | 2838 +++---------
 .../clientpositive/llap/subquery_multi.q.out    | 2788 +++---------
 .../clientpositive/llap/subquery_notin.q.out    | 4222 +++++-------------
 .../clientpositive/llap/subquery_scalar.q.out   | 2679 +++--------
 .../clientpositive/llap/subquery_views.q.out    |  558 +--
 .../clientpositive/llap/union_top_level.q.out   |    4 +-
 .../clientpositive/llap/vector_between_in.q.out |   22 +-
 .../llap/vector_decimal_mapjoin.q.out           |    4 +-
 .../llap/vector_mapjoin_reduce.q.out            |  196 +-
 .../llap/vector_reduce_groupby_decimal.q.out    |    2 +-
 .../vectorized_dynamic_semijoin_reduction.q.out |  141 +
 ...vectorized_dynamic_semijoin_reduction2.q.out | 1192 +++++
 .../results/clientpositive/llap_reader.q.out    |   86 +
 .../clientpositive/load_dyn_part14_win.q.out    |  298 --
 .../results/clientpositive/masking_10.q.out     |  244 +
 .../clientpositive/metadata_only_queries.q.out  |    4 +-
 ql/src/test/results/clientpositive/mm_all.q.out |   10 +-
 .../clientpositive/msck_repair_batchsize.q.out  |   25 +
 .../results/clientpositive/parallel_join0.q.out |    2 +-
 .../results/clientpositive/parenthese.q.out     |  168 +
 .../partition_timestamp2_win.q.out              |  399 --
 .../partition_timestamp_win.q.out               |  316 --
 .../results/clientpositive/perf/query1.q.out    |  190 +-
 .../results/clientpositive/perf/query14.q.out   |    8 +-
 .../results/clientpositive/perf/query16.q.out   |  222 +-
 .../results/clientpositive/perf/query23.q.out   |    4 +-
 .../results/clientpositive/perf/query30.q.out   |  375 +-
 .../results/clientpositive/perf/query6.q.out    |  403 +-
 .../results/clientpositive/perf/query69.q.out   |  499 +--
 .../results/clientpositive/perf/query81.q.out   |  375 +-
 .../test/results/clientpositive/plan_json.q.out |    2 +-
 .../clientpositive/position_alias_test_1.q.out  |  148 +
 .../clientpositive/scriptfile1_win.q.out        |   55 -
 .../test/results/clientpositive/semijoin5.q.out |  138 +-
 .../spark/constprog_partitioner.q.out           |   87 +-
 .../spark/metadata_only_queries.q.out           |    4 +-
 .../spark/spark_use_file_size_for_mapjoin.q.out |  257 ++
 .../spark/spark_use_op_stats.q.out              |  331 ++
 .../clientpositive/spark/subquery_exists.q.out  |  167 +-
 .../clientpositive/spark/subquery_in.q.out      | 2597 +++--------
 .../clientpositive/spark/union_remove_25.q.out  |    2 +-
 .../clientpositive/spark/union_top_level.q.out  |    4 +-
 .../spark/vector_between_in.q.out               |   22 +-
 .../spark/vector_decimal_mapjoin.q.out          |    4 +-
 .../spark/vector_mapjoin_reduce.q.out           |  216 +-
 .../clientpositive/subquery_exists.q.out        |  221 +-
 .../clientpositive/subquery_exists_having.q.out |  232 +-
 .../clientpositive/subquery_in_having.q.out     |  641 +--
 .../clientpositive/subquery_notexists.q.out     |  209 +-
 .../subquery_notexists_having.q.out             |  223 +-
 .../clientpositive/subquery_notin_having.q.out  |  644 +--
 .../subquery_unqualcolumnrefs.q.out             |  687 +--
 .../clientpositive/tez/explainanalyze_3.q.out   |    4 +-
 .../clientpositive/tez/explainuser_3.q.out      |    4 +-
 .../udaf_percentile_approx_23.q.out             |    9 +
 .../results/clientpositive/udf_between.q.out    |    2 +-
 ql/src/test/results/clientpositive/udf_in.q.out |    4 +-
 .../test/results/clientpositive/udf_lpad.q.out  |    6 +-
 .../test/results/clientpositive/udf_notop.q.out |    4 +-
 .../test/results/clientpositive/udf_rpad.q.out  |    6 +-
 .../clientpositive/union_pos_alias.q.out        |  308 ++
 .../clientpositive/vector_decimal_mapjoin.q.out |    2 +-
 .../clientpositive/vector_mapjoin_reduce.q.out  |  322 +-
 .../clientpositive/vector_outer_join3.q.out     |    6 +-
 .../clientpositive/vector_outer_join4.q.out     |    6 +-
 .../clientpositive/vector_outer_join6.q.out     |    4 +-
 .../vector_partitioned_date_time_win.q.out      | 2036 ---------
 .../vector_reduce_groupby_decimal.q.out         |    2 +-
 .../results/clientpositive/view_alias.q.out     |   12 +-
 .../test/results/clientpositive/view_cbo.q.out  |  823 ++++
 .../lazy/fast/LazySimpleDeserializeRead.java    |    2 +-
 .../gen/thrift/gen-cpp/TCLIService_types.cpp    |    8 +-
 .../src/gen/thrift/gen-cpp/TCLIService_types.h  |   11 +-
 .../service/rpc/thrift/TOpenSessionReq.java     |    4 +-
 .../service/rpc/thrift/TOpenSessionResp.java    |    4 +-
 .../service/rpc/thrift/TProtocolVersion.java    |    5 +-
 service-rpc/src/gen/thrift/gen-php/Types.php    |    6 +-
 .../src/gen/thrift/gen-py/TCLIService/ttypes.py |    7 +-
 .../gen/thrift/gen-rb/t_c_l_i_service_types.rb  |    9 +-
 .../org/apache/hive/service/cli/CLIService.java |   45 +-
 .../hive/service/cli/operation/Operation.java   |   16 +-
 .../service/cli/operation/SQLOperation.java     |    4 +-
 .../service/cli/thrift/ThriftCLIService.java    |   24 +-
 .../cli/thrift/ThriftHttpCLIService.java        |    2 +-
 .../apache/hadoop/fs/ProxyLocalFileSystem.java  |    8 -
 .../org/apache/hive/spark/client/rpc/Rpc.java   |    2 +-
 .../common/io/encoded/EncodedColumnBatch.java   |    1 +
 .../hive/ql/exec/vector/BytesColumnVector.java  |   11 +
 .../hive/ql/exec/vector/ColumnVector.java       |   12 +
 .../ql/exec/vector/DecimalColumnVector.java     |   11 +-
 .../hive/ql/exec/vector/DoubleColumnVector.java |    8 +-
 .../vector/IntervalDayTimeColumnVector.java     |    8 +
 .../hive/ql/exec/vector/LongColumnVector.java   |    8 +-
 .../ql/exec/vector/MultiValuedColumnVector.java |    4 +
 .../hive/ql/exec/vector/StructColumnVector.java |    5 +
 .../ql/exec/vector/TimestampColumnVector.java   |    8 +
 .../hive/ql/exec/vector/UnionColumnVector.java  |    5 +
 .../hive/serde2/io/HiveDecimalWritable.java     |    5 +-
 testutils/hadoop.cmd                            |  252 --
 vector-code-gen/pom.xml                         |   69 +
 .../apache/hadoop/hive/tools/GenVectorCode.java | 3327 ++++++++++++++
 .../hadoop/hive/tools/GenVectorTestCode.java    |  261 ++
 496 files changed, 23684 insertions(+), 31973 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/2014ece9/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/2014ece9/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/2014ece9/metastore/if/hive_metastore.thrift
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/2014ece9/metastore/scripts/upgrade/derby/hive-schema-2.2.0.derby.sql
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/2014ece9/metastore/scripts/upgrade/derby/upgrade-2.1.0-to-2.2.0.derby.sql
----------------------------------------------------------------------
diff --cc metastore/scripts/upgrade/derby/upgrade-2.1.0-to-2.2.0.derby.sql
index f59b37f,e5a144c..cbf5be1
--- a/metastore/scripts/upgrade/derby/upgrade-2.1.0-to-2.2.0.derby.sql
+++ b/metastore/scripts/upgrade/derby/upgrade-2.1.0-to-2.2.0.derby.sql
@@@ -1,6 -1,5 +1,7 @@@
  -- Upgrade MetaStore schema from 2.1.0 to 2.2.0
  RUN '037-HIVE-14496.derby.sql';
+ RUN '038-HIVE-10562.derby.sql';
  
 +RUN '037-HIVE-14637.derby.sql';
 +
  UPDATE "APP".VERSION SET SCHEMA_VERSION='2.2.0', VERSION_COMMENT='Hive release version 2.2.0' where VER_ID=1;

http://git-wip-us.apache.org/repos/asf/hive/blob/2014ece9/metastore/scripts/upgrade/mssql/hive-schema-2.2.0.mssql.sql
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/2014ece9/metastore/scripts/upgrade/mssql/upgrade-2.1.0-to-2.2.0.mssql.sql
----------------------------------------------------------------------
diff --cc metastore/scripts/upgrade/mssql/upgrade-2.1.0-to-2.2.0.mssql.sql
index 2dcdedc,a4b8fda..99024c2
--- a/metastore/scripts/upgrade/mssql/upgrade-2.1.0-to-2.2.0.mssql.sql
+++ b/metastore/scripts/upgrade/mssql/upgrade-2.1.0-to-2.2.0.mssql.sql
@@@ -1,7 -1,7 +1,8 @@@
  SELECT 'Upgrading MetaStore schema from 2.1.0 to 2.2.0' AS MESSAGE;
  
  :r 022-HIVE-14496.mssql.sql
 +:r 023-HIVE-14637.mssql.sql
+ :r 023-HIVE-10562.mssql.sql
  
  UPDATE VERSION SET SCHEMA_VERSION='2.2.0', VERSION_COMMENT='Hive release version 2.2.0' where VER_ID=1;
  SELECT 'Finished upgrading MetaStore schema from 2.1.0 to 2.2.0' AS MESSAGE;

http://git-wip-us.apache.org/repos/asf/hive/blob/2014ece9/metastore/scripts/upgrade/mysql/hive-schema-2.2.0.mysql.sql
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/2014ece9/metastore/scripts/upgrade/mysql/upgrade-2.1.0-to-2.2.0.mysql.sql
----------------------------------------------------------------------
diff --cc metastore/scripts/upgrade/mysql/upgrade-2.1.0-to-2.2.0.mysql.sql
index afe17f9,509c532..68300d3
--- a/metastore/scripts/upgrade/mysql/upgrade-2.1.0-to-2.2.0.mysql.sql
+++ b/metastore/scripts/upgrade/mysql/upgrade-2.1.0-to-2.2.0.mysql.sql
@@@ -1,7 -1,7 +1,8 @@@
  SELECT 'Upgrading MetaStore schema from 2.1.0 to 2.2.0' AS ' ';
  
  SOURCE 037-HIVE-14496.mysql.sql;
 +SOURCE 038-HIVE-14637.mysql.sql;
+ SOURCE 038-HIVE-10562.mysql.sql;
  
  UPDATE VERSION SET SCHEMA_VERSION='2.2.0', VERSION_COMMENT='Hive release version 2.2.0' where VER_ID=1;
  SELECT 'Finished upgrading MetaStore schema from 2.1.0 to 2.2.0' AS ' ';

http://git-wip-us.apache.org/repos/asf/hive/blob/2014ece9/metastore/scripts/upgrade/oracle/hive-schema-2.2.0.oracle.sql
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/2014ece9/metastore/scripts/upgrade/oracle/upgrade-2.1.0-to-2.2.0.oracle.sql
----------------------------------------------------------------------
diff --cc metastore/scripts/upgrade/oracle/upgrade-2.1.0-to-2.2.0.oracle.sql
index dd1b97c,f31fda9..058c0d5
--- a/metastore/scripts/upgrade/oracle/upgrade-2.1.0-to-2.2.0.oracle.sql
+++ b/metastore/scripts/upgrade/oracle/upgrade-2.1.0-to-2.2.0.oracle.sql
@@@ -1,7 -1,7 +1,8 @@@
  SELECT 'Upgrading MetaStore schema from 2.1.0 to 2.2.0' AS Status from dual;
  
  @037-HIVE-14496.oracle.sql;
 +@038-HIVE-14637.oracle.sql;
+ @038-HIVE-10562.oracle.sql;
  
  UPDATE VERSION SET SCHEMA_VERSION='2.2.0', VERSION_COMMENT='Hive release version 2.2.0' where VER_ID=1;
  SELECT 'Finished upgrading MetaStore schema from 2.1.0 to 2.2.0' AS Status from dual;

http://git-wip-us.apache.org/repos/asf/hive/blob/2014ece9/metastore/scripts/upgrade/postgres/hive-schema-2.2.0.postgres.sql
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/2014ece9/metastore/scripts/upgrade/postgres/upgrade-2.1.0-to-2.2.0.postgres.sql
----------------------------------------------------------------------
diff --cc metastore/scripts/upgrade/postgres/upgrade-2.1.0-to-2.2.0.postgres.sql
index a316383,0f64a90..ae4adf7
--- a/metastore/scripts/upgrade/postgres/upgrade-2.1.0-to-2.2.0.postgres.sql
+++ b/metastore/scripts/upgrade/postgres/upgrade-2.1.0-to-2.2.0.postgres.sql
@@@ -1,7 -1,7 +1,8 @@@
  SELECT 'Upgrading MetaStore schema from 2.1.0 to 2.2.0';
  
  \i 036-HIVE-14496.postgres.sql;
 +\i 037-HIVE-14637.postgres.sql;
+ \i 037-HIVE-10562.postgres.sql;
  
  UPDATE "VERSION" SET "SCHEMA_VERSION"='2.2.0', "VERSION_COMMENT"='Hive release version 2.2.0' where "VER_ID"=1;
  SELECT 'Finished upgrading MetaStore schema from 2.1.0 to 2.2.0';

http://git-wip-us.apache.org/repos/asf/hive/blob/2014ece9/metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/2014ece9/metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/2014ece9/metastore/src/gen/thrift/gen-php/metastore/Types.php
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/2014ece9/metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/2014ece9/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/2014ece9/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/2014ece9/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/2014ece9/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/2014ece9/metastore/src/model/package.jdo
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/2014ece9/ql/src/java/org/apache/hadoop/hive/ql/Context.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/2014ece9/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/2014ece9/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index d7db991,3484493..966b2b5
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@@ -200,9 -141,9 +200,10 @@@ import org.apache.hadoop.mapred.RecordR
  import org.apache.hadoop.mapred.Reporter;
  import org.apache.hadoop.mapred.SequenceFileInputFormat;
  import org.apache.hadoop.mapred.SequenceFileOutputFormat;
 +import org.apache.hadoop.mapred.TextInputFormat;
  import org.apache.hadoop.util.Progressable;
  import org.apache.hadoop.util.Shell;
+ import org.apache.hive.common.util.ACLConfigurationParser;
  import org.apache.hive.common.util.ReflectionUtil;
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
@@@ -3966,331 -3796,25 +3951,352 @@@ public final class Utilities 
      return String.format("%.2f%sB", bytes / Math.pow(unit, exp), suffix);
    }
  
 +  private static final String MANIFEST_EXTENSION = ".manifest";
 +
 +  private static void tryDelete(FileSystem fs, Path path) {
 +    try {
 +      fs.delete(path, true);
 +    } catch (IOException ex) {
 +      LOG.error("Failed to delete " + path, ex);
 +    }
 +  }
 +
 +  public static Path[] getMmDirectoryCandidates(FileSystem fs, Path path, int dpLevels,
 +      int lbLevels, PathFilter filter, long mmWriteId, Configuration conf) throws IOException {
 +    int skipLevels = dpLevels + lbLevels;
 +    if (filter == null) {
 +      filter = new ValidWriteIds.IdPathFilter(mmWriteId, true);
 +    }
 +    if (skipLevels == 0) {
 +      return statusToPath(fs.listStatus(path, filter));
 +    }
 +    if (HiveConf.getBoolVar(conf, ConfVars.HIVE_MM_AVOID_GLOBSTATUS_ON_S3) && isS3(fs)) {
 +      return getMmDirectoryCandidatesRecursive(fs, path, skipLevels, filter);
 +    }
 +    return getMmDirectoryCandidatesGlobStatus(fs, path, skipLevels, filter, mmWriteId);
 +  }
 +
 +  private static boolean isS3(FileSystem fs) {
 +    try {
 +      return fs.getScheme().equalsIgnoreCase("s3a");
 +    } catch (UnsupportedOperationException ex) {
 +      // Some FS-es do not implement getScheme, e.g. ProxyLocalFileSystem.
 +      return false;
 +    }
 +  }
 +
 +  private static Path[] statusToPath(FileStatus[] statuses) {
 +    if (statuses == null) return null;
 +    Path[] paths = new Path[statuses.length];
 +    for (int i = 0; i < statuses.length; ++i) {
 +      paths[i] = statuses[i].getPath();
 +    }
 +    return paths;
 +  }
 +
 +  private static Path[] getMmDirectoryCandidatesRecursive(FileSystem fs,
 +      Path path, int skipLevels, PathFilter filter) throws IOException {
 +    String lastRelDir = null;
 +    HashSet<Path> results = new HashSet<Path>();
 +    String relRoot = Path.getPathWithoutSchemeAndAuthority(path).toString();
 +    if (!relRoot.endsWith(Path.SEPARATOR)) {
 +      relRoot += Path.SEPARATOR;
 +    }
 +    RemoteIterator<LocatedFileStatus> allFiles = fs.listFiles(path, true);
 +    while (allFiles.hasNext()) {
 +      LocatedFileStatus lfs = allFiles.next();
 +      Path dirPath = Path.getPathWithoutSchemeAndAuthority(lfs.getPath());
 +      String dir = dirPath.toString();
 +      if (!dir.startsWith(relRoot)) {
 +        throw new IOException("Path " + lfs.getPath() + " is not under " + relRoot
 +            + " (when shortened to " + dir + ")");
 +      }
 +      String subDir = dir.substring(relRoot.length());
 +      Utilities.LOG14535.info("Looking at " + subDir + " from " + lfs.getPath());
 +      // If sorted, we'll skip a bunch of files.
 +      if (lastRelDir != null && subDir.startsWith(lastRelDir)) continue;
 +      int startIx = skipLevels > 0 ? -1 : 0;
 +      for (int i = 0; i < skipLevels; ++i) {
 +        startIx = subDir.indexOf(Path.SEPARATOR_CHAR, startIx + 1);
 +        if (startIx == -1) {
 +          Utilities.LOG14535.info("Expected level of nesting (" + skipLevels + ") is not "
 +              + " present in " + subDir + " (from " + lfs.getPath() + ")");
 +          break;
 +        }
 +      }
 +      if (startIx == -1) continue;
 +      int endIx = subDir.indexOf(Path.SEPARATOR_CHAR, startIx + 1);
 +      if (endIx == -1) {
 +        Utilities.LOG14535.info("Expected level of nesting (" + (skipLevels + 1) + ") is not "
 +            + " present in " + subDir + " (from " + lfs.getPath() + ")");
 +        continue;
 +      }
 +      lastRelDir = subDir = subDir.substring(0, endIx);
 +      Path candidate = new Path(relRoot, subDir);
 +      Utilities.LOG14535.info("Considering MM directory candidate " + candidate);
 +      if (!filter.accept(candidate)) continue;
 +      results.add(fs.makeQualified(candidate));
 +    }
 +    return results.toArray(new Path[results.size()]);
 +  }
 +
 +  private static Path[] getMmDirectoryCandidatesGlobStatus(FileSystem fs,
 +      Path path, int skipLevels, PathFilter filter, long mmWriteId) throws IOException {
 +    StringBuilder sb = new StringBuilder(path.toUri().getPath());
 +    for (int i = 0; i < skipLevels; i++) {
 +      sb.append(Path.SEPARATOR).append("*");
 +    }
 +    sb.append(Path.SEPARATOR).append(ValidWriteIds.getMmFilePrefix(mmWriteId));
 +    Path pathPattern = new Path(path, sb.toString());
 +    Utilities.LOG14535.info("Looking for files via: " + pathPattern);
 +    return statusToPath(fs.globStatus(pathPattern, filter));
 +  }
 +
 +  private static void tryDeleteAllMmFiles(FileSystem fs, Path specPath, Path manifestDir,
 +      int dpLevels, int lbLevels, String unionSuffix, ValidWriteIds.IdPathFilter filter,
 +      long mmWriteId, Configuration conf) throws IOException {
 +    Path[] files = getMmDirectoryCandidates(
 +        fs, specPath, dpLevels, lbLevels, filter, mmWriteId, conf);
 +    if (files != null) {
 +      for (Path path : files) {
 +        Utilities.LOG14535.info("Deleting " + path + " on failure");
 +        tryDelete(fs, path);
 +      }
 +    }
 +    Utilities.LOG14535.info("Deleting " + manifestDir + " on failure");
 +    fs.delete(manifestDir, true);
 +  }
 +
 +
 +  public static void writeMmCommitManifest(List<Path> commitPaths, Path specPath, FileSystem fs,
 +      String taskId, Long mmWriteId, String unionSuffix) throws HiveException {
 +    if (commitPaths.isEmpty()) return;
 +    // We assume one FSOP per task (per specPath), so we create it in specPath.
 +    Path manifestPath = getManifestDir(specPath, mmWriteId, unionSuffix);
 +    manifestPath = new Path(manifestPath, taskId + MANIFEST_EXTENSION);
 +    Utilities.LOG14535.info("Writing manifest to " + manifestPath + " with " + commitPaths);
 +    try {
 +      // Don't overwrite the manifest... should fail if we have collisions.
 +      try (FSDataOutputStream out = fs.create(manifestPath, false)) {
 +        if (out == null) {
 +          throw new HiveException("Failed to create manifest at " + manifestPath);
 +        }
 +        out.writeInt(commitPaths.size());
 +        for (Path path : commitPaths) {
 +          out.writeUTF(path.toString());
 +        }
 +      }
 +    } catch (IOException e) {
 +      throw new HiveException(e);
 +    }
 +  }
 +
 +  private static Path getManifestDir(Path specPath, long mmWriteId, String unionSuffix) {
 +    Path manifestPath = new Path(specPath, "_tmp." + ValidWriteIds.getMmFilePrefix(mmWriteId));
 +    return (unionSuffix == null) ? manifestPath : new Path(manifestPath, unionSuffix);
 +  }
 +
 +  public static final class MissingBucketsContext {
 +    public final TableDesc tableInfo;
 +    public final int numBuckets;
 +    public final boolean isCompressed;
 +    public MissingBucketsContext(TableDesc tableInfo, int numBuckets, boolean isCompressed) {
 +      this.tableInfo = tableInfo;
 +      this.numBuckets = numBuckets;
 +      this.isCompressed = isCompressed;
 +    }
 +  }
 +
 +  public static void handleMmTableFinalPath(Path specPath, String unionSuffix, Configuration hconf,
 +      boolean success, int dpLevels, int lbLevels, MissingBucketsContext mbc, long mmWriteId,
 +      Reporter reporter, boolean isMmCtas) throws IOException, HiveException {
 +    FileSystem fs = specPath.getFileSystem(hconf);
 +    Path manifestDir = getManifestDir(specPath, mmWriteId, unionSuffix);
 +    if (!success) {
 +      ValidWriteIds.IdPathFilter filter = new ValidWriteIds.IdPathFilter(mmWriteId, true);
 +      tryDeleteAllMmFiles(fs, specPath, manifestDir, dpLevels, lbLevels,
 +          unionSuffix, filter, mmWriteId, hconf);
 +      return;
 +    }
 +
 +    Utilities.LOG14535.info("Looking for manifests in: " + manifestDir + " (" + mmWriteId + ")");
 +    // TODO# may be wrong if there are no splits (empty insert/CTAS)
 +    List<Path> manifests = new ArrayList<>();
 +    if (fs.exists(manifestDir)) {
 +      FileStatus[] manifestFiles = fs.listStatus(manifestDir);
 +      if (manifestFiles != null) {
 +        for (FileStatus status : manifestFiles) {
 +          Path path = status.getPath();
 +          if (path.getName().endsWith(MANIFEST_EXTENSION)) {
 +            Utilities.LOG14535.info("Reading manifest " + path);
 +            manifests.add(path);
 +          }
 +        }
 +      }
 +    } else {
 +      Utilities.LOG14535.info("No manifests found - query produced no output");
 +      manifestDir = null;
 +    }
 +
 +    Utilities.LOG14535.info("Looking for files in: " + specPath);
 +    ValidWriteIds.IdPathFilter filter = new ValidWriteIds.IdPathFilter(mmWriteId, true);
 +    if (isMmCtas && !fs.exists(specPath)) {
 +      // TODO: do we also need to do this when creating an empty partition from select?
 +      Utilities.LOG14535.info("Creating table directory for CTAS with no output at " + specPath);
 +      FileUtils.mkdir(fs, specPath, hconf);
 +    }
 +    Path[] files = getMmDirectoryCandidates(
 +        fs, specPath, dpLevels, lbLevels, filter, mmWriteId, hconf);
 +    ArrayList<Path> mmDirectories = new ArrayList<>();
 +    if (files != null) {
 +      for (Path path : files) {
 +        Utilities.LOG14535.info("Looking at path: " + path);
 +        mmDirectories.add(path);
 +      }
 +    }
 +
 +    HashSet<String> committed = new HashSet<>();
 +    for (Path mfp : manifests) {
 +      try (FSDataInputStream mdis = fs.open(mfp)) {
 +        int fileCount = mdis.readInt();
 +        for (int i = 0; i < fileCount; ++i) {
 +          String nextFile = mdis.readUTF();
 +          if (!committed.add(nextFile)) {
 +            throw new HiveException(nextFile + " was specified in multiple manifests");
 +          }
 +        }
 +      }
 +    }
 +
 +    if (manifestDir != null) {
 +      Utilities.LOG14535.info("Deleting manifest directory " + manifestDir);
 +      tryDelete(fs, manifestDir);
 +      if (unionSuffix != null) {
 +        // Also delete the parent directory if we are the last union FSOP to execute.
 +        manifestDir = manifestDir.getParent();
 +        FileStatus[] remainingFiles = fs.listStatus(manifestDir);
 +        if (remainingFiles == null || remainingFiles.length == 0) {
 +          Utilities.LOG14535.info("Deleting manifest directory " + manifestDir);
 +          tryDelete(fs, manifestDir);
 +        }
 +      }
 +    }
 +
 +    for (Path path : mmDirectories) {
 +      cleanMmDirectory(path, fs, unionSuffix, committed);
 +    }
 +
 +    if (!committed.isEmpty()) {
 +      throw new HiveException("The following files were committed but not found: " + committed);
 +    }
 +
 +    if (mmDirectories.isEmpty()) return;
 +
 +    // TODO: see HIVE-14886 - removeTempOrDuplicateFiles is broken for list bucketing,
 +    //       so maintain parity here by not calling it at all.
 +    if (lbLevels != 0) return;
 +    // Create fake file statuses to avoid querying the file system. removeTempOrDuplicateFiles
 +    // doesn't need tocheck anything except path and directory status for MM directories.
 +    FileStatus[] finalResults = new FileStatus[mmDirectories.size()];
 +    for (int i = 0; i < mmDirectories.size(); ++i) {
 +      finalResults[i] = new PathOnlyFileStatus(mmDirectories.get(i));
 +    }
 +    List<Path> emptyBuckets = Utilities.removeTempOrDuplicateFiles(
 +        fs, finalResults, dpLevels, mbc == null ? 0 : mbc.numBuckets, hconf, mmWriteId);
 +    // create empty buckets if necessary
 +    if (emptyBuckets.size() > 0) {
 +      assert mbc != null;
 +      Utilities.createEmptyBuckets(hconf, emptyBuckets, mbc.isCompressed, mbc.tableInfo, reporter);
 +    }
 +  }
 +
 +  private static final class PathOnlyFileStatus extends FileStatus {
 +    public PathOnlyFileStatus(Path path) {
 +      super(0, true, 0, 0, 0, path);
 +    }
 +  }
 +
 +  private static void cleanMmDirectory(Path dir, FileSystem fs,
 +      String unionSuffix, HashSet<String> committed) throws IOException, HiveException {
 +    for (FileStatus child : fs.listStatus(dir)) {
 +      Path childPath = child.getPath();
 +      if (unionSuffix == null) {
 +        if (committed.remove(childPath.toString())) continue; // A good file.
 +        deleteUncommitedFile(childPath, fs);
 +      } else if (!child.isDirectory()) {
 +        if (committed.contains(childPath.toString())) {
 +          throw new HiveException("Union FSOP has commited "
 +              + childPath + " outside of union directory" + unionSuffix);
 +        }
 +        deleteUncommitedFile(childPath, fs);
 +      } else if (childPath.getName().equals(unionSuffix)) {
 +        // Found the right union directory; treat it as "our" MM directory.
 +        cleanMmDirectory(childPath, fs, null, committed);
 +      } else {
 +        Utilities.LOG14535.info("FSOP for " + unionSuffix
 +            + " is ignoring the other side of the union " + childPath.getName());
 +      }
 +    }
 +  }
 +
 +  private static void deleteUncommitedFile(Path childPath, FileSystem fs)
 +      throws IOException, HiveException {
 +    Utilities.LOG14535.info("Deleting " + childPath + " that was not committed");
 +    // We should actually succeed here - if we fail, don't commit the query.
 +    if (!fs.delete(childPath, true)) {
 +      throw new HiveException("Failed to delete an uncommitted path " + childPath);
 +    }
 +  }
 +
 +  /**
 +   * @return the complete list of valid MM directories under a table/partition path; null
 +   * if the entire directory is valid (has no uncommitted/temporary files).
 +   */
 +  public static List<Path> getValidMmDirectoriesFromTableOrPart(Path path, Configuration conf,
 +      ValidWriteIds ids, int lbLevels) throws IOException {
 +    Utilities.LOG14535.info("Looking for valid MM paths under " + path);
 +    // NULL means this directory is entirely valid.
 +    List<Path> result = null;
 +    FileSystem fs = path.getFileSystem(conf);
 +    FileStatus[] children = (lbLevels == 0) ? fs.listStatus(path)
 +        : fs.globStatus(new Path(path, StringUtils.repeat("*" + Path.SEPARATOR, lbLevels) + "*"));
 +    for (int i = 0; i < children.length; ++i) {
 +      FileStatus file = children[i];
 +      Path childPath = file.getPath();
 +      Long writeId = ValidWriteIds.extractWriteId(childPath);
 +      if (!file.isDirectory() || writeId == null || !ids.isValid(writeId)) {
 +        Utilities.LOG14535.info("Skipping path " + childPath);
 +        if (result == null) {
 +          result = new ArrayList<>(children.length - 1);
 +          for (int j = 0; j < i; ++j) {
 +            result.add(children[j].getPath());
 +          }
 +        }
 +      } else if (result != null) {
 +        result.add(childPath);
 +      }
 +    }
 +    return result;
 +  }
+ 
+   public static String getAclStringWithHiveModification(Configuration tezConf,
+                                                         String propertyName,
+                                                         boolean addHs2User,
+                                                         String user,
+                                                         String hs2User) throws
+       IOException {
+ 
+     // Start with initial ACLs
+     ACLConfigurationParser aclConf =
+         new ACLConfigurationParser(tezConf, propertyName);
+ 
+     // Always give access to the user
+     aclConf.addAllowedUser(user);
+ 
+     // Give access to the process user if the config is set.
+     if (addHs2User && hs2User != null) {
+       aclConf.addAllowedUser(hs2User);
+     }
+     return aclConf.toAclString();
+   }
  }

http://git-wip-us.apache.org/repos/asf/hive/blob/2014ece9/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/2014ece9/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/2014ece9/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/2014ece9/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java
----------------------------------------------------------------------


Mime
View raw message