hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ser...@apache.org
Subject [30/31] hive git commit: Merge branch 'master' into hive-14535
Date Fri, 15 Sep 2017 19:52:31 GMT
Merge branch 'master' into hive-14535


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9a9f2540
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9a9f2540
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9a9f2540

Branch: refs/heads/hive-14535
Commit: 9a9f25409f4db9b3798aa529bb5531264cecd282
Parents: bb1c0f2 50fb6f3
Author: sergey <sershe@apache.org>
Authored: Fri Sep 15 12:47:20 2017 -0700
Committer: sergey <sershe@apache.org>
Committed: Fri Sep 15 12:47:20 2017 -0700

----------------------------------------------------------------------
 .gitignore                                      |    1 +
 .../org/apache/hive/beeline/HiveSchemaTool.java |    5 +-
 .../apache/hive/beeline/TestHiveSchemaTool.java |    9 +-
 .../common/classification/RetrySemantics.java   |   54 -
 .../org/apache/hadoop/hive/conf/HiveConf.java   |   34 +-
 .../listener/DbNotificationListener.java        |   14 +-
 .../hive/hcatalog/streaming/TestStreaming.java  |    2 +-
 .../src/test/queries/clientpositive/load_data.q |   20 +
 .../test/results/clientpositive/load_data.q.out |   59 +
 .../hadoop/hive/metastore/TestFilterHooks.java  |    3 +-
 .../hive/ql/parse/TestReplicationScenarios.java |  124 +-
 ...TestReplicationScenariosAcrossInstances.java |    9 +-
 .../hive/ql/txn/compactor/TestCompactor.java    |   40 +-
 .../test/resources/testconfiguration.properties |    1 +
 .../impl/LlapZookeeperRegistryImpl.java         |   17 +-
 .../hive/registry/impl/TezAmInstance.java       |   76 +
 .../hive/registry/impl/TezAmRegistryImpl.java   |  118 ++
 .../hive/registry/impl/ZkRegistryBase.java      |   37 +-
 .../hive/llap/cache/EvictionDispatcher.java     |   27 +-
 .../hive/llap/io/api/impl/LlapIoImpl.java       |   42 +-
 .../llap/io/decode/OrcColumnVectorProducer.java |    6 +-
 .../llap/io/encoded/OrcEncodedDataReader.java   |  345 +++--
 .../hive/llap/io/metadata/MetadataCache.java    |  537 +++++++
 .../llap/io/metadata/OrcFileEstimateErrors.java |    5 +-
 .../hive/llap/io/metadata/OrcFileMetadata.java  |  114 +-
 .../hive/llap/io/metadata/OrcMetadataCache.java |  163 ---
 .../llap/io/metadata/OrcStripeMetadata.java     |  123 +-
 .../io/metadata/ParquetMetadataCacheImpl.java   |  353 -----
 .../TestIncrementalObjectSizeEstimator.java     |    4 +-
 .../hive/llap/cache/TestOrcMetadataCache.java   |  159 ++-
 llap-tez/pom.xml                                |   34 +
 .../tezplugins/LlapTaskSchedulerService.java    |   83 +-
 .../endpoint/LlapPluginServerImpl.java          |   11 +-
 .../TestLlapTaskSchedulerService.java           |    1 +
 metastore/pom.xml                               |   39 -
 .../upgrade/derby/045-HIVE-16886.derby.sql      |    1 +
 .../upgrade/derby/hive-schema-3.0.0.derby.sql   |    6 +
 .../derby/upgrade-2.3.0-to-3.0.0.derby.sql      |    1 +
 .../upgrade/mssql/030-HIVE-16886.mssql.sql      |    1 +
 .../upgrade/mssql/hive-schema-3.0.0.mssql.sql   |    2 +
 .../mssql/upgrade-2.3.0-to-3.0.0.mssql.sql      |    1 +
 .../upgrade/mysql/045-HIVE-16886.mysql.sql      |    1 +
 .../upgrade/mysql/hive-schema-3.0.0.mysql.sql   |    2 +
 .../mysql/upgrade-2.3.0-to-3.0.0.mysql.sql      |    1 +
 .../upgrade/oracle/045-HIVE-16886.oracle.sql    |    1 +
 .../upgrade/oracle/hive-schema-3.0.0.oracle.sql |    1 +
 .../oracle/upgrade-2.3.0-to-3.0.0.oracle.sql    |    1 +
 .../postgres/044-HIVE-16886.postgres.sql        |    1 +
 .../postgres/hive-schema-3.0.0.postgres.sql     |    2 +
 .../upgrade-2.3.0-to-3.0.0.postgres.sql         |    1 +
 .../apache/hadoop/hive/metastore/Metastore.java | 1331 ------------------
 .../hive/metastore/AggregateStatsCache.java     |  573 --------
 .../hadoop/hive/metastore/DatabaseProduct.java  |   75 -
 .../hive/metastore/DefaultHiveMetaHook.java     |   51 -
 .../DefaultMetaStoreFilterHookImpl.java         |  102 --
 .../hadoop/hive/metastore/FileFormatProxy.java  |   64 -
 .../hive/metastore/HiveMetaException.java       |   42 -
 .../hadoop/hive/metastore/HiveMetaHook.java     |   96 --
 .../hive/metastore/HiveMetaHookLoader.java      |   39 -
 .../hadoop/hive/metastore/HiveMetaStore.java    |   11 +-
 .../hive/metastore/HiveMetaStoreClient.java     |    5 +-
 .../hive/metastore/HiveMetaStoreFsImpl.java     |   52 -
 .../hive/metastore/IExtrapolatePartStatus.java  |   85 --
 .../hadoop/hive/metastore/IHMSHandler.java      |   28 -
 .../hadoop/hive/metastore/IMetaStoreClient.java |    2 +-
 .../hive/metastore/IMetaStoreSchemaInfo.java    |  107 --
 .../metastore/LinearExtrapolatePartStatus.java  |  106 --
 .../hive/metastore/LockComponentBuilder.java    |  121 --
 .../hive/metastore/LockRequestBuilder.java      |  168 ---
 .../hive/metastore/MetaStoreDirectSql.java      |   21 +-
 .../metastore/MetaStoreEndFunctionContext.java  |   59 -
 .../metastore/MetaStoreEndFunctionListener.java |   58 -
 .../hadoop/hive/metastore/MetaStoreFS.java      |   43 -
 .../hive/metastore/MetaStoreFilterHook.java     |  132 --
 .../hadoop/hive/metastore/MetaStoreInit.java    |  112 --
 .../hive/metastore/MetaStoreSchemaInfo.java     |  234 ---
 .../metastore/MetaStoreSchemaInfoFactory.java   |   65 -
 .../hadoop/hive/metastore/MetaStoreThread.java  |   63 -
 .../hadoop/hive/metastore/ObjectStore.java      |  102 +-
 .../hive/metastore/PartitionDropOptions.java    |   54 -
 .../metastore/PartitionExpressionProxy.java     |   73 -
 .../hive/metastore/TServerSocketKeepAlive.java  |   47 -
 .../apache/hadoop/hive/metastore/TableType.java |   26 -
 .../hive/metastore/annotation/NoReconnect.java  |   29 -
 .../metastore/hooks/JDOConnectionURLHook.java   |   53 -
 .../spec/CompositePartitionSpecProxy.java       |  228 ---
 .../spec/PartitionListComposingSpecProxy.java   |  171 ---
 .../partition/spec/PartitionSpecProxy.java      |  199 ---
 .../spec/PartitionSpecWithSharedSDProxy.java    |  172 ---
 .../hive/metastore/repl/DumpDirCleanerTask.java |   45 +
 .../hive/metastore/tools/HiveSchemaHelper.java  |  640 ---------
 .../hive/metastore/tools/SQLGenerator.java      |  172 +++
 .../hadoop/hive/metastore/txn/TxnHandler.java   |  134 +-
 .../hadoop/hive/metastore/metastore.proto       |   29 -
 .../hive/metastore/TestAggregateStatsCache.java |  266 ----
 .../hive/metastore/TestLockRequestBuilder.java  |  584 --------
 .../metastore/TestMetaStoreSchemaFactory.java   |   67 -
 .../hive/metastore/TestMetaStoreSchemaInfo.java |   52 -
 .../hadoop/hive/metastore/TestObjectStore.java  |  163 ++-
 .../hadoop/hive/metastore/txn/TestTxnUtils.java |    9 +-
 .../org/apache/hadoop/hive/ql/ErrorMsg.java     |    9 +-
 .../hive/ql/exec/AbstractFileMergeOperator.java |   60 +-
 .../org/apache/hadoop/hive/ql/exec/DDLTask.java |   24 +-
 .../apache/hadoop/hive/ql/exec/Utilities.java   |    5 +
 .../hadoop/hive/ql/exec/repl/ReplDumpTask.java  |   32 +
 .../ql/exec/repl/bootstrap/ReplLoadTask.java    |   32 +-
 .../ql/exec/repl/bootstrap/ReplLoadWork.java    |    9 +-
 .../repl/bootstrap/events/BootstrapEvent.java   |    2 +-
 .../repl/bootstrap/events/ConstraintEvent.java  |   24 +
 .../filesystem/ConstraintEventsIterator.java    |   90 ++
 .../events/filesystem/FSConstraintEvent.java    |   39 +
 .../repl/bootstrap/load/LoadConstraint.java     |  119 ++
 .../bootstrap/load/table/LoadPartitions.java    |  110 +-
 .../repl/bootstrap/load/table/LoadTable.java    |    3 +-
 .../exec/repl/bootstrap/load/util/Context.java  |    4 +-
 .../exec/repl/bootstrap/load/util/PathInfo.java |   90 ++
 .../repl/bootstrap/load/util/PathUtils.java     |   68 +-
 .../ql/exec/tez/SessionExpirationTracker.java   |    3 +-
 .../hadoop/hive/ql/exec/tez/TezSessionPool.java |   93 +-
 .../hive/ql/exec/tez/TezSessionPoolManager.java |   34 +-
 .../hive/ql/exec/tez/TezSessionPoolSession.java |    8 +-
 .../hive/ql/exec/tez/TezSessionState.java       |   40 +-
 .../apache/hadoop/hive/ql/exec/tez/TezTask.java |    5 +-
 .../org/apache/hadoop/hive/ql/io/AcidUtils.java |    4 +-
 .../hadoop/hive/ql/io/orc/LocalCache.java       |    8 +-
 .../hadoop/hive/ql/io/orc/OrcInputFormat.java   |   11 +
 .../hive/ql/io/orc/encoded/EncodedReader.java   |   14 +
 .../ql/io/orc/encoded/EncodedReaderImpl.java    |  534 +++++--
 .../hadoop/hive/ql/io/orc/encoded/Reader.java   |    4 +
 .../hive/ql/io/orc/encoded/ReaderImpl.java      |   11 +-
 .../apache/hadoop/hive/ql/metadata/Hive.java    |   43 +-
 .../hive/ql/parse/DDLSemanticAnalyzer.java      |   25 +-
 .../apache/hadoop/hive/ql/parse/EximUtil.java   |    5 +-
 .../hadoop/hive/ql/parse/GenTezUtils.java       |    3 +-
 .../ql/parse/ReplicationSemanticAnalyzer.java   |    1 +
 .../hive/ql/parse/repl/dump/TableExport.java    |    9 +-
 .../repl/dump/io/ConstraintsSerializer.java     |   71 +
 .../repl/load/message/AddForeignKeyHandler.java |   16 +-
 .../message/AddNotNullConstraintHandler.java    |    7 +-
 .../repl/load/message/AddPrimaryKeyHandler.java |    7 +-
 .../message/AddUniqueConstraintHandler.java     |    6 +-
 .../hadoop/hive/ql/plan/DropTableDesc.java      |    7 +
 .../AuthorizationMetaStoreFilterHook.java       |    4 +-
 .../hadoop/hive/ql/session/SessionState.java    |   10 +-
 .../hive/ql/txn/compactor/CompactorThread.java  |   23 +-
 .../apache/hadoop/hive/ql/TestTxnCommands2.java |   18 +-
 .../hive/ql/exec/tez/SampleTezSessionState.java |   12 +-
 .../hive/ql/exec/tez/TestTezSessionPool.java    |   14 +-
 .../hadoop/hive/ql/exec/tez/TestTezTask.java    |    3 +-
 .../hive/ql/txn/compactor/CompactorTest.java    |    2 +-
 .../queries/clientnegative/merge_negative_4.q   |    6 +
 .../queries/clientnegative/merge_negative_5.q   |   14 +
 .../clientpositive/dynamic_semijoin_reduction.q |    1 +
 .../test/queries/clientpositive/mapjoin_hint.q  |    1 +
 .../test/queries/clientpositive/orc_merge13.q   |   44 +
 .../tez_union_dynamic_partition_2.q             |   27 +
 .../clientnegative/merge_negative_3.q.out       |    2 +-
 .../clientnegative/merge_negative_4.q.out       |   19 +
 .../clientnegative/merge_negative_5.q.out       |   67 +
 .../llap/dynamic_semijoin_reduction.q.out       |   48 +-
 .../llap/dynamic_semijoin_reduction_2.q.out     |   24 +-
 .../llap/dynamic_semijoin_reduction_3.q.out     |   14 +-
 .../llap/dynamic_semijoin_reduction_sw.q.out    |   16 +-
 .../llap/dynamic_semijoin_user_level.q.out      |   32 +-
 .../clientpositive/llap/mapjoin_hint.q.out      |    4 +-
 .../results/clientpositive/llap/mergejoin.q.out |   12 +-
 .../clientpositive/llap/orc_llap_counters.q.out |   98 +-
 .../llap/orc_llap_counters1.q.out               |   11 +-
 .../clientpositive/llap/orc_ppd_basic.q.out     |  147 +-
 .../llap/orc_ppd_schema_evol_3a.q.out           |  156 +-
 .../clientpositive/llap/semijoin_hint.q.out     |   12 +-
 .../llap/tez_union_dynamic_partition_2.q.out    |  219 +++
 .../vectorized_dynamic_semijoin_reduction.q.out |   36 +-
 ...vectorized_dynamic_semijoin_reduction2.q.out |   32 +-
 .../results/clientpositive/orc_merge13.q.out    |  248 ++++
 .../hive/serde2/avro/AvroDeserializer.java      |   22 +-
 .../lazy/fast/LazySimpleSerializeWrite.java     |    4 +-
 .../ObjectInspectorConverters.java              |    8 +-
 standalone-metastore/pom.xml                    |   50 +
 .../gen/thrift/gen-cpp/ThriftHiveMetastore.cpp  |   32 +-
 .../gen/thrift/gen-cpp/ThriftHiveMetastore.h    |   12 +-
 .../hive/metastore/api/ThriftHiveMetastore.java |  140 +-
 .../gen-php/metastore/ThriftHiveMetastore.php   |   31 +-
 .../hive_metastore/ThriftHiveMetastore.py       |   25 +-
 .../gen/thrift/gen-rb/thrift_hive_metastore.rb  |    7 +-
 .../common/classification/RetrySemantics.java   |   57 +
 .../hive/metastore/AggregateStatsCache.java     |  567 ++++++++
 .../hadoop/hive/metastore/DatabaseProduct.java  |   75 +
 .../hive/metastore/DefaultHiveMetaHook.java     |   51 +
 .../DefaultMetaStoreFilterHookImpl.java         |  102 ++
 .../hadoop/hive/metastore/FileFormatProxy.java  |   64 +
 .../hive/metastore/HiveMetaException.java       |   42 +
 .../hadoop/hive/metastore/HiveMetaHook.java     |   95 ++
 .../hive/metastore/HiveMetaHookLoader.java      |   39 +
 .../hive/metastore/HiveMetaStoreFsImpl.java     |   53 +
 .../hive/metastore/IExtrapolatePartStatus.java  |   85 ++
 .../hadoop/hive/metastore/IHMSHandler.java      |   28 +
 .../hive/metastore/IMetaStoreSchemaInfo.java    |  108 ++
 .../metastore/LinearExtrapolatePartStatus.java  |  106 ++
 .../hive/metastore/LockComponentBuilder.java    |  121 ++
 .../hive/metastore/LockRequestBuilder.java      |  168 +++
 .../metastore/MetaStoreEndFunctionContext.java  |   59 +
 .../metastore/MetaStoreEndFunctionListener.java |   58 +
 .../hadoop/hive/metastore/MetaStoreFS.java      |   43 +
 .../hive/metastore/MetaStoreFilterHook.java     |  132 ++
 .../hadoop/hive/metastore/MetaStoreInit.java    |  109 ++
 .../hive/metastore/MetaStoreSchemaInfo.java     |  234 +++
 .../metastore/MetaStoreSchemaInfoFactory.java   |   64 +
 .../hadoop/hive/metastore/MetaStoreThread.java  |   58 +
 .../hive/metastore/PartitionDropOptions.java    |   54 +
 .../metastore/PartitionExpressionProxy.java     |   73 +
 .../hive/metastore/TServerSocketKeepAlive.java  |   47 +
 .../apache/hadoop/hive/metastore/TableType.java |   26 +
 .../annotation/MetastoreVersionAnnotation.java  |   85 ++
 .../hive/metastore/annotation/NoReconnect.java  |   29 +
 .../metastore/hooks/JDOConnectionURLHook.java   |   53 +
 .../spec/CompositePartitionSpecProxy.java       |  226 +++
 .../spec/PartitionListComposingSpecProxy.java   |  171 +++
 .../partition/spec/PartitionSpecProxy.java      |  199 +++
 .../spec/PartitionSpecWithSharedSDProxy.java    |  172 +++
 .../hive/metastore/tools/HiveSchemaHelper.java  |  641 +++++++++
 .../hadoop/hive/metastore/utils/FileUtils.java  |   66 +
 .../hadoop/hive/metastore/utils/JavaUtils.java  |   37 +
 .../hive/metastore/utils/MetaStoreUtils.java    |   41 +
 .../metastore/utils/MetastoreVersionInfo.java   |  133 ++
 .../hadoop/hive/metastore/metastore.proto       |   29 +
 .../src/main/resources/saveVersion.sh           |   91 ++
 .../src/main/thrift/hive_metastore.thrift       |    2 +-
 .../hive/metastore/TestAggregateStatsCache.java |  267 ++++
 .../hive/metastore/TestLockRequestBuilder.java  |  584 ++++++++
 .../metastore/TestMetaStoreSchemaFactory.java   |   69 +
 .../hive/metastore/TestMetaStoreSchemaInfo.java |   52 +
 .../hive/common/io/FileMetadataCache.java       |    8 +-
 233 files changed, 10377 insertions(+), 8544 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/9a9f2540/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/9a9f2540/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/9a9f2540/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/9a9f2540/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/9a9f2540/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
----------------------------------------------------------------------
diff --cc metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index d2038c5,3053dcb..c65baa4
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@@ -241,8 -242,8 +244,9 @@@ public class ObjectStore implements Raw
  
    private boolean isInitialized = false;
    private PersistenceManager pm = null;
+   private SQLGenerator sqlGenerator = null;
    private MetaStoreDirectSql directSql = null;
 +  private DatabaseProduct dbType = null;
    private PartitionExpressionProxy expressionProxy = null;
    private Configuration hiveConf;
    private volatile int openTrasactionCalls = 0;
@@@ -430,9 -431,17 +434,18 @@@
      LOG.info("ObjectStore, initialize called");
      prop = dsProps;
      pm = getPersistenceManager();
+     try {
+       String productName = MetaStoreDirectSql.getProductName(pm);
+       sqlGenerator = new SQLGenerator(
+           DatabaseProduct.determineDatabaseProduct(productName),
+           new HiveConf(hiveConf, ObjectStore.class));
+     } catch (SQLException e) {
+       LOG.error("error trying to figure out the database product", e);
+       throw new RuntimeException(e);
+     }
      isInitialized = pm != null;
      if (isInitialized) {
 +      dbType = determineDatabaseProduct();
        expressionProxy = createExpressionProxy(hiveConf);
        if (HiveConf.getBoolVar(getConf(), ConfVars.METASTORE_TRY_DIRECT_SQL)) {
          String schema = prop.getProperty("javax.jdo.mapping.Schema");

http://git-wip-us.apache.org/repos/asf/hive/blob/9a9f2540/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
----------------------------------------------------------------------
diff --cc metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
index 75eba79,7882dad..176fc4c
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
@@@ -59,9 -57,21 +57,23 @@@ import org.mockito.Mockito
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
  
 +import com.google.common.base.Supplier;
 +import com.google.common.collect.ImmutableList;
  import javax.jdo.Query;
+ import java.util.Arrays;
+ import java.util.HashMap;
+ import java.util.HashSet;
+ import java.util.List;
+ import java.util.Set;
+ import java.util.concurrent.BrokenBarrierException;
+ import java.util.concurrent.CyclicBarrier;
+ import java.util.concurrent.ExecutionException;
+ import java.util.concurrent.ExecutorService;
+ import java.util.concurrent.Executors;
+ import java.util.concurrent.TimeUnit;
+ 
+ import static org.junit.Assert.assertEquals;
+ import static org.junit.Assert.assertTrue;
  
  public class TestObjectStore {
    private ObjectStore objectStore = null;
@@@ -287,8 -282,8 +299,8 @@@
    public void testPartitionOps() throws MetaException, InvalidObjectException, NoSuchObjectException,
InvalidInputException {
      Database db1 = new Database(DB1, "description", "locationurl", null);
      objectStore.createDatabase(db1);
 -    StorageDescriptor sd = new StorageDescriptor(null, "location", null, null, false, 0,
new SerDeInfo("SerDeName", "serializationLib", null), null, null, null);
 +    StorageDescriptor sd = createFakeSd("location");
-     HashMap<String,String> tableParams = new HashMap<String,String>();
+     HashMap<String, String> tableParams = new HashMap<>();
      tableParams.put("EXTERNAL", "false");
      FieldSchema partitionKey1 = new FieldSchema("Country", ColumnType.STRING_TYPE_NAME,
"");
      FieldSchema partitionKey2 = new FieldSchema("State", ColumnType.STRING_TYPE_NAME, "");

http://git-wip-us.apache.org/repos/asf/hive/blob/9a9f2540/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractFileMergeOperator.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractFileMergeOperator.java
index 70e764e,6bba057..6b98ecd
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractFileMergeOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractFileMergeOperator.java
@@@ -224,50 -216,63 +232,78 @@@ public abstract class AbstractFileMerge
    @Override
    public void closeOp(boolean abort) throws HiveException {
      try {
 -      if (!abort) {
 -        // if outPath does not exist, then it means all paths within combine split are skipped
as
 -        // they are incompatible for merge (for example: files without stripe stats).
 -        // Those files will be added to incompatFileSet
 -        if (fs.exists(outPath)) {
 -          FileStatus fss = fs.getFileStatus(outPath);
 +      if (abort) {
 +        if (!autoDelete || isMmTable) {
 +          fs.delete(outPath, true);
 +        }
 +        return;
 +      }
 +      // if outPath does not exist, then it means all paths within combine split are skipped
as
 +      // they are incompatible for merge (for example: files without stripe stats).
 +      // Those files will be added to incompatFileSet
 +      if (fs.exists(outPath)) {
 +        FileStatus fss = fs.getFileStatus(outPath);
 +        if (!isMmTable) {
            if (!fs.rename(outPath, finalPath)) {
 -            throw new IOException(
 -                "Unable to rename " + outPath + " to " + finalPath);
 +            throw new IOException("Unable to rename " + outPath + " to " + finalPath);
            }
 -          LOG.info("renamed path " + outPath + " to " + finalPath + " . File" +
 -              " size is "
 -              + fss.getLen());
 +          LOG.info("Renamed path " + outPath + " to " + finalPath
 +              + "(" + fss.getLen() + " bytes).");
 +        } else {
 +          assert finalPath.equals(outPath);
 +          // There's always just one file that we have merged.
 +          // The union/DP/etc. should already be account for in the path.
 +          Utilities.writeMmCommitManifest(Lists.newArrayList(outPath),
 +              tmpPath.getParent(), fs, taskId, conf.getTxnId(), conf.getStmtId(), null);
 +          LOG.info("Merged into " + finalPath + "(" + fss.getLen() + " bytes).");
          }
 +      }
  
 +      // move any incompatible files to final path
 +      if (incompatFileSet != null && !incompatFileSet.isEmpty()) {
 +        if (isMmTable) {
 +          // We only support query-time merge for MM tables, so don't handle this.
 +          throw new HiveException("Incompatible files should not happen in MM tables.");
 +        }
-         for (Path incompatFile : incompatFileSet) {
-           Path destDir = finalPath.getParent();
-           try {
-             Utilities.renameOrMoveFiles(fs, incompatFile, destDir);
-             LOG.info("Moved incompatible file " + incompatFile + " to " +
-                 destDir);
-           } catch (HiveException e) {
-             LOG.error("Unable to move " + incompatFile + " to " + destDir);
-             throw new IOException(e);
+         Path destDir = finalPath.getParent();
+         Path destPath = destDir;
+         // move any incompatible files to final path
+         if (incompatFileSet != null && !incompatFileSet.isEmpty()) {
+           for (Path incompatFile : incompatFileSet) {
+             // check if path conforms to Hive's file name convention. Hive expects filenames
to be in specific format
+             // like 000000_0, but "LOAD DATA" commands can let you add any files to any
partitions/tables without
+             // renaming. This can cause MoveTask to remove files in some cases where MoveTask
assumes the files are
+             // are generated by speculatively executed tasks.
+             // Example: MoveTask thinks the following files are same
+             // part-m-00000_1417075294718
+             // part-m-00001_1417075294718
+             // Assumes 1417075294718 as taskId and retains only large file supposedly generated
by speculative execution.
+             // This can result in data loss in case of CONCATENATE/merging. Filter out files
that does not match Hive's
+             // filename convention.
+             if (!Utilities.isHiveManagedFile(incompatFile)) {
+               // rename un-managed files to conform to Hive's naming standard
+               // Example:
+               // /warehouse/table/part-m-00000_1417075294718 will get renamed to /warehouse/table/.hive-staging/000000_0
+               // If staging directory already contains the file, taskId_copy_N naming will
be used.
+               final String taskId = Utilities.getTaskId(jc);
+               Path destFilePath = new Path(destDir, new Path(taskId));
+               for (int counter = 1; fs.exists(destFilePath); counter++) {
+                 destFilePath = new Path(destDir, taskId + (Utilities.COPY_KEYWORD + counter));
+               }
+               LOG.warn("Path doesn't conform to Hive's expectation. Renaming {} to {}",
incompatFile, destFilePath);
+               destPath = destFilePath;
+             }
+ 
+             try {
+               Utilities.renameOrMoveFiles(fs, incompatFile, destPath);
+               LOG.info("Moved incompatible file " + incompatFile + " to " + destPath);
+             } catch (HiveException e) {
+               LOG.error("Unable to move " + incompatFile + " to " + destPath);
+               throw new IOException(e);
+             }
            }
          }
 -      } else {
 -        if (!autoDelete) {
 -          fs.delete(outPath, true);
 -        }
++
        }
      } catch (IOException e) {
        throw new HiveException("Failed to close AbstractFileMergeOperator", e);

http://git-wip-us.apache.org/repos/asf/hive/blob/9a9f2540/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/9a9f2540/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/9a9f2540/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java
index 2ae18cf,acf9746..522444e
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java
@@@ -44,8 -45,7 +45,9 @@@ import org.apache.hadoop.hive.ql.plan.D
  import org.apache.hadoop.hive.ql.plan.ImportTableDesc;
  import org.apache.hadoop.hive.ql.plan.LoadTableDesc;
  import org.apache.hadoop.hive.ql.plan.MoveWork;
 +import org.apache.hadoop.hive.ql.session.SessionState;
 +import org.mortbay.jetty.servlet.AbstractSessionManager;
+ import org.datanucleus.util.StringUtils;
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
  

http://git-wip-us.apache.org/repos/asf/hive/blob/9a9f2540/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/9a9f2540/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/9a9f2540/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/9a9f2540/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/9a9f2540/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/9a9f2540/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/9a9f2540/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
index 8f94679,40d4fad..cf1d14c
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
@@@ -314,9 -315,9 +315,9 @@@ public class GenTezUtils 
          linked = context.linkedFileSinks.get(path);
          linked.add(desc);
  
-         desc.setDirName(new Path(path, "" + linked.size()));
+         desc.setDirName(new Path(path, AbstractFileMergeOperator.UNION_SUDBIR_PREFIX + linked.size()));
 +        Utilities.LOG14535.info("removing union - new desc with " + desc.getDirName() +
"; parent " + path);
          desc.setLinkedFileSink(true);
 -        desc.setParentDir(path);
          desc.setLinkedFileSinkDesc(linked);
        }
  

http://git-wip-us.apache.org/repos/asf/hive/blob/9a9f2540/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/9a9f2540/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/9a9f2540/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreThread.java
----------------------------------------------------------------------
diff --cc standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreThread.java
index 0000000,10f7732..6ef2e35
mode 000000,100644..100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreThread.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreThread.java
@@@ -1,0 -1,57 +1,58 @@@
+ /*
+  * Licensed to the Apache Software Foundation (ASF) under one
+  * or more contributor license agreements.  See the NOTICE file
+  * distributed with this work for additional information
+  * regarding copyright ownership.  The ASF licenses this file
+  * to you under the Apache License, Version 2.0 (the
+  * "License"); you may not use this file except in compliance
+  * with the License.  You may obtain a copy of the License at
+  *
+  *     http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+ package org.apache.hadoop.hive.metastore;
+ 
+ import org.apache.hadoop.conf.Configurable;
+ import org.apache.hadoop.conf.Configuration;
+ import org.apache.hadoop.hive.metastore.api.MetaException;
+ 
+ import java.util.concurrent.atomic.AtomicBoolean;
+ 
+ /**
+  * A thread that runs in the metastore, separate from the threads in the thrift service.
+  */
+ public interface MetaStoreThread extends Configurable {
+ 
+   /**
+    * Set the id for this thread.
+    * @param threadId id of the thread
+    */
+   void setThreadId(int threadId);
+ 
+   /**
+    * Initialize the thread.  This must not be called until after
+    * {@link #setConf(Configuration)}  and  {@link #setThreadId(int)}
+    * have been called.
+    * @param stop a flag to watch for when to stop.  If this value is set to true,
+    *             the thread will terminate the next time through its main loop.
+    * @param looped a flag that is set to true everytime a thread goes through it's main
loop.
+    *               This is purely for testing so that tests can assure themselves that the
thread
+    *               has run through it's loop once.  The test can set this value to false.
 The
+    *               thread should then assure that the loop has been gone completely through
at
+    *               least once.
+    */
++  // TODO: move these test parameters to more specific places... there's no need to have
them here
+   void init(AtomicBoolean stop, AtomicBoolean looped) throws MetaException;
+ 
+   /**
+    * Run the thread in the background.  This must not be called until
+    * {@link MetaStoreThread#init(java.util.concurrent.atomic.AtomicBoolean,java.util.concurrent.atomic.AtomicBoolean)}
has
+    * been called.
+    */
+   void start();
+ }

http://git-wip-us.apache.org/repos/asf/hive/blob/9a9f2540/standalone-metastore/src/main/thrift/hive_metastore.thrift
----------------------------------------------------------------------


Mime
View raw message