hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From lium...@apache.org
Subject [50/50] [abbrv] hadoop git commit: Merge branch 'trunk' into HADOOP-13345
Date Tue, 06 Jun 2017 08:28:40 GMT
Merge branch 'trunk' into HADOOP-13345


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5f02de2e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5f02de2e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5f02de2e

Branch: refs/heads/HADOOP-13345
Commit: 5f02de2e029fd739c8504740467b838d1d336e40
Parents: 2d06842 6aeda55
Author: Mingliang Liu <liuml07@apache.org>
Authored: Mon Jun 5 13:10:05 2017 -0700
Committer: Mingliang Liu <liuml07@apache.org>
Committed: Mon Jun 5 13:30:29 2017 -0700

----------------------------------------------------------------------
 BUILDING.txt                                    |    2 -
 LICENSE.txt                                     |   47 +-
 dev-support/docker/Dockerfile                   |    2 +
 hadoop-assemblies/pom.xml                       |    4 +-
 .../resources/assemblies/hadoop-yarn-dist.xml   |    5 +-
 hadoop-build-tools/pom.xml                      |   12 +-
 .../main/resources/checkstyle/checkstyle.xml    |    4 +-
 hadoop-client-modules/hadoop-client-api/pom.xml |    4 +-
 .../hadoop-client-check-invariants/pom.xml      |    4 +-
 .../hadoop-client-check-test-invariants/pom.xml |    4 +-
 .../hadoop-client-integration-tests/pom.xml     |    4 +-
 .../hadoop-client-minicluster/pom.xml           |    8 +-
 .../hadoop-client-runtime/pom.xml               |    5 +-
 hadoop-client-modules/hadoop-client/pom.xml     |    4 +-
 hadoop-client-modules/pom.xml                   |    2 +-
 .../hadoop-cloud-storage/pom.xml                |    4 +-
 hadoop-cloud-storage-project/pom.xml            |    4 +-
 .../hadoop-annotations/pom.xml                  |    4 +-
 .../hadoop-auth-examples/pom.xml                |    4 +-
 hadoop-common-project/hadoop-auth/pom.xml       |   19 +-
 .../authentication/util/KerberosName.java       |    5 +-
 .../authentication/util/TestKerberosName.java   |    4 +-
 hadoop-common-project/hadoop-common/pom.xml     |    9 +-
 .../src/main/bin/hadoop-functions.sh            |   24 -
 .../src/main/conf/log4j.properties              |   37 +-
 .../org/apache/hadoop/conf/ConfigRedactor.java  |    5 +-
 .../org/apache/hadoop/conf/Configuration.java   |   37 +-
 .../hadoop/crypto/CryptoOutputStream.java       |    4 +-
 .../org/apache/hadoop/crypto/OpensslCipher.java |    4 +-
 .../hadoop/crypto/key/kms/ValueQueue.java       |    2 +-
 .../org/apache/hadoop/fs/BlockLocation.java     |   21 +-
 .../apache/hadoop/fs/CachingGetSpaceUsed.java   |    3 +
 .../fs/CommonConfigurationKeysPublic.java       |   17 +-
 .../java/org/apache/hadoop/fs/CreateFlag.java   |    8 +-
 .../src/main/java/org/apache/hadoop/fs/DU.java  |    3 +-
 .../apache/hadoop/fs/FSDataOutputStream.java    |   10 +-
 .../hadoop/fs/FSDataOutputStreamBuilder.java    |    8 +-
 .../java/org/apache/hadoop/fs/FileSystem.java   |    8 +-
 .../java/org/apache/hadoop/fs/FileUtil.java     |    5 +-
 .../org/apache/hadoop/fs/FilterFileSystem.java  |    2 +-
 .../java/org/apache/hadoop/fs/FilterFs.java     |    4 +-
 .../main/java/org/apache/hadoop/fs/Options.java |    2 +-
 .../apache/hadoop/fs/RawLocalFileSystem.java    |    2 +-
 .../apache/hadoop/fs/StreamCapabilities.java    |   67 +
 .../hadoop/fs/http/AbstractHttpFileSystem.java  |  153 +
 .../apache/hadoop/fs/http/HttpFileSystem.java   |   28 +
 .../apache/hadoop/fs/http/HttpsFileSystem.java  |   28 +
 .../org/apache/hadoop/fs/http/package-info.java |   23 +
 .../apache/hadoop/fs/sftp/SFTPFileSystem.java   |    2 +-
 .../hadoop/fs/shell/CommandWithDestination.java |    2 +-
 .../org/apache/hadoop/fs/shell/FsUsage.java     |   14 +-
 .../apache/hadoop/fs/shell/XAttrCommands.java   |    7 +-
 .../hadoop/fs/viewfs/ChRootedFileSystem.java    |    2 +-
 .../org/apache/hadoop/fs/viewfs/ChRootedFs.java |    2 +-
 .../apache/hadoop/fs/viewfs/ViewFileSystem.java |    8 +-
 .../apache/hadoop/ha/ActiveStandbyElector.java  |    4 +-
 .../org/apache/hadoop/ha/HAServiceProtocol.java |    2 +-
 .../java/org/apache/hadoop/ha/NodeFencer.java   |    3 +-
 .../org/apache/hadoop/ha/PowerShellFencer.java  |  154 +
 .../org/apache/hadoop/http/HttpServer2.java     |   48 +-
 .../main/java/org/apache/hadoop/io/IOUtils.java |   55 +-
 .../java/org/apache/hadoop/io/SequenceFile.java |    2 +-
 .../apache/hadoop/io/compress/BZip2Codec.java   |   20 +-
 .../apache/hadoop/io/compress/CodecPool.java    |   10 +-
 .../io/compress/CompressionInputStream.java     |   11 +-
 .../io/compress/CompressionOutputStream.java    |   16 +-
 .../hadoop/io/compress/CompressorStream.java    |    3 +-
 .../hadoop/io/compress/DecompressorStream.java  |    7 +-
 .../compress/zlib/BuiltInGzipDecompressor.java  |    2 +-
 .../hadoop/io/compress/zlib/ZlibCompressor.java |    6 +-
 .../io/compress/zlib/ZlibDecompressor.java      |    2 +-
 .../hadoop/io/erasurecode/CodecRegistry.java    |  182 +
 .../apache/hadoop/io/erasurecode/CodecUtil.java |   86 +-
 .../apache/hadoop/io/erasurecode/ECSchema.java  |   40 +-
 .../io/erasurecode/ErasureCodeConstants.java    |    1 +
 .../rawcoder/DummyRawErasureCoderFactory.java   |   12 +
 .../NativeRSRawErasureCoderFactory.java         |   13 +
 .../NativeXORRawErasureCoderFactory.java        |   13 +
 .../rawcoder/RSLegacyRawDecoder.java            |  270 +
 .../rawcoder/RSLegacyRawEncoder.java            |  129 +
 .../RSLegacyRawErasureCoderFactory.java         |   51 +
 .../rawcoder/RSRawDecoderLegacy.java            |  270 -
 .../rawcoder/RSRawEncoderLegacy.java            |  129 -
 .../rawcoder/RSRawErasureCoderFactory.java      |   13 +
 .../RSRawErasureCoderFactoryLegacy.java         |   38 -
 .../rawcoder/RawErasureCoderFactory.java        |   12 +
 .../rawcoder/XORRawErasureCoderFactory.java     |   13 +
 .../hadoop/io/file/tfile/Compression.java       |    2 +-
 .../org/apache/hadoop/io/nativeio/NativeIO.java |    2 +-
 .../org/apache/hadoop/ipc/CallQueueManager.java |    1 +
 .../apache/hadoop/ipc/DecayRpcScheduler.java    |   33 +-
 .../apache/hadoop/ipc/DefaultRpcScheduler.java  |    4 +
 .../org/apache/hadoop/ipc/RpcScheduler.java     |    2 +
 .../main/java/org/apache/hadoop/ipc/Server.java |   22 +-
 .../apache/hadoop/ipc/metrics/RpcMetrics.java   |    4 +
 .../apache/hadoop/metrics2/AbstractMetric.java  |   16 +-
 .../org/apache/hadoop/metrics2/MetricsTag.java  |   13 +-
 .../metrics2/impl/AbstractMetricsRecord.java    |   16 +-
 .../org/apache/hadoop/metrics2/impl/MsInfo.java |    9 +-
 .../hadoop/metrics2/lib/MetricsInfoImpl.java    |   11 +-
 .../hadoop/metrics2/lib/MetricsRegistry.java    |   20 +-
 .../apache/hadoop/metrics2/package-info.java    |    2 +-
 .../hadoop/metrics2/source/JvmMetricsInfo.java  |    9 +-
 .../hadoop/metrics2/util/MetricsCache.java      |   18 +-
 .../apache/hadoop/metrics2/util/Servers.java    |    4 +-
 .../org/apache/hadoop/net/NetworkTopology.java  |   12 +-
 .../JniBasedUnixGroupsNetgroupMapping.java      |    2 +-
 .../apache/hadoop/security/SaslRpcServer.java   |    4 +-
 .../ShellBasedUnixGroupsNetgroupMapping.java    |    2 +-
 .../hadoop/security/UserGroupInformation.java   |    2 +-
 .../apache/hadoop/security/ssl/SSLFactory.java  |   15 +-
 .../security/ssl/SSLHostnameVerifier.java       |   11 -
 .../web/DelegationTokenAuthenticator.java       |    2 +-
 .../hadoop/service/ServiceStateException.java   |   66 +-
 .../launcher/AbstractLaunchableService.java     |   78 +
 .../HadoopUncaughtExceptionHandler.java         |  129 +
 .../service/launcher/InterruptEscalator.java    |  216 +
 .../hadoop/service/launcher/IrqHandler.java     |  178 +
 .../service/launcher/LaunchableService.java     |   95 +
 .../service/launcher/LauncherArguments.java     |   59 +
 .../service/launcher/LauncherExitCodes.java     |  183 +
 .../launcher/ServiceLaunchException.java        |   81 +
 .../service/launcher/ServiceLauncher.java       | 1044 ++++
 .../service/launcher/ServiceShutdownHook.java   |  112 +
 .../hadoop/service/launcher/package-info.java   |  462 ++
 .../apache/hadoop/util/ComparableVersion.java   |   29 +-
 .../org/apache/hadoop/util/DataChecksum.java    |   14 +-
 .../apache/hadoop/util/ExitCodeProvider.java    |   35 +
 .../java/org/apache/hadoop/util/ExitUtil.java   |  257 +-
 .../hadoop/util/GenericOptionsParser.java       |  125 +-
 .../org/apache/hadoop/util/HostsFileReader.java |  263 +-
 .../org/apache/hadoop/util/StringInterner.java  |   37 +-
 .../org/apache/hadoop/util/StringUtils.java     |   54 +-
 .../org/apache/hadoop/util/SysInfoWindows.java  |    2 +-
 .../org/apache/hadoop/io/nativeio/NativeIO.c    |    2 +-
 .../services/org.apache.hadoop.fs.FileSystem    |    2 +
 ....erasurecode.rawcoder.RawErasureCoderFactory |   18 +
 .../src/main/resources/core-default.xml         |   35 +-
 .../src/site/markdown/DeprecatedProperties.md   |    1 +
 .../src/site/markdown/FileSystemShell.md        |    6 +-
 .../hadoop-common/src/site/markdown/Metrics.md  |   24 +-
 .../src/site/markdown/RackAwareness.md          |   12 +-
 .../3.0.0-alpha3/CHANGES.3.0.0-alpha3.md        |   71 +
 .../3.0.0-alpha3/RELEASENOTES.3.0.0-alpha3.md   |   22 +
 .../apache/hadoop/conf/TestConfigRedactor.java  |   22 +-
 .../apache/hadoop/conf/TestConfiguration.java   |   23 +-
 .../crypto/TestCryptoStreamsForLocalFS.java     |   10 +-
 .../hadoop/crypto/TestCryptoStreamsNormal.java  |   16 +-
 .../kms/TestLoadBalancingKMSClientProvider.java |    2 +-
 .../apache/hadoop/fs/FileContextTestHelper.java |    2 +-
 .../hadoop/fs/FileSystemContractBaseTest.java   |  142 +-
 .../apache/hadoop/fs/FileSystemTestHelper.java  |    2 +-
 .../java/org/apache/hadoop/fs/TestFsShell.java  |    3 +-
 .../org/apache/hadoop/fs/TestGlobPattern.java   |    2 +-
 .../apache/hadoop/fs/TestLocalFileSystem.java   |   37 +-
 .../apache/hadoop/fs/TestSymlinkLocalFS.java    |    6 +-
 .../hadoop/fs/TestSymlinkLocalFSFileSystem.java |    8 +-
 .../fs/contract/AbstractContractMkdirTest.java  |   31 +-
 .../contract/rawlocal/RawlocalFSContract.java   |    2 +-
 .../hadoop/fs/http/TestHttpFileSystem.java      |   67 +
 .../hadoop/fs/sftp/TestSFTPFileSystem.java      |   14 +
 .../fs/viewfs/ViewFileSystemBaseTest.java       |    6 +-
 .../apache/hadoop/fs/viewfs/ViewFsBaseTest.java |    2 +-
 .../apache/hadoop/http/TestSSLHttpServer.java   |   21 +-
 .../java/org/apache/hadoop/io/TestIOUtils.java  |    2 +-
 .../apache/hadoop/io/TestSortedMapWritable.java |    2 +-
 .../apache/hadoop/io/compress/TestCodec.java    |  103 +-
 .../hadoop/io/compress/TestCodecPool.java       |   12 +-
 .../erasurecode/TestCodecRawCoderMapping.java   |   20 +-
 .../io/erasurecode/TestCodecRegistry.java       |  173 +
 .../hadoop/io/erasurecode/TestECSchema.java     |   47 +-
 .../coder/TestHHXORErasureCoder.java            |    2 +-
 .../erasurecode/coder/TestRSErasureCoder.java   |    2 +-
 .../rawcoder/RawErasureCoderBenchmark.java      |    2 +-
 .../erasurecode/rawcoder/TestDummyRawCoder.java |    4 +-
 .../rawcoder/TestNativeRSRawCoder.java          |    4 +-
 .../rawcoder/TestNativeXORRawCoder.java         |    4 +-
 .../rawcoder/TestRSLegacyRawCoder.java          |   33 +
 .../io/erasurecode/rawcoder/TestRSRawCoder.java |    4 +-
 .../rawcoder/TestRSRawCoderInteroperable1.java  |    4 +-
 .../rawcoder/TestRSRawCoderInteroperable2.java  |    4 +-
 .../rawcoder/TestRSRawCoderLegacy.java          |   33 -
 .../erasurecode/rawcoder/TestRawCoderBase.java  |   16 +-
 .../erasurecode/rawcoder/TestXORRawCoder.java   |    4 +-
 .../rawcoder/TestXORRawCoderInteroperable1.java |    4 +-
 .../rawcoder/TestXORRawCoderInteroperable2.java |    4 +-
 .../apache/hadoop/io/nativeio/TestNativeIO.java |    6 +-
 .../io/retry/UnreliableImplementation.java      |    2 +-
 .../java/org/apache/hadoop/ipc/TestIPC.java     |   10 +-
 .../org/apache/hadoop/net/TestNetUtils.java     |   10 +-
 .../security/TestUserGroupInformation.java      |    2 +-
 .../hadoop/security/ssl/TestSSLFactory.java     |   10 +-
 .../apache/hadoop/service/BreakableService.java |   23 +-
 .../hadoop/service/TestCompositeService.java    |   66 +-
 .../AbstractServiceLauncherTestBase.java        |  317 ++
 .../launcher/ExitTrackingServiceLauncher.java   |   59 +
 .../service/launcher/TestServiceConf.java       |  146 +
 .../launcher/TestServiceInterruptHandling.java  |  118 +
 .../service/launcher/TestServiceLauncher.java   |  213 +
 .../TestServiceLauncherCreationFailures.java    |   83 +
 .../TestServiceLauncherInnerMethods.java        |   95 +
 .../ExceptionInExecuteLaunchableService.java    |   96 +
 .../testservices/FailInConstructorService.java  |   33 +
 .../testservices/FailInInitService.java         |   38 +
 .../testservices/FailInStartService.java        |   37 +
 .../testservices/FailingStopInStartService.java |   47 +
 .../testservices/FailureTestService.java        |   55 +
 .../InitInConstructorLaunchableService.java     |   63 +
 .../testservices/LaunchableRunningService.java  |  111 +
 .../testservices/NoArgsAllowedService.java      |   64 +
 .../testservices/NullBindLaunchableService.java |   46 +
 .../launcher/testservices/RunningService.java   |   84 +
 .../StoppingInStartLaunchableService.java       |   49 +
 .../StringConstructorOnlyService.java           |   39 +
 .../apache/hadoop/test/GenericTestUtils.java    |   34 +
 .../org/apache/hadoop/util/TestClassUtil.java   |    2 +-
 .../apache/hadoop/util/TestHostsFileReader.java |   19 +-
 .../util/bloom/BloomFilterCommonTester.java     |    4 +-
 .../hadoop/util/bloom/TestBloomFilters.java     |    4 +-
 .../src/test/resources/contract/rawlocal.xml    |   32 +-
 .../src/test/resources/testConf.xml             |    2 +-
 .../scripts/hadoop_finalize_catalina_opts.bats  |   56 -
 hadoop-common-project/hadoop-kms/pom.xml        |    8 +-
 .../hadoop/crypto/key/kms/server/KMS.java       |    2 +-
 .../crypto/key/kms/server/KMSWebServer.java     |    7 +-
 hadoop-common-project/hadoop-minikdc/pom.xml    |    4 +-
 hadoop-common-project/hadoop-nfs/pom.xml        |    4 +-
 .../org/apache/hadoop/mount/MountInterface.java |    2 +-
 .../apache/hadoop/nfs/nfs3/Nfs3Constant.java    |    2 +-
 .../hadoop/nfs/nfs3/request/SetAttr3.java       |    2 +-
 .../org/apache/hadoop/oncrpc/RpcMessage.java    |    2 +-
 .../org/apache/hadoop/portmap/TestPortmap.java  |    4 +-
 hadoop-common-project/pom.xml                   |    4 +-
 hadoop-dist/pom.xml                             |    4 +-
 hadoop-hdfs-project/hadoop-hdfs-client/pom.xml  |    4 +-
 .../java/org/apache/hadoop/hdfs/DFSClient.java  |   37 +-
 .../hadoop/hdfs/DFSClientFaultInjector.java     |    4 +
 .../org/apache/hadoop/hdfs/DFSOutputStream.java |   15 +-
 .../hadoop/hdfs/DFSStripedOutputStream.java     |   43 +-
 .../org/apache/hadoop/hdfs/DataStreamer.java    |  105 +-
 .../hadoop/hdfs/DistributedFileSystem.java      |   98 +-
 .../apache/hadoop/hdfs/StripedDataStreamer.java |   10 +-
 .../apache/hadoop/hdfs/client/HdfsAdmin.java    |   12 +
 .../hdfs/client/HdfsClientConfigKeys.java       |    4 +
 .../hdfs/protocol/AddECPolicyResponse.java      |   66 +
 .../hdfs/protocol/ClientDatanodeProtocol.java   |    5 +
 .../hadoop/hdfs/protocol/ClientProtocol.java    |   37 +-
 .../hdfs/protocol/DatanodeVolumeInfo.java       |  122 +
 .../hadoop/hdfs/protocol/DirectoryListing.java  |    4 +-
 .../hdfs/protocol/ErasureCodingPolicy.java      |   62 +-
 .../hadoop/hdfs/protocol/ExtendedBlock.java     |   16 +-
 .../hadoop/hdfs/protocol/HdfsConstants.java     |    2 +-
 .../hadoop/hdfs/protocol/HdfsFileStatus.java    |   22 +-
 .../hdfs/protocol/IllegalECPolicyException.java |   34 +
 .../datatransfer/DataTransferProtocol.java      |   19 +-
 .../hdfs/protocol/datatransfer/Sender.java      |   29 +-
 .../ClientDatanodeProtocolTranslatorPB.java     |   29 +
 .../ClientNamenodeProtocolTranslatorPB.java     |   48 +
 .../hadoop/hdfs/protocolPB/PBHelperClient.java  |   65 +-
 .../token/block/BlockTokenIdentifier.java       |   66 +-
 .../hdfs/server/protocol/SlowDiskReports.java   |    5 +-
 .../apache/hadoop/hdfs/util/ECPolicyLoader.java |  325 ++
 .../src/main/proto/ClientDatanodeProtocol.proto |    9 +
 .../src/main/proto/ClientNamenodeProtocol.proto |    7 +
 .../src/main/proto/datatransfer.proto           |    4 +
 .../src/main/proto/erasurecoding.proto          |   23 +
 .../src/main/proto/hdfs.proto                   |   21 +
 .../hdfs/protocol/TestErasureCodingPolicy.java  |   86 +
 .../hadoop/hdfs/util/TestECPolicyLoader.java    |  313 ++
 hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml  |   27 +-
 .../hadoop/fs/http/client/HttpFSFileSystem.java |    4 +-
 .../fs/http/server/HttpFSServerWebServer.java   |    7 +-
 .../org/apache/hadoop/lib/server/Server.java    |    2 +-
 .../hadoop/lib/server/ServerException.java      |    2 +-
 .../apache/hadoop/lib/lang/TestXException.java  |    2 +-
 .../org/apache/hadoop/lib/wsrs/TestParam.java   |    2 +-
 .../hadoop-hdfs-native-client/pom.xml           |    4 +-
 .../src/main/native/libhdfs-tests/expect.h      |   18 +
 .../libhdfs-tests/test_libhdfs_threaded.c       |   16 +
 .../src/main/native/libhdfs/exception.c         |   75 +-
 .../src/main/native/libhdfs/exception.h         |   16 +-
 .../src/main/native/libhdfs/hdfs.c              |   90 +
 .../src/main/native/libhdfs/include/hdfs/hdfs.h |   61 +
 .../src/main/native/libhdfs/jni_helper.c        |   95 +-
 .../src/main/native/libhdfs/jni_helper.h        |   35 +
 .../libhdfs/os/posix/thread_local_storage.c     |   54 +-
 .../native/libhdfs/os/thread_local_storage.h    |   61 +-
 .../libhdfs/os/windows/thread_local_storage.c   |   50 +-
 hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml     |    4 +-
 .../hadoop/hdfs/nfs/nfs3/OpenFileCtx.java       |    2 +-
 .../apache/hadoop/hdfs/nfs/nfs3/WriteCtx.java   |    2 +-
 .../hdfs/nfs/nfs3/TestRpcProgramNfs3.java       |    2 +-
 .../dev-support/findbugsExcludeFile.xml         |    5 +
 .../jdiff/Apache_Hadoop_HDFS_3.0.0-alpha3.xml   |  326 ++
 hadoop-hdfs-project/hadoop-hdfs/pom.xml         |    4 +-
 .../src/main/conf/user_ec_policies.xml.template |   72 +
 .../org/apache/hadoop/hdfs/DFSConfigKeys.java   |   19 +-
 .../hadoop/hdfs/net/DFSNetworkTopology.java     |   24 +-
 .../hadoop/hdfs/net/DFSTopologyNodeImpl.java    |  137 +
 .../hadoop/hdfs/protocol/LayoutVersion.java     |    2 +-
 .../hdfs/protocol/datatransfer/Receiver.java    |   20 +-
 ...tDatanodeProtocolServerSideTranslatorPB.java |   28 +
 ...tNamenodeProtocolServerSideTranslatorPB.java |   54 +-
 .../hdfs/qjournal/client/IPCLoggerChannel.java  |    2 +-
 .../qjournal/server/GetJournalEditServlet.java  |   16 +-
 .../hadoop/hdfs/qjournal/server/JNStorage.java  |   25 +-
 .../hadoop/hdfs/qjournal/server/Journal.java    |   21 +-
 .../hdfs/qjournal/server/JournalNode.java       |   25 +-
 .../hdfs/qjournal/server/JournalNodeSyncer.java |  154 +-
 .../block/BlockPoolTokenSecretManager.java      |   34 +-
 .../token/block/BlockTokenSecretManager.java    |   93 +-
 .../hadoop/hdfs/server/balancer/Dispatcher.java |   47 +-
 .../hadoop/hdfs/server/balancer/KeyManager.java |   41 +-
 .../hdfs/server/balancer/NameNodeConnector.java |   16 +-
 .../hdfs/server/blockmanagement/BlockInfo.java  |   18 +-
 .../server/blockmanagement/BlockManager.java    |  248 +-
 .../BlockPlacementPolicyDefault.java            |   40 +-
 .../BlockUnderConstructionFeature.java          |    9 +-
 .../blockmanagement/CorruptReplicasMap.java     |    2 +-
 .../blockmanagement/DatanodeDescriptor.java     |   68 +-
 .../server/blockmanagement/DatanodeManager.java |   36 +-
 .../blockmanagement/DatanodeStorageInfo.java    |    9 +-
 .../blockmanagement/InvalidateBlocks.java       |   17 +-
 .../blockmanagement/LowRedundancyBlocks.java    |    6 +-
 .../hdfs/server/common/HdfsServerConstants.java |    7 +-
 .../apache/hadoop/hdfs/server/common/Util.java  |   12 +-
 .../hdfs/server/datanode/BPOfferService.java    |    3 +-
 .../hdfs/server/datanode/BPServiceActor.java    |    5 +-
 .../hdfs/server/datanode/BlockReceiver.java     |   25 +-
 .../hadoop/hdfs/server/datanode/DNConf.java     |    7 +-
 .../hadoop/hdfs/server/datanode/DataNode.java   |   80 +-
 .../server/datanode/DataNodeFaultInjector.java  |    6 +-
 .../server/datanode/DataNodeLayoutVersion.java  |    2 +-
 .../hdfs/server/datanode/DataStorage.java       |   22 +-
 .../hdfs/server/datanode/DataXceiver.java       |  113 +-
 .../hdfs/server/datanode/FileIoProvider.java    |   23 +-
 .../hdfs/server/datanode/LocalReplica.java      |   13 +
 .../server/datanode/ProfilingFileIoEvents.java  |   19 +-
 .../erasurecode/ErasureCodingWorker.java        |    3 +-
 .../erasurecode/StripedBlockReader.java         |    4 +-
 .../erasurecode/StripedBlockWriter.java         |   11 +-
 .../erasurecode/StripedReconstructionInfo.java  |   16 +-
 .../datanode/erasurecode/StripedWriter.java     |    5 +-
 .../AvailableSpaceVolumeChoosingPolicy.java     |   20 +-
 .../server/datanode/fsdataset/FsDatasetSpi.java |   12 +-
 .../RoundRobinVolumeChoosingPolicy.java         |    2 +-
 .../fsdataset/VolumeChoosingPolicy.java         |    5 +-
 .../impl/FsDatasetAsyncDiskService.java         |   14 +-
 .../datanode/fsdataset/impl/FsDatasetImpl.java  |  137 +-
 .../datanode/fsdataset/impl/FsVolumeList.java   |   19 +-
 .../datanode/web/SimpleHttpProxyHandler.java    |    4 +-
 .../hdfs/server/namenode/AclTransformation.java |    7 +-
 .../hdfs/server/namenode/BackupImage.java       |    2 +-
 .../hdfs/server/namenode/CacheManager.java      |    2 +-
 .../hadoop/hdfs/server/namenode/Content.java    |    2 +-
 .../ContentSummaryComputationContext.java       |   94 +-
 .../namenode/ErasureCodingPolicyManager.java    |  143 +-
 .../hadoop/hdfs/server/namenode/FSDirAclOp.java |   13 +-
 .../hdfs/server/namenode/FSDirAttrOp.java       |   14 +-
 .../hdfs/server/namenode/FSDirConcatOp.java     |    4 +-
 .../server/namenode/FSDirEncryptionZoneOp.java  |   10 +-
 .../server/namenode/FSDirErasureCodingOp.java   |   39 +-
 .../hdfs/server/namenode/FSDirMkdirOp.java      |    4 +-
 .../hdfs/server/namenode/FSDirRenameOp.java     |    8 +-
 .../hdfs/server/namenode/FSDirSnapshotOp.java   |    7 +-
 .../hdfs/server/namenode/FSDirSymlinkOp.java    |    4 +-
 .../hdfs/server/namenode/FSDirTruncateOp.java   |   10 +-
 .../hdfs/server/namenode/FSDirWriteFileOp.java  |   31 +-
 .../hdfs/server/namenode/FSDirXAttrOp.java      |   11 +-
 .../hdfs/server/namenode/FSDirectory.java       |   43 +-
 .../hdfs/server/namenode/FSEditLogLoader.java   |    8 +-
 .../hadoop/hdfs/server/namenode/FSImage.java    |   52 +-
 .../server/namenode/FSImageFormatPBINode.java   |    3 +-
 .../hdfs/server/namenode/FSNamesystem.java      |  153 +-
 .../hdfs/server/namenode/FSNamesystemLock.java  |  121 +-
 .../server/namenode/FileJournalManager.java     |   18 +-
 .../hadoop/hdfs/server/namenode/INode.java      |    1 -
 .../hdfs/server/namenode/INodeDirectory.java    |   19 +-
 .../hadoop/hdfs/server/namenode/INodeFile.java  |    8 +-
 .../hdfs/server/namenode/INodeReference.java    |    2 -
 .../hdfs/server/namenode/INodeSymlink.java      |    1 -
 .../namenode/INodeWithAdditionalFields.java     |    2 +-
 .../hdfs/server/namenode/INodesInPath.java      |   78 +-
 .../hadoop/hdfs/server/namenode/JournalSet.java |   33 +-
 .../hdfs/server/namenode/LeaseManager.java      |  118 +-
 .../namenode/NNStorageRetentionManager.java     |   27 +-
 .../hadoop/hdfs/server/namenode/NameNode.java   |    2 +-
 .../server/namenode/NameNodeLayoutVersion.java  |    2 +-
 .../hdfs/server/namenode/NameNodeRpcServer.java |   18 +-
 .../server/namenode/ha/StandbyCheckpointer.java |    7 +-
 .../snapshot/DirectorySnapshottableFeature.java |   24 +-
 .../snapshot/DirectoryWithSnapshotFeature.java  |   19 +-
 .../hdfs/server/namenode/snapshot/Snapshot.java |    1 -
 .../namenode/snapshot/SnapshotManager.java      |   24 +-
 .../protocol/ReceivedDeletedBlockInfo.java      |    2 +-
 .../org/apache/hadoop/hdfs/tools/DFSAdmin.java  |  122 +-
 .../org/apache/hadoop/hdfs/tools/ECAdmin.java   |  122 +-
 .../OfflineEditsXmlLoader.java                  |    2 +-
 .../offlineImageViewer/ImageLoaderCurrent.java  |   10 +-
 .../java/org/apache/hadoop/hdfs/util/Diff.java  |    2 +-
 .../src/main/resources/hdfs-default.xml         |   51 +-
 .../src/main/webapps/hdfs/dfshealth.html        |    2 +-
 .../src/main/webapps/hdfs/dfshealth.js          |    4 +-
 .../src/site/markdown/HDFSCommands.md           |    6 +-
 .../src/site/markdown/HDFSErasureCoding.md      |   34 +-
 .../hadoop-hdfs/src/site/markdown/HdfsDesign.md |    2 +-
 .../hadoop-hdfs/src/site/markdown/ViewFs.md     |    2 +-
 .../org/apache/hadoop/TestRefreshCallQueue.java |   88 +-
 .../hadoop/fs/TestSymlinkHdfsFileContext.java   |    2 +-
 .../hadoop/fs/TestSymlinkHdfsFileSystem.java    |    4 +-
 .../org/apache/hadoop/hdfs/DFSTestUtil.java     |   14 +-
 .../apache/hadoop/hdfs/StripedFileTestUtil.java |   12 +
 .../hadoop/hdfs/TestBlockStoragePolicy.java     |  143 +
 .../TestClientProtocolForPipelineRecovery.java  |   92 +
 .../apache/hadoop/hdfs/TestDFSOutputStream.java |   25 +
 .../TestDFSRSDefault10x4StripedInputStream.java |   34 -
 ...TestDFSRSDefault10x4StripedOutputStream.java |   35 -
 ...fault10x4StripedOutputStreamWithFailure.java |   35 -
 .../org/apache/hadoop/hdfs/TestDFSShell.java    |    2 +-
 .../hadoop/hdfs/TestDFSStripedInputStream.java  |    2 +-
 ...DFSStripedInputStreamWithRandomECPolicy.java |   45 +
 .../hadoop/hdfs/TestDFSStripedOutputStream.java |   37 +-
 .../TestDFSStripedOutputStreamWithFailure.java  |    2 +-
 ...tputStreamWithFailureWithRandomECPolicy.java |   45 +
 ...FSStripedOutputStreamWithRandomECPolicy.java |   45 +
 .../org/apache/hadoop/hdfs/TestDFSUtil.java     |    2 +-
 .../hdfs/TestDFSXORStripedInputStream.java      |   32 -
 .../hdfs/TestDFSXORStripedOutputStream.java     |   34 -
 ...estDFSXORStripedOutputStreamWithFailure.java |   35 -
 .../hadoop/hdfs/TestDataTransferProtocol.java   |    3 +-
 .../apache/hadoop/hdfs/TestEncryptionZones.java |   10 +-
 .../hadoop/hdfs/TestErasureCodingPolicies.java  |  129 +
 .../apache/hadoop/hdfs/TestFileCorruption.java  |    1 +
 .../apache/hadoop/hdfs/TestFileCreation.java    |    2 +-
 .../org/apache/hadoop/hdfs/TestGetBlocks.java   |   84 +-
 .../hadoop/hdfs/TestHDFSFileSystemContract.java |   12 +-
 .../org/apache/hadoop/hdfs/TestHdfsAdmin.java   |   33 +
 .../hadoop/hdfs/TestMaintenanceState.java       |  222 +-
 .../hadoop/hdfs/TestReconstructStripedFile.java |    2 +-
 .../TestUnsetAndChangeDirectoryEcPolicy.java    |    2 +-
 .../hdfs/TestWriteBlockGetsBlockLengthHint.java |    6 +-
 .../apache/hadoop/hdfs/UpgradeUtilities.java    |    4 +-
 .../net/TestDFSNetworkTopologyPerformance.java  |  524 ++
 .../hadoop/hdfs/protocolPB/TestPBHelper.java    |   37 +-
 .../hdfs/qjournal/TestJournalNodeSync.java      |    1 +
 .../qjournal/client/DirectExecutorService.java  |  154 +
 .../hdfs/qjournal/client/TestQJMWithFaults.java |    3 +-
 .../client/TestQuorumJournalManager.java        |    3 +-
 .../security/token/block/TestBlockToken.java    |  186 +-
 .../hdfs/server/balancer/TestBalancer.java      |  108 +-
 .../server/balancer/TestBalancerRPCDelay.java   |   32 +
 .../hdfs/server/balancer/TestKeyManager.java    |   87 +
 .../blockmanagement/BlockManagerTestUtil.java   |   21 +
 .../blockmanagement/TestBlockManager.java       |   73 +-
 .../TestBlockUnderConstructionFeature.java      |    8 +-
 .../blockmanagement/TestSlowDiskTracker.java    |    4 +-
 .../server/datanode/BlockReportTestBase.java    |    2 +-
 .../server/datanode/SimulatedFSDataset.java     |   24 +-
 .../hdfs/server/datanode/TestBlockRecovery.java |    6 +-
 .../server/datanode/TestBlockReplacement.java   |    2 +-
 .../datanode/TestDataNodeHotSwapVolumes.java    |   40 +-
 .../server/datanode/TestDataNodeMXBean.java     |    4 +-
 .../server/datanode/TestDataNodeMetrics.java    |    7 +-
 .../datanode/TestDataNodeVolumeMetrics.java     |    4 +-
 .../TestDataXceiverLazyPersistHint.java         |    4 +-
 .../hdfs/server/datanode/TestDiskError.java     |    5 +-
 .../server/datanode/TestFsDatasetCache.java     |   62 +-
 .../server/datanode/TestSimulatedFSDataset.java |    8 +-
 .../extdataset/ExternalDatasetImpl.java         |   14 +-
 .../TestAvailableSpaceVolumeChoosingPolicy.java |   76 +-
 .../TestRoundRobinVolumeChoosingPolicy.java     |   29 +-
 .../fsdataset/impl/TestFsDatasetImpl.java       |    6 +-
 .../fsdataset/impl/TestFsVolumeList.java        |    2 +-
 .../fsdataset/impl/TestSpaceReservation.java    |   67 +-
 .../fsdataset/impl/TestWriteToReplica.java      |   33 +-
 .../TestDataNodeOutlierDetectionViaMetrics.java |    8 +
 .../hdfs/server/namenode/NameNodeAdapter.java   |   31 +-
 .../namenode/TestBlockUnderConstruction.java    |   45 +
 .../hdfs/server/namenode/TestCheckpoint.java    |    4 +-
 .../TestCommitBlockSynchronization.java         |    2 +-
 .../namenode/TestDecommissioningStatus.java     |  113 +-
 .../TestDefaultBlockPlacementPolicy.java        |   46 +
 .../server/namenode/TestEnabledECPolicies.java  |   11 +-
 .../server/namenode/TestFSNamesystemLock.java   |   16 +-
 .../hadoop/hdfs/server/namenode/TestFsck.java   |   32 +
 .../hdfs/server/namenode/TestLeaseManager.java  |  286 +-
 .../namenode/TestMetadataVersionOutput.java     |    4 +-
 .../namenode/TestNameNodeStatusMXBean.java      |    5 +-
 .../namenode/TestNamenodeStorageDirectives.java |  330 ++
 .../namenode/TestReconstructStripedBlocks.java  |    2 +
 .../hdfs/server/namenode/TestStartup.java       |    5 +-
 .../namenode/ha/TestFailureToReadEdits.java     |    2 +-
 .../namenode/ha/TestStandbyCheckpoints.java     |   60 +
 .../namenode/metrics/TestNameNodeMetrics.java   |    2 +-
 .../snapshot/TestOpenFilesWithSnapshot.java     |  298 ++
 .../snapshot/TestRenameWithSnapshots.java       |  201 -
 .../namenode/snapshot/TestSnapshotDeletion.java |   75 -
 .../snapshot/TestSnapshotDiffReport.java        |  188 +-
 .../namenode/snapshot/TestSnapshotManager.java  |   11 +-
 .../shortcircuit/TestShortCircuitCache.java     |    8 +-
 .../apache/hadoop/hdfs/tools/TestDFSAdmin.java  |   20 +
 .../hdfs/web/TestWebHdfsFileSystemContract.java |   18 +-
 .../hadoop/hdfs/web/TestWebHdfsTokens.java      |    8 +-
 .../hadoop/tools/TestHdfsConfigFields.java      |    2 -
 .../java/org/apache/hadoop/tools/TestTools.java |    5 +-
 .../test/resources/testErasureCodingConf.xml    |   97 +
 .../src/test/resources/testHDFSConf.xml         |    2 +-
 .../src/test/resources/test_ec_policies.xml     |   65 +
 hadoop-hdfs-project/pom.xml                     |    4 +-
 hadoop-mapreduce-project/.gitignore             |    1 -
 hadoop-mapreduce-project/conf/mapred-site.xml   |   21 +
 .../conf/mapred-site.xml.template               |   21 -
 .../hadoop-mapreduce-client-app/pom.xml         |    4 +-
 .../hadoop/mapred/LocalContainerLauncher.java   |   46 +-
 .../hadoop/mapreduce/v2/app/MRAppMaster.java    |   16 +-
 .../v2/app/launcher/ContainerLauncherImpl.java  |    2 +-
 .../v2/app/job/impl/TestShuffleProvider.java    |    1 -
 .../app/webapp/TestAMWebServicesAttempts.java   |    1 -
 .../hadoop-mapreduce-client-common/pom.xml      |    4 +-
 .../apache/hadoop/mapreduce/v2/util/MRApps.java |    8 +-
 .../mapred/TestMRWithDistributedCache.java      |    2 +-
 .../hadoop-mapreduce-client-core/pom.xml        |    4 +-
 .../org/apache/hadoop/mapred/AuditLogger.java   |  154 -
 .../apache/hadoop/mapred/FileInputFormat.java   |    2 +-
 .../apache/hadoop/mapred/FileOutputFormat.java  |    2 +-
 .../java/org/apache/hadoop/mapred/JVMId.java    |    2 +-
 .../org/apache/hadoop/mapred/JobClient.java     |    2 +-
 .../java/org/apache/hadoop/mapred/JobConf.java  |    7 +-
 .../apache/hadoop/mapred/JobContextImpl.java    |    3 +-
 .../org/apache/hadoop/mapred/JobInProgress.java |    2 +-
 .../org/apache/hadoop/mapred/JobTracker.java    |    2 +-
 .../org/apache/hadoop/mapred/Operation.java     |   14 +-
 .../java/org/apache/hadoop/mapred/Task.java     |    2 +-
 .../hadoop/mapred/TaskAttemptContextImpl.java   |    3 +-
 .../java/org/apache/hadoop/mapred/TaskLog.java  |    2 +-
 .../org/apache/hadoop/mapred/TaskStatus.java    |    4 +-
 .../hadoop/mapred/pipes/BinaryProtocol.java     |    2 +-
 .../org/apache/hadoop/mapreduce/Cluster.java    |    2 +-
 .../java/org/apache/hadoop/mapreduce/Job.java   |    4 +-
 .../org/apache/hadoop/mapreduce/JobStatus.java  |    2 +-
 .../apache/hadoop/mapreduce/MRJobConfig.java    |    6 +
 .../mapreduce/lib/input/FileInputFormat.java    |    2 +-
 .../mapreduce/lib/jobcontrol/ControlledJob.java |    2 +-
 .../mapreduce/lib/jobcontrol/JobControl.java    |    2 +-
 .../mapreduce/lib/output/FileOutputFormat.java  |    2 +-
 .../hadoop/mapreduce/task/reduce/Fetcher.java   |    2 +-
 .../hadoop/mapreduce/task/reduce/MapHost.java   |    2 +-
 .../org/apache/hadoop/mapred/TestClock.java     |    2 +-
 .../apache/hadoop/mapred/TestClusterStatus.java |    4 +-
 .../org/apache/hadoop/mapred/TestJobConf.java   |    2 +-
 .../hadoop-mapreduce-client-hs-plugins/pom.xml  |    4 +-
 .../hadoop-mapreduce-client-hs/pom.xml          |    4 +-
 .../hadoop/mapreduce/v2/hs/HSAuditLogger.java   |    2 +-
 .../mapreduce/v2/hs/HistoryFileManager.java     |    7 +-
 .../mapreduce/v2/hs/JobHistoryServer.java       |    3 -
 .../hadoop-mapreduce-client-jobclient/pom.xml   |    4 +-
 .../org/apache/hadoop/mapred/YARNRunner.java    |  132 +-
 .../org/apache/hadoop/RandomTextWriterJob.java  |    2 +-
 .../org/apache/hadoop/fs/JHLogAnalyzer.java     |    2 +-
 .../java/org/apache/hadoop/fs/TestDFSIO.java    |    4 +-
 .../apache/hadoop/fs/slive/OperationOutput.java |    2 +-
 .../org/apache/hadoop/fs/slive/PathFinder.java  |    2 +-
 .../apache/hadoop/hdfs/NNBenchWithoutMR.java    |    6 +-
 .../hadoop/mapred/GenericMRLoadGenerator.java   |    2 +-
 .../java/org/apache/hadoop/mapred/MRBench.java  |    2 +-
 .../apache/hadoop/mapred/TestAuditLogger.java   |  161 -
 .../mapred/TestMRTimelineEventHandling.java     |    2 +-
 .../apache/hadoop/mapred/TestYARNRunner.java    |  167 +
 .../hadoop/mapred/ThreadedMapBenchmark.java     |    2 +-
 .../mapreduce/GenericMRLoadGenerator.java       |    2 +-
 .../hadoop/mapreduce/GrowingSleepJob.java       |   68 +
 .../apache/hadoop/mapreduce/LargeSorter.java    |    2 +-
 .../hadoop/mapreduce/RandomTextWriter.java      |    2 +-
 .../apache/hadoop/mapreduce/RandomWriter.java   |    2 +-
 .../mapreduce/TimelineServicePerformance.java   |    2 +-
 .../mapreduce/security/TestMRCredentials.java   |    3 +-
 .../hadoop/mapreduce/v2/MiniMRYarnCluster.java  |    6 +-
 .../apache/hadoop/test/MapredTestDriver.java    |    3 +
 .../hadoop-mapreduce-client-nativetask/pom.xml  |    4 +-
 .../src/CMakeLists.txt                          |    2 +-
 .../hadoop-mapreduce-client-shuffle/pom.xml     |    4 +-
 .../hadoop-mapreduce-client/pom.xml             |    4 +-
 .../hadoop-mapreduce-examples/pom.xml           |    4 +-
 .../hadoop/examples/RandomTextWriter.java       |    2 +-
 .../apache/hadoop/examples/RandomWriter.java    |    2 +-
 .../org/apache/hadoop/examples/WordMean.java    |    4 +-
 .../org/apache/hadoop/examples/WordMedian.java  |    4 +-
 .../hadoop/examples/WordStandardDeviation.java  |    4 +-
 .../org/apache/hadoop/examples/pi/Parser.java   |    8 +-
 .../hadoop/examples/terasort/TeraGen.java       |    2 +-
 hadoop-mapreduce-project/pom.xml                |    4 +-
 hadoop-maven-plugins/pom.xml                    |    2 +-
 .../resource/ServicesResourceTransformer.java   |    4 +-
 hadoop-minicluster/pom.xml                      |    4 +-
 hadoop-project-dist/pom.xml                     |   24 +-
 hadoop-project/pom.xml                          |   36 +-
 hadoop-tools/hadoop-aliyun/pom.xml              |   10 +-
 .../oss/TestAliyunOSSFileSystemContract.java    |  142 +-
 hadoop-tools/hadoop-archive-logs/pom.xml        |    9 +-
 hadoop-tools/hadoop-archives/pom.xml            |    4 +-
 hadoop-tools/hadoop-aws/pom.xml                 |   22 +-
 .../org/apache/hadoop/fs/s3a/Constants.java     |   29 +-
 .../hadoop/fs/s3a/S3AEncryptionMethods.java     |   13 +-
 .../org/apache/hadoop/fs/s3a/S3AFileSystem.java |   93 +-
 .../java/org/apache/hadoop/fs/s3a/S3AUtils.java |  120 +-
 .../src/site/markdown/tools/hadoop-aws/index.md |  111 +-
 .../site/markdown/tools/hadoop-aws/testing.md   |    4 +-
 .../fs/s3a/AbstractTestS3AEncryption.java       |    4 +
 .../fs/s3a/ITestS3ACopyFromLocalFile.java       |  154 +
 .../ITestS3AEncryptionAlgorithmValidation.java  |    4 +-
 .../hadoop/fs/s3a/ITestS3AEncryptionSSEC.java   |  308 +-
 .../hadoop/fs/s3a/ITestS3AFailureHandling.java  |   51 +
 .../fs/s3a/ITestS3AFileSystemContract.java      |   21 +-
 .../hadoop/fs/s3a/TestSSEConfiguration.java     |  223 +
 .../NativeS3FileSystemContractBaseTest.java     |   26 +-
 hadoop-tools/hadoop-azure-datalake/pom.xml      |   12 +-
 .../src/site/markdown/index.md                  |    4 +-
 .../fs/adl/live/AdlStorageConfiguration.java    |    8 +-
 .../adl/live/TestAdlFileSystemContractLive.java |   25 +-
 hadoop-tools/hadoop-azure/pom.xml               |   12 +-
 .../hadoop/fs/azure/NativeAzureFileSystem.java  |  195 +-
 .../fs/azure/RemoteSASKeyGeneratorImpl.java     |   26 +-
 .../fs/azure/RemoteWasbAuthorizerImpl.java      |   45 +-
 .../fs/azure/SyncableDataOutputStream.java      |   12 +-
 .../fs/azure/WasbAuthorizationOperations.java   |    2 -
 .../fs/azure/WasbAuthorizerInterface.java       |    3 +-
 .../fs/azure/security/WasbTokenRenewer.java     |    6 -
 .../hadoop-azure/src/site/markdown/index.md     |   90 +-
 .../hadoop/fs/azure/AbstractWasbTestBase.java   |    7 +-
 .../hadoop/fs/azure/MockWasbAuthorizerImpl.java |   73 +-
 .../TestNativeAzureFileSystemAuthorization.java |  576 +-
 ...veAzureFileSystemAuthorizationWithOwner.java |  122 +
 ...stNativeAzureFileSystemContractEmulator.java |   20 +-
 .../TestNativeAzureFileSystemContractLive.java  |   32 +-
 ...TestNativeAzureFileSystemContractMocked.java |   11 +-
 ...tiveAzureFileSystemContractPageBlobLive.java |   25 +-
 .../fs/azure/TestWasbRemoteCallHelper.java      |   18 +-
 .../TestAzureFileSystemInstrumentation.java     |    3 +-
 hadoop-tools/hadoop-datajoin/pom.xml            |    4 +-
 hadoop-tools/hadoop-distcp/pom.xml              |    4 +-
 .../apache/hadoop/tools/DistCpConstants.java    |   10 +-
 .../org/apache/hadoop/tools/DistCpContext.java  |    4 +
 .../apache/hadoop/tools/DistCpOptionSwitch.java |    8 +
 .../org/apache/hadoop/tools/DistCpOptions.java  |   23 +-
 .../org/apache/hadoop/tools/OptionsParser.java  |   12 +
 .../tools/mapred/RetriableFileCopyCommand.java  |   11 +-
 .../src/site/markdown/DistCp.md.vm              |    1 +
 .../apache/hadoop/tools/TestDistCpOptions.java  |   19 +-
 hadoop-tools/hadoop-extras/pom.xml              |    4 +-
 hadoop-tools/hadoop-gridmix/pom.xml             |    4 +-
 .../hadoop/mapred/gridmix/InputStriper.java     |    6 +-
 .../TotalHeapUsageEmulatorPlugin.java           |   16 +-
 .../hadoop/mapred/gridmix/GridmixTestUtils.java |   31 +-
 .../gridmix/TestGridmixMemoryEmulation.java     |    2 +-
 .../mapred/gridmix/TestGridmixSubmission.java   |    3 +-
 hadoop-tools/hadoop-kafka/pom.xml               |   12 +-
 .../hadoop/metrics2/impl/TestKafkaMetrics.java  |   10 +-
 hadoop-tools/hadoop-openstack/pom.xml           |   19 +-
 .../SwiftAuthenticationFailedException.java     |    6 +-
 .../exceptions/SwiftBadRequestException.java    |    6 +-
 .../SwiftInvalidResponseException.java          |    9 +-
 .../SwiftThrottledRequestException.java         |    6 +-
 .../apache/hadoop/fs/swift/http/CopyMethod.java |   41 -
 .../hadoop/fs/swift/http/CopyRequest.java       |   41 +
 .../swift/http/HttpInputStreamWithRelease.java  |   25 +-
 .../hadoop/fs/swift/http/SwiftRestClient.java   |  805 +--
 .../snative/SwiftNativeFileSystemStore.java     |   11 +-
 .../hadoop/fs/swift/util/HttpResponseUtils.java |  121 +
 .../fs/swift/TestSwiftFileSystemContract.java   |   19 +-
 .../TestSwiftFileSystemPartitionedUploads.java  |    2 +-
 .../fs/swift/http/TestSwiftRestClient.java      |    2 +-
 .../src/test/resources/log4j.properties         |    3 -
 hadoop-tools/hadoop-pipes/pom.xml               |    4 +-
 hadoop-tools/hadoop-rumen/pom.xml               |    4 +-
 .../org/apache/hadoop/tools/rumen/TaskInfo.java |   29 +-
 .../apache/hadoop/tools/rumen/ZombieJob.java    |    9 +-
 .../util/MapReduceJobPropertiesParser.java      |   10 +-
 hadoop-tools/hadoop-sls/pom.xml                 |   23 +-
 hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh  |   46 +-
 .../src/main/data/2jobs2min-rumen-jh.json       |  587 +--
 .../hadoop/yarn/sls/ReservationClientUtil.java  |   78 +
 .../org/apache/hadoop/yarn/sls/SLSRunner.java   |  885 +++-
 .../hadoop/yarn/sls/appmaster/AMSimulator.java  |  131 +-
 .../yarn/sls/appmaster/MRAMSimulator.java       |    9 +-
 .../sls/resourcemanager/MockAMLauncher.java     |    5 +
 .../sls/scheduler/FairSchedulerMetrics.java     |   20 +-
 .../sls/scheduler/SLSCapacityScheduler.java     |   33 +-
 .../yarn/sls/scheduler/SLSFairScheduler.java    |   31 +-
 .../yarn/sls/scheduler/SchedulerWrapper.java    |    4 +
 .../hadoop/yarn/sls/scheduler/TaskRunner.java   |    9 +-
 .../hadoop/yarn/sls/synthetic/SynthJob.java     |  306 ++
 .../yarn/sls/synthetic/SynthJobClass.java       |  180 +
 .../sls/synthetic/SynthTraceJobProducer.java    |  316 ++
 .../hadoop/yarn/sls/synthetic/SynthUtils.java   |  101 +
 .../yarn/sls/synthetic/SynthWorkload.java       |  121 +
 .../hadoop/yarn/sls/synthetic/package-info.java |   22 +
 .../apache/hadoop/yarn/sls/utils/SLSUtils.java  |   52 +-
 .../apache/hadoop/yarn/sls/web/SLSWebApp.java   |   37 +-
 .../src/site/markdown/SchedulerLoadSimulator.md |  150 +-
 .../hadoop/yarn/sls/BaseSLSRunnerTest.java      |  120 +
 .../sls/TestReservationSystemInvariants.java    |   73 +
 .../apache/hadoop/yarn/sls/TestSLSRunner.java   |   90 +-
 .../hadoop/yarn/sls/TestSynthJobGeneration.java |   96 +
 .../yarn/sls/appmaster/TestAMSimulator.java     |    2 +-
 .../yarn/sls/scheduler/TestTaskRunner.java      |    2 +-
 .../hadoop/yarn/sls/utils/TestSLSUtils.java     |   30 +
 .../hadoop/yarn/sls/web/TestSLSWebApp.java      |   28 +-
 .../src/test/resources/capacity-scheduler.xml   |   10 +
 .../src/test/resources/fair-scheduler.xml       |    8 +-
 .../hadoop-sls/src/test/resources/inputsls.json |   55 +
 .../hadoop-sls/src/test/resources/nodes.json    |   84 +
 .../src/test/resources/sls-runner.xml           |    6 +-
 .../hadoop-sls/src/test/resources/syn.json      |   53 +
 .../hadoop-sls/src/test/resources/yarn-site.xml |   10 +-
 hadoop-tools/hadoop-streaming/pom.xml           |    4 +-
 hadoop-tools/hadoop-tools-dist/pom.xml          |    4 +-
 hadoop-tools/pom.xml                            |    4 +-
 .../hadoop-yarn/hadoop-yarn-api/pom.xml         |    4 +-
 .../yarn/api/ContainerManagementProtocol.java   |    8 +-
 .../yarn/api/records/QueueConfigurations.java   |  150 +
 .../hadoop/yarn/api/records/QueueInfo.java      |   42 +
 .../records/timelineservice/TimelineMetric.java |    2 +-
 .../hadoop/yarn/conf/YarnConfiguration.java     |    8 +
 .../yarn/exceptions/ConfigurationException.java |   45 +
 .../src/main/proto/yarn_protos.proto            |   14 +
 .../pom.xml                                     |    4 +-
 .../distributedshell/ApplicationMaster.java     |    4 +-
 .../pom.xml                                     |    4 +-
 .../hadoop-yarn-applications/pom.xml            |    4 +-
 .../hadoop-yarn/hadoop-yarn-client/pom.xml      |    4 +-
 .../apache/hadoop/yarn/client/api/NMClient.java |   86 +-
 .../yarn/client/api/async/NMClientAsync.java    |   98 +-
 .../api/async/impl/NMClientAsyncImpl.java       |  264 +-
 .../yarn/client/api/impl/NMClientImpl.java      |   91 +
 .../yarn/client/api/impl/YarnClientImpl.java    |   10 +-
 .../apache/hadoop/yarn/client/cli/LogsCLI.java  |   58 +-
 .../hadoop/yarn/client/ProtocolHATestBase.java  |    6 +-
 .../hadoop/yarn/client/TestRMFailover.java      |    5 +-
 .../api/async/impl/TestNMClientAsync.java       |  359 +-
 .../yarn/client/api/impl/TestAMRMClient.java    |    2 +-
 .../yarn/client/api/impl/TestNMClient.java      |  141 +
 .../hadoop/yarn/client/cli/TestLogsCLI.java     |    2 +-
 .../hadoop/yarn/client/cli/TestYarnCLI.java     |    4 +-
 .../hadoop-yarn/hadoop-yarn-common/pom.xml      |    4 +-
 .../pb/ApplicationSubmissionContextPBImpl.java  |    3 +-
 .../impl/pb/QueueConfigurationsPBImpl.java      |  137 +
 .../api/records/impl/pb/QueueInfoPBImpl.java    |   98 +-
 .../hadoop/yarn/client/ClientRMProxy.java       |    4 +-
 .../ConfiguredRMFailoverProxyProvider.java      |    2 +-
 .../org/apache/hadoop/yarn/client/RMProxy.java  |   45 +-
 .../RequestHedgingRMFailoverProxyProvider.java  |    2 +-
 .../apache/hadoop/yarn/client/ServerProxy.java  |    5 +
 .../yarn/client/api/impl/TimelineConnector.java |    9 +-
 .../hadoop/yarn/event/AsyncDispatcher.java      |   12 +-
 .../apache/hadoop/yarn/event/Dispatcher.java    |   10 -
 .../hadoop/yarn/event/EventDispatcher.java      |   17 +-
 .../logaggregation/AggregatedLogFormat.java     |    4 +
 .../yarn/util/ProcfsBasedProcessTree.java       |   37 +-
 .../util/ResourceCalculatorProcessTree.java     |   54 -
 .../yarn/util/WindowsBasedProcessTree.java      |   12 -
 .../resource/DefaultResourceCalculator.java     |    5 +
 .../resource/DominantResourceCalculator.java    |    5 +
 .../yarn/util/resource/ResourceCalculator.java  |    9 +
 .../hadoop/yarn/util/resource/Resources.java    |    5 +
 .../org/apache/hadoop/yarn/webapp/WebApp.java   |    6 +-
 .../hadoop/yarn/webapp/util/WebAppUtils.java    |   17 +-
 .../src/main/resources/yarn-default.xml         |   22 +-
 .../hadoop/yarn/api/TestPBImplRecords.java      |   21 +-
 .../hadoop/yarn/client/TestClientRMProxy.java   |  109 +
 .../client/api/impl/TestTimelineClient.java     |    2 +
 .../hadoop/yarn/event/DrainDispatcher.java      |   11 +-
 .../yarn/util/TestProcfsBasedProcessTree.java   |   72 +-
 .../yarn/util/TestWindowsBasedProcessTree.java  |   12 -
 .../hadoop-yarn/hadoop-yarn-registry/pom.xml    |    4 +-
 .../pom.xml                                     |   16 +-
 ...pplicationHistoryManagerOnTimelineStore.java |   37 +-
 .../timeline/RollingLevelDBTimelineStore.java   |   29 +-
 .../TimelineDelegationTokenIdentifierData.java  |    4 +-
 ...pplicationHistoryManagerOnTimelineStore.java |   32 +-
 .../hadoop-yarn-server-common/pom.xml           |    4 +-
 .../hadoop/yarn/server/api/ServerRMProxy.java   |    4 +-
 .../hadoop-yarn-server-nodemanager/pom.xml      |    4 +-
 .../nodemanager/CMgrCompletedAppsEvent.java     |    2 +-
 .../CMgrCompletedContainersEvent.java           |    2 +-
 .../server/nodemanager/ContainerExecutor.java   |    4 +-
 .../nodemanager/DefaultContainerExecutor.java   |    7 +-
 .../server/nodemanager/DeletionService.java     |  468 +-
 .../nodemanager/LinuxContainerExecutor.java     |  171 +-
 .../yarn/server/nodemanager/NMAuditLogger.java  |    2 +-
 .../nodemanager/NodeHealthCheckerService.java   |   54 +-
 .../yarn/server/nodemanager/NodeManager.java    |    5 -
 .../server/nodemanager/NodeStatusUpdater.java   |    6 +
 .../nodemanager/NodeStatusUpdaterImpl.java      |  398 +-
 .../nodemanager/amrmproxy/AMRMProxyService.java |   63 +-
 .../nodemanager/api/impl/pb/NMProtoUtils.java   |  110 +
 .../nodemanager/api/impl/pb/package-info.java   |   25 +
 .../containermanager/ContainerLocalization.java |   21 -
 .../ContainerLocalizationImpl.java              |   29 -
 .../containermanager/ContainerManager.java      |    2 -
 .../containermanager/ContainerManagerImpl.java  |  105 +-
 .../application/ApplicationImpl.java            |   27 +-
 .../container/ContainerImpl.java                |    9 +-
 .../recovery/DeletionTaskRecoveryInfo.java      |   73 +
 .../deletion/recovery/package-info.java         |   25 +
 .../deletion/task/DeletionTask.java             |  258 +
 .../deletion/task/DeletionTaskType.java         |   24 +
 .../deletion/task/FileDeletionTask.java         |  202 +
 .../deletion/task/package-info.java             |   25 +
 .../launcher/ContainerLaunch.java               |  270 +-
 .../launcher/ContainerRelaunch.java             |    9 +
 .../linux/resources/CGroupsHandler.java         |    6 +-
 .../linux/resources/CGroupsHandlerImpl.java     |  248 +-
 .../DelegatingLinuxContainerRuntime.java        |    9 +-
 .../runtime/DockerLinuxContainerRuntime.java    |  109 +-
 .../JavaSandboxLinuxContainerRuntime.java       |   77 +-
 .../runtime/docker/DockerCommandExecutor.java   |  191 +
 .../linux/runtime/docker/package-info.java      |   26 +
 .../localizer/LocalResourcesTrackerImpl.java    |   26 +-
 .../localizer/ResourceLocalizationService.java  |   50 +-
 .../logaggregation/AppLogAggregatorImpl.java    |   60 +-
 .../loghandler/NonAggregatingLogHandler.java    |    7 +-
 .../monitor/ContainersMonitorImpl.java          |   49 +-
 .../util/CgroupsLCEResourcesHandler.java        |   32 +-
 .../impl/container-executor.c                   |  231 +-
 .../impl/container-executor.h                   |   28 +-
 .../container-executor/impl/get_executable.c    |    2 +-
 .../test/test-container-executor.c              |   34 +
 .../yarn_server_nodemanager_recovery.proto      |    8 +
 .../nodemanager/DummyContainerManager.java      |    6 +-
 .../TestDefaultContainerExecutor.java           |    3 +-
 .../server/nodemanager/TestDeletionService.java |   57 +-
 .../nodemanager/TestLinuxContainerExecutor.java |    9 +-
 .../TestLinuxContainerExecutorWithMocks.java    |  217 +-
 .../nodemanager/TestNodeHealthService.java      |   11 +-
 .../nodemanager/TestNodeManagerReboot.java      |   99 +-
 .../nodemanager/TestNodeManagerResync.java      |  164 -
 .../amrmproxy/BaseAMRMProxyTest.java            |   21 +-
 .../amrmproxy/TestAMRMProxyService.java         |   65 +
 .../TestAMRMProxyTokenSecretManager.java        |   81 +
 .../api/impl/pb/TestNMProtoUtils.java           |   91 +
 .../BaseContainerManagerTest.java               |   20 +-
 .../containermanager/TestContainerManager.java  |    5 -
 .../TestContainerManagerRecovery.java           |   56 +-
 .../containermanager/TestNMProxy.java           |   30 +-
 .../deletion/task/FileDeletionMatcher.java      |   84 +
 .../deletion/task/TestFileDeletionTask.java     |   85 +
 .../launcher/TestContainerLaunch.java           |   65 +-
 .../MockPrivilegedOperationCaptor.java          |   68 +
 .../linux/resources/TestCGroupsHandlerImpl.java |  407 +-
 .../resources/TestResourceHandlerModule.java    |    3 -
 .../runtime/TestDockerContainerRuntime.java     |   42 +-
 .../TestJavaSandboxLinuxContainerRuntime.java   |  126 +-
 .../docker/TestDockerCommandExecutor.java       |  218 +
 .../runtime/docker/TestDockerLoadCommand.java   |   48 +
 .../runtime/docker/TestDockerRunCommand.java    |   63 +
 .../TestLocalResourcesTrackerImpl.java          |   13 +-
 .../TestResourceLocalizationService.java        |  106 +-
 .../TestAppLogAggregatorImpl.java               |   15 +-
 .../TestLogAggregationService.java              |   17 +-
 .../TestNonAggregatingLogHandler.java           |    8 +-
 .../TestContainersMonitorResourceChange.java    |    3 +-
 .../TestContainerSchedulerQueuing.java          |    9 +-
 .../TestNMLeveldbStateStoreService.java         |    2 +-
 .../util/TestCgroupsLCEResourcesHandler.java    |   70 +-
 ...-container-executer-with-configuration-error |   20 +
 .../hadoop-yarn-server-resourcemanager/pom.xml  |    5 +-
 ...ActiveStandbyElectorBasedElectorService.java |    5 +-
 .../server/resourcemanager/AdminService.java    |    6 +-
 .../ApplicationMasterService.java               |    2 +-
 .../resourcemanager/NodesListManager.java       |   30 +-
 .../server/resourcemanager/RMAppManager.java    |   21 +-
 .../server/resourcemanager/RMAuditLogger.java   |    2 +-
 ...MCriticalThreadUncaughtExceptionHandler.java |   19 +-
 .../server/resourcemanager/RMFatalEvent.java    |   67 +-
 .../resourcemanager/RMFatalEventType.java       |    1 +
 .../server/resourcemanager/ResourceManager.java |  199 +-
 .../metrics/AbstractSystemMetricsPublisher.java |    2 +-
 .../monitor/SchedulingMonitor.java              |   55 +-
 .../CapacitySchedulerPreemptionContext.java     |    5 +
 .../CapacitySchedulerPreemptionUtils.java       |    9 +-
 .../FifoIntraQueuePreemptionPlugin.java         |  329 +-
 .../capacity/IntraQueueCandidatesSelector.java  |  110 +-
 .../IntraQueuePreemptionComputePlugin.java      |   10 +-
 .../ProportionalCapacityPreemptionPolicy.java   |   42 +-
 ...QueuePriorityContainerCandidateSelector.java |    2 +-
 .../monitor/capacity/TempAppPerPartition.java   |    6 +-
 .../monitor/capacity/TempQueuePerPartition.java |   14 +
 .../monitor/capacity/TempUserPerPartition.java  |   88 +
 .../invariants/InvariantViolationException.java |   35 +
 .../monitor/invariants/InvariantsChecker.java   |   96 +
 .../invariants/MetricsInvariantChecker.java     |  195 +
 .../ReservationInvariantsChecker.java           |   63 +
 .../monitor/invariants/package-info.java        |   22 +
 .../resourcemanager/recovery/RMStateStore.java  |   12 +-
 .../recovery/ZKRMStateStore.java                |  506 +-
 .../reservation/InMemoryPlan.java               |   11 +
 .../PeriodicRLESparseResourceAllocation.java    |  167 +
 .../resourcemanager/reservation/PlanView.java   |    9 +
 .../RLESparseResourceAllocation.java            |   51 +
 .../planning/AlignedPlannerWithGreedy.java      |   15 +-
 .../planning/GreedyReservationAgent.java        |   13 +-
 .../reservation/planning/IterativePlanner.java  |  196 +-
 .../reservation/planning/ReservationAgent.java  |   23 +-
 .../planning/SimpleCapacityReplanner.java       |    8 +-
 .../reservation/planning/StageAllocator.java    |   10 +-
 .../planning/StageAllocatorGreedy.java          |    4 +-
 .../planning/StageAllocatorGreedyRLE.java       |    4 +-
 .../planning/StageAllocatorLowCostAligned.java  |  279 +-
 .../planning/StageEarliestStart.java            |   46 -
 .../planning/StageEarliestStartByDemand.java    |  106 -
 .../StageEarliestStartByJobArrival.java         |   39 -
 .../planning/StageExecutionInterval.java        |   47 +
 .../StageExecutionIntervalByDemand.java         |  144 +
 .../StageExecutionIntervalUnconstrained.java    |   73 +
 .../server/resourcemanager/rmapp/RMAppImpl.java |   24 +-
 .../rmapp/attempt/RMAppAttemptImpl.java         |    6 +-
 .../rmapp/attempt/RMAppAttemptMetrics.java      |    2 +-
 .../scheduler/AbstractYarnScheduler.java        |    2 +-
 .../resourcemanager/scheduler/NodeType.java     |   12 +-
 .../resourcemanager/scheduler/QueueMetrics.java |   21 +-
 .../scheduler/capacity/AbstractCSQueue.java     |   25 +
 .../scheduler/capacity/CSQueueMetrics.java      |    4 +-
 .../CapacitySchedulerConfiguration.java         |    8 +
 .../capacity/CapacitySchedulerQueueManager.java |    6 +-
 .../scheduler/capacity/LeafQueue.java           |   11 +-
 .../scheduler/capacity/UsersManager.java        |   11 +-
 .../allocator/AbstractContainerAllocator.java   |    6 +-
 .../fair/AllocationFileLoaderService.java       |    3 +-
 .../scheduler/fair/FSAppAttempt.java            |   44 +-
 .../scheduler/fair/FSLeafQueue.java             |    9 +-
 .../scheduler/fair/FSParentQueue.java           |    4 +-
 .../resourcemanager/scheduler/fair/FSQueue.java |   22 +-
 .../scheduler/fair/FSQueueMetrics.java          |    4 +-
 .../scheduler/fair/FSSchedulerNode.java         |    8 +-
 .../scheduler/fair/FairScheduler.java           |   27 +-
 .../scheduler/fair/FairSchedulerUtilities.java  |   69 +
 .../scheduler/fair/QueueManager.java            |    6 +-
 .../scheduler/fair/SchedulingPolicy.java        |   38 +-
 .../fair/policies/FairSharePolicy.java          |    2 +-
 .../resourcemanager/webapp/RMAppsBlock.java     |   17 +-
 .../resourcemanager/webapp/RMWebAppUtil.java    |  149 +
 .../yarn/server/resourcemanager/HATestUtil.java |    9 +-
 .../yarn/server/resourcemanager/MockRM.java     |   10 +-
 .../server/resourcemanager/TestAppManager.java  |    2 +
 .../resourcemanager/TestClientRMService.java    |   25 +
 .../resourcemanager/TestRMDispatcher.java       |    1 +
 .../resourcemanager/TestRMStoreCommands.java    |   15 +-
 .../applicationsmanager/TestAMRestart.java      |  140 +-
 .../monitor/TestSchedulingMonitor.java          |   19 +
 ...alCapacityPreemptionPolicyMockFramework.java |   89 +-
 ...ionalCapacityPreemptionPolicyIntraQueue.java |   30 +-
 ...cityPreemptionPolicyIntraQueueUserLimit.java |  899 ++++
 ...pacityPreemptionPolicyIntraQueueWithDRF.java |  178 +
 .../invariants/TestMetricsInvariantChecker.java |   99 +
 .../recovery/RMStateStoreTestBase.java          |   10 +-
 .../recovery/TestFSRMStateStore.java            |    1 +
 .../recovery/TestLeveldbRMStateStore.java       |    1 +
 .../recovery/TestMemoryRMStateStore.java        |    2 +
 .../recovery/TestZKRMStateStore.java            |  573 +-
 .../reservation/ReservationSystemTestUtil.java  |   14 +
 ...TestPeriodicRLESparseResourceAllocation.java |  142 +
 .../TestRLESparseResourceAllocation.java        |   56 +-
 .../planning/TestAlignedPlanner.java            |  417 +-
 .../planning/TestGreedyReservationAgent.java    |   14 +-
 .../planning/TestSimpleCapacityReplanner.java   |    4 +-
 .../scheduler/TestAbstractYarnScheduler.java    |    2 +-
 .../capacity/TestCapacityScheduler.java         |   15 +-
 .../scheduler/fair/TestFairScheduler.java       |   15 +
 .../fair/TestFairSchedulerPreemption.java       |    4 +
 .../fair/TestFairSchedulerUtilities.java        |   67 +
 .../fair/policies/TestEmptyQueues.java          |    6 +-
 .../src/test/resources/invariants.txt           |   54 +
 .../pom.xml                                     |    4 +-
 .../hadoop-yarn-server-tests/pom.xml            |    4 +-
 .../hadoop/yarn/server/MiniYARNCluster.java     |   22 +-
 .../server/TestContainerManagerSecurity.java    |   14 +-
 .../hadoop/yarn/server/TestMiniYarnCluster.java |   34 +-
 .../pom.xml                                     |    4 +-
 .../pom.xml                                     |    4 +-
 .../pom.xml                                     |    2 +-
 .../storage/TimelineSchemaCreator.java          |  144 +-
 .../storage/application/ApplicationTable.java   |   12 +-
 .../storage/entity/EntityTable.java             |   12 +-
 .../hadoop-yarn-server-timelineservice/pom.xml  |    4 +-
 .../collector/AppLevelTimelineCollector.java    |    8 +-
 .../PerNodeTimelineCollectorsAuxService.java    |    7 +-
 .../collector/TimelineCollector.java            |   27 +-
 .../collector/TimelineCollectorManager.java     |    6 +-
 .../reader/filter/TimelineFilterList.java       |    2 +-
 .../storage/FileSystemTimelineReaderImpl.java   |  129 +-
 .../collector/TestTimelineCollector.java        |   95 +-
 .../hadoop-yarn-server-web-proxy/pom.xml        |    4 +-
 .../server/webproxy/amfilter/AmIpFilter.java    |    6 +
 .../server/webproxy/amfilter/TestAmFilter.java  |    8 +
 .../hadoop-yarn/hadoop-yarn-server/pom.xml      |    4 +-
 .../hadoop-yarn/hadoop-yarn-site/pom.xml        |    4 +-
 .../src/site/markdown/FairScheduler.md          |    6 +
 .../src/site/markdown/ResourceManagerHA.md      |    4 +-
 .../src/site/markdown/ResourceManagerRestart.md |    2 +-
 .../src/site/markdown/TimelineServiceV2.md      |    5 +-
 .../hadoop-yarn/hadoop-yarn-ui/README.md        |   64 +
 .../hadoop-yarn/hadoop-yarn-ui/pom.xml          |  156 +-
 .../hadoop-yarn-ui/src/main/webapp/.bowerrc     |    5 +-
 .../src/main/webapp/WEB-INF/wro.xml             |    9 +
 .../webapp/app/adapters/yarn-app-attempt.js     |    4 +-
 .../webapp/app/adapters/yarn-app-flowrun.js     |   33 +
 .../webapp/app/adapters/yarn-app-timeline.js    |   35 +
 .../src/main/webapp/app/adapters/yarn-app.js    |    2 +-
 .../main/webapp/app/adapters/yarn-container.js  |   18 +-
 .../src/main/webapp/app/adapters/yarn-entity.js |   35 +
 .../webapp/app/adapters/yarn-flow-activity.js   |   30 +
 .../webapp/app/adapters/yarn-flowrun-brief.js   |   32 +
 .../main/webapp/app/adapters/yarn-flowrun.js    |   38 +
 .../app/adapters/yarn-timeline-appattempt.js    |   41 +
 .../app/adapters/yarn-timeline-container.js     |   40 +
 .../webapp/app/components/em-table-html-cell.js |   23 +
 .../main/webapp/app/components/nodes-heatmap.js |   18 +-
 .../webapp/app/components/simple-bar-chart.js   |  206 +
 .../webapp/app/components/sunburst-chart.js     |  261 +
 .../main/webapp/app/components/timeline-view.js |   13 +-
 .../webapp/app/controllers/app-table-columns.js |  213 +-
 .../webapp/app/controllers/yarn-app-attempt.js  |   32 +-
 .../webapp/app/controllers/yarn-app-attempts.js |   40 -
 .../src/main/webapp/app/controllers/yarn-app.js |   53 +-
 .../webapp/app/controllers/yarn-app/attempts.js |   24 +
 .../webapp/app/controllers/yarn-app/charts.js   |   28 +
 .../webapp/app/controllers/yarn-app/info.js     |   32 +
 .../webapp/app/controllers/yarn-apps/apps.js    |   10 +-
 .../app/controllers/yarn-apps/services.js       |   25 -
 .../app/controllers/yarn-flow-activity.js       |  125 +
 .../main/webapp/app/controllers/yarn-flow.js    |   36 +
 .../webapp/app/controllers/yarn-flow/info.js    |   66 +
 .../webapp/app/controllers/yarn-flow/runs.js    |  178 +
 .../main/webapp/app/controllers/yarn-flowrun.js |   50 +
 .../webapp/app/controllers/yarn-flowrun/info.js |  157 +
 .../app/controllers/yarn-flowrun/metrics.js     |  126 +
 .../webapp/app/controllers/yarn-nodes/table.js  |    8 +
 .../webapp/app/controllers/yarn-services.js     |   89 +-
 .../src/main/webapp/app/mixins/app-attempt.js   |   66 +
 .../main/webapp/app/models/yarn-app-attempt.js  |   11 +-
 .../main/webapp/app/models/yarn-app-flowrun.js  |   52 +
 .../main/webapp/app/models/yarn-app-timeline.js |  105 +
 .../src/main/webapp/app/models/yarn-app.js      |    6 +-
 .../main/webapp/app/models/yarn-container.js    |    9 +-
 .../src/main/webapp/app/models/yarn-entity.js   |   26 +
 .../webapp/app/models/yarn-flow-activity.js     |   28 +
 .../webapp/app/models/yarn-flowrun-brief.js     |   70 +
 .../src/main/webapp/app/models/yarn-flowrun.js  |   32 +
 .../app/models/yarn-timeline-appattempt.js      |  143 +
 .../app/models/yarn-timeline-container.js       |   63 +
 .../src/main/webapp/app/router.js               |   21 +-
 .../main/webapp/app/routes/timeline-error.js    |   29 +
 .../main/webapp/app/routes/yarn-app-attempt.js  |   32 +-
 .../main/webapp/app/routes/yarn-app-attempts.js |   34 -
 .../src/main/webapp/app/routes/yarn-app.js      |   35 +-
 .../main/webapp/app/routes/yarn-app/attempts.js |   37 +
 .../main/webapp/app/routes/yarn-app/charts.js   |   53 +
 .../src/main/webapp/app/routes/yarn-app/info.js |   37 +
 .../webapp/app/routes/yarn-apps/services.js     |   33 -
 .../webapp/app/routes/yarn-flow-activity.js     |   37 +
 .../src/main/webapp/app/routes/yarn-flow.js     |   53 +
 .../main/webapp/app/routes/yarn-flow/info.js    |   22 +
 .../main/webapp/app/routes/yarn-flow/runs.js    |   22 +
 .../webapp/app/routes/yarn-flowrun-metric.js    |  107 +
 .../src/main/webapp/app/routes/yarn-flowrun.js  |   58 +
 .../main/webapp/app/routes/yarn-flowrun/info.js |   22 +
 .../webapp/app/routes/yarn-flowrun/metrics.js   |   22 +
 .../src/main/webapp/app/routes/yarn-services.js |   34 +
 .../webapp/app/serializers/yarn-app-flowrun.js  |   64 +
 .../webapp/app/serializers/yarn-app-timeline.js |   76 +
 .../src/main/webapp/app/serializers/yarn-app.js |    3 +-
 .../main/webapp/app/serializers/yarn-entity.js  |   51 +
 .../app/serializers/yarn-flow-activity.js       |   54 +
 .../app/serializers/yarn-flowrun-brief.js       |   64 +
 .../main/webapp/app/serializers/yarn-flowrun.js |   63 +
 .../main/webapp/app/serializers/yarn-queue.js   |   22 +-
 .../app/serializers/yarn-timeline-appattempt.js |   69 +
 .../app/serializers/yarn-timeline-container.js  |   71 +
 .../src/main/webapp/app/styles/app.css          |   92 +
 .../main/webapp/app/templates/application.hbs   |   19 +-
 .../templates/components/app-attempt-table.hbs  |    8 +-
 .../app/templates/components/app-table.hbs      |    6 +-
 .../templates/components/container-table.hbs    |    2 +-
 .../templates/components/em-table-html-cell.hbs |   23 +
 .../app/templates/components/timeline-view.hbs  |    8 +-
 .../webapp/app/templates/timeline-error.hbs     |   22 +
 .../webapp/app/templates/yarn-app-attempt.hbs   |   46 +-
 .../webapp/app/templates/yarn-app-attempts.hbs  |   55 -
 .../src/main/webapp/app/templates/yarn-app.hbs  |  232 +-
 .../webapp/app/templates/yarn-app/attempts.hbs  |   29 +
 .../webapp/app/templates/yarn-app/charts.hbs    |   43 +
 .../main/webapp/app/templates/yarn-app/info.hbs |  167 +
 .../webapp/app/templates/yarn-app/loading.hbs   |   23 +
 .../src/main/webapp/app/templates/yarn-apps.hbs |   11 +-
 .../webapp/app/templates/yarn-apps/apps.hbs     |    2 +-
 .../webapp/app/templates/yarn-apps/services.hbs |   25 -
 .../webapp/app/templates/yarn-flow-activity.hbs |   45 +
 .../src/main/webapp/app/templates/yarn-flow.hbs |   49 +
 .../webapp/app/templates/yarn-flow/info.hbs     |   53 +
 .../webapp/app/templates/yarn-flow/runs.hbs     |  131 +
 .../app/templates/yarn-flowrun-metric.hbs       |   34 +
 .../main/webapp/app/templates/yarn-flowrun.hbs  |   49 +
 .../webapp/app/templates/yarn-flowrun/info.hbs  |  128 +
 .../app/templates/yarn-flowrun/metrics.hbs      |   34 +
 .../webapp/app/templates/yarn-nodes/table.hbs   |    2 +-
 .../main/webapp/app/templates/yarn-services.hbs |   85 +
 .../src/main/webapp/app/utils/converter.js      |   44 +-
 .../src/main/webapp/app/utils/error-utils.js    |   58 +
 .../src/main/webapp/bower-shrinkwrap.json       |   66 +
 .../src/main/webapp/config/default-config.js    |    3 +-
 .../src/main/webapp/ember-cli-build.js          |   16 +-
 .../hadoop-yarn-ui/src/main/webapp/package.json |   11 +-
 .../components/em-table-html-cell-test.js       |   43 +
 .../components/simple-bar-chart-test.js         |   43 +
 .../unit/adapters/yarn-app-timeline-test.js     |   30 +
 .../adapters/yarn-timeline-appattempt-test.js   |   30 +
 .../adapters/yarn-timeline-container-test.js    |   30 +
 .../unit/controllers/yarn-app-attempts-test.js  |   30 -
 .../unit/controllers/yarn-app/attempts-test.js  |   30 +
 .../unit/controllers/yarn-app/charts-test.js    |   30 +
 .../unit/controllers/yarn-app/info-test.js      |   30 +
 .../unit/controllers/yarn-apps/services-test.js |   30 -
 .../unit/controllers/yarn-flow/info-test.js     |   30 +
 .../unit/controllers/yarn-flow/runs-test.js     |   30 +
 .../unit/controllers/yarn-flowrun/info-test.js  |   30 +
 .../controllers/yarn-flowrun/metrics-test.js    |   30 +
 .../tests/unit/mixins/app-attempt-test.js       |   30 +
 .../webapp/tests/unit/mixins/charts-test.js     |   30 -
 .../tests/unit/models/yarn-app-timeline-test.js |   30 +
 .../models/yarn-timeline-appattempt-test.js     |   30 +
 .../unit/models/yarn-timeline-container-test.js |   30 +
 .../tests/unit/routes/timeline-error-test.js    |   29 +
 .../tests/unit/routes/yarn-app-attempts-test.js |   29 -
 .../tests/unit/routes/yarn-app/attempts-test.js |   29 +
 .../tests/unit/routes/yarn-app/charts-test.js   |   29 +
 .../tests/unit/routes/yarn-app/info-test.js     |   29 +
 .../tests/unit/routes/yarn-flow/info-test.js    |   29 +
 .../tests/unit/routes/yarn-flow/runs-test.js    |   29 +
 .../tests/unit/routes/yarn-flowrun/info-test.js |   29 +
 .../unit/routes/yarn-flowrun/metrics-test.js    |   29 +
 .../tests/unit/routes/yarn-services-test.js     |   29 +
 .../unit/serializers/yarn-app-timeline-test.js  |   33 +
 .../yarn-timeline-appattempt-test.js            |   33 +
 .../serializers/yarn-timeline-container-test.js |   33 +
 .../hadoop-yarn-ui/src/main/webapp/yarn.lock    | 4983 ++++++++++++++++++
 hadoop-yarn-project/hadoop-yarn/pom.xml         |    4 +-
 hadoop-yarn-project/pom.xml                     |    4 +-
 pom.xml                                         |    2 +-
 1148 files changed, 45884 insertions(+), 10731 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f02de2e/hadoop-common-project/hadoop-common/pom.xml
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f02de2e/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f02de2e/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java
----------------------------------------------------------------------
diff --cc hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java
index 17d2404,92e2135..5b2ba65
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java
@@@ -705,26 -748,13 +748,27 @@@ public abstract class FileSystemContrac
  
    /**
     * This a sanity check to make sure that any filesystem's handling of
 -   * renames doesn't cause any regressions
 +   * renames empty dirs doesn't cause any regressions.
 +   */
 +  public void testRenameEmptyToDirWithSamePrefixAllowed() throws Throwable {
 +    if (!renameSupported()) return;
 +    Path parentdir = path("testRenameEmptyToDirWithSamePrefixAllowed");
 +    fs.mkdirs(parentdir);
 +    Path dest = path("testRenameEmptyToDirWithSamePrefixAllowedDest");
 +    rename(parentdir, dest, true, false, true);
 +  }
 +
 +  /**
 +   * This a sanity check to make sure that any filesystem's handling of
 +   * renames non-empty dirs doesn't cause any regressions.
     */
+   @Test
    public void testRenameToDirWithSamePrefixAllowed() throws Throwable {
-     if (!renameSupported()) return;
+     assumeTrue(renameSupported());
      final Path parentdir = path("testRenameToDirWithSamePrefixAllowed");
      fs.mkdirs(parentdir);
 +    // Before renaming, we create one file under the source parent directory
 +    createFile(new Path(parentdir, "mychild"));
      final Path dest = path("testRenameToDirWithSamePrefixAllowedDest");
      rename(parentdir, dest, true, false, true);
    }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f02de2e/hadoop-project/pom.xml
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f02de2e/hadoop-tools/hadoop-aws/pom.xml
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f02de2e/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
----------------------------------------------------------------------
diff --cc hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
index e14e341,9e15b3f..0fb1197
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
@@@ -313,123 -320,4 +325,114 @@@ public final class Constants 
    @InterfaceAudience.Private
    public static final int MAX_MULTIPART_COUNT = 10000;
  
 +  /**
 +   * Classname of the S3A-specific output committer factory. This
 +   * is what must be declared when attempting to use
 +   */
 +  @InterfaceStability.Unstable
 +  public static final String S3A_OUTPUT_COMMITTER_FACTORY =
 +      "org.apache.hadoop.fs.s3a.commit.S3AOutputCommitterFactory";
 +
 +  /* Constants. */
 +  public static final String S3_METADATA_STORE_IMPL =
 +      "fs.s3a.metadatastore.impl";
 +
 +  /** Minimum period of time (in milliseconds) to keep metadata (may only be
 +   * applied when a prune command is manually run).
 +   */
 +  @InterfaceStability.Unstable
 +  public static final String S3GUARD_CLI_PRUNE_AGE =
 +      "fs.s3a.s3guard.cli.prune.age";
 +
 +  /**
 +   * The region of the DynamoDB service.
 +   *
 +   * This config has no default value. If the user does not set this, the
 +   * S3Guard will operate table in the associated S3 bucket region.
 +   */
 +  @InterfaceStability.Unstable
 +  public static final String S3GUARD_DDB_REGION_KEY =
 +      "fs.s3a.s3guard.ddb.region";
 +
 +  /**
 +   * The DynamoDB table name to use.
 +   *
 +   * This config has no default value. If the user does not set this, the
 +   * S3Guard implementation will use the respective S3 bucket name.
 +   */
 +  @InterfaceStability.Unstable
 +  public static final String S3GUARD_DDB_TABLE_NAME_KEY =
 +      "fs.s3a.s3guard.ddb.table";
 +
 +  /**
 +   * Whether to create the DynamoDB table if the table does not exist.
 +   */
 +  @InterfaceStability.Unstable
 +  public static final String S3GUARD_DDB_TABLE_CREATE_KEY =
 +      "fs.s3a.s3guard.ddb.table.create";
 +
 +  @InterfaceStability.Unstable
 +  public static final String S3GUARD_DDB_TABLE_CAPACITY_READ_KEY =
 +      "fs.s3a.s3guard.ddb.table.capacity.read";
 +  public static final long S3GUARD_DDB_TABLE_CAPACITY_READ_DEFAULT = 500;
 +  @InterfaceStability.Unstable
 +  public static final String S3GUARD_DDB_TABLE_CAPACITY_WRITE_KEY =
 +      "fs.s3a.s3guard.ddb.table.capacity.write";
 +  public static final long S3GUARD_DDB_TABLE_CAPACITY_WRITE_DEFAULT = 100;
 +
 +  /**
 +   * The maximum put or delete requests per BatchWriteItem request.
 +   *
 +   * Refer to Amazon API reference for this limit.
 +   */
 +  public static final int S3GUARD_DDB_BATCH_WRITE_REQUEST_LIMIT = 25;
 +
 +  @InterfaceStability.Unstable
 +  public static final String S3GUARD_DDB_MAX_RETRIES =
 +      "fs.s3a.s3guard.ddb.max.retries";
 +  /**
 +   * Max retries on batched DynamoDB operations before giving up and
 +   * throwing an IOException.  Default is {@value}. See core-default.xml for
 +   * more detail.
 +   */
 +  public static final int S3GUARD_DDB_MAX_RETRIES_DEFAULT = 9;
 +
 +  /**
 +   * Period of time (in milliseconds) to sleep between batches of writes.
 +   * Currently only applies to prune operations, as they are naturally a
 +   * lower priority than other operations.
 +   */
 +  @InterfaceStability.Unstable
 +  public static final String S3GUARD_DDB_BACKGROUND_SLEEP_MSEC_KEY =
 +      "fs.s3a.s3guard.ddb.background.sleep";
 +  public static final int S3GUARD_DDB_BACKGROUND_SLEEP_MSEC_DEFAULT = 25;
 +
 +  /**
 +   * V1 committer.
 +   */
 +  @InterfaceStability.Unstable
 +  public static final String S3A_OUTPUT_COMMITTER_MRV1 =
 +      "org.apache.hadoop.fs.s3a.commit.S3OutputCommitterMRv1";
 +
 +  /**
 +   * The default "Null" metadata store: {@value}.
 +   */
 +  @InterfaceStability.Unstable
 +  public static final String S3GUARD_METASTORE_NULL
 +      = "org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore";
 +
 +  /**
 +   * Use Local memory for the metadata: {@value}.
 +   * This is not coherent across processes and must be used for testing only.
 +   */
 +  @InterfaceStability.Unstable
 +  public static final String S3GUARD_METASTORE_LOCAL
 +      = "org.apache.hadoop.fs.s3a.s3guard.LocalMetadataStore";
 +
 +  /**
 +   * Use DynamoDB for the metadata: {@value}.
 +   */
 +  @InterfaceStability.Unstable
 +  public static final String S3GUARD_METASTORE_DYNAMO
 +      = "org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore";
- 
-   @InterfaceAudience.Private
-   public static final String SSE_C_NO_KEY_ERROR = S3AEncryptionMethods.SSE_C
-       .getMethod() +" is enabled and no encryption key is provided.";
- 
- 
-   @InterfaceAudience.Private
-   public static final String SSE_S3_WITH_KEY_ERROR = S3AEncryptionMethods.SSE_S3
-       .getMethod() +" is configured and an " + "encryption key is provided";
  }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f02de2e/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --cc hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 7de94cc,872dd5f..4eb94ad
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@@ -1675,65 -1553,45 +1702,67 @@@ public class S3AFileSystem extends File
     * @throws IOException other IO problems
     * @throws AmazonClientException on failures inside the AWS SDK
     */
 -  private boolean innerMkdirs(Path f, FsPermission permission)
 +  private boolean innerMkdirs(Path p, FsPermission permission)
        throws IOException, FileAlreadyExistsException, AmazonClientException {
 +    boolean createOnS3 = false;
 +    Path f = qualify(p);
      LOG.debug("Making directory: {}", f);
      incrementStatistic(INVOCATION_MKDIRS);
 -    FileStatus fileStatus;
 -    try {
 -      fileStatus = getFileStatus(f);
 +    List<Path> metadataStoreDirs = null;
 +    if (hasMetadataStore()) {
 +      metadataStoreDirs = new ArrayList<>();
 +    }
  
 -      if (fileStatus.isDirectory()) {
 -        return true;
 -      } else {
 -        throw new FileAlreadyExistsException("Path is a file: " + f);
 +    DirectoryStatus status = checkPathForDirectory(f);
 +    if (status == DirectoryStatus.DOES_NOT_EXIST) {
 +      createOnS3 = true;
 +      if (metadataStoreDirs != null) {
 +        metadataStoreDirs.add(f);
        }
 -    } catch (FileNotFoundException e) {
 -      Path fPart = f.getParent();
 -      do {
 -        try {
 -          fileStatus = getFileStatus(fPart);
 -          if (fileStatus.isDirectory()) {
 -            break;
 -          }
 -          if (fileStatus.isFile()) {
 -            throw new FileAlreadyExistsException(String.format(
 -                "Can't make directory for path '%s' since it is a file.",
 -                fPart));
 -          }
 -        } catch (FileNotFoundException fnfe) {
 -          instrumentation.errorIgnored();
 +    } else if (status == DirectoryStatus.EXISTS_AND_IS_DIRECTORY_ON_S3_ONLY) {
 +      if (metadataStoreDirs != null) {
 +        metadataStoreDirs.add(f);
 +      }
 +    } else if (status == DirectoryStatus
 +        .EXISTS_AND_IS_DIRECTORY_ON_METADATASTORE) {
 +      return true;
 +    } else if (status == DirectoryStatus.EXISTS_AND_IS_FILE) {
 +      throw new FileAlreadyExistsException("Path is a file: " + f);
 +    }
 +
 +    // Walk path to root, ensuring closest ancestor is a directory, not file
 +    Path fPart = f.getParent();
 +    do {
 +      status = checkPathForDirectory(fPart);
 +      // The fake directory on S3 may not be visible immediately, but
 +      // only the leaf node has a fake directory on S3 so we can treat both
 +      // cases the same and just create a metadata store entry.
 +      if (status == DirectoryStatus.DOES_NOT_EXIST || status ==
 +          DirectoryStatus.EXISTS_AND_IS_DIRECTORY_ON_S3_ONLY) {
 +        if (metadataStoreDirs != null) {
 +          metadataStoreDirs.add(fPart);
          }
 -        fPart = fPart.getParent();
 -      } while (fPart != null);
 +      } else if (status == DirectoryStatus
 +          .EXISTS_AND_IS_DIRECTORY_ON_METADATASTORE) {
 +        // do nothing - just break out of the loop, make whatever child
 +        // directories are needed on the metadata store, and return the
 +        // result.
 +        break;
 +      } else if (status == DirectoryStatus.EXISTS_AND_IS_FILE) {
 +        throw new FileAlreadyExistsException("Path is a file: " + f);
 +      }
 +      fPart = fPart.getParent();
 +    } while (fPart != null);
  
 +    if (createOnS3) {
        String key = pathToKey(f);
        createFakeDirectory(key);
 -      // this is complicated because getParent(a/b/c/) returns a/b/c, but
 -      // we want a/b. See HADOOP-14428 for more details.
 -      deleteUnnecessaryFakeDirectories(new Path(f.toString()).getParent());
 -      return true;
      }
 +    S3Guard.makeDirsOrdered(metadataStore, metadataStoreDirs, username);
-     deleteUnnecessaryFakeDirectories(f.getParent());
++    // this is complicated because getParent(a/b/c/) returns a/b/c, but
++    // we want a/b. See HADOOP-14428 for more details.
++    deleteUnnecessaryFakeDirectories(new Path(f.toString()).getParent());
 +    return true;
    }
  
    /**
@@@ -2025,16 -1756,33 +2051,34 @@@
      // Since we have a local file, we don't need to stream into a temporary file
      LocalFileSystem local = getLocal(getConf());
      File srcfile = local.pathToFile(src);
+     if (!srcfile.exists()) {
+       throw new FileNotFoundException("No file: " + src);
+     }
+     if (!srcfile.isFile()) {
+       throw new FileNotFoundException("Not a file: " + src);
+     }
  
+     try {
+       FileStatus status = getFileStatus(dst);
+       if (!status.isFile()) {
+         throw new FileAlreadyExistsException(dst + " exists and is not a file");
+       }
+       if (!overwrite) {
+         throw new FileAlreadyExistsException(dst + " already exists");
+       }
+     } catch (FileNotFoundException e) {
+       // no destination, all is well
+     }
+     final String key = pathToKey(dst);
      final ObjectMetadata om = newObjectMetadata(srcfile.length());
      PutObjectRequest putObjectRequest = newPutObjectRequest(key, om, srcfile);
 -    Upload up = putObject(putObjectRequest);
 +    UploadInfo info = putObject(putObjectRequest);
 +    Upload upload = info.getUpload();
      ProgressableProgressListener listener = new ProgressableProgressListener(
 -        this, key, up, null);
 -    up.addProgressListener(listener);
 +        this, key, upload, null);
 +    upload.addProgressListener(listener);
      try {
 -      up.waitForUploadResult();
 +      upload.waitForUploadResult();
      } catch (InterruptedException e) {
        throw new InterruptedIOException("Interrupted copying " + src
            + " to "  + dst + ", cancelling");

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f02de2e/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f02de2e/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f02de2e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACopyFromLocalFile.java
----------------------------------------------------------------------
diff --cc hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACopyFromLocalFile.java
index 0000000,71776ac..7dc286d
mode 000000,100644..100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACopyFromLocalFile.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACopyFromLocalFile.java
@@@ -1,0 -1,153 +1,154 @@@
+ /*
+  * Licensed to the Apache Software Foundation (ASF) under one
+  *  or more contributor license agreements.  See the NOTICE file
+  *  distributed with this work for additional information
+  *  regarding copyright ownership.  The ASF licenses this file
+  *  to you under the Apache License, Version 2.0 (the
+  *  "License"); you may not use this file except in compliance
+  *  with the License.  You may obtain a copy of the License at
+  *
+  *       http://www.apache.org/licenses/LICENSE-2.0
+  *
+  *  Unless required by applicable law or agreed to in writing, software
+  *  distributed under the License is distributed on an "AS IS" BASIS,
+  *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  *  See the License for the specific language governing permissions and
+  *  limitations under the License.
+  */
+ 
+ package org.apache.hadoop.fs.s3a;
+ 
+ import java.io.File;
+ import java.io.FileNotFoundException;
+ import java.io.IOException;
+ import java.nio.charset.Charset;
+ 
+ import org.junit.Test;
+ 
+ import org.apache.commons.io.Charsets;
+ import org.apache.commons.io.FileUtils;
+ import org.apache.commons.io.IOUtils;
+ import org.apache.hadoop.fs.FileAlreadyExistsException;
++import org.apache.hadoop.fs.FileStatus;
+ import org.apache.hadoop.fs.Path;
+ 
+ import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+ 
+ /**
+  * Test {@link S3AFileSystem#copyFromLocalFile(boolean, boolean, Path, Path)}.
+  */
+ public class ITestS3ACopyFromLocalFile extends AbstractS3ATestBase {
+   private static final Charset ASCII = Charsets.US_ASCII;
+ 
+   private File file;
+ 
+   @Override
+   public void teardown() throws Exception {
+     super.teardown();
+     if (file != null) {
+       file.delete();
+     }
+   }
+ 
+   @Test
+   public void testCopyEmptyFile() throws Throwable {
+     file = File.createTempFile("test", ".txt");
+     Path dest = upload(file, true);
+     assertPathExists("uploaded file", dest);
+   }
+ 
+   @Test
+   public void testCopyFile() throws Throwable {
+     String message = "hello";
+     file = createTempFile(message);
+     Path dest = upload(file, true);
+     assertPathExists("uploaded file not found", dest);
+     S3AFileSystem fs = getFileSystem();
 -    S3AFileStatus status = fs.getFileStatus(dest);
++    FileStatus status = fs.getFileStatus(dest);
+     assertEquals("File length of " + status,
+         message.getBytes(ASCII).length, status.getLen());
+     assertFileTextEquals(dest, message);
+   }
+ 
+   public void assertFileTextEquals(Path path, String expected)
+       throws IOException {
+     assertEquals("Wrong data in " + path,
+         expected, IOUtils.toString(getFileSystem().open(path), ASCII));
+   }
+ 
+   @Test
+   public void testCopyFileNoOverwrite() throws Throwable {
+     file = createTempFile("hello");
+     Path dest = upload(file, true);
+     intercept(FileAlreadyExistsException.class,
+         () -> upload(file, false));
+   }
+ 
+   @Test
+   public void testCopyFileOverwrite() throws Throwable {
+     file = createTempFile("hello");
+     Path dest = upload(file, true);
+     String updated = "updated";
+     FileUtils.write(file, updated, ASCII);
+     upload(file, true);
+     assertFileTextEquals(dest, updated);
+   }
+ 
+   @Test
+   public void testCopyFileNoOverwriteDirectory() throws Throwable {
+     file = createTempFile("hello");
+     Path dest = upload(file, true);
+     S3AFileSystem fs = getFileSystem();
+     fs.delete(dest, false);
+     fs.mkdirs(dest);
+     intercept(FileAlreadyExistsException.class,
+         () -> upload(file, true));
+   }
+ 
+   @Test
+   public void testCopyMissingFile() throws Throwable {
+     file = File.createTempFile("test", ".txt");
+     file.delete();
+     // first upload to create
+     intercept(FileNotFoundException.class, "No file",
+         () -> upload(file, true));
+   }
+ 
+   @Test
+   public void testCopyDirectoryFile() throws Throwable {
+     file = File.createTempFile("test", ".txt");
+     // first upload to create
+     intercept(FileNotFoundException.class, "Not a file",
+         () -> upload(file.getParentFile(), true));
+   }
+ 
+ 
+   @Test
+   public void testLocalFilesOnly() throws Throwable {
+     Path dst = path("testLocalFilesOnly");
+     intercept(IllegalArgumentException.class,
+         () -> {
+           getFileSystem().copyFromLocalFile(false, true, dst, dst);
+           return "copy successful";
+         });
+   }
+ 
+   public Path upload(File srcFile, boolean overwrite) throws IOException {
+     Path src = new Path(srcFile.toURI());
+     Path dst = path(srcFile.getName());
+     getFileSystem().copyFromLocalFile(false, overwrite, src, dst);
+     return dst;
+   }
+ 
+   /**
+    * Create a temp file with some text.
+    * @param text text for the file
+    * @return the file
+    * @throws IOException on a failure
+    */
+   public File createTempFile(String text) throws IOException {
+     File f = File.createTempFile("test", ".txt");
+     FileUtils.write(f, text, ASCII);
+     return f;
+   }
+ }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f02de2e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java
----------------------------------------------------------------------
diff --cc hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java
index 6a0f878,1b49d07..2d47bb3
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java
@@@ -26,9 -27,9 +27,11 @@@ import org.slf4j.LoggerFactory
  import org.apache.hadoop.conf.Configuration;
  import org.apache.hadoop.fs.FileSystemContractBaseTest;
  import org.apache.hadoop.fs.Path;
+ import static org.junit.Assume.*;
+ import static org.junit.Assert.*;
  
 +import java.io.IOException;
 +
  /**
   *  Tests a live S3 system. If your keys and bucket aren't specified, all tests
   *  are marked as passed.


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message