hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ste...@apache.org
Subject [50/50] [abbrv] hadoop git commit: Merge branch 'trunk' into s3guard/HADOOP-13345
Date Wed, 11 Jan 2017 18:20:06 GMT
Merge branch 'trunk' into s3guard/HADOOP-13345

# Conflicts:
#	hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0c61010d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0c61010d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0c61010d

Branch: refs/heads/HADOOP-13345
Commit: 0c61010deec872a56149e58055fc90aba08fbb83
Parents: a5cc315 e648b6e
Author: Steve Loughran <stevel@apache.org>
Authored: Wed Jan 11 18:18:49 2017 +0000
Committer: Steve Loughran <stevel@apache.org>
Committed: Wed Jan 11 18:18:49 2017 +0000

----------------------------------------------------------------------
 .gitignore                                      |    1 +
 BUILDING.txt                                    |   25 +
 LICENSE.txt                                     | 1322 +++++++++++-------
 NOTICE.txt                                      |  129 +-
 dev-support/bin/dist-copynativelibs             |   13 +
 dev-support/bin/dist-layout-stitching           |    8 +-
 .../resources/assemblies/hadoop-kms-dist.xml    |   25 +-
 hadoop-client-modules/hadoop-client-api/pom.xml |  216 +++
 .../hadoop-client-check-invariants/pom.xml      |  124 ++
 .../hadoop-client-check-test-invariants/pom.xml |  132 ++
 .../hadoop-client-integration-tests/pom.xml     |  130 ++
 .../apache/hadoop/example/ITUseMiniCluster.java |  113 ++
 .../src/test/resources/hdfs-site.xml            |   34 +
 .../src/test/resources/log4j.properties         |   24 +
 .../hadoop-client-minicluster/pom.xml           |  745 ++++++++++
 .../hadoop-client-runtime/pom.xml               |  318 +++++
 hadoop-client-modules/hadoop-client/pom.xml     |  337 +++++
 hadoop-client-modules/pom.xml                   |   45 +
 hadoop-client/pom.xml                           |  319 -----
 .../server/AuthenticationFilter.java            |    6 +-
 .../server/KerberosAuthenticationHandler.java   |  254 +++-
 .../authentication/util/KerberosName.java       |    4 +-
 .../authentication/util/TestKerberosName.java   |   22 +
 hadoop-common-project/hadoop-common/pom.xml     |   24 +-
 .../hadoop-common/src/CMakeLists.txt            |   29 +
 .../hadoop-common/src/config.h.cmake            |    1 +
 .../src/main/bin/hadoop-functions.sh            |   51 +-
 .../hadoop/conf/ConfigurationWithLogging.java   |  113 ++
 .../hadoop/fs/CommonConfigurationKeys.java      |   16 +
 .../main/java/org/apache/hadoop/fs/FsShell.java |    7 +-
 .../org/apache/hadoop/fs/ftp/FTPFileSystem.java |   70 +-
 .../hadoop/fs/viewfs/ChRootedFileSystem.java    |    5 +
 .../apache/hadoop/fs/viewfs/ViewFileSystem.java |   11 +
 .../apache/hadoop/ha/ZKFailoverController.java  |    7 +-
 .../org/apache/hadoop/http/HttpServer2.java     |  163 ++-
 .../apache/hadoop/io/compress/Decompressor.java |    2 +-
 .../hadoop/io/compress/ZStandardCodec.java      |  242 ++++
 .../io/compress/zstd/ZStandardCompressor.java   |  305 ++++
 .../io/compress/zstd/ZStandardDecompressor.java |  323 +++++
 .../hadoop/io/compress/zstd/package-info.java   |   22 +
 .../org/apache/hadoop/io/nativeio/NativeIO.java |   40 +-
 .../hadoop/metrics2/MetricsJsonBuilder.java     |  125 ++
 .../hadoop/metrics2/lib/MutableQuantiles.java   |   10 +
 .../lib/MutableRatesWithAggregation.java        |   40 +-
 .../hadoop/metrics2/lib/RollingAverages.java    |  251 ++++
 .../apache/hadoop/security/ssl/SSLFactory.java  |   64 +-
 .../hadoop/util/DiskValidatorFactory.java       |    5 +-
 .../hadoop/util/GenericOptionsParser.java       |   14 +-
 .../apache/hadoop/util/NativeCodeLoader.java    |    5 +
 .../hadoop/util/NativeLibraryChecker.java       |   13 +-
 .../hadoop/util/ReadWriteDiskValidator.java     |   95 ++
 .../util/ReadWriteDiskValidatorMetrics.java     |  170 +++
 .../main/java/org/apache/hadoop/util/Shell.java |   24 +
 .../io/compress/zstd/ZStandardCompressor.c      |  259 ++++
 .../io/compress/zstd/ZStandardDecompressor.c    |  218 +++
 .../zstd/org_apache_hadoop_io_compress_zstd.h   |   34 +
 .../org/apache/hadoop/util/NativeCodeLoader.c   |   11 +
 ...g.apache.hadoop.io.compress.CompressionCodec |    1 +
 .../src/main/resources/core-default.xml         |   68 +-
 .../src/site/markdown/ClusterSetup.md           |    1 +
 .../src/site/markdown/CommandsManual.md         |   15 +-
 .../hadoop-common/src/site/markdown/Metrics.md  |   37 +
 .../src/site/markdown/NativeLibraries.md.vm     |    1 +
 .../src/site/markdown/SingleCluster.md.vm       |   12 +-
 .../conf/TestCommonConfigurationFields.java     |   11 +-
 .../java/org/apache/hadoop/fs/TestFsShell.java  |   67 +-
 .../apache/hadoop/fs/ftp/TestFTPFileSystem.java |   56 +-
 .../fs/viewfs/ViewFileSystemBaseTest.java       |   73 +
 .../apache/hadoop/io/compress/TestCodec.java    |   12 +
 .../io/compress/TestCompressionStreamReuse.java |    8 +
 .../TestZStandardCompressorDecompressor.java    |  485 +++++++
 .../hadoop/metrics2/impl/MetricsRecords.java    |   16 +
 .../metrics2/lib/TestRollingAverages.java       |  123 ++
 .../delegation/web/TestWebDelegationToken.java  |    4 +
 .../apache/hadoop/test/GenericTestUtils.java    |   72 +-
 .../org/apache/hadoop/test/MetricsAsserts.java  |    7 +
 .../hadoop/util/TestReadWriteDiskValidator.java |  161 +++
 .../java/org/apache/hadoop/util/TestShell.java  |   52 +
 .../src/test/resources/zstd/test_file.txt       |   71 +
 .../src/test/resources/zstd/test_file.txt.zst   |  Bin 0 -> 3690 bytes
 .../src/test/scripts/hadoop_mkdir.bats          |   42 +
 .../src/test/scripts/hadoop_using_envvar.bats   |   33 +
 .../dev-support/findbugsExcludeFile.xml         |    2 +-
 hadoop-common-project/hadoop-kms/pom.xml        |  160 +--
 .../hadoop-kms/src/main/conf/kms-env.sh         |   53 +-
 .../src/main/conf/kms-log4j.properties          |    3 +-
 .../hadoop-kms/src/main/conf/kms-site.xml       |  167 +--
 .../hadoop/crypto/key/kms/server/KMSACLs.java   |   36 +-
 .../hadoop/crypto/key/kms/server/KMSAudit.java  |   30 +-
 .../crypto/key/kms/server/KMSAuditLogger.java   |   21 +-
 .../crypto/key/kms/server/KMSConfiguration.java |   16 +
 .../crypto/key/kms/server/KMSJMXServlet.java    |   36 -
 .../hadoop/crypto/key/kms/server/KMSWebApp.java |   23 +-
 .../crypto/key/kms/server/KMSWebServer.java     |  155 ++
 .../hadoop-kms/src/main/libexec/kms-config.sh   |   77 -
 .../main/libexec/shellprofile.d/hadoop-kms.sh   |   57 +
 .../src/main/resources/kms-default.xml          |  248 ++++
 .../main/resources/webapps/kms/WEB-INF/web.xml  |   68 +
 .../main/resources/webapps/static/index.html    |   35 +
 .../hadoop-kms/src/main/sbin/kms.sh             |  123 +-
 .../src/main/tomcat/ROOT/WEB-INF/web.xml        |   16 -
 .../hadoop-kms/src/main/tomcat/ROOT/index.html  |   27 -
 .../src/main/tomcat/logging.properties          |   67 -
 .../hadoop-kms/src/main/tomcat/server.xml       |  155 --
 .../src/main/tomcat/ssl-server.xml.conf         |  137 --
 .../hadoop-kms/src/main/webapp/WEB-INF/web.xml  |   78 --
 .../hadoop-kms/src/site/configuration.xsl       |   49 +
 .../hadoop-kms/src/site/markdown/index.md.vm    |  122 +-
 .../hadoop/crypto/key/kms/server/MiniKMS.java   |  118 +-
 hadoop-dist/pom.xml                             |   17 +
 .../java/org/apache/hadoop/hdfs/DFSClient.java  |   16 +-
 .../hadoop/hdfs/DistributedFileSystem.java      |   29 +
 .../apache/hadoop/hdfs/client/HdfsAdmin.java    |   10 +
 .../hadoop/hdfs/protocol/ClientProtocol.java    |    7 +
 .../datatransfer/BlockPinningException.java     |   33 +
 .../datatransfer/DataTransferProtoUtil.java     |   17 +-
 .../ClientNamenodeProtocolTranslatorPB.java     |   15 +
 .../server/datanode/BlockMetadataHeader.java    |   29 +-
 .../server/datanode/DiskBalancerWorkItem.java   |    2 +-
 .../server/datanode/DiskBalancerWorkStatus.java |    4 +-
 .../hdfs/util/CombinedHostsFileReader.java      |    2 +-
 .../apache/hadoop/hdfs/web/JsonUtilClient.java  |   55 +-
 .../hadoop/hdfs/web/WebHdfsFileSystem.java      |   73 +-
 ...onfRefreshTokenBasedAccessTokenProvider.java |    2 +-
 .../CredentialBasedAccessTokenProvider.java     |    2 +-
 .../hdfs/web/resources/CreateParentParam.java   |    2 +-
 .../hadoop/hdfs/web/resources/GetOpParam.java   |   12 +-
 .../hdfs/web/resources/OverwriteParam.java      |    2 +-
 .../src/main/proto/ClientNamenodeProtocol.proto |    2 +
 .../src/main/proto/datatransfer.proto           |    1 +
 .../src/main/proto/erasurecoding.proto          |    7 +
 .../dev-support/findbugsExcludeFile.xml         |   27 +
 .../org/apache/hadoop/hdfs/DFSConfigKeys.java   |   25 +-
 ...tNamenodeProtocolServerSideTranslatorPB.java |   14 +
 .../apache/hadoop/hdfs/protocolPB/PBHelper.java |   15 +-
 .../hadoop/hdfs/server/balancer/Dispatcher.java |   62 +-
 .../blockmanagement/DatanodeDescriptor.java     |   17 +-
 .../server/blockmanagement/DatanodeManager.java |    8 +-
 .../blockmanagement/DecommissionManager.java    |    9 +-
 .../server/blockmanagement/NumberReplicas.java  |    2 +-
 .../hdfs/server/common/HdfsServerConstants.java |    9 +-
 .../server/common/HttpGetFailedException.java   |   39 +
 .../server/common/HttpPutFailedException.java   |   37 +
 .../hadoop/hdfs/server/common/Storage.java      |   12 +-
 .../apache/hadoop/hdfs/server/common/Util.java  |  258 +++-
 .../hdfs/server/datanode/BPServiceActor.java    |   10 +-
 .../server/datanode/BlockPoolSliceStorage.java  |    8 +-
 .../hdfs/server/datanode/BlockReceiver.java     |   84 +-
 .../hdfs/server/datanode/BlockScanner.java      |   10 +-
 .../hdfs/server/datanode/BlockSender.java       |   30 +-
 .../hadoop/hdfs/server/datanode/DataNode.java   |  207 +--
 .../server/datanode/DataNodeFaultInjector.java  |   25 +-
 .../hdfs/server/datanode/DataNodeMXBean.java    |   18 +-
 .../hdfs/server/datanode/DataStorage.java       |   16 +-
 .../hdfs/server/datanode/DataXceiver.java       |   17 +-
 .../hdfs/server/datanode/DatanodeUtil.java      |   19 +-
 .../datanode/FaultInjectorFileIoEvents.java     |   55 +
 .../hdfs/server/datanode/FileIoProvider.java    | 1051 ++++++++++++++
 .../hdfs/server/datanode/FinalizedReplica.java  |   29 +
 .../hdfs/server/datanode/LocalReplica.java      |  133 +-
 .../server/datanode/LocalReplicaInPipeline.java |   41 +-
 .../server/datanode/ProfilingFileIoEvents.java  |  124 ++
 .../hdfs/server/datanode/ReplicaInPipeline.java |    4 +-
 .../hdfs/server/datanode/ReplicaInfo.java       |   17 +-
 .../server/datanode/ReplicaUnderRecovery.java   |    2 +-
 .../hdfs/server/datanode/VolumeScanner.java     |    9 +-
 .../server/datanode/checker/AsyncChecker.java   |    5 +-
 .../datanode/checker/DatasetVolumeChecker.java  |  178 ++-
 .../checker/StorageLocationChecker.java         |    8 +-
 .../datanode/checker/ThrottledAsyncChecker.java |   19 +-
 .../erasurecode/StripedBlockReader.java         |   10 +-
 .../erasurecode/StripedBlockReconstructor.java  |    7 +-
 .../erasurecode/StripedBlockWriter.java         |    1 +
 .../datanode/erasurecode/StripedReader.java     |    4 +
 .../erasurecode/StripedReconstructor.java       |   31 +
 .../datanode/erasurecode/StripedWriter.java     |    4 +
 .../fsdataset/DataNodeVolumeMetrics.java        |  289 ++++
 .../server/datanode/fsdataset/FsDatasetSpi.java |    3 +-
 .../server/datanode/fsdataset/FsVolumeSpi.java  |   16 +
 .../datanode/fsdataset/ReplicaInputStreams.java |   11 +-
 .../fsdataset/ReplicaOutputStreams.java         |   72 +-
 .../datanode/fsdataset/impl/BlockPoolSlice.java |  123 +-
 .../impl/FsDatasetAsyncDiskService.java         |    6 +-
 .../datanode/fsdataset/impl/FsDatasetImpl.java  |   53 +-
 .../datanode/fsdataset/impl/FsDatasetUtil.java  |    5 +-
 .../datanode/fsdataset/impl/FsVolumeImpl.java   |  133 +-
 .../fsdataset/impl/FsVolumeImplBuilder.java     |   12 +-
 .../datanode/fsdataset/impl/FsVolumeList.java   |   25 +-
 .../datanode/metrics/DataNodeMetrics.java       |   24 +-
 .../datanode/metrics/DataNodePeerMetrics.java   |  117 ++
 .../server/datanode/web/webhdfs/HdfsWriter.java |   19 +-
 .../server/diskbalancer/command/Command.java    |    2 +-
 .../connectors/JsonNodeConnector.java           |    2 +-
 .../datamodel/DiskBalancerCluster.java          |    2 +-
 .../datamodel/DiskBalancerVolume.java           |    2 +-
 .../server/diskbalancer/planner/NodePlan.java   |    2 +-
 .../apache/hadoop/hdfs/server/mover/Mover.java  |   26 +-
 .../server/namenode/EditLogFileInputStream.java |    3 +-
 .../server/namenode/EncryptionZoneManager.java  |    8 +-
 .../hadoop/hdfs/server/namenode/FSDirAclOp.java |    6 +-
 .../hdfs/server/namenode/FSDirAttrOp.java       |   16 +-
 .../server/namenode/FSDirErasureCodingOp.java   |  115 +-
 .../hadoop/hdfs/server/namenode/FSEditLog.java  |   18 +-
 .../hadoop/hdfs/server/namenode/FSImage.java    |    3 -
 .../server/namenode/FSImageFormatProtobuf.java  |    4 +-
 ...FSImagePreTransactionalStorageInspector.java |    2 +-
 .../hdfs/server/namenode/FSNamesystem.java      |   65 +-
 .../hdfs/server/namenode/ImageServlet.java      |   15 +-
 .../hadoop/hdfs/server/namenode/NNStorage.java  |   14 +-
 .../hadoop/hdfs/server/namenode/NameNode.java   |    5 +-
 .../hdfs/server/namenode/NameNodeMXBean.java    |   15 +-
 .../hdfs/server/namenode/NameNodeRpcServer.java |   20 +-
 .../server/namenode/NameNodeStatusMXBean.java   |    8 +-
 .../hdfs/server/namenode/NamenodeFsck.java      |   61 +-
 .../hdfs/server/namenode/TransferFsImage.java   |  295 +---
 .../server/namenode/ha/BootstrapStandby.java    |   10 +-
 .../web/resources/NamenodeWebHdfsMethods.java   |   16 +
 .../apache/hadoop/hdfs/tools/CacheAdmin.java    |    8 +-
 .../apache/hadoop/hdfs/tools/CryptoAdmin.java   |    8 +-
 .../org/apache/hadoop/hdfs/tools/DFSck.java     |    4 +
 .../hadoop/hdfs/tools/StoragePolicyAdmin.java   |    6 +-
 .../hdfs/tools/erasurecode/ECCommand.java       |   34 +
 .../org/apache/hadoop/hdfs/web/JsonUtil.java    |   33 +
 .../src/main/proto/DatanodeProtocol.proto       |   10 -
 .../hadoop-hdfs/src/main/proto/HdfsServer.proto |   15 +-
 .../src/main/resources/hdfs-default.xml         |   25 +
 .../src/main/webapps/hdfs/dfshealth.html        |   36 +-
 .../src/main/webapps/hdfs/dfshealth.js          |   11 +-
 .../src/site/markdown/HDFSCommands.md           |    6 +-
 .../src/site/markdown/HDFSErasureCoding.md      |    5 +
 .../src/site/markdown/HdfsRollingUpgrade.md     |    8 +-
 .../hadoop-hdfs/src/site/markdown/LibHdfs.md    |    2 +-
 .../hadoop-hdfs/src/site/markdown/WebHDFS.md    |  197 ++-
 .../apache/hadoop/cli/TestCryptoAdminCLI.java   |    3 +-
 .../org/apache/hadoop/hdfs/MiniDFSCluster.java  |   15 +-
 .../apache/hadoop/hdfs/StripedFileTestUtil.java |   24 +
 .../TestClientProtocolForPipelineRecovery.java  |    3 +-
 .../hadoop/hdfs/TestErasureCodingPolicies.java  |    5 +-
 .../org/apache/hadoop/hdfs/TestFileAppend.java  |    2 +-
 .../org/apache/hadoop/hdfs/TestSafeMode.java    |    1 +
 .../TestUnsetAndChangeDirectoryEcPolicy.java    |  366 +++++
 .../blockmanagement/BlockManagerTestUtil.java   |    4 +-
 .../blockmanagement/TestBlockStatsMXBean.java   |    4 +
 .../hdfs/server/common/StorageAdapter.java      |    2 +-
 .../hdfs/server/datanode/DataNodeTestUtils.java |   82 ++
 .../server/datanode/SimulatedFSDataset.java     |   56 +-
 .../server/datanode/TestBPOfferService.java     |    1 +
 .../datanode/TestBlockPoolSliceStorage.java     |    2 +-
 .../hdfs/server/datanode/TestBlockRecovery.java |    2 +-
 .../server/datanode/TestBlockReplacement.java   |   70 +-
 .../hdfs/server/datanode/TestBlockScanner.java  |  100 ++
 .../TestDataNodeErasureCodingMetrics.java       |  168 ++-
 .../datanode/TestDataNodeFaultInjector.java     |  173 +++
 .../datanode/TestDataNodeHotSwapVolumes.java    |  177 ++-
 .../server/datanode/TestDataNodeLifeline.java   |   33 +-
 .../datanode/TestDataNodePeerMetrics.java       |   92 ++
 .../datanode/TestDataNodeVolumeFailure.java     |   20 +-
 .../TestDataNodeVolumeFailureReporting.java     |   66 +-
 .../TestDataNodeVolumeFailureToleration.java    |    3 +
 .../datanode/TestDataNodeVolumeMetrics.java     |  182 +++
 .../server/datanode/TestDirectoryScanner.java   |   17 +
 .../hdfs/server/datanode/TestDiskError.java     |   24 +-
 .../server/datanode/TestSimulatedFSDataset.java |    2 +-
 .../checker/TestDatasetVolumeChecker.java       |   97 +-
 .../TestDatasetVolumeCheckerFailures.java       |   60 +-
 .../checker/TestThrottledAsyncChecker.java      |  128 +-
 .../extdataset/ExternalDatasetImpl.java         |    5 +-
 .../extdataset/ExternalReplicaInPipeline.java   |    6 +-
 .../datanode/extdataset/ExternalVolumeImpl.java |   19 +
 .../fsdataset/impl/TestFsDatasetImpl.java       |   84 +-
 .../fsdataset/impl/TestFsVolumeList.java        |   37 -
 .../hadoop/hdfs/server/mover/TestMover.java     |  163 ++-
 .../namenode/TestEncryptionZoneManager.java     |  138 ++
 .../hdfs/server/namenode/TestFSDirAttrOp.java   |   76 +
 .../hdfs/server/namenode/TestFSImage.java       |   12 +-
 .../hdfs/server/namenode/TestFileTruncate.java  |   31 +-
 .../hadoop/hdfs/server/namenode/TestFsck.java   |  340 ++++-
 .../server/namenode/TestNameNodeMXBean.java     |  225 ++-
 .../apache/hadoop/hdfs/web/TestJsonUtil.java    |    8 +-
 .../org/apache/hadoop/hdfs/web/TestWebHDFS.java |  315 +++++
 .../hdfs/web/TestWebHdfsFileSystemContract.java |   40 +
 .../hadoop/tools/TestHdfsConfigFields.java      |    4 +
 .../src/test/resources/testCryptoConf.xml       |   98 +-
 .../test/resources/testErasureCodingConf.xml    |   64 +-
 .../jobhistory/JobHistoryCopyService.java       |    5 +-
 .../hadoop/mapreduce/v2/app/AppContext.java     |    4 +-
 .../hadoop/mapreduce/v2/app/MRAppMaster.java    |    2 +-
 .../mapreduce/v2/app/job/impl/JobImpl.java      |    4 +-
 .../v2/app/rm/RMContainerAllocator.java         |   20 +-
 .../v2/app/speculate/DefaultSpeculator.java     |    3 +-
 .../mapred/TestLocalContainerLauncher.java      |    2 +-
 .../mapred/TestTaskAttemptFinishingMonitor.java |    2 +-
 .../mapred/TestTaskAttemptListenerImpl.java     |   17 +-
 .../jobhistory/TestJobHistoryEventHandler.java  |   10 +-
 .../hadoop/mapreduce/v2/app/MockAppContext.java |    4 +-
 .../mapreduce/v2/app/MockEventHandler.java      |    6 +-
 .../v2/app/TestCheckpointPreemptionPolicy.java  |    4 +-
 .../mapreduce/v2/app/TestFetchFailure.java      |   17 +-
 .../v2/app/TestKillAMPreemptionPolicy.java      |    4 +-
 .../mapreduce/v2/app/TestRuntimeEstimators.java |    3 +-
 .../app/commit/TestCommitterEventHandler.java   |    2 +-
 .../mapreduce/v2/app/job/impl/TestJobImpl.java  |    8 +
 .../app/launcher/TestContainerLauncherImpl.java |   12 +-
 .../app/local/TestLocalContainerAllocator.java  |    4 +-
 .../v2/app/rm/TestRMContainerAllocator.java     |   12 +-
 .../apache/hadoop/mapreduce/TypeConverter.java  |    7 +-
 .../org/apache/hadoop/mapred/CleanupQueue.java  |    7 +-
 .../java/org/apache/hadoop/mapred/JobConf.java  |    2 +-
 .../java/org/apache/hadoop/mapred/MapTask.java  |   10 +-
 .../java/org/apache/hadoop/mapred/Mapper.java   |    2 +-
 .../java/org/apache/hadoop/mapred/Reducer.java  |    2 +-
 .../org/apache/hadoop/mapreduce/JobStatus.java  |   18 +-
 .../apache/hadoop/mapreduce/JobSubmitter.java   |    2 +-
 .../apache/hadoop/mapreduce/MRJobConfig.java    |    6 +-
 .../mapreduce/lib/output/TextOutputFormat.java  |   15 +-
 .../task/reduce/ShuffleSchedulerImpl.java       |   40 +-
 .../src/site/markdown/MapredCommands.md         |    6 +
 .../hadoop/mapreduce/v2/hs/JobHistory.java      |    3 +-
 .../resources/job_1329348432655_0001_conf.xml   |    2 -
 .../apache/hadoop/mapred/JobClientUnitTest.java |    6 +-
 .../hadoop/mapred/TestMROpportunisticMaps.java  |    2 +-
 .../mapred/pipes/TestPipeApplication.java       |    4 +-
 .../hadoop-mapreduce-client-nativetask/pom.xml  |    8 +
 .../hadoop/examples/dancing/Pentomino.java      |    5 +
 .../hadoop/examples/terasort/TeraScheduler.java |   16 +-
 hadoop-maven-plugins/pom.xml                    |   22 +
 .../resource/ServicesResourceTransformer.java   |  164 +++
 .../plugin/shade/resource/package-info.java     |   23 +
 hadoop-project-dist/pom.xml                     |    8 +-
 hadoop-project/pom.xml                          |  130 +-
 .../hadoop/tools/TestHadoopArchiveLogs.java     |   12 +-
 .../tools/TestHadoopArchiveLogsRunner.java      |   11 +-
 .../org/apache/hadoop/fs/s3a/Constants.java     |   14 +
 .../org/apache/hadoop/fs/s3a/S3AFileSystem.java |   16 +-
 .../apache/hadoop/fs/s3a/S3AInputStream.java    |   33 +-
 .../java/org/apache/hadoop/fs/s3a/S3AUtils.java |  100 +-
 .../src/site/markdown/tools/hadoop-aws/index.md |  153 +-
 .../tools/hadoop-aws/troubleshooting_s3a.md     |   52 +
 .../fs/s3a/ITestS3AAWSCredentialsProvider.java  |    1 -
 .../hadoop/fs/s3a/ITestS3AConfiguration.java    |  130 ++
 .../apache/hadoop/fs/s3a/S3ATestConstants.java  |   12 -
 .../org/apache/hadoop/fs/s3a/S3ATestUtils.java  |   61 +-
 .../scale/ITestS3AInputStreamPerformance.java   |   57 +-
 .../hadoop/fs/s3a/scale/S3AScaleTestBase.java   |   28 +-
 hadoop-tools/hadoop-azure-datalake/pom.xml      |   10 +-
 .../fs/azure/AzureNativeFileSystemStore.java    |   91 +-
 .../org/apache/hadoop/fs/azure/KeyProvider.java |    3 +-
 .../fs/azure/LocalSASKeyGeneratorImpl.java      |  263 ++++
 .../hadoop/fs/azure/NativeAzureFileSystem.java  |   31 +-
 .../hadoop/fs/azure/NativeFileSystemStore.java  |    5 +-
 .../fs/azure/RemoteSASKeyGeneratorImpl.java     |  296 ++++
 .../fs/azure/SASKeyGenerationException.java     |   40 +
 .../hadoop/fs/azure/SASKeyGeneratorImpl.java    |   61 +
 .../fs/azure/SASKeyGeneratorInterface.java      |   66 +
 .../hadoop/fs/azure/SecureModeException.java    |   40 +
 .../fs/azure/SecureStorageInterfaceImpl.java    |  565 ++++++++
 .../hadoop/fs/azure/SelfRenewingLease.java      |    2 +-
 .../hadoop/fs/azure/SendRequestIntercept.java   |    3 +-
 .../hadoop/fs/azure/StorageInterface.java       |   28 +-
 .../fs/azure/WasbRemoteCallException.java       |   41 +
 .../hadoop/fs/azure/WasbRemoteCallHelper.java   |   93 ++
 .../metrics/AzureFileSystemInstrumentation.java |    2 +
 .../hadoop-azure/src/site/markdown/index.md     |   48 +
 .../hadoop/fs/azure/TestContainerChecks.java    |   12 +-
 .../fs/azure/TestWasbUriAndConfiguration.java   |   17 +-
 .../src/test/resources/azure-test.xml           |   15 +-
 .../src/site/markdown/DistCp.md.vm              |    1 +
 hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh  |    4 +-
 .../src/main/data/2jobs2min-rumen-jh.json       |    6 -
 .../sls/scheduler/ResourceSchedulerWrapper.java |   11 +-
 .../sls/scheduler/SLSCapacityScheduler.java     |   53 +-
 .../hadoop-yarn/bin/stop-yarn.sh                |   16 +-
 .../jdiff/Apache_Hadoop_YARN_API_2.6.0.xml      |    2 +-
 .../jdiff/Apache_Hadoop_YARN_API_2.7.2.xml      |    2 +-
 .../api/records/AbstractResourceRequest.java    |   55 -
 .../yarn/api/records/ApplicationTimeout.java    |   12 +-
 .../hadoop/yarn/api/records/ContainerState.java |    2 +
 .../yarn/api/records/ContainerUpdateType.java   |    9 +-
 .../hadoop/yarn/api/records/QueueState.java     |   11 +-
 .../yarn/api/records/ResourceRequest.java       |   19 +-
 .../yarn/api/records/UpdateContainerError.java  |    6 +
 .../api/records/UpdateContainerRequest.java     |   39 +-
 .../records/timelineservice/FlowRunEntity.java  |    4 +-
 .../org/apache/hadoop/yarn/conf/HAUtil.java     |    4 +-
 .../hadoop/yarn/conf/YarnConfiguration.java     |  136 +-
 .../src/main/proto/yarn_protos.proto            |    1 +
 .../src/main/proto/yarn_service_protos.proto    |    3 +-
 .../distributedshell/ApplicationMaster.java     |    2 +-
 .../UnmanagedAMLauncher.java                    |   10 +-
 .../api/async/impl/AMRMClientAsyncImpl.java     |   30 +-
 .../hadoop/yarn/client/cli/ApplicationCLI.java  |   16 +-
 .../apache/hadoop/yarn/client/cli/LogsCLI.java  |    2 +-
 .../hadoop/yarn/client/cli/RMAdminCLI.java      |    2 +-
 .../apache/hadoop/yarn/client/cli/TopCLI.java   |   15 +-
 ...stHedgingRequestRMFailoverProxyProvider.java |   57 +-
 ...nagerAdministrationProtocolPBClientImpl.java |   33 +-
 .../api/async/impl/TestAMRMClientAsync.java     |    2 +-
 .../yarn/client/api/impl/TestAMRMClient.java    |    2 +-
 .../api/impl/TestAMRMClientOnRMRestart.java     |   11 +-
 .../yarn/client/api/impl/TestAMRMProxy.java     |   37 +-
 .../api/impl/TestDistributedScheduling.java     |    1 -
 .../hadoop/yarn/client/cli/TestLogsCLI.java     |   15 +
 .../hadoop/yarn/client/cli/TestYarnCLI.java     |   77 +-
 .../impl/pb/AllocateResponsePBImpl.java         |    4 +-
 .../client/api/impl/TimelineClientImpl.java     |    2 +-
 .../hadoop/yarn/event/AsyncDispatcher.java      |   20 +-
 .../apache/hadoop/yarn/event/Dispatcher.java    |    2 +-
 .../nodelabels/CommonNodeLabelsManager.java     |    2 +-
 .../apache/hadoop/yarn/security/AccessType.java |    2 +
 .../org/apache/hadoop/yarn/state/Graph.java     |    2 +-
 .../yarn/state/VisualizeStateMachine.java       |    4 +-
 .../yarn/util/ProcfsBasedProcessTree.java       |    5 +-
 .../src/main/resources/yarn-default.xml         |   95 +-
 .../hadoop/yarn/api/BasePBImplRecordsTest.java  |    2 +-
 .../hadoop/yarn/event/DrainDispatcher.java      |    6 +-
 .../hadoop/yarn/event/InlineDispatcher.java     |    2 +-
 .../nodelabels/TestCommonNodeLabelsManager.java |    2 +-
 .../server/timeline/LeveldbTimelineStore.java   |   33 +-
 .../timeline/TestLeveldbTimelineStore.java      |   39 +
 .../server/api/protocolrecords/RemoteNode.java  |    7 +
 .../OpportunisticContainerAllocator.java        |   77 +-
 .../OpportunisticContainerContext.java          |   40 +-
 .../server/scheduler/SchedulerRequestKey.java   |  162 +++
 .../hadoop/yarn/server/webapp/AppBlock.java     |    9 +
 .../src/CMakeLists.txt                          |   16 +
 .../server/nodemanager/ContainerExecutor.java   |   11 +-
 .../yarn/server/nodemanager/NodeManager.java    |    4 +-
 .../nodemanager/amrmproxy/AMRMProxyService.java |    2 +-
 .../containermanager/ContainerManagerImpl.java  |    6 +-
 .../application/ApplicationImpl.java            |    2 +-
 .../container/ContainerImpl.java                |    2 +-
 .../linux/resources/TrafficController.java      |   13 +-
 .../localizer/LocalizedResource.java            |    2 +-
 .../localizer/ResourceLocalizationService.java  |    1 +
 .../logaggregation/AppLogAggregatorImpl.java    |    4 +-
 .../scheduler/ContainerScheduler.java           |   12 +-
 .../nodemanager/metrics/NodeManagerMetrics.java |   65 +-
 .../scheduler/DistributedScheduler.java         |   11 +-
 .../timelineservice/NMTimelinePublisher.java    |    2 +-
 .../nodemanager/webapp/NMWebAppFilter.java      |    8 +
 .../server/nodemanager/webapp/WebServer.java    |   35 +-
 .../impl/container-executor.c                   |   12 +-
 .../nodemanager/TestNodeHealthService.java      |    2 +-
 .../nodemanager/TestNodeStatusUpdater.java      |    4 +-
 .../launcher/TestContainerLaunch.java           |   73 +-
 .../TestResourceLocalizationService.java        |    2 +-
 .../TestAppLogAggregatorImpl.java               |    4 +-
 .../TestLogAggregationService.java              |    2 +-
 .../nodemanager/webapp/TestNMWebFilter.java     |  176 +++
 .../conf/capacity-scheduler.xml                 |    9 +
 ...ActiveStandbyElectorBasedElectorService.java |  271 ++++
 .../server/resourcemanager/AdminService.java    |   83 +-
 .../ApplicationMasterService.java               |  444 +++---
 .../server/resourcemanager/ClientRMService.java |   13 +-
 .../CuratorBasedElectorService.java             |  139 ++
 .../server/resourcemanager/EmbeddedElector.java |   41 +
 .../resourcemanager/EmbeddedElectorService.java |  260 ----
 .../resourcemanager/LeaderElectorService.java   |  129 --
 .../resourcemanager/NMLivelinessMonitor.java    |    3 +-
 ...pportunisticContainerAllocatorAMService.java |  137 +-
 .../server/resourcemanager/RMAppManager.java    |   40 +-
 .../yarn/server/resourcemanager/RMContext.java  |    6 +-
 .../server/resourcemanager/RMContextImpl.java   |   15 +-
 .../server/resourcemanager/RMServerUtils.java   |  144 +-
 .../server/resourcemanager/ResourceManager.java |   41 +-
 .../ahs/RMApplicationHistoryWriter.java         |    4 +-
 .../metrics/AbstractSystemMetricsPublisher.java |    4 +-
 .../recovery/FileSystemRMStateStore.java        |    2 +-
 .../recovery/MemoryRMStateStore.java            |    2 +-
 .../resourcemanager/recovery/RMStateStore.java  |    2 +-
 .../reservation/InMemoryPlan.java               |    4 +-
 .../reservation/ReservationInputValidator.java  |    2 +-
 .../server/resourcemanager/rmapp/RMAppImpl.java |   31 +-
 .../rmapp/attempt/AMLivelinessMonitor.java      |    3 +-
 .../rmapp/attempt/RMAppAttemptImpl.java         |   33 +-
 .../rmapp/monitor/RMAppLifetimeMonitor.java     |    4 +-
 .../rmcontainer/RMContainer.java                |    3 +-
 .../rmcontainer/RMContainerImpl.java            |   34 +-
 .../rmcontainer/RMContainerReservedEvent.java   |    2 +-
 .../resourcemanager/rmnode/RMNodeImpl.java      |    2 +-
 .../scheduler/AbstractYarnScheduler.java        |  138 +-
 .../resourcemanager/scheduler/Allocation.java   |   26 +-
 .../scheduler/AppSchedulingInfo.java            |  563 +++-----
 .../scheduler/ContainerUpdateContext.java       |  267 ++++
 .../scheduler/ContainerUpdates.java             |   68 +
 .../scheduler/QueueStateManager.java            |  100 ++
 .../scheduler/SchedulerApplicationAttempt.java  |  216 ++-
 .../scheduler/SchedulerNode.java                |   89 +-
 .../scheduler/SchedulerQueue.java               |   69 +
 .../scheduler/SchedulerQueueManager.java        |    3 +-
 .../scheduler/SchedulerRequestKey.java          |  122 --
 .../scheduler/SchedulerUtils.java               |   42 +-
 .../scheduler/YarnScheduler.java                |   24 +-
 .../scheduler/capacity/AbstractCSQueue.java     |  164 ++-
 .../AppPriorityACLConfigurationParser.java      |  219 +++
 .../scheduler/capacity/AppPriorityACLGroup.java |  108 ++
 .../scheduler/capacity/CSQueue.java             |    4 +-
 .../scheduler/capacity/CapacityScheduler.java   |  178 ++-
 .../CapacitySchedulerConfiguration.java         |   44 +-
 .../capacity/CapacitySchedulerContext.java      |    9 +
 .../capacity/CapacitySchedulerQueueManager.java |   36 +-
 .../scheduler/capacity/LeafQueue.java           |   44 +-
 .../scheduler/capacity/ParentQueue.java         |   27 +-
 .../allocator/AbstractContainerAllocator.java   |    4 +
 .../allocator/IncreaseContainerAllocator.java   |    2 +-
 .../allocator/RegularContainerAllocator.java    |  138 +-
 .../scheduler/common/PendingAsk.java            |   57 +
 .../scheduler/common/SchedulerContainer.java    |    2 +-
 .../scheduler/common/fica/FiCaSchedulerApp.java |   71 +-
 .../common/fica/FiCaSchedulerNode.java          |    6 +-
 .../distributed/NodeQueueLoadMonitor.java       |   16 +-
 .../fair/AllocationFileLoaderService.java       |    2 +-
 .../scheduler/fair/FSAppAttempt.java            |  166 +--
 .../scheduler/fair/FSPreemptionThread.java      |   11 +-
 .../scheduler/fair/FSQueueMetrics.java          |    2 +-
 .../scheduler/fair/FSSchedulerNode.java         |    2 +-
 .../scheduler/fair/FairScheduler.java           |   50 +-
 .../fair/policies/FairSharePolicy.java          |   19 +-
 .../scheduler/fifo/FifoAppAttempt.java          |   18 +-
 .../scheduler/fifo/FifoScheduler.java           |  141 +-
 .../LocalitySchedulingPlacementSet.java         |  414 ++++++
 .../placement/SchedulingPlacementSet.java       |   94 +-
 .../security/AppPriorityACLsManager.java        |  230 +++
 .../webapp/CapacitySchedulerPage.java           |    2 +-
 .../resourcemanager/webapp/NodesPage.java       |    6 +-
 .../server/resourcemanager/webapp/RMWebApp.java |    3 +-
 .../resourcemanager/webapp/dao/AppInfo.java     |    8 +-
 .../resourcemanager/webapp/dao/ClusterInfo.java |    2 +-
 .../resourcemanager/webapp/dao/NodeInfo.java    |   10 +-
 .../server/resourcemanager/ACLsTestBase.java    |    5 +-
 .../server/resourcemanager/Application.java     |    6 +-
 .../yarn/server/resourcemanager/MockAM.java     |    7 +
 .../yarn/server/resourcemanager/MockNM.java     |    6 +
 .../yarn/server/resourcemanager/MockRM.java     |   35 +-
 .../server/resourcemanager/RMHATestBase.java    |    4 +-
 .../yarn/server/resourcemanager/Task.java       |    2 +-
 .../TestApplicationMasterService.java           |    5 +-
 .../resourcemanager/TestClientRMService.java    |   13 +-
 .../TestContainerResourceUsage.java             |    8 +-
 .../TestLeaderElectorService.java               |   17 +-
 ...pportunisticContainerAllocatorAMService.java |  456 +++++-
 .../yarn/server/resourcemanager/TestRM.java     |    7 +-
 .../resourcemanager/TestRMAdminService.java     |   56 +-
 .../resourcemanager/TestRMEmbeddedElector.java  |   49 +-
 .../yarn/server/resourcemanager/TestRMHA.java   |   45 +-
 .../resourcemanager/TestRMNodeTransitions.java  |    2 +-
 .../server/resourcemanager/TestRMRestart.java   |   14 +-
 ...alCapacityPreemptionPolicyMockFramework.java |    8 +-
 ...estProportionalCapacityPreemptionPolicy.java |    6 +-
 .../recovery/RMStateStoreTestBase.java          |    2 +-
 .../TestReservationInputValidator.java          |   16 +-
 .../rmapp/TestNodesListManager.java             |    3 +-
 .../attempt/TestRMAppAttemptTransitions.java    |   19 +-
 .../rmcontainer/TestRMContainerImpl.java        |    7 +-
 .../scheduler/TestAbstractYarnScheduler.java    |  129 +-
 .../scheduler/TestAppSchedulingInfo.java        |    5 +-
 .../TestSchedulerApplicationAttempt.java        |    5 +-
 .../scheduler/TestSchedulerUtils.java           |    8 +-
 .../TestApplicationLimitsByPartition.java       |    6 +
 .../capacity/TestApplicationPriority.java       |   25 +-
 ...TestApplicationPriorityACLConfiguration.java |  120 ++
 .../capacity/TestApplicationPriorityACLs.java   |  206 +++
 .../capacity/TestCapacityScheduler.java         |  279 ++--
 .../scheduler/capacity/TestChildQueueOrder.java |   12 +-
 .../capacity/TestContainerAllocation.java       |   50 +
 .../scheduler/capacity/TestLeafQueue.java       |   80 +-
 .../TestNodeLabelContainerAllocation.java       |   65 +-
 .../scheduler/capacity/TestParentQueue.java     |    6 +-
 .../scheduler/capacity/TestQueueState.java      |  104 ++
 .../capacity/TestQueueStateManager.java         |  162 +++
 .../scheduler/capacity/TestReservations.java    |   33 +-
 .../scheduler/capacity/TestUtils.java           |    5 +-
 .../scheduler/fair/FairSchedulerTestBase.java   |   14 +-
 .../fair/TestContinuousScheduling.java          |   11 +-
 .../scheduler/fair/TestFSAppAttempt.java        |    2 +-
 .../scheduler/fair/TestFairScheduler.java       |   35 +-
 .../scheduler/fifo/TestFifoScheduler.java       |   48 +-
 .../resourcemanager/webapp/TestRMWebApp.java    |    2 +-
 .../webapp/TestRMWebServiceAppsNodelabel.java   |    2 +-
 .../webapp/TestRMWebServicesNodes.java          |   10 +-
 .../hadoop/yarn/server/MiniYARNCluster.java     |   10 +-
 .../hadoop/yarn/server/TestMiniYarnCluster.java |   80 +-
 .../pom.xml                                     |  142 +-
 ...stTimelineReaderWebServicesHBaseStorage.java |    6 +-
 ...TestPhoenixOfflineAggregationWriterImpl.java |  161 ---
 .../flow/TestHBaseStorageFlowActivity.java      |   12 +-
 .../storage/flow/TestHBaseStorageFlowRun.java   |   14 +-
 .../flow/TestHBaseStorageFlowRunCompaction.java |   44 +-
 .../hadoop-yarn-server-timelineservice/pom.xml  |   30 +-
 .../collector/TimelineCollectorManager.java     |   26 +-
 .../reader/TimelineReaderServer.java            |   26 +-
 .../PhoenixOfflineAggregationWriterImpl.java    |  358 -----
 .../storage/TimelineSchemaCreator.java          |   22 -
 .../storage/common/AppIdKeyConverter.java       |    5 +-
 .../common/HBaseTimelineStorageUtils.java       |  243 ++++
 .../storage/common/TimelineStorageUtils.java    |  207 ---
 .../storage/flow/FlowActivityColumnPrefix.java  |   10 +-
 .../storage/flow/FlowActivityRowKey.java        |    4 +-
 .../storage/flow/FlowRunColumn.java             |    6 +-
 .../storage/flow/FlowRunColumnPrefix.java       |    6 +-
 .../storage/flow/FlowRunCoprocessor.java        |    6 +-
 .../storage/flow/FlowScanner.java               |   13 +-
 .../collector/TestTimelineCollectorManager.java |   74 +
 .../reader/TestTimelineReaderServer.java        |   43 +
 .../storage/common/TestRowKeys.java             |    2 +-
 .../site/markdown/OpportunisticContainers.md    |  225 +++
 .../src/site/markdown/ResourceManagerRest.md    |  252 +++-
 .../src/site/markdown/TimelineServiceV2.md      |    2 +-
 .../src/site/markdown/YarnCommands.md           |    7 +
 .../webapp/app/components/app-timeout-bar.js    |   60 +
 .../src/main/webapp/app/models/yarn-app.js      |    2 +
 .../src/main/webapp/app/serializers/yarn-app.js |   21 +-
 .../templates/components/app-timeout-bar.hbs    |   34 +
 .../src/main/webapp/app/templates/yarn-app.hbs  |   10 +-
 .../src/main/webapp/app/utils/converter.js      |    3 +
 .../components/app-timeout-bar-test.js          |   34 +
 pom.xml                                         |    4 +-
 617 files changed, 26395 insertions(+), 8108 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c61010d/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c61010d/hadoop-project/pom.xml
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c61010d/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c61010d/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c61010d/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c61010d/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
----------------------------------------------------------------------
diff --cc hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
index 15604c6,567bacb..fa55586
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
@@@ -21,13 -21,12 +21,15 @@@ package org.apache.hadoop.fs.s3a
  import org.apache.commons.lang.StringUtils;
  import org.apache.hadoop.conf.Configuration;
  import org.apache.hadoop.fs.FileContext;
 +import org.apache.hadoop.fs.FileStatus;
  import org.apache.hadoop.fs.Path;
 +import org.apache.hadoop.fs.permission.FsPermission;
 +import org.apache.hadoop.fs.s3a.scale.S3AScaleTestBase;
  import org.junit.Assert;
+ import org.junit.Assume;
  import org.junit.internal.AssumptionViolatedException;
  import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
  
  import java.io.IOException;
  import java.net.URI;
@@@ -532,61 -502,46 +522,104 @@@ public final class S3ATestUtils 
    private S3ATestUtils() {
    }
  
 +  public static void verifyFileStatus(FileStatus status, long size,
 +      long blockSize, long modTime) {
 +    verifyFileStatus(status, size, 0, modTime, 0, blockSize, null, null, null);
 +  }
 +
 +  public static void verifyFileStatus(FileStatus status, long size,
 +      int replication,
 +      long modTime,
 +      long accessTime,
 +      long blockSize,
 +      String owner,
 +      String group,
 +      FsPermission permission) {
 +    String details = status.toString();
 +    assertFalse("Not a dir: " + details, status.isDirectory());
 +    assertEquals("Mod time: " + details, modTime, status.getModificationTime());
 +    assertEquals("File size: " + details, size, status.getLen());
 +    assertEquals("Block size: " + details, blockSize, status.getBlockSize());
 +    if (replication > 0) {
 +      assertEquals("Replication value: " + details, replication,
 +          status.getReplication());
 +    }
 +    if (accessTime != 0) {
 +      assertEquals("Access time: " + details, accessTime,
 +          status.getAccessTime());
 +    }
 +    if (owner != null) {
 +      assertEquals("Owner: " + details, owner, status.getOwner());
 +    }
 +    if (group != null) {
 +      assertEquals("Group: " + details, group, status.getGroup());
 +    }
 +    if (permission != null) {
 +      assertEquals("Permission: " + details, permission,
 +          status.getPermission());
 +    }
 +  }
 +
 +  public static void verifyDirStatus(FileStatus status,
 +      int replication,
 +      long modTime,
 +      long accessTime,
 +      String owner,
 +      String group,
 +      FsPermission permission) {
 +    String details = status.toString();
 +    assertTrue("Is a dir: " + details, status.isDirectory());
 +    assertEquals("zero length: " + details, 0, status.getLen());
 +
 +    assertEquals("Mod time: " + details, modTime, status.getModificationTime());
 +    assertEquals("Replication value: " + details, replication,
 +        status.getReplication());
 +    assertEquals("Access time: " + details, accessTime, status.getAccessTime());
 +    assertEquals("Owner: " + details, owner, status.getOwner());
 +    assertEquals("Group: " + details, group, status.getGroup());
 +    assertEquals("Permission: " + details, permission, status.getPermission());
 +  }
++
+   /**
+    * Set a bucket specific property to a particular value.
+    * If the generic key passed in has an {@code fs.s3a. prefix},
+    * that's stripped off, so that when the the bucket properties are propagated
+    * down to the generic values, that value gets copied down.
+    * @param conf configuration to set
+    * @param bucket bucket name
+    * @param genericKey key; can start with "fs.s3a."
+    * @param value value to set
+    */
+   public static void setBucketOption(Configuration conf, String bucket,
+       String genericKey, String value) {
+     final String baseKey = genericKey.startsWith(FS_S3A_PREFIX) ?
+         genericKey.substring(FS_S3A_PREFIX.length())
+         : genericKey;
+     conf.set(FS_S3A_BUCKET_PREFIX + bucket + '.' + baseKey, value);
+   }
+ 
+   /**
+    * Assert that a configuration option matches the expected value.
+    * @param conf configuration
+    * @param key option key
+    * @param expected expected value
+    */
+   public static void assertOptionEquals(Configuration conf,
+       String key,
+       String expected) {
+     assertEquals("Value of " + key, expected, conf.get(key));
+   }
+ 
+   /**
+    * Assume that a condition is met. If not: log at WARN and
+    * then throw an {@link AssumptionViolatedException}.
+    * @param message
+    * @param condition
+    */
+   public static void assume(String message, boolean condition) {
+     if (!condition) {
+       LOG.warn(message);
+     }
+     Assume.assumeTrue(message, condition);
+   }
  }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message