hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aengin...@apache.org
Subject [50/50] [abbrv] hadoop git commit: Merge branch 'trunk' into HDFS-7240
Date Fri, 10 Jun 2016 03:36:01 GMT
Merge branch 'trunk' into HDFS-7240


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/23923086
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/23923086
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/23923086

Branch: refs/heads/HDFS-7240
Commit: 23923086d1aff381fcf3a0d0e1dd19ed029bda18
Parents: 54f7975 9581fb7
Author: Anu Engineer <aengineer@apache.org>
Authored: Thu Jun 9 20:34:39 2016 -0700
Committer: Anu Engineer <aengineer@apache.org>
Committed: Thu Jun 9 20:34:39 2016 -0700

----------------------------------------------------------------------
 LICENSE.txt                                     | 1017 +++++++++++++++-
 NOTICE.txt                                      |  266 +++++
 dev-support/bin/qbt                             |   18 +
 dev-support/bin/yetus-wrapper                   |    2 +-
 dev-support/checkstyle/suppressions.xml         |   21 +
 hadoop-build-tools/pom.xml                      |   41 +
 hadoop-common-project/hadoop-auth/pom.xml       |   27 +-
 .../authentication/util/KerberosUtil.java       |   18 +-
 .../TestKerberosAuthenticationHandler.java      |    1 -
 .../authentication/util/TestKerberosUtil.java   |   26 +-
 .../hadoop-common/HadoopCommon.cmake            |    9 +-
 .../dev-support/findbugsExcludeFile.xml         |    8 +-
 hadoop-common-project/hadoop-common/pom.xml     |    4 +
 .../KeyProviderDelegationTokenExtension.java    |   30 +-
 .../org/apache/hadoop/crypto/key/KeyShell.java  |  182 +--
 .../crypto/key/kms/KMSClientProvider.java       |  158 ++-
 .../key/kms/LoadBalancingKMSClientProvider.java |   41 +-
 .../hadoop/crypto/key/kms/ValueQueue.java       |    2 +-
 .../hadoop/fs/CommonConfigurationKeys.java      |    3 +
 .../hadoop/fs/InvalidRequestException.java      |    4 +
 .../org/apache/hadoop/fs/LocalDirAllocator.java |  153 ++-
 .../hadoop/fs/PathAccessDeniedException.java    |   12 +-
 .../apache/hadoop/fs/PathNotFoundException.java |   20 +-
 .../hadoop/fs/PathPermissionException.java      |   16 +-
 .../apache/hadoop/io/FastByteComparisons.java   |    2 +-
 .../apache/hadoop/io/erasurecode/CodecUtil.java |  113 +-
 .../io/erasurecode/ErasureCoderOptions.java     |   89 ++
 .../erasurecode/coder/HHXORErasureDecoder.java  |   18 +-
 .../erasurecode/coder/HHXORErasureEncoder.java  |   15 +-
 .../io/erasurecode/coder/RSErasureDecoder.java  |    6 +-
 .../io/erasurecode/coder/RSErasureEncoder.java  |    6 +-
 .../io/erasurecode/coder/XORErasureDecoder.java |    6 +-
 .../io/erasurecode/coder/XORErasureEncoder.java |    6 +-
 .../rawcoder/AbstractRawErasureCoder.java       |  220 ----
 .../rawcoder/AbstractRawErasureDecoder.java     |  181 ---
 .../rawcoder/AbstractRawErasureEncoder.java     |  146 ---
 .../rawcoder/ByteArrayDecodingState.java        |  111 ++
 .../rawcoder/ByteArrayEncodingState.java        |   81 ++
 .../rawcoder/ByteBufferDecodingState.java       |  134 +++
 .../rawcoder/ByteBufferEncodingState.java       |   98 ++
 .../io/erasurecode/rawcoder/CoderOption.java    |   43 -
 .../io/erasurecode/rawcoder/CoderUtil.java      |  199 ++++
 .../io/erasurecode/rawcoder/DecodingState.java  |   55 +
 .../erasurecode/rawcoder/DummyRawDecoder.java   |   16 +-
 .../erasurecode/rawcoder/DummyRawEncoder.java   |   15 +-
 .../rawcoder/DummyRawErasureCoderFactory.java   |   10 +-
 .../io/erasurecode/rawcoder/EncodingState.java  |   44 +
 .../io/erasurecode/rawcoder/RSRawDecoder.java   |   48 +-
 .../rawcoder/RSRawDecoderLegacy.java            |   66 +-
 .../io/erasurecode/rawcoder/RSRawEncoder.java   |   45 +-
 .../rawcoder/RSRawEncoderLegacy.java            |   82 +-
 .../rawcoder/RSRawErasureCoderFactory.java      |    9 +-
 .../RSRawErasureCoderFactoryLegacy.java         |    9 +-
 .../erasurecode/rawcoder/RawErasureCoder.java   |   73 --
 .../rawcoder/RawErasureCoderFactory.java        |   11 +-
 .../erasurecode/rawcoder/RawErasureDecoder.java |  137 ++-
 .../erasurecode/rawcoder/RawErasureEncoder.java |  135 ++-
 .../io/erasurecode/rawcoder/XORRawDecoder.java  |   51 +-
 .../io/erasurecode/rawcoder/XORRawEncoder.java  |   57 +-
 .../rawcoder/XORRawErasureCoderFactory.java     |    9 +-
 .../io/erasurecode/rawcoder/package-info.java   |   38 +
 .../io/erasurecode/rawcoder/util/CoderUtil.java |   83 --
 .../erasurecode/rawcoder/util/GaloisField.java  |    4 +-
 .../hadoop/io/retry/AsyncCallHandler.java       |  321 +++++
 .../org/apache/hadoop/io/retry/CallReturn.java  |   75 ++
 .../hadoop/io/retry/RetryInvocationHandler.java |  134 ++-
 .../apache/hadoop/io/retry/RetryPolicies.java   |    4 +-
 .../main/java/org/apache/hadoop/ipc/Client.java |  124 +-
 .../apache/hadoop/ipc/DecayRpcScheduler.java    |  130 +-
 .../apache/hadoop/ipc/ProtobufRpcEngine.java    |   65 +-
 .../java/org/apache/hadoop/log/LogLevel.java    |  285 ++++-
 .../hadoop/metrics2/impl/MetricsSystemImpl.java |    1 +
 .../metrics2/lib/DefaultMetricsSystem.java      |    9 +
 .../metrics2/sink/RollingFileSystemSink.java    |  353 +++++-
 .../java/org/apache/hadoop/security/KDiag.java  |   34 +-
 .../hadoop/security/LdapGroupsMapping.java      |   12 +
 .../hadoop/security/ShellBasedIdMapping.java    |    2 +-
 .../hadoop/security/alias/CredentialShell.java  |  203 ++--
 .../hadoop/security/token/DtFileOperations.java |   23 +
 .../hadoop/security/token/DtUtilShell.java      |   47 +-
 .../DelegationTokenAuthenticationHandler.java   |    7 +
 .../web/DelegationTokenAuthenticator.java       |   19 +
 .../org/apache/hadoop/tools/CommandShell.java   |    4 +-
 .../org/apache/hadoop/tracing/TraceAdmin.java   |   16 +-
 .../org/apache/hadoop/util/HostsFileReader.java |  159 ++-
 .../java/org/apache/hadoop/util/KMSUtil.java    |   76 ++
 .../org/apache/hadoop/util/NativeCrc32.java     |    6 +-
 .../apache/hadoop/util/concurrent/AsyncGet.java |   65 +
 .../hadoop/util/concurrent/AsyncGetFuture.java  |   73 ++
 ...rg.apache.hadoop.security.token.TokenRenewer |   14 +
 .../src/main/resources/core-default.xml         |  109 +-
 .../src/site/markdown/CommandsManual.md         |   32 +-
 .../hadoop-common/src/site/markdown/Metrics.md  |    1 +
 .../src/site/markdown/SingleCluster.md.vm       |   15 +-
 .../hadoop-common/src/site/markdown/Tracing.md  |    9 +
 .../conf/TestCommonConfigurationFields.java     |    6 +
 .../kms/TestLoadBalancingKMSClientProvider.java |   94 ++
 .../fs/FileContextCreateMkdirBaseTest.java      |   73 +-
 .../hadoop/fs/contract/ContractTestUtils.java   |  420 +++++++
 .../erasurecode/TestCodecRawCoderMapping.java   |   29 +-
 .../hadoop/io/erasurecode/TestCoderBase.java    |   14 +-
 .../erasurecode/rawcoder/TestDummyRawCoder.java |    2 +-
 .../erasurecode/rawcoder/TestRawCoderBase.java  |   50 +-
 .../org/apache/hadoop/ipc/TestAsyncIPC.java     |  130 +-
 .../java/org/apache/hadoop/ipc/TestRPC.java     |   31 +-
 .../org/apache/hadoop/log/TestLogLevel.java     |  479 ++++++--
 .../sink/RollingFileSystemSinkTestBase.java     |   24 +-
 .../sink/TestRollingFileSystemSink.java         |  285 +++--
 .../TestRollingFileSystemSinkWithLocal.java     |  157 +++
 .../org/apache/hadoop/net/ServerSocketUtil.java |   39 +
 .../hadoop/security/TestLdapGroupsMapping.java  |  140 +++
 .../security/TestShellBasedIdMapping.java       |   16 +-
 .../hadoop/security/token/TestDtUtilShell.java  |   24 +
 .../delegation/web/TestWebDelegationToken.java  |  114 +-
 .../apache/hadoop/util/TestHostsFileReader.java |   26 +
 .../hadoop-common/src/test/resources/krb5.conf  |   23 +-
 .../key/kms/server/KMSAuthenticationFilter.java |    2 +-
 .../hadoop/crypto/key/kms/server/TestKMS.java   |  134 ++-
 .../dev-support/findbugsExcludeFile.xml         |   28 +
 hadoop-common-project/hadoop-minikdc/pom.xml    |  119 +-
 .../java/org/apache/hadoop/minikdc/MiniKdc.java |  381 ++----
 .../src/main/resources/minikdc-krb5.conf        |   25 -
 .../src/main/resources/minikdc.ldiff            |   47 -
 .../org/apache/hadoop/minikdc/TestMiniKdc.java  |   19 +-
 .../hadoop/hdfs/AsyncDistributedFileSystem.java |   86 +-
 .../org/apache/hadoop/hdfs/DFSInputStream.java  |   39 +-
 .../hadoop/hdfs/DFSStripedInputStream.java      |    7 +-
 .../hadoop/hdfs/DFSStripedOutputStream.java     |    7 +-
 .../org/apache/hadoop/hdfs/DFSUtilClient.java   |   38 +-
 .../hadoop/hdfs/DistributedFileSystem.java      |    3 +
 .../apache/hadoop/hdfs/FileChecksumHelper.java  |    3 +-
 .../hadoop/hdfs/protocol/StripedBlockInfo.java  |   10 +-
 .../hdfs/protocol/datatransfer/Sender.java      |    2 +
 .../ClientNamenodeProtocolTranslatorPB.java     |   73 +-
 .../hadoop/hdfs/protocolPB/PBHelperClient.java  |   16 +
 .../hdfs/web/resources/NoRedirectParam.java     |   49 +
 .../src/main/proto/datatransfer.proto           |    1 +
 .../hadoop/fs/http/server/HttpFSServer.java     |  218 ++--
 .../org/apache/hadoop/hdfs/DFSConfigKeys.java   |   10 +
 .../java/org/apache/hadoop/hdfs/DFSUtil.java    |   21 +-
 .../apache/hadoop/hdfs/HDFSPolicyProvider.java  |    6 +-
 .../hdfs/protocol/datatransfer/Receiver.java    |    1 +
 .../qjournal/server/JournalNodeRpcServer.java   |    6 +-
 .../hadoop/hdfs/server/balancer/Balancer.java   |    2 +-
 .../server/blockmanagement/BlockManager.java    |   72 +-
 .../blockmanagement/DecommissionManager.java    |    8 +-
 .../PendingReconstructionBlocks.java            |   16 +-
 .../hdfs/server/common/HdfsServerConstants.java |    1 -
 .../server/datanode/BlockChecksumHelper.java    |  172 ++-
 .../hadoop/hdfs/server/datanode/DataNode.java   |    3 +-
 .../server/datanode/ShortCircuitRegistry.java   |    6 +-
 .../erasurecode/ErasureCodingWorker.java        |   15 +-
 .../StripedBlockChecksumReconstructor.java      |  129 ++
 .../erasurecode/StripedBlockReconstructor.java  |  119 ++
 .../datanode/erasurecode/StripedReader.java     |   22 +-
 .../erasurecode/StripedReconstructionInfo.java  |   99 ++
 .../erasurecode/StripedReconstructor.java       |  176 +--
 .../datanode/erasurecode/StripedWriter.java     |   29 +-
 .../datanode/web/webhdfs/ParameterParser.java   |    5 +
 .../datanode/web/webhdfs/WebHdfsHandler.java    |   26 +-
 .../apache/hadoop/hdfs/server/mover/Mover.java  |    2 +-
 .../server/namenode/EncryptionZoneManager.java  |   35 +-
 .../server/namenode/FSDirEncryptionZoneOp.java  |    2 +-
 .../hadoop/hdfs/server/namenode/FSEditLog.java  |   11 +-
 .../hdfs/server/namenode/FSNamesystem.java      |   49 +-
 .../server/namenode/FSPermissionChecker.java    |   24 +-
 .../hdfs/server/namenode/LeaseManager.java      |   21 +-
 .../hdfs/server/namenode/NameNodeRpcServer.java |    5 +-
 .../hdfs/server/namenode/NamenodeFsck.java      |    2 +-
 .../web/resources/NamenodeWebHdfsMethods.java   |   82 +-
 .../org/apache/hadoop/hdfs/tools/DFSAdmin.java  |   42 +-
 .../hadoop/hdfs/tools/NNHAServiceTarget.java    |    2 +-
 .../src/main/resources/hdfs-default.xml         |   18 +
 .../src/main/webapps/hdfs/dfshealth.html        |    2 +-
 .../src/main/webapps/hdfs/explorer.html         |   25 +-
 .../src/main/webapps/hdfs/explorer.js           |   53 +-
 .../src/site/markdown/HdfsPermissionsGuide.md   |    2 +-
 .../hadoop-hdfs/src/site/markdown/WebHDFS.md    |   57 +-
 .../org/apache/hadoop/hdfs/MiniDFSCluster.java  |    5 +
 .../apache/hadoop/hdfs/StripedFileTestUtil.java |    8 +-
 .../org/apache/hadoop/hdfs/TestAsyncDFS.java    |  454 +++++++
 .../apache/hadoop/hdfs/TestAsyncDFSRename.java  |  572 ++-------
 .../apache/hadoop/hdfs/TestAsyncHDFSWithHA.java |  181 +++
 .../apache/hadoop/hdfs/TestDFSPermission.java   |   13 +
 .../org/apache/hadoop/hdfs/TestDFSShell.java    |    4 +-
 .../hadoop/hdfs/TestDFSStripedInputStream.java  |   23 +-
 .../hadoop/hdfs/TestDFSUpgradeFromImage.java    |   16 +-
 .../org/apache/hadoop/hdfs/TestDFSUtil.java     |   47 +-
 .../hdfs/TestDecommissionWithStriped.java       |   47 +
 .../hadoop/hdfs/TestDistributedFileSystem.java  |    1 -
 .../apache/hadoop/hdfs/TestFileChecksum.java    |   41 +-
 .../hadoop/hdfs/TestHDFSPolicyProvider.java     |  126 ++
 .../apache/hadoop/hdfs/TestMiniDFSCluster.java  |   31 -
 .../java/org/apache/hadoop/hdfs/TestRead.java   |   87 ++
 .../hdfs/server/balancer/TestBalancer.java      |   17 +-
 .../balancer/TestBalancerWithHANameNodes.java   |    2 +-
 .../TestBalancerWithMultipleNameNodes.java      |    2 +-
 .../balancer/TestBalancerWithNodeGroup.java     |    4 +-
 .../TestPendingReconstruction.java              |   20 +-
 .../server/datanode/SimulatedFSDataset.java     |    4 +-
 .../TestDataNodeErasureCodingMetrics.java       |   16 +
 .../hadoop/hdfs/server/mover/TestMover.java     |   12 +-
 .../hdfs/server/mover/TestStorageMover.java     |    2 +-
 .../hdfs/server/namenode/FSAclBaseTest.java     |   12 +-
 .../hdfs/server/namenode/TestLeaseManager.java  |   24 +-
 .../hdfs/server/namenode/ha/HATestUtil.java     |    9 +-
 .../server/namenode/ha/TestNNHealthCheck.java   |   12 +-
 .../shortcircuit/TestShortCircuitCache.java     |   34 +-
 .../shortcircuit/TestShortCircuitLocalRead.java |    6 +-
 .../org/apache/hadoop/hdfs/web/TestWebHDFS.java |   84 +-
 .../sink/TestRollingFileSystemSinkWithHdfs.java |  106 +-
 ...TestRollingFileSystemSinkWithSecureHdfs.java |  201 ++--
 .../apache/hadoop/tracing/TestTraceAdmin.java   |   69 +-
 .../hadoop-hdfs/src/test/resources/krb5.conf    |   19 +-
 .../src/test/resources/testHDFSConf.xml         |    2 +-
 .../hadoop/mapred/LocalContainerLauncher.java   |   28 +
 .../jobhistory/JobHistoryEventHandler.java      |    4 +-
 .../hadoop/mapreduce/v2/app/MRAppMaster.java    |   32 +-
 .../v2/app/job/impl/TaskAttemptImpl.java        |    7 +-
 .../v2/app/launcher/ContainerLauncherEvent.java |   21 +-
 .../v2/app/launcher/ContainerLauncherImpl.java  |   19 +-
 .../v2/app/rm/RMContainerAllocator.java         |  192 +--
 .../v2/app/rm/RMContainerRequestor.java         |   19 +-
 .../v2/app/rm/ResourceCalculatorUtils.java      |   17 +-
 .../CheckpointAMPreemptionPolicy.java           |    8 +-
 .../v2/app/TestCheckpointPreemptionPolicy.java  |   16 +-
 .../hadoop/mapreduce/v2/app/TestRecovery.java   |    2 +-
 .../mapreduce/v2/app/TestStagingCleanup.java    |  120 ++
 .../v2/app/job/impl/TestTaskAttempt.java        |   29 +-
 .../v2/app/launcher/TestContainerLauncher.java  |   10 +-
 .../app/launcher/TestContainerLauncherImpl.java |    8 +
 .../app/local/TestLocalContainerAllocator.java  |    2 +-
 .../v2/app/rm/TestRMContainerAllocator.java     |    2 +-
 .../v2/app/rm/TestResourceCalculatorUtils.java  |    4 +-
 .../apache/hadoop/mapreduce/TypeConverter.java  |    6 +-
 .../mapreduce/v2/jobhistory/JHAdminConfig.java  |    8 +
 .../java/org/apache/hadoop/mapred/JobConf.java  |    6 -
 .../org/apache/hadoop/mapreduce/Cluster.java    |   15 +-
 .../org/apache/hadoop/mapreduce/JobStatus.java  |   18 +-
 .../apache/hadoop/mapreduce/JobSubmitter.java   |    7 +
 .../apache/hadoop/mapreduce/MRJobConfig.java    |   18 +
 .../jobhistory/NormalizedResourceEvent.java     |    6 +-
 .../org/apache/hadoop/mapreduce/tools/CLI.java  |    6 +-
 .../src/main/resources/mapred-default.xml       |   16 +
 .../mapreduce/v2/hs/HistoryClientService.java   |    1 +
 .../hadoop/mapred/ResourceMgrDelegate.java      |   12 +-
 .../apache/hadoop/mapred/JobClientUnitTest.java |    6 +-
 .../hadoop/mapred/TestClientRedirect.java       |   10 +-
 .../hadoop/mapred/TestLocalJobSubmission.java   |   24 +-
 .../hadoop/mapred/TestMROpportunisticMaps.java  |  281 +++++
 .../hadoop/mapred/TestMiniMRChildTask.java      |  233 +---
 .../TestClientProtocolProviderImpls.java        |   26 +-
 .../apache/hadoop/mapreduce/v2/TestMRJobs.java  |  119 ++
 hadoop-project-dist/pom.xml                     |    2 +
 hadoop-project/pom.xml                          |   48 +-
 hadoop-project/src/site/site.xml                |    2 +
 hadoop-tools/hadoop-ant/pom.xml                 |   56 -
 .../java/org/apache/hadoop/ant/DfsTask.java     |  220 ----
 .../ant/condition/DfsBaseConditional.java       |   68 --
 .../apache/hadoop/ant/condition/DfsExists.java  |   24 -
 .../apache/hadoop/ant/condition/DfsIsDir.java   |   24 -
 .../apache/hadoop/ant/condition/DfsZeroLen.java |   24 -
 .../resources/org/apache/hadoop/ant/antlib.xml  |   29 -
 hadoop-tools/hadoop-aws/pom.xml                 |   16 +
 .../hadoop/fs/s3a/AWSClientIOException.java     |   50 +
 .../apache/hadoop/fs/s3a/AWSS3IOException.java  |   61 +
 .../hadoop/fs/s3a/AWSServiceIOException.java    |   72 ++
 .../fs/s3a/AnonymousAWSCredentialsProvider.java |   11 +
 .../fs/s3a/BasicAWSCredentialsProvider.java     |   11 +-
 .../org/apache/hadoop/fs/s3a/Constants.java     |   18 +-
 .../s3a/CredentialInitializationException.java  |   46 +
 .../fs/s3a/ProgressableProgressListener.java    |   94 ++
 .../hadoop/fs/s3a/S3AFastOutputStream.java      |  152 ++-
 .../org/apache/hadoop/fs/s3a/S3AFileStatus.java |    7 +
 .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 1032 +++++++++++-----
 .../apache/hadoop/fs/s3a/S3AInputStream.java    |   33 +-
 .../hadoop/fs/s3a/S3AInstrumentation.java       |  218 ++--
 .../apache/hadoop/fs/s3a/S3AOutputStream.java   |  116 +-
 .../hadoop/fs/s3a/S3AStorageStatistics.java     |  104 ++
 .../java/org/apache/hadoop/fs/s3a/S3AUtils.java |  237 ++++
 .../org/apache/hadoop/fs/s3a/Statistic.java     |  143 +++
 .../fs/s3a/TemporaryAWSCredentialsProvider.java |   70 ++
 .../org/apache/hadoop/fs/s3a/package-info.java  |   28 +
 .../src/site/markdown/tools/hadoop-aws/index.md |  164 ++-
 .../fs/contract/s3a/TestS3AContractDistCp.java  |   46 +
 .../hadoop/fs/s3a/AbstractS3ATestBase.java      |  103 ++
 .../apache/hadoop/fs/s3a/S3ATestConstants.java  |   76 ++
 .../org/apache/hadoop/fs/s3a/S3ATestUtils.java  |  293 ++++-
 .../fs/s3a/TestS3AAWSCredentialsProvider.java   |  171 +++
 .../hadoop/fs/s3a/TestS3AConfiguration.java     |   23 +-
 .../apache/hadoop/fs/s3a/TestS3AEncryption.java |  104 ++
 .../TestS3AEncryptionAlgorithmPropagation.java  |   82 ++
 .../s3a/TestS3AEncryptionFastOutputStream.java  |   35 +
 .../hadoop/fs/s3a/TestS3AFailureHandling.java   |  194 +++
 .../hadoop/fs/s3a/TestS3AFileOperationCost.java |  191 +++
 .../fs/s3a/TestS3AFileSystemContract.java       |    1 -
 .../fs/s3a/TestS3ATemporaryCredentials.java     |  150 +++
 .../hadoop/fs/s3a/scale/S3AScaleTestBase.java   |  130 +-
 .../fs/s3a/scale/TestS3ADeleteManyFiles.java    |   10 +-
 .../s3a/scale/TestS3ADirectoryPerformance.java  |  189 +++
 .../scale/TestS3AInputStreamPerformance.java    |    6 +-
 .../src/test/resources/log4j.properties         |    4 +-
 .../dev-support/findbugs-exclude.xml            |   24 +
 hadoop-tools/hadoop-azure-datalake/pom.xml      |  180 +++
 .../main/java/org/apache/hadoop/fs/adl/Adl.java |   52 +
 .../org/apache/hadoop/fs/adl/AdlFileSystem.java |   41 +
 ...hedRefreshTokenBasedAccessTokenProvider.java |  135 +++
 .../hadoop/fs/adl/oauth2/package-info.java      |   23 +
 .../org/apache/hadoop/fs/adl/package-info.java  |   23 +
 .../org/apache/hadoop/hdfs/web/ADLConfKeys.java |   61 +
 .../apache/hadoop/hdfs/web/BufferManager.java   |  180 +++
 .../web/PrivateAzureDataLakeFileSystem.java     | 1108 ++++++++++++++++++
 ...hedRefreshTokenBasedAccessTokenProvider.java |   37 +
 .../hadoop/hdfs/web/oauth2/package-info.java    |   24 +
 .../apache/hadoop/hdfs/web/package-info.java    |   25 +
 .../hadoop/hdfs/web/resources/ADLFlush.java     |   49 +
 .../hdfs/web/resources/ADLGetOpParam.java       |   96 ++
 .../hdfs/web/resources/ADLPostOpParam.java      |   97 ++
 .../hdfs/web/resources/ADLPutOpParam.java       |   94 ++
 .../hdfs/web/resources/ADLVersionInfo.java      |   51 +
 .../web/resources/AppendADLNoRedirectParam.java |   45 +
 .../web/resources/CreateADLNoRedirectParam.java |   44 +
 .../hadoop/hdfs/web/resources/LeaseParam.java   |   53 +
 .../web/resources/ReadADLNoRedirectParam.java   |   44 +
 .../hadoop/hdfs/web/resources/package-info.java |   27 +
 .../src/site/markdown/index.md                  |  219 ++++
 ...hedRefreshTokenBasedAccessTokenProvider.java |  147 +++
 hadoop-tools/hadoop-azure/pom.xml               |   19 +
 .../contract/TestAzureNativeContractDistCp.java |   33 +
 hadoop-tools/hadoop-distcp/pom.xml              |   16 +
 .../apache/hadoop/tools/util/DistCpUtils.java   |   10 +-
 .../src/site/markdown/DistCp.md.vm              |    1 +
 .../contract/AbstractContractDistCpTest.java    |  204 ++++
 .../sls/scheduler/FairSchedulerMetrics.java     |   60 +-
 .../sls/scheduler/ResourceSchedulerWrapper.java |   36 +-
 .../sls/scheduler/SLSCapacityScheduler.java     |   36 +-
 hadoop-tools/hadoop-tools-dist/pom.xml          |    4 +-
 hadoop-tools/pom.xml                            |    2 +-
 hadoop-yarn-project/hadoop-yarn/bin/yarn        |    2 +
 .../dev-support/findbugs-exclude.xml            |    5 +
 .../yarn/api/ApplicationClientProtocol.java     |   27 +-
 .../yarn/api/ContainerManagementProtocol.java   |    5 +
 .../GetNewReservationRequest.java               |   41 +
 .../GetNewReservationResponse.java              |   71 ++
 .../ReservationSubmissionRequest.java           |   25 +-
 .../ReservationSubmissionResponse.java          |   46 +-
 .../SignalContainerResponse.java                |    2 +-
 .../yarn/api/records/AMBlackListingRequest.java |   67 --
 .../records/ApplicationSubmissionContext.java   |   23 -
 .../hadoop/yarn/api/records/Container.java      |   26 +
 .../yarn/api/records/ExecutionTypeRequest.java  |  124 ++
 .../hadoop/yarn/api/records/Resource.java       |   34 +-
 .../yarn/api/records/ResourceRequest.java       |   34 +-
 .../hadoop/yarn/conf/YarnConfiguration.java     |   67 +-
 .../main/proto/applicationclient_protocol.proto |    3 +-
 .../proto/containermanagement_protocol.proto    |    1 +
 .../src/main/proto/yarn_protos.proto            |   17 +-
 .../src/main/proto/yarn_service_protos.proto    |    9 +-
 .../yarn/conf/TestYarnConfigurationFields.java  |    7 +
 .../distributedshell/ApplicationMaster.java     |   10 +-
 .../applications/distributedshell/Client.java   |    4 +-
 .../hadoop/yarn/client/api/YarnClient.java      |   25 +-
 .../yarn/client/api/impl/AMRMClientImpl.java    |   16 +-
 .../yarn/client/api/impl/YarnClientImpl.java    |   14 +-
 .../hadoop/yarn/client/cli/ApplicationCLI.java  |    6 +-
 .../apache/hadoop/yarn/client/cli/LogsCLI.java  |  376 +++---
 .../apache/hadoop/yarn/client/cli/NodeCLI.java  |    4 +-
 .../apache/hadoop/yarn/client/cli/TopCLI.java   |    4 +-
 .../api/impl/TestDistributedScheduling.java     |    9 +-
 .../yarn/client/api/impl/TestYarnClient.java    |  374 ++++--
 .../hadoop/yarn/client/cli/TestLogsCLI.java     |  179 ++-
 .../yarn/api/ContainerManagementProtocolPB.java |    7 +
 .../ApplicationClientProtocolPBClientImpl.java  |   23 +-
 ...ContainerManagementProtocolPBClientImpl.java |   19 +
 .../ApplicationClientProtocolPBServiceImpl.java |   26 +-
 ...ontainerManagementProtocolPBServiceImpl.java |   20 +
 .../impl/pb/GetNewReservationRequestPBImpl.java |   77 ++
 .../pb/GetNewReservationResponsePBImpl.java     |  144 +++
 .../pb/ReservationSubmissionRequestPBImpl.java  |   30 +
 .../pb/ReservationSubmissionResponsePBImpl.java |   59 -
 .../impl/pb/AMBlackListingRequestPBImpl.java    |  104 --
 .../pb/ApplicationSubmissionContextPBImpl.java  |   40 -
 .../api/records/impl/pb/ContainerPBImpl.java    |   26 +-
 .../impl/pb/ExecutionTypeRequestPBImpl.java     |   93 ++
 .../yarn/api/records/impl/pb/ProtoUtils.java    |   15 +
 .../api/records/impl/pb/ResourcePBImpl.java     |   30 +-
 .../records/impl/pb/ResourceRequestPBImpl.java  |   52 +-
 .../logaggregation/AggregatedLogFormat.java     |  175 ++-
 .../logaggregation/ContainerLogsRequest.java    |  135 +++
 .../yarn/logaggregation/LogCLIHelpers.java      |  160 ++-
 .../nodelabels/CommonNodeLabelsManager.java     |   18 +-
 .../yarn/security/SchedulerSecurityInfo.java    |    2 +-
 .../apache/hadoop/yarn/util/AdHocLogDumper.java |    9 +-
 .../yarn/util/ProcfsBasedProcessTree.java       |   19 +-
 .../util/ResourceCalculatorProcessTree.java     |    4 +-
 .../yarn/util/WindowsBasedProcessTree.java      |    8 +
 .../resource/DefaultResourceCalculator.java     |   36 +-
 .../resource/DominantResourceCalculator.java    |   46 +-
 .../yarn/util/resource/ResourceCalculator.java  |   10 +-
 .../hadoop/yarn/util/resource/Resources.java    |   62 +-
 .../org/apache/hadoop/yarn/webapp/WebApps.java  |   53 +-
 .../hadoop/yarn/webapp/util/WebAppUtils.java    |   18 +
 .../src/main/resources/yarn-default.xml         |   73 +-
 .../hadoop/yarn/TestContainerLaunchRPC.java     |   10 +
 .../yarn/TestContainerResourceIncreaseRPC.java  |    8 +
 .../java/org/apache/hadoop/yarn/TestRPC.java    |   10 +
 .../hadoop/yarn/api/TestPBImplRecords.java      |   12 +-
 .../yarn/util/TestProcfsBasedProcessTree.java   |   66 +-
 .../yarn/util/resource/TestResources.java       |    8 +-
 .../client/impl/zk/RegistrySecurity.java        |    2 +-
 .../ApplicationHistoryServer.java               |    1 +
 .../webapp/AHSWebServices.java                  |  187 +--
 .../webapp/TestAHSWebServices.java              |   66 ++
 .../api/DistributedSchedulerProtocolPB.java     |   10 +-
 .../pb/DistSchedRegisterResponsePBImpl.java     |    6 +-
 .../hadoop/yarn/server/utils/BuilderUtils.java  |   12 +-
 .../hadoop/yarn/server/webapp/dao/AppInfo.java  |   10 +-
 .../yarn/server/webapp/dao/ContainerInfo.java   |   10 +-
 .../proto/distributed_scheduler_protocol.proto  |    1 +
 .../hadoop/yarn/TestYarnServerApiClasses.java   |    6 +-
 .../server/nodemanager/ContainerExecutor.java   |    2 +-
 .../nodemanager/DefaultContainerExecutor.java   |   22 +-
 .../containermanager/ContainerManagerImpl.java  |   40 +-
 .../application/ApplicationImpl.java            |   80 +-
 .../container/ContainerImpl.java                |   20 +-
 .../launcher/RecoveredContainerLaunch.java      |    7 +-
 .../CGroupsMemoryResourceHandlerImpl.java       |    4 +-
 .../runtime/DockerLinuxContainerRuntime.java    |   46 +-
 .../localizer/ContainerLocalizer.java           |   40 +-
 .../logaggregation/AppLogAggregatorImpl.java    |  101 +-
 .../logaggregation/LogAggregationService.java   |   60 +-
 .../event/LogHandlerAppStartedEvent.java        |   21 +-
 .../monitor/ContainerMetrics.java               |   18 +-
 .../monitor/ContainersMonitorImpl.java          |   41 +-
 .../queuing/QueuingContainerManagerImpl.java    |   41 +-
 .../nodemanager/metrics/NodeManagerMetrics.java |   12 +-
 .../nodemanager/scheduler/LocalScheduler.java   |    3 +-
 .../OpportunisticContainerAllocator.java        |    3 +-
 .../nodemanager/webapp/NMWebServices.java       |   89 +-
 .../server/nodemanager/webapp/WebServer.java    |    5 +-
 .../nodemanager/webapp/dao/ContainerInfo.java   |    2 +-
 .../impl/container-executor.c                   |   19 +-
 .../test/test-container-executor.c              |   49 -
 .../yarn_server_nodemanager_recovery.proto      |    1 +
 .../TestDefaultContainerExecutor.java           |   89 +-
 .../yarn/server/nodemanager/TestEventFlow.java  |    3 +
 .../nodemanager/TestNodeStatusUpdater.java      |    4 +-
 .../TestNodeStatusUpdaterForLabels.java         |    7 +
 .../amrmproxy/MockResourceManagerFacade.java    |   12 +-
 .../BaseContainerManagerTest.java               |   67 ++
 .../containermanager/TestContainerManager.java  |   76 +-
 .../application/TestApplication.java            |   45 +-
 .../container/TestContainer.java                |   34 +
 .../runtime/TestDockerContainerRuntime.java     |  298 ++++-
 .../MockLocalizerHeartbeatResponse.java         |    7 +-
 .../localizer/TestContainerLocalizer.java       |   41 +-
 .../TestAppLogAggregatorImpl.java               |  436 +++++++
 .../TestLogAggregationService.java              |    7 +-
 .../MockCPUResourceCalculatorProcessTree.java   |   70 ++
 .../MockResourceCalculatorProcessTree.java      |    5 +
 .../monitor/TestContainerMetrics.java           |    4 +-
 .../monitor/TestContainersMonitor.java          |    2 +-
 .../TestContainersMonitorResourceChange.java    |   62 +-
 .../queuing/TestQueuingContainerManager.java    |  551 +++++++--
 .../scheduler/TestLocalScheduler.java           |   11 +-
 .../nodemanager/webapp/TestNMWebServices.java   |   63 +-
 .../server/resourcemanager/ClientRMService.java |   38 +-
 .../resourcemanager/NodesListManager.java       |   69 +-
 .../server/resourcemanager/RMAuditLogger.java   |    2 +
 .../yarn/server/resourcemanager/RMNMInfo.java   |    4 +-
 .../server/resourcemanager/RMServerUtils.java   |   14 +-
 .../server/resourcemanager/ResourceManager.java |    1 +
 .../resourcemanager/ResourceTrackerService.java |    2 +-
 .../blacklist/BlacklistManager.java             |    9 +-
 .../blacklist/BlacklistUpdates.java             |   47 -
 .../blacklist/DisabledBlacklistManager.java     |   12 +-
 .../blacklist/SimpleBlacklistManager.java       |   17 +-
 .../metrics/SystemMetricsPublisher.java         |    2 +-
 .../monitor/capacity/TempQueuePerPartition.java |   12 +-
 .../nodelabels/RMNodeLabelsManager.java         |    9 +-
 .../AbstractSchedulerPlanFollower.java          |    2 +-
 .../reservation/CapacityOverTimePolicy.java     |    6 +-
 .../reservation/NoOverCommitPolicy.java         |   38 +-
 .../RLESparseResourceAllocation.java            |    6 +-
 .../reservation/ReservationInputValidator.java  |    7 +
 .../reservation/ReservationSystemUtil.java      |    2 +-
 .../planning/StageAllocatorGreedyRLE.java       |   16 +-
 .../planning/StageEarliestStartByDemand.java    |    2 +-
 .../server/resourcemanager/rmapp/RMAppImpl.java |   79 +-
 .../rmapp/attempt/RMAppAttempt.java             |    2 +-
 .../rmapp/attempt/RMAppAttemptImpl.java         |  113 +-
 .../rmcontainer/RMContainerImpl.java            |    2 +-
 .../scheduler/AbstractYarnScheduler.java        |   10 +-
 .../scheduler/AppSchedulingInfo.java            |   74 +-
 .../scheduler/ClusterNodeTracker.java           |   10 +-
 .../resourcemanager/scheduler/QueueMetrics.java |   61 +-
 .../scheduler/SchedulerAppUtils.java            |   16 +-
 .../scheduler/SchedulerApplicationAttempt.java  |   39 +-
 .../scheduler/SchedulerUtils.java               |    8 +-
 .../scheduler/capacity/CSQueueMetrics.java      |   24 +-
 .../capacity/CapacityHeadroomProvider.java      |    2 +-
 .../scheduler/capacity/CapacityScheduler.java   |   11 +-
 .../CapacitySchedulerConfiguration.java         |    6 +-
 .../scheduler/capacity/LeafQueue.java           |    6 +-
 .../allocator/RegularContainerAllocator.java    |    4 +-
 .../scheduler/fair/FSAppAttempt.java            |    5 +-
 .../scheduler/fair/FSLeafQueue.java             |   25 +-
 .../scheduler/fair/FSParentQueue.java           |    2 +-
 .../resourcemanager/scheduler/fair/FSQueue.java |   35 +-
 .../scheduler/fair/FSQueueMetrics.java          |   66 +-
 .../scheduler/fair/FairScheduler.java           |   47 +-
 .../fair/FairSchedulerConfiguration.java        |   12 +
 .../scheduler/fair/QueueManager.java            |    5 +-
 .../fair/policies/ComputeFairShares.java        |   18 +-
 .../DominantResourceFairnessPolicy.java         |   10 +-
 .../fair/policies/FairSharePolicy.java          |   20 +-
 .../scheduler/fair/policies/FifoPolicy.java     |    8 +-
 .../scheduler/fifo/FifoScheduler.java           |   27 +-
 .../scheduler/policy/FairOrderingPolicy.java    |    5 +-
 .../security/DelegationTokenRenewer.java        |   27 +-
 .../webapp/DefaultSchedulerPage.java            |    4 +-
 .../webapp/FairSchedulerAppsBlock.java          |    2 +-
 .../webapp/RMAppAttemptBlock.java               |    9 +-
 .../resourcemanager/webapp/RMAppBlock.java      |   13 +-
 .../resourcemanager/webapp/RMWebServices.java   |  128 +-
 .../webapp/dao/AMBlackListingRequestInfo.java   |   61 -
 .../webapp/dao/AppAttemptInfo.java              |    8 +-
 .../resourcemanager/webapp/dao/AppInfo.java     |   20 +-
 .../dao/ApplicationSubmissionContextInfo.java   |   13 -
 .../webapp/dao/FairSchedulerInfo.java           |    4 +-
 .../webapp/dao/FairSchedulerQueueInfo.java      |   14 +-
 .../webapp/dao/FifoSchedulerInfo.java           |   18 +-
 .../webapp/dao/NewReservation.java              |   55 +
 .../resourcemanager/webapp/dao/NodeInfo.java    |    4 +-
 .../dao/ReservationSubmissionRequestInfo.java   |   11 +
 .../dao/ReservationSubmissionResponseInfo.java  |   54 -
 .../webapp/dao/ResourceInfo.java                |   10 +-
 .../server/resourcemanager/Application.java     |    2 +-
 .../yarn/server/resourcemanager/MockNM.java     |    8 +-
 .../yarn/server/resourcemanager/MockNodes.java  |    4 +-
 .../yarn/server/resourcemanager/MockRM.java     |   20 +-
 .../server/resourcemanager/NodeManager.java     |   17 +-
 .../ReservationACLsTestBase.java                |   63 +-
 .../resourcemanager/TestAMAuthorization.java    |    8 +
 .../resourcemanager/TestApplicationACLs.java    |    6 +-
 .../resourcemanager/TestApplicationCleanup.java |    2 +-
 .../TestApplicationMasterLauncher.java          |    8 +
 .../resourcemanager/TestClientRMService.java    |  287 +++--
 .../TestContainerResourceUsage.java             |   28 +-
 .../TestDistributedSchedulingService.java       |  166 ++-
 .../TestNodeBlacklistingOnAMFailures.java       |  251 ++++
 .../yarn/server/resourcemanager/TestRM.java     |    8 +-
 .../yarn/server/resourcemanager/TestRMHA.java   |    4 +-
 .../resourcemanager/TestRMNodeTransitions.java  |   22 +-
 .../TestReservationSystemWithRMHA.java          |   94 +-
 .../TestResourceTrackerService.java             |    2 +-
 .../resourcemanager/TestSignalContainer.java    |    2 +-
 .../TestWorkPreservingRMRestart.java            |  121 +-
 .../applicationsmanager/TestAMRestart.java      |  177 +--
 .../blacklist/TestBlacklistManager.java         |   29 +-
 .../metrics/TestSystemMetricsPublisher.java     |   10 +-
 ...alCapacityPreemptionPolicyMockFramework.java |    4 +-
 ...estProportionalCapacityPreemptionPolicy.java |   39 +-
 ...alCapacityPreemptionPolicyMockFramework.java |    6 +-
 .../nodelabels/TestRMNodeLabelsManager.java     |   69 +-
 .../reservation/ReservationSystemTestUtil.java  |    7 +-
 .../TestRLESparseResourceAllocation.java        |   40 +-
 .../planning/TestSimpleCapacityReplanner.java   |    6 +-
 .../rmapp/TestRMAppTransitions.java             |   58 -
 .../attempt/TestRMAppAttemptTransitions.java    |    3 +-
 .../scheduler/TestAbstractYarnScheduler.java    |   16 +-
 .../scheduler/TestAppSchedulingInfo.java        |   12 +-
 .../scheduler/TestQueueMetrics.java             |    8 +-
 .../scheduler/TestResourceUsage.java            |    2 +-
 .../scheduler/TestSchedulerUtils.java           |   20 +-
 .../capacity/TestApplicationLimits.java         |   15 +-
 .../capacity/TestApplicationPriority.java       |   52 +-
 .../capacity/TestCapacityScheduler.java         |  129 +-
 .../TestCapacitySchedulerLazyPreemption.java    |   30 +-
 .../TestCapacitySchedulerNodeLabelUpdate.java   |   52 +-
 ...TestCapacitySchedulerSurgicalPreemption.java |    4 +-
 .../scheduler/capacity/TestChildQueueOrder.java |    4 +-
 .../capacity/TestContainerAllocation.java       |   53 +-
 .../capacity/TestContainerResizing.java         |  150 +--
 .../capacity/TestIncreaseAllocationExpirer.java |   36 +-
 .../scheduler/capacity/TestLeafQueue.java       |  492 ++++----
 .../TestNodeLabelContainerAllocation.java       |   15 +-
 .../scheduler/capacity/TestParentQueue.java     |    4 +-
 .../scheduler/capacity/TestQueueCapacities.java |    2 +-
 .../scheduler/capacity/TestReservations.java    |  406 +++----
 ...TestWorkPreservingRMRestartForNodeLabel.java |    4 +-
 .../scheduler/fair/FairSchedulerTestBase.java   |    4 +-
 .../scheduler/fair/TestComputeFairShares.java   |    2 +-
 .../fair/TestContinuousScheduling.java          |    2 +-
 .../scheduler/fair/TestFSAppAttempt.java        |   44 +-
 .../scheduler/fair/TestFSLeafQueue.java         |   27 +-
 .../scheduler/fair/TestFairScheduler.java       |  403 ++++---
 .../fair/TestFairSchedulerFairShare.java        |   32 +-
 .../fair/TestFairSchedulerPreemption.java       |   62 +-
 .../scheduler/fifo/TestFifoScheduler.java       |   63 +-
 .../security/TestDelegationTokenRenewer.java    |   98 ++
 .../webapp/TestRMWebServices.java               |   17 +-
 .../webapp/TestRMWebServicesApps.java           |   46 +-
 .../TestRMWebServicesAppsModification.java      |   39 +-
 .../webapp/TestRMWebServicesCapacitySched.java  |    5 +-
 .../webapp/TestRMWebServicesNodes.java          |    4 +-
 .../webapp/TestRMWebServicesReservation.java    |  351 ++++--
 .../hadoop/yarn/webapp/TestRMWithXFSFilter.java |  158 +++
 .../src/test/resources/krb5.conf                |   23 +-
 .../src/test/resources/submit-reservation.json  |    1 +
 .../yarn/server/timeline/EntityCacheItem.java   |   66 +-
 .../timeline/EntityGroupFSTimelineStore.java    |   82 +-
 .../timeline/EntityGroupPlugInForTest.java      |   19 +-
 .../TestEntityGroupFSTimelineStore.java         |  248 +++-
 .../src/site/markdown/CapacityScheduler.md      |    2 +-
 .../src/site/markdown/FairScheduler.md          |    7 +-
 .../src/site/markdown/ReservationSystem.md      |    4 +-
 .../src/site/markdown/ResourceManagerRest.md    |   90 +-
 hadoop-yarn-project/hadoop-yarn/pom.xml         |    7 +
 pom.xml                                         |   44 +
 620 files changed, 27022 insertions(+), 9621 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/23923086/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/23923086/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/23923086/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
----------------------------------------------------------------------


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message