hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bha...@apache.org
Subject [21/37] hadoop git commit: Merge remote-tracking branch 'apache-commit/trunk' into HDDS-48
Date Mon, 09 Jul 2018 20:33:30 GMT
Merge remote-tracking branch 'apache-commit/trunk' into HDDS-48


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c1045251
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c1045251
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c1045251

Branch: refs/heads/trunk
Commit: c10452516804eed793e575dedcd1ee7758ec1f4c
Parents: 52d1d96 d3fa83a
Author: Bharat Viswanadham <bharat@apache.org>
Authored: Thu Jun 28 14:28:23 2018 -0700
Committer: Bharat Viswanadham <bharat@apache.org>
Committed: Thu Jun 28 14:28:23 2018 -0700

----------------------------------------------------------------------
 NOTICE.txt                                      |   8 +
 dev-support/bin/ozone-dist-layout-stitching     |   4 +-
 .../hadoop-client-minicluster/pom.xml           |   8 -
 hadoop-common-project/hadoop-common/pom.xml     |   5 -
 .../org/apache/hadoop/conf/Configuration.java   |   8 +-
 .../hadoop/conf/ReconfigurationServlet.java     |  42 +-
 .../apache/hadoop/crypto/key/KeyProvider.java   |   4 +-
 .../org/apache/hadoop/crypto/key/KeyShell.java  |   2 +-
 .../crypto/key/kms/KMSClientProvider.java       |  16 +-
 .../java/org/apache/hadoop/fs/BBPartHandle.java |  58 ++
 .../org/apache/hadoop/fs/BBUploadHandle.java    |  57 ++
 .../fs/CommonConfigurationKeysPublic.java       |   4 +-
 .../hadoop/fs/FSDataOutputStreamBuilder.java    |  22 +
 .../java/org/apache/hadoop/fs/FileContext.java  |  66 ++
 .../hadoop/fs/FileSystemMultipartUploader.java  | 132 ++++
 .../main/java/org/apache/hadoop/fs/FsShell.java |   3 +-
 .../hadoop/fs/LocalFileSystemPathHandle.java    | 100 +++
 .../org/apache/hadoop/fs/MultipartUploader.java |  90 +++
 .../hadoop/fs/MultipartUploaderFactory.java     |  65 ++
 .../main/java/org/apache/hadoop/fs/Options.java |   3 +
 .../java/org/apache/hadoop/fs/PartHandle.java   |  45 ++
 .../main/java/org/apache/hadoop/fs/Path.java    |   2 +-
 .../apache/hadoop/fs/RawLocalFileSystem.java    |  61 +-
 .../UnsupportedMultipartUploaderException.java  |  41 ++
 .../java/org/apache/hadoop/fs/UploadHandle.java |  47 ++
 .../java/org/apache/hadoop/fs/shell/Count.java  |   2 +-
 .../apache/hadoop/io/ElasticByteBufferPool.java |   2 +-
 .../apache/hadoop/io/FastByteComparisons.java   |  44 +-
 .../apache/hadoop/io/erasurecode/ECSchema.java  |   4 +-
 .../io/erasurecode/ErasureCodeNative.java       |   2 +-
 .../nativeio/SharedFileDescriptorFactory.java   |   2 +-
 .../org/apache/hadoop/ipc/CallerContext.java    |   4 +-
 .../apache/hadoop/ipc/DecayRpcScheduler.java    |   4 +-
 .../org/apache/hadoop/ipc/FairCallQueue.java    |   4 +-
 .../hadoop/metrics2/MetricsJsonBuilder.java     |   2 +-
 .../hadoop/metrics2/impl/MetricsConfig.java     |  15 +-
 .../hadoop/metrics2/lib/MethodMetric.java       |   2 +-
 .../metrics2/lib/MutableMetricsFactory.java     |   2 +-
 .../hadoop/metrics2/lib/MutableQuantiles.java   |   2 +-
 .../metrics2/lib/MutableRollingAverages.java    |   2 +-
 .../apache/hadoop/metrics2/lib/MutableStat.java |   2 +-
 .../metrics2/sink/RollingFileSystemSink.java    |   2 +-
 .../org/apache/hadoop/net/TableMapping.java     |   2 +-
 .../apache/hadoop/net/unix/DomainSocket.java    |   2 +-
 .../hadoop/net/unix/DomainSocketWatcher.java    |   2 +-
 .../security/ShellBasedUnixGroupsMapping.java   |  12 +-
 .../hadoop/security/UserGroupInformation.java   |   4 +-
 .../hadoop/security/alias/CredentialShell.java  |   2 +-
 .../hadoop/security/http/CrossOriginFilter.java |   2 +-
 .../hadoop/security/token/DtFileOperations.java |   2 +-
 .../web/DelegationTokenAuthenticator.java       |   8 +-
 .../org/apache/hadoop/tools/TableListing.java   |   6 +-
 .../apache/hadoop/util/HttpExceptionUtils.java  |  12 +-
 .../apache/hadoop/util/JsonSerialization.java   |  24 +
 .../main/java/org/apache/hadoop/util/Shell.java |   2 +-
 .../org/apache/hadoop/util/StringUtils.java     |  64 +-
 .../org/apache/hadoop/util/SysInfoLinux.java    |  19 +-
 .../hadoop-common/src/main/proto/FSProtos.proto |   8 +
 ...rg.apache.hadoop.fs.MultipartUploaderFactory |  16 +
 .../apache/hadoop/conf/TestConfiguration.java   |   2 +-
 .../conf/TestConfigurationFieldsBase.java       |   2 +-
 .../crypto/random/TestOsSecureRandom.java       |   2 +-
 .../fs/AbstractSystemMultipartUploaderTest.java | 143 ++++
 .../org/apache/hadoop/fs/FSTestWrapper.java     |   2 +-
 .../fs/FileContextMainOperationsBaseTest.java   |  44 +-
 .../apache/hadoop/fs/FileContextTestHelper.java |   1 -
 .../hadoop/fs/TestDFCachingGetSpaceUsed.java    |   2 +-
 .../fs/TestFileSystemStorageStatistics.java     |  18 +-
 .../apache/hadoop/fs/TestLocalFileSystem.java   |  23 +-
 .../TestLocalFileSystemMultipartUploader.java   |  65 ++
 .../AbstractContractPathHandleTest.java         |   6 +
 .../TestRawlocalContractPathHandle.java         |  40 ++
 .../hadoop/fs/shell/TestCopyFromLocal.java      |  10 +-
 .../TestSharedFileDescriptorFactory.java        |   2 +-
 .../org/apache/hadoop/ipc/TestProtoBufRpc.java  |   2 +-
 .../java/org/apache/hadoop/ipc/TestSaslRPC.java |   2 +-
 .../org/apache/hadoop/net/TestNetUtils.java     |   2 +-
 .../hadoop/net/unix/TestDomainSocket.java       |   2 +-
 .../TestShellBasedUnixGroupsMapping.java        |  39 +-
 ...tionTokenAuthenticationHandlerWithMocks.java |   2 +-
 .../AbstractServiceLauncherTestBase.java        |   2 +-
 .../apache/hadoop/test/GenericTestUtils.java    |   4 +-
 .../hadoop/util/TestShutdownHookManager.java    |   2 +-
 .../apache/hadoop/util/TestSignalLogger.java    |   2 +-
 .../apache/hadoop/util/TestSysInfoLinux.java    |  60 ++
 .../src/test/resources/contract/rawlocal.xml    |   5 +
 .../test/scripts/process_with_sigterm_trap.sh   |  24 +
 .../crypto/key/kms/server/KMSJSONWriter.java    |   6 +-
 hadoop-dist/pom.xml                             |   5 +
 .../src/main/compose/ozone/docker-compose.yaml  |  12 -
 .../src/main/compose/ozone/docker-config        |   5 -
 .../main/compose/ozoneperf/docker-compose.yaml  |  13 -
 .../src/main/compose/ozoneperf/docker-config    |   5 -
 .../apache/hadoop/hdds/scm/ScmConfigKeys.java   |  26 -
 .../container/common/helpers/ContainerInfo.java |  32 +
 .../scm/container/common/helpers/Pipeline.java  | 142 ++--
 .../common/helpers/PipelineChannel.java         | 124 ----
 .../org/apache/hadoop/ozone/OzoneConsts.java    |   1 -
 .../apache/hadoop/utils/BackgroundService.java  |   2 +-
 hadoop-hdds/common/src/main/proto/hdds.proto    |   8 +-
 .../common/src/main/resources/ozone-default.xml |  47 --
 .../hadoop/ozone/HddsDatanodeService.java       |  13 +-
 .../statemachine/DatanodeStateMachine.java      |   3 +
 .../background/BlockDeletingService.java        |   8 +-
 .../ReplicateContainerCommandHandler.java       |  67 ++
 .../states/endpoint/HeartbeatEndpointTask.java  |  12 +
 .../container/common/utils/HddsVolumeUtil.java  |   2 +-
 .../container/replication/ReplicationQueue.java |  76 ++
 .../replication/ReplicationRequest.java         | 106 +++
 .../container/replication/package-info.java     |  23 +
 .../protocol/commands/CommandForDatanode.java   |  45 ++
 .../commands/ReplicateContainerCommand.java     |  94 +++
 .../StorageContainerDatanodeProtocol.proto      |  12 +-
 .../replication/TestReplicationQueue.java       | 134 ++++
 .../container/replication/package-info.java     |  23 +
 .../testutils/BlockDeletingServiceTestImpl.java |   4 +-
 .../hadoop/hdds/server/events/EventWatcher.java | 157 +++++
 .../server/events/IdentifiableEventPayload.java |  30 +
 .../hdds/server/events/EventHandlerStub.java    |  38 +
 .../hdds/server/events/TestEventWatcher.java    | 220 ++++++
 .../hdds/scm/container/ContainerMapping.java    |  10 +-
 .../scm/container/closer/ContainerCloser.java   |   6 +-
 .../replication/ContainerSupervisor.java        | 340 ---------
 .../container/replication/InProgressPool.java   | 255 -------
 .../scm/container/replication/PeriodicPool.java | 119 ----
 .../scm/container/replication/package-info.java |  23 -
 .../hadoop/hdds/scm/node/NodeManager.java       |   6 -
 .../hadoop/hdds/scm/node/NodePoolManager.java   |  71 --
 .../hadoop/hdds/scm/node/SCMNodeManager.java    |  41 +-
 .../hdds/scm/node/SCMNodePoolManager.java       | 269 --------
 .../hdds/scm/pipelines/PipelineManager.java     |  67 +-
 .../hdds/scm/pipelines/PipelineSelector.java    |  11 +-
 .../scm/pipelines/ratis/RatisManagerImpl.java   |  13 +-
 .../standalone/StandaloneManagerImpl.java       |   8 +-
 .../server/SCMDatanodeHeartbeatDispatcher.java  | 126 ++++
 .../scm/server/SCMDatanodeProtocolServer.java   |  25 +-
 .../scm/server/StorageContainerManager.java     |  10 +-
 .../SCMDatanodeContainerReportHandler.java      |  76 --
 .../report/SCMDatanodeHeartbeatDispatcher.java  | 189 -----
 .../report/SCMDatanodeNodeReportHandler.java    |  43 --
 .../server/report/SCMDatanodeReportHandler.java |  83 ---
 .../report/SCMDatanodeReportHandlerFactory.java |  82 ---
 .../hdds/scm/server/report/package-info.java    |  57 --
 .../hdds/scm/block/TestDeletedBlockLog.java     |   8 +-
 .../hdds/scm/container/MockNodeManager.java     |   6 -
 .../hadoop/hdds/scm/node/TestNodeManager.java   |  39 ++
 .../hdds/scm/node/TestSCMNodePoolManager.java   | 160 -----
 .../TestSCMDatanodeHeartbeatDispatcher.java     | 119 ++++
 .../TestSCMDatanodeContainerReportHandler.java  |  34 -
 .../TestSCMDatanodeHeartbeatDispatcher.java     | 138 ----
 .../TestSCMDatanodeNodeReportHandler.java       |  36 -
 .../TestSCMDatanodeReportHandlerFactory.java    |  51 --
 .../hdds/scm/server/report/package-info.java    |  21 -
 .../testutils/ReplicationNodeManagerMock.java   |   5 -
 .../ReplicationNodePoolManagerMock.java         | 133 ----
 .../main/java/org/apache/hadoop/fs/XAttr.java   |   4 +-
 .../hdfs/DFSMultipartUploaderFactory.java       |  40 ++
 .../org/apache/hadoop/hdfs/ExtendedBlockId.java |   4 +-
 .../hdfs/client/impl/BlockReaderFactory.java    |   2 +-
 .../AddErasureCodingPolicyResponse.java         |   4 +-
 .../hdfs/protocol/CacheDirectiveInfo.java       |   4 +-
 .../hadoop/hdfs/protocol/CachePoolInfo.java     |   4 +-
 .../hadoop/hdfs/protocol/EncryptionZone.java    |   4 +-
 .../hdfs/protocol/ErasureCodingPolicy.java      |   4 +-
 .../hdfs/protocol/ErasureCodingPolicyInfo.java  |   4 +-
 .../hadoop/hdfs/protocol/ExtendedBlock.java     |   2 +-
 .../hdfs/shortcircuit/DfsClientShmManager.java  |   2 +-
 .../hdfs/shortcircuit/ShortCircuitCache.java    |   2 +-
 .../hdfs/shortcircuit/ShortCircuitShm.java      |   4 +-
 .../hadoop/hdfs/web/WebHdfsFileSystem.java      |   7 +-
 ...onfRefreshTokenBasedAccessTokenProvider.java |   8 +-
 .../CredentialBasedAccessTokenProvider.java     |   8 +-
 ...rg.apache.hadoop.fs.MultipartUploaderFactory |  16 +
 .../ha/TestRequestHedgingProxyProvider.java     |   2 +-
 hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml     |   5 -
 .../federation/router/ConnectionPoolId.java     |   2 +-
 .../router/RemoteLocationContext.java           |   2 +-
 .../store/driver/impl/StateStoreFileImpl.java   |   2 +-
 .../federation/store/records/MountTable.java    |   2 +-
 hadoop-hdfs-project/hadoop-hdfs/pom.xml         |   5 -
 .../org/apache/hadoop/hdfs/DFSConfigKeys.java   |   4 +-
 .../hadoop/hdfs/protocol/CacheDirective.java    |   2 +-
 .../qjournal/server/GetJournalEditServlet.java  |   4 +-
 .../hadoop/hdfs/qjournal/server/Journal.java    |  12 +-
 .../hadoop/hdfs/server/balancer/Balancer.java   |   6 +-
 .../hadoop/hdfs/server/balancer/Dispatcher.java |  30 +-
 .../server/blockmanagement/BlockManager.java    |   2 +-
 .../hdfs/server/datanode/DirectoryScanner.java  |   4 +-
 .../datanode/fsdataset/impl/FsDatasetCache.java |   2 +-
 .../server/diskbalancer/command/Command.java    |   4 +-
 .../diskbalancer/command/PlanCommand.java       |   4 +-
 .../diskbalancer/command/ReportCommand.java     |   4 +-
 .../server/namenode/EncryptionZoneManager.java  |   4 +-
 .../server/namenode/FSDirErasureCodingOp.java   |   2 +-
 .../hdfs/server/namenode/FSNamesystem.java      |   6 +-
 .../hdfs/server/namenode/FSNamesystemLock.java  |  17 +-
 .../server/namenode/INodeAttributeProvider.java |   2 +-
 .../server/namenode/startupprogress/Step.java   |   6 +-
 .../server/namenode/top/metrics/TopMetrics.java |   2 +-
 .../apache/hadoop/hdfs/tools/CacheAdmin.java    |   2 +-
 .../apache/hadoop/hdfs/util/EnumCounters.java   |   2 +-
 .../src/main/resources/hdfs-default.xml         |  10 +
 .../hadoop/fs/TestEnhancedByteBufferAccess.java |   4 +-
 .../org/apache/hadoop/fs/TestGlobPaths.java     |   2 +-
 .../hadoop/fs/TestHDFSMultipartUploader.java    |  76 ++
 .../TestWebHdfsFileContextMainOperations.java   |   2 +-
 .../org/apache/hadoop/hdfs/DFSTestUtil.java     |   4 +-
 .../org/apache/hadoop/hdfs/TestDFSShell.java    |   2 +-
 .../apache/hadoop/hdfs/TestDecommission.java    |   2 +-
 .../hadoop/hdfs/TestFsShellPermission.java      |   2 +-
 .../hadoop/hdfs/TestHDFSPolicyProvider.java     |   2 +-
 .../hadoop/hdfs/TestLeaseRecoveryStriped.java   |   2 +-
 .../hdfs/TestSecureEncryptionZoneWithKMS.java   |  34 +-
 .../sasl/SaslDataTransferTestCase.java          |   2 +-
 .../hdfs/server/balancer/TestBalancer.java      |  81 ++-
 .../server/datanode/SimulatedFSDataset.java     |   2 +-
 .../command/TestDiskBalancerCommand.java        |   2 +-
 .../hadoop/hdfs/server/mover/TestMover.java     |  46 ++
 .../hdfs/server/namenode/FSImageTestUtil.java   |   2 +-
 .../hdfs/server/namenode/NameNodeAdapter.java   |   2 +-
 .../hdfs/server/namenode/TestAuditLogger.java   |   2 +-
 .../server/namenode/TestCacheDirectives.java    |   2 +-
 .../namenode/TestEditLogJournalFailures.java    |   2 +-
 .../hdfs/server/namenode/TestEditLogRace.java   |   6 +-
 .../server/namenode/TestFSNamesystemLock.java   |  10 +-
 .../hdfs/server/namenode/TestStartup.java       |   9 +-
 .../namenode/metrics/TestNameNodeMetrics.java   |   4 +-
 .../shortcircuit/TestShortCircuitCache.java     |   2 +-
 .../apache/hadoop/hdfs/tools/TestDFSAdmin.java  |   2 +-
 .../hadoop/hdfs/tools/TestDFSAdminWithHA.java   |  12 +
 .../org/apache/hadoop/tracing/TestTracing.java  |   2 +-
 .../v2/app/job/impl/TestTaskAttempt.java        |   4 +-
 .../apache/hadoop/mapreduce/JobSubmitter.java   |   8 +-
 .../lib/input/CombineFileInputFormat.java       |   4 +-
 .../mapred/uploader/FrameworkUploader.java      |   2 +-
 .../mapred/uploader/TestFrameworkUploader.java  |  13 +
 hadoop-maven-plugins/pom.xml                    |  38 +-
 .../maven/plugin/resourcegz/ResourceGzMojo.java |   2 +-
 .../dev-support/bin/robot-all.sh                |   2 +-
 .../dev-support/bin/robot-dnd-all.sh            |  63 ++
 .../acceptance-test/dev-support/bin/robot.sh    |   9 +-
 .../dev-support/docker/Dockerfile               |  21 +
 .../dev-support/docker/docker-compose.yaml      |  23 +
 hadoop-ozone/acceptance-test/pom.xml            |   1 +
 .../src/test/acceptance/basic/.env              |  17 +
 .../src/test/acceptance/basic/basic.robot       |  50 ++
 .../test/acceptance/basic/docker-compose.yaml   |  50 ++
 .../src/test/acceptance/basic/docker-config     |  33 +
 .../src/test/acceptance/basic/ozone-shell.robot |  85 +++
 .../src/test/acceptance/commonlib.robot         |  78 +++
 .../src/test/acceptance/ozonefs/.env            |  17 +
 .../test/acceptance/ozonefs/docker-compose.yaml |  59 ++
 .../src/test/acceptance/ozonefs/docker-config   |  34 +
 .../src/test/acceptance/ozonefs/ozonefs.robot   |  39 ++
 .../acceptance-test/src/test/compose/.env       |  17 -
 .../src/test/compose/docker-compose.yaml        |  62 --
 .../src/test/compose/docker-config              |  36 -
 .../robotframework/acceptance/ozone-shell.robot | 256 -------
 .../test/robotframework/acceptance/ozone.robot  | 104 ---
 hadoop-ozone/common/src/main/bin/ozone          |   9 +-
 .../hadoop/ozone/TestMiniOzoneCluster.java      |   8 +-
 .../ozone/container/ContainerTestHelper.java    |  19 +-
 .../common/TestBlockDeletingService.java        |  11 +-
 .../TestReplicateContainerHandler.java          |  71 ++
 ...TestGenerateOzoneRequiredConfigurations.java |   2 +-
 .../hadoop/ozone/scm/TestContainerSQLCli.java   |  31 -
 .../apache/hadoop/ozone/scm/TestSCMMetrics.java | 253 -------
 .../ozone-manager/src/main/webapps/ksm/ksm.js   |   4 +-
 hadoop-ozone/ozonefs/pom.xml                    | 211 ++++++
 .../org/apache/hadoop/fs/ozone/Constants.java   |  42 ++
 .../java/org/apache/hadoop/fs/ozone/OzFs.java   |  44 ++
 .../hadoop/fs/ozone/OzoneFSInputStream.java     |  79 +++
 .../hadoop/fs/ozone/OzoneFSOutputStream.java    |  59 ++
 .../apache/hadoop/fs/ozone/OzoneFileSystem.java | 689 +++++++++++++++++++
 .../apache/hadoop/fs/ozone/package-info.java    |  30 +
 .../hadoop/fs/ozone/TestOzoneFSInputStream.java | 157 +++++
 .../fs/ozone/TestOzoneFileInterfaces.java       | 231 +++++++
 .../contract/ITestOzoneContractCreate.java      |  48 ++
 .../contract/ITestOzoneContractDelete.java      |  48 ++
 .../contract/ITestOzoneContractDistCp.java      |  50 ++
 .../ITestOzoneContractGetFileStatus.java        |  61 ++
 .../ozone/contract/ITestOzoneContractMkdir.java |  48 ++
 .../ozone/contract/ITestOzoneContractOpen.java  |  47 ++
 .../contract/ITestOzoneContractRename.java      |  49 ++
 .../contract/ITestOzoneContractRootDir.java     |  51 ++
 .../ozone/contract/ITestOzoneContractSeek.java  |  47 ++
 .../hadoop/fs/ozone/contract/OzoneContract.java | 123 ++++
 .../src/test/resources/contract/ozone.xml       | 113 +++
 .../ozonefs/src/test/resources/log4j.properties |  23 +
 hadoop-ozone/pom.xml                            |   1 +
 .../genesis/BenchMarkContainerStateMap.java     |  11 +-
 .../genesis/BenchMarkDatanodeDispatcher.java    |   6 +-
 .../org/apache/hadoop/ozone/scm/cli/SQLCLI.java |  78 +--
 hadoop-project/pom.xml                          |  16 +-
 .../aliyun/oss/AliyunCredentialsProvider.java   |   2 +-
 .../fs/aliyun/oss/AliyunOSSFileSystem.java      |   2 +-
 .../fs/aliyun/oss/AliyunOSSFileSystemStore.java |   2 +-
 .../hadoop/fs/aliyun/oss/AliyunOSSUtils.java    |   2 +-
 .../fs/aliyun/oss/AliyunOSSTestUtils.java       |   2 +-
 hadoop-tools/hadoop-aws/pom.xml                 |  42 +-
 .../fs/s3a/AWSCredentialProviderList.java       |   2 +-
 .../fs/s3a/BasicAWSCredentialsProvider.java     |   2 +-
 .../org/apache/hadoop/fs/s3a/Constants.java     |  11 +
 .../org/apache/hadoop/fs/s3a/S3AFileSystem.java |   4 +-
 .../apache/hadoop/fs/s3a/S3AInputStream.java    |   2 +-
 .../hadoop/fs/s3a/S3AMultipartUploader.java     | 150 ++++
 .../java/org/apache/hadoop/fs/s3a/S3AUtils.java |   2 +-
 .../fs/s3a/SimpleAWSCredentialsProvider.java    |   2 +-
 .../fs/s3a/TemporaryAWSCredentialsProvider.java |   2 +-
 .../s3a/auth/AssumedRoleCredentialProvider.java |   2 +-
 .../s3a/commit/files/SinglePendingCommit.java   |   2 +-
 .../hadoop/fs/s3a/commit/files/SuccessData.java |   2 +-
 .../fs/s3a/commit/magic/MagicCommitTracker.java |   2 +-
 .../hadoop/fs/s3a/commit/staging/Paths.java     |   2 +-
 .../fs/s3a/s3guard/DynamoDBClientFactory.java   |   2 +-
 .../fs/s3a/s3guard/DynamoDBMetadataStore.java   |   3 +-
 .../fs/s3a/s3guard/LocalMetadataEntry.java      |  81 +++
 .../fs/s3a/s3guard/LocalMetadataStore.java      | 249 ++++---
 .../hadoop/fs/s3a/s3guard/S3GuardTool.java      |   6 +-
 .../hadoop/fs/s3native/S3xLoginHelper.java      |   4 +-
 ...rg.apache.hadoop.fs.MultipartUploaderFactory |  15 +
 .../org.apache.hadoop.fs.MultipartUploader      |  16 +
 .../hadoop/fs/s3a/AbstractS3ATestBase.java      |  21 +-
 .../hadoop/fs/s3a/ITestS3AConfiguration.java    |   4 +-
 .../ITestS3AEncryptionSSEKMSUserDefinedKey.java |   2 +-
 ...onSSEKMSUserDefinedKeyBlockOutputStream.java |   2 +-
 .../apache/hadoop/fs/s3a/S3ATestConstants.java  |   1 -
 .../org/apache/hadoop/fs/s3a/S3ATestUtils.java  |  36 +-
 .../fs/s3a/commit/AbstractCommitITest.java      |   2 +-
 .../fs/s3a/commit/staging/StagingTestBase.java  |  30 +-
 .../s3a/s3guard/DynamoDBLocalClientFactory.java | 160 -----
 .../fs/s3a/s3guard/MetadataStoreTestBase.java   |   6 +-
 .../s3a/s3guard/TestDynamoDBMetadataStore.java  | 589 ----------------
 .../fs/s3a/s3guard/TestLocalMetadataStore.java  |  33 +-
 .../org/apache/hadoop/fs/adl/AdlFileSystem.java |   2 +-
 .../hadoop/fs/adl/TestAzureADTokenProvider.java |   2 +-
 .../fs/azure/AzureNativeFileSystemStore.java    |   2 +-
 .../hadoop/fs/azure/BlockBlobAppendStream.java  |   2 +-
 .../fs/azure/ClientThrottlingAnalyzer.java      |   2 +-
 .../hadoop/fs/azure/NativeAzureFileSystem.java  |   4 +-
 .../hadoop/fs/azure/PageBlobOutputStream.java   |   2 +-
 .../fs/azure/RemoteWasbAuthorizerImpl.java      |   2 +-
 .../fs/azure/SecureWasbRemoteCallHelper.java    |   2 +-
 .../hadoop/fs/azure/security/JsonUtils.java     |   4 +-
 .../fs/azure/ITestBlobDataValidation.java       |   2 +-
 .../azure/ITestNativeAzureFileSystemAppend.java |   2 +-
 .../hadoop/fs/azure/MockStorageInterface.java   |  12 +-
 ...estNativeAzureFileSystemBlockCompaction.java |   2 +-
 .../fs/azure/integration/AzureTestUtils.java    |   2 +-
 .../org/apache/hadoop/tools/DistCpOptions.java  |   2 +-
 .../org/apache/hadoop/tools/OptionsParser.java  |   2 +-
 .../apache/hadoop/tools/mapred/CopyMapper.java  |   2 +-
 .../mapred/gridmix/ClusterSummarizer.java       |   2 +-
 .../mapred/gridmix/ExecutionSummarizer.java     |   4 +-
 .../hadoop/mapred/gridmix/JobFactory.java       |   2 +-
 .../mapred/gridmix/RandomTextDataGenerator.java |   2 +-
 hadoop-tools/hadoop-ozone/pom.xml               | 174 -----
 .../org/apache/hadoop/fs/ozone/Constants.java   |  42 --
 .../java/org/apache/hadoop/fs/ozone/OzFs.java   |  44 --
 .../hadoop/fs/ozone/OzoneFSInputStream.java     |  79 ---
 .../hadoop/fs/ozone/OzoneFSOutputStream.java    |  59 --
 .../apache/hadoop/fs/ozone/OzoneFileSystem.java | 689 -------------------
 .../apache/hadoop/fs/ozone/package-info.java    |  30 -
 .../hadoop/fs/ozone/TestOzoneFSInputStream.java | 157 -----
 .../fs/ozone/TestOzoneFileInterfaces.java       | 231 -------
 .../contract/ITestOzoneContractCreate.java      |  48 --
 .../contract/ITestOzoneContractDelete.java      |  48 --
 .../contract/ITestOzoneContractDistCp.java      |  50 --
 .../ITestOzoneContractGetFileStatus.java        |  61 --
 .../ozone/contract/ITestOzoneContractMkdir.java |  48 --
 .../ozone/contract/ITestOzoneContractOpen.java  |  47 --
 .../contract/ITestOzoneContractRename.java      |  49 --
 .../contract/ITestOzoneContractRootDir.java     |  51 --
 .../ozone/contract/ITestOzoneContractSeek.java  |  47 --
 .../hadoop/fs/ozone/contract/OzoneContract.java | 123 ----
 .../src/test/resources/contract/ozone.xml       | 113 ---
 .../src/test/resources/log4j.properties         |  23 -
 .../translator/impl/BaseLogParser.java          |   1 -
 .../WordListAnonymizerUtility.java              |   2 +-
 .../util/MapReduceJobPropertiesParser.java      |   2 +-
 .../hadoop/streaming/TestUnconsumedInput.java   |   2 +-
 hadoop-tools/hadoop-tools-dist/pom.xml          |  15 -
 hadoop-tools/pom.xml                            |  11 -
 .../yarn/api/records/ResourceRequest.java       |  20 +
 .../hadoop/yarn/conf/YarnConfiguration.java     |   2 +-
 .../exceptions/RestApiErrorMessages.java        |   2 +
 .../yarn/service/utils/ServiceApiUtil.java      |   5 +
 .../hadoop/yarn/service/TestServiceApiUtil.java |  18 +
 .../yarn/client/api/impl/AMRMClientImpl.java    |  11 +-
 .../api/impl/TestAMRMClientOnRMRestart.java     |   6 +-
 .../yarn/client/api/impl/TestYarnClient.java    | 337 ++-------
 .../client/api/impl/TestYarnClientImpl.java     | 254 +++++++
 .../hadoop-yarn/hadoop-yarn-common/pom.xml      |   2 +-
 .../org/apache/hadoop/yarn/webapp/WebApps.java  |  14 +-
 .../hadoop/yarn/webapp/view/JQueryUI.java       |   2 +-
 .../jquery/jquery-ui-1.12.1.custom.min.js       |  13 +
 .../static/jquery/jquery-ui-1.9.1.custom.min.js |   6 -
 .../registry/client/api/RegistryConstants.java  |   2 +-
 .../hadoop/registry/server/dns/RegistryDNS.java |  29 +-
 .../registry/server/dns/TestRegistryDNS.java    |  23 +-
 .../hadoop/yarn/server/AMRMClientRelayer.java   |   8 +-
 .../LocalityMulticastAMRMProxyPolicy.java       |  10 +-
 .../server/scheduler/ResourceRequestSet.java    |  14 +-
 .../hadoop/yarn/server/utils/BuilderUtils.java  |  12 -
 .../src/CMakeLists.txt                          |   4 +-
 .../server/nodemanager/ContainerExecutor.java   |   4 +-
 .../containermanager/container/Container.java   |   6 +
 .../container/ContainerImpl.java                |  11 +
 .../launcher/ContainerLaunch.java               |  46 +-
 .../launcher/ContainerRelaunch.java             |   1 -
 .../CGroupsMemoryResourceHandlerImpl.java       |  24 +
 .../linux/resources/MemoryResourceHandler.java  |  10 +
 .../resources/gpu/GpuResourceAllocator.java     |  68 +-
 .../resources/gpu/GpuResourceHandlerImpl.java   |   1 -
 .../localizer/ContainerLocalizer.java           |   7 +-
 .../localizer/ResourceLocalizationService.java  |   5 +-
 .../monitor/ContainersMonitorImpl.java          | 108 +--
 .../oom-listener/test/oom_listener_test_main.cc |  33 +-
 .../container/TestContainer.java                |   8 +-
 .../launcher/TestContainerLaunch.java           |   2 -
 .../TestCGroupsMemoryResourceHandlerImpl.java   |  43 ++
 .../nodemanager/webapp/MockContainer.java       |   3 +
 .../server/resourcemanager/ResourceManager.java |   2 +-
 .../scheduler/capacity/CSQueueUtils.java        |  18 +-
 .../scheduler/capacity/CapacityScheduler.java   |  18 +-
 .../fair/AllocationFileLoaderService.java       |   4 +-
 .../LocalityAppPlacementAllocator.java          |  10 +-
 .../webapp/dao/ClusterMetricsInfo.java          |   2 +-
 .../server/resourcemanager/Application.java     |  10 +-
 .../server/resourcemanager/TestAppManager.java  |  18 +-
 .../src/site/markdown/FairScheduler.md          |   2 +-
 .../site/markdown/yarn-service/RegistryDNS.md   |  10 +-
 .../markdown/yarn-service/ServiceDiscovery.md   |   6 +-
 .../hadoop-yarn-ui/src/main/webapp/.bowerrc     |   3 +-
 .../src/main/webapp/WEB-INF/web.xml             |   6 +-
 pom.xml                                         |   2 +-
 start-build-env.sh                              |   8 +-
 437 files changed, 7828 insertions(+), 7627 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1045251/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1045251/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1045251/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1045251/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
----------------------------------------------------------------------
diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
index b6a9bb9,b073d7b..245d76f
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
@@@ -93,8 -95,9 +95,9 @@@ public class DatanodeStateMachine imple
       // trick.
      commandDispatcher = CommandDispatcher.newBuilder()
          .addHandler(new CloseContainerCommandHandler())
 -        .addHandler(new DeleteBlocksCommandHandler(
 -            container.getContainerManager(), conf))
 +        .addHandler(new DeleteBlocksCommandHandler(container.getContainerSet(),
 +            conf))
+         .addHandler(new ReplicateContainerCommandHandler())
          .setConnectionManager(connectionManager)
          .setContainer(container)
          .setContext(context)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1045251/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/BlockDeletingService.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1045251/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/HddsVolumeUtil.java
----------------------------------------------------------------------
diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/HddsVolumeUtil.java
index 6809d57,0000000..5d6fc0a
mode 100644,000000..100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/HddsVolumeUtil.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/HddsVolumeUtil.java
@@@ -1,163 -1,0 +1,163 @@@
 +/**
 + * Licensed to the Apache Software Foundation (ASF) under one
 + * or more contributor license agreements.  See the NOTICE file
 + * distributed with this work for additional information
 + * regarding copyright ownership.  The ASF licenses this file
 + * to you under the Apache License, Version 2.0 (the
 + * "License"); you may not use this file except in compliance
 + * with the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +
 +package org.apache.hadoop.ozone.container.common.utils;
 +
 +import com.google.common.annotations.VisibleForTesting;
- import org.apache.commons.lang.StringUtils;
++import org.apache.commons.lang3.StringUtils;
 +import org.apache.hadoop.ozone.OzoneConsts;
 +import org.apache.hadoop.ozone.common.InconsistentStorageStateException;
 +import org.apache.hadoop.ozone.container.common.DataNodeLayoutVersion;
 +import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
 +import org.apache.hadoop.util.Time;
 +
 +import java.io.File;
 +import java.util.Properties;
 +import java.util.UUID;
 +
 +/**
 + * A util class for {@link HddsVolume}.
 + */
 +public final class HddsVolumeUtil {
 +
 +  // Private constructor for Utility class. Unused.
 +  private HddsVolumeUtil() {
 +  }
 +
 +  private static final String VERSION_FILE   = "VERSION";
 +  private static final String STORAGE_ID_PREFIX = "DS-";
 +
 +  public static File getVersionFile(File rootDir) {
 +    return new File(rootDir, VERSION_FILE);
 +  }
 +
 +  public static String generateUuid() {
 +    return STORAGE_ID_PREFIX + UUID.randomUUID();
 +  }
 +
 +  /**
 +   * Get hddsRoot from volume root. If volumeRoot points to hddsRoot, it is
 +   * returned as is.
 +   * For a volumeRoot /data/disk1, the hddsRoot is /data/disk1/hdds.
 +   * @param volumeRoot root of the volume.
 +   * @return hddsRoot of the volume.
 +   */
 +  public static String getHddsRoot(String volumeRoot) {
 +    if (volumeRoot.endsWith(HddsVolume.HDDS_VOLUME_DIR)) {
 +      return volumeRoot;
 +    } else {
 +      File hddsRoot = new File(volumeRoot, HddsVolume.HDDS_VOLUME_DIR);
 +      return hddsRoot.getPath();
 +    }
 +  }
 +
 +  /**
 +   * Returns storageID if it is valid. Throws an exception otherwise.
 +   */
 +  @VisibleForTesting
 +  public static String getStorageID(Properties props, File versionFile)
 +      throws InconsistentStorageStateException {
 +    return getProperty(props, OzoneConsts.STORAGE_ID, versionFile);
 +  }
 +
 +  /**
 +   * Returns clusterID if it is valid. It should match the clusterID from the
 +   * Datanode. Throws an exception otherwise.
 +   */
 +  @VisibleForTesting
 +  public static String getClusterID(Properties props, File versionFile,
 +      String clusterID) throws InconsistentStorageStateException {
 +    String cid = getProperty(props, OzoneConsts.CLUSTER_ID, versionFile);
 +
 +    if (clusterID == null) {
 +      return cid;
 +    }
 +    if (!clusterID.equals(cid)) {
 +      throw new InconsistentStorageStateException("Mismatched " +
 +          "ClusterIDs. Version File : " + versionFile + " has clusterID: " +
 +          cid + " and Datanode has clusterID: " + clusterID);
 +    }
 +    return cid;
 +  }
 +
 +  /**
 +   * Returns datanodeUuid if it is valid. It should match the UUID of the
 +   * Datanode. Throws an exception otherwise.
 +   */
 +  @VisibleForTesting
 +  public static String getDatanodeUUID(Properties props, File versionFile,
 +      String datanodeUuid)
 +      throws InconsistentStorageStateException {
 +    String datanodeID = getProperty(props, OzoneConsts.DATANODE_UUID,
 +        versionFile);
 +
 +    if (datanodeUuid != null && !datanodeUuid.equals(datanodeID)) {
 +      throw new InconsistentStorageStateException("Mismatched " +
 +          "DatanodeUUIDs. Version File : " + versionFile + " has datanodeUuid: "
 +          + datanodeID + " and Datanode has datanodeUuid: " + datanodeUuid);
 +    }
 +    return datanodeID;
 +  }
 +
 +  /**
 +   * Returns creationTime if it is valid. Throws an exception otherwise.
 +   */
 +  @VisibleForTesting
 +  public static long getCreationTime(Properties props, File versionFile)
 +      throws InconsistentStorageStateException {
 +    String cTimeStr = getProperty(props, OzoneConsts.CTIME, versionFile);
 +
 +    long cTime = Long.parseLong(cTimeStr);
 +    long currentTime = Time.now();
 +    if (cTime > currentTime || cTime < 0) {
 +      throw new InconsistentStorageStateException("Invalid Creation time in " +
 +          "Version File : " + versionFile + " - " + cTime + ". Current system" +
 +          " time is " + currentTime);
 +    }
 +    return cTime;
 +  }
 +
 +  /**
 +   * Returns layOutVersion if it is valid. Throws an exception otherwise.
 +   */
 +  @VisibleForTesting
 +  public static int getLayOutVersion(Properties props, File versionFile) throws
 +      InconsistentStorageStateException {
 +    String lvStr = getProperty(props, OzoneConsts.LAYOUTVERSION, versionFile);
 +
 +    int lv = Integer.parseInt(lvStr);
 +    if(DataNodeLayoutVersion.getLatestVersion().getVersion() != lv) {
 +      throw new InconsistentStorageStateException("Invalid layOutVersion. " +
 +          "Version file has layOutVersion as " + lv + " and latest Datanode " +
 +          "layOutVersion is " +
 +          DataNodeLayoutVersion.getLatestVersion().getVersion());
 +    }
 +    return lv;
 +  }
 +
 +  private static String getProperty(Properties props, String propName, File
 +      versionFile)
 +      throws InconsistentStorageStateException {
 +    String value = props.getProperty(propName);
 +    if (StringUtils.isBlank(value)) {
 +      throw new InconsistentStorageStateException("Invalid " + propName +
 +          ". Version File : " + versionFile + " has null or empty " + propName);
 +    }
 +    return value;
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1045251/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
----------------------------------------------------------------------
diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
index 9ac9930,adca8ea..ff5b9f1
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
@@@ -40,9 -44,9 +44,10 @@@ import org.apache.hadoop.hdds.protocol.
      .StorageContainerDatanodeProtocolProtos.SCMVersionRequestProto;
  import org.apache.hadoop.ipc.Server;
  import org.apache.hadoop.metrics2.util.MBeans;
 +import org.apache.hadoop.ozone.OzoneConsts;
  import org.apache.hadoop.ozone.protocol.StorageContainerNodeProtocol;
  import org.apache.hadoop.ozone.protocol.VersionResponse;
+ import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode;
  import org.apache.hadoop.ozone.protocol.commands.RegisteredCommand;
  import org.apache.hadoop.ozone.protocol.commands.ReregisterCommand;
  import org.apache.hadoop.ozone.protocol.commands.SCMCommand;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1045251/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1045251/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
----------------------------------------------------------------------


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message