From ozone-commits-return-360-archive-asf-public=cust-asf.ponee.io@hadoop.apache.org Thu Dec 12 16:30:42 2019 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [207.244.88.153]) by mx-eu-01.ponee.io (Postfix) with SMTP id CDB9B18067C for ; Thu, 12 Dec 2019 17:30:41 +0100 (CET) Received: (qmail 5725 invoked by uid 500); 12 Dec 2019 16:30:41 -0000 Mailing-List: contact ozone-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ozone-commits@hadoop.apache.org Delivered-To: mailing list ozone-commits@hadoop.apache.org Received: (qmail 5663 invoked by uid 99); 12 Dec 2019 16:30:41 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 12 Dec 2019 16:30:41 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id 0892E8D80F; Thu, 12 Dec 2019 16:30:41 +0000 (UTC) Date: Thu, 12 Dec 2019 16:30:41 +0000 To: "ozone-commits@hadoop.apache.org" Subject: [hadoop-ozone] 01/01: Merge remote-tracking branch 'origin/master' into HDDS-1880-Decom MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit From: elek@apache.org In-Reply-To: <157616824044.15003.4670330067752660405@gitbox.apache.org> References: <157616824044.15003.4670330067752660405@gitbox.apache.org> X-Git-Host: gitbox.apache.org X-Git-Repo: hadoop-ozone X-Git-Refname: refs/heads/HDDS-1880-Decom X-Git-Reftype: branch X-Git-Rev: 92e8d47b7c8bbacc1ab2bea478ac08c7b9f6ab4c X-Git-NotificationType: diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated Message-Id: <20191212163041.0892E8D80F@gitbox.apache.org> This is an automated email from the ASF dual-hosted git repository. elek pushed a commit to branch HDDS-1880-Decom in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git commit 92e8d47b7c8bbacc1ab2bea478ac08c7b9f6ab4c Merge: 920b8c5 e14f709 Author: Márton Elek AuthorDate: Thu Dec 12 17:29:25 2019 +0100 Merge remote-tracking branch 'origin/master' into HDDS-1880-Decom .github/workflows/post-commit.yml | 1 + .../hadoop/hdds/scm/client/HddsClientUtils.java | 75 +-- .../hadoop/hdds/scm/storage/BlockOutputStream.java | 3 +- .../apache/hadoop/hdds/scm/storage/BufferPool.java | 4 +- .../hadoop/hdds/scm/storage/CommitWatcher.java | 20 +- hadoop-hdds/common/pom.xml | 9 +- .../org/apache/hadoop/hdds/HddsConfigKeys.java | 10 +- .../java/org/apache/hadoop/hdds/HddsUtils.java | 216 ++++---- .../org/apache/hadoop/hdds/client/OzoneQuota.java | 12 +- .../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 13 +- .../org/apache/hadoop/hdds/scm/net/NetUtils.java | 19 +- .../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 8 +- .../ContainerCommandResponseBuilders.java | 269 +++++++++ .../org/apache/hadoop/hdds/utils/Scheduler.java | 7 +- .../apache/hadoop/hdds/utils/db/DBCheckpoint.java | 12 + .../hadoop/hdds/utils/db/RocksDBCheckpoint.java | 11 + .../org/apache/hadoop/hdds/utils/db/Table.java | 10 +- .../apache/hadoop/hdds/utils/db/TypedTable.java | 5 +- .../hadoop/hdds/utils/db/cache/EpochEntry.java | 2 +- .../hadoop/hdds/utils/db/cache/TableCache.java | 14 +- .../hadoop/hdds/utils/db/cache/TableCacheImpl.java | 58 +- .../org/apache/hadoop/ozone/OzoneConfigKeys.java | 5 + .../java/org/apache/hadoop/ozone/OzoneConsts.java | 2 + .../org/apache/hadoop/ozone/OzoneSecurityUtil.java | 53 +- .../apache/hadoop/ozone/audit/AuditMessage.java | 51 +- .../apache/hadoop/ozone/common/ChunkBuffer.java | 49 +- .../common/ChunkBufferImplWithByteBuffer.java | 10 +- .../ozone/common/IncrementalChunkBuffer.java | 267 +++++++++ .../ozone/container/common/helpers/BlockData.java | 5 - .../apache/hadoop/ozone/lease/LeaseManager.java | 4 +- .../apache/hadoop/ozone/web/utils/JsonUtils.java | 15 +- .../common/src/main/resources/ozone-default.xml | 69 +-- .../apache/hadoop/hdds/scm/net/TestNetUtils.java | 32 +- .../hdds/utils/db/TestTypedRDBTableStore.java | 7 +- .../hdds/utils/db/cache/TestTableCacheImpl.java | 244 ++++++++- .../hadoop/ozone/audit/TestOzoneAuditLogger.java | 22 +- .../hadoop/ozone/common/TestChunkBuffer.java | 18 + .../org/apache/hadoop/hdds/scm/HddsServerUtil.java | 117 ++-- .../container/common/helpers/ContainerUtils.java | 85 +-- .../ozone/container/common/impl/ContainerSet.java | 2 +- .../container/common/impl/HddsDispatcher.java | 18 +- .../common/statemachine/DatanodeStateMachine.java | 6 + .../CloseContainerCommandHandler.java | 11 +- .../ClosePipelineCommandHandler.java | 120 ++++ .../commandhandler/CommandHandler.java | 2 +- .../CreatePipelineCommandHandler.java | 135 +++++ .../states/endpoint/HeartbeatEndpointTask.java | 22 + .../common/transport/server/XceiverServerSpi.java | 18 + .../server/ratis/ContainerStateMachine.java | 30 +- .../transport/server/ratis/XceiverServerRatis.java | 78 ++- .../ozone/container/common/volume/HddsVolume.java | 3 +- .../ozone/container/common/volume/VolumeSet.java | 1 + .../ozone/container/keyvalue/KeyValueHandler.java | 124 ++--- .../container/keyvalue/helpers/BlockUtils.java | 88 +-- .../container/keyvalue/helpers/ChunkUtils.java | 62 +-- .../keyvalue/helpers/KeyValueContainerUtil.java | 27 - .../container/keyvalue/helpers/SmallFileUtils.java | 91 --- .../container/keyvalue/impl/BlockManagerImpl.java | 65 +-- .../container/keyvalue/impl/ChunkManagerImpl.java | 43 +- .../container/ozoneimpl/ContainerController.java | 9 + .../ozoneimpl/ContainerDataScrubberMetrics.java | 8 +- .../ozoneimpl/ContainerMetadataScanner.java | 18 +- .../ContainerMetadataScrubberMetrics.java | 8 +- .../replication/ReplicationSupervisor.java | 2 +- .../protocol/commands/ClosePipelineCommand.java | 73 +++ .../protocol/commands/CreatePipelineCommand.java | 100 ++++ .../proto/StorageContainerDatanodeProtocol.proto | 23 + .../ozone/container/ContainerTestHelper.java | 325 +---------- .../container/common/helpers/TestBlockData.java | 0 .../impl/TestContainerDeletionChoosingPolicy.java | 0 .../common/impl/TestContainerPersistence.java | 2 +- .../container/common/impl/TestContainerSet.java | 39 +- .../container/common/impl/TestHddsDispatcher.java | 6 +- .../container/common/volume/TestVolumeSet.java | 24 + .../container/keyvalue/TestChunkManagerImpl.java | 3 +- .../container/keyvalue/helpers/TestChunkUtils.java | 39 +- .../ozoneimpl/TestContainerScrubberMetrics.java | 35 +- .../apache/hadoop/hdds/server/BaseHttpServer.java | 19 +- .../apache/hadoop/hdds/server/ProfileServlet.java | 7 +- .../hadoop/hdds/server/PrometheusMetricsSink.java | 5 +- .../org/apache/hadoop/hdds/server/ServerUtils.java | 15 + .../hdds/server/TestPrometheusMetricsSink.java | 2 +- .../hadoop/hdds/scm/block/BlockManagerImpl.java | 3 + .../hdds/scm/container/ContainerStateManager.java | 1 + .../apache/hadoop/hdds/scm/events/SCMEvents.java | 12 +- .../scm/pipeline/BackgroundPipelineCreator.java | 23 +- .../hadoop/hdds/scm/pipeline/PipelineFactory.java | 13 +- .../hadoop/hdds/scm/pipeline/PipelineManager.java | 15 +- .../hadoop/hdds/scm/pipeline/PipelineProvider.java | 2 + .../hdds/scm/pipeline/PipelineReportHandler.java | 54 +- .../hdds/scm/pipeline/PipelineStateManager.java | 4 +- .../hdds/scm/pipeline/RatisPipelineProvider.java | 137 ++--- .../hdds/scm/pipeline/RatisPipelineUtils.java | 103 ---- .../hdds/scm/pipeline/SCMPipelineManager.java | 83 ++- .../hdds/scm/pipeline/SCMPipelineMetrics.java | 10 + .../hdds/scm/pipeline/SimplePipelineProvider.java | 5 + .../scm/safemode/HealthyPipelineSafeModeRule.java | 80 +-- .../safemode/OneReplicaPipelineSafeModeRule.java | 65 +-- .../hdds/scm/safemode/SCMSafeModeManager.java | 16 +- .../hadoop/hdds/scm/safemode/SafeModeHandler.java | 5 +- .../hdds/scm/server/SCMBlockProtocolServer.java | 26 +- .../hdds/scm/server/SCMClientProtocolServer.java | 30 +- .../scm/server/SCMDatanodeHeartbeatDispatcher.java | 1 + .../hdds/scm/server/SCMDatanodeProtocolServer.java | 46 +- .../hdds/scm/server/StorageContainerManager.java | 10 +- .../java/org/apache/hadoop/hdds/scm/TestUtils.java | 12 + .../hadoop/hdds/scm/block/TestBlockManager.java | 44 +- .../container/TestCloseContainerEventHandler.java | 11 +- .../scm/container/TestSCMContainerManager.java | 4 +- .../hdds/scm/node/TestContainerPlacement.java | 2 +- .../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 11 +- .../hadoop/hdds/scm/node/TestSCMNodeManager.java | 9 +- .../scm/pipeline/MockRatisPipelineProvider.java | 10 +- .../scm/pipeline/TestPipelineStateManager.java | 0 .../scm/pipeline/TestRatisPipelineProvider.java | 26 +- .../hdds/scm/pipeline/TestSCMPipelineManager.java | 93 +++- .../scm/pipeline/TestSimplePipelineProvider.java | 0 .../hdds/scm/pipeline/pipeline/package-info.java | 22 + .../safemode/TestHealthyPipelineSafeModeRule.java | 44 +- .../TestOneReplicaPipelineSafeModeRule.java | 36 +- .../hdds/scm/safemode/TestSCMSafeModeManager.java | 50 +- .../hadoop/hdds/scm/cli/SafeModeCommands.java | 1 + .../hdds/scm/cli/SafeModeWaitSubcommand.java | 93 ++++ .../apache/hadoop/ozone/client/OzoneBucket.java | 11 +- .../hadoop/ozone/client/OzoneClientFactory.java | 68 +-- .../org/apache/hadoop/ozone/client/OzoneKey.java | 14 +- .../apache/hadoop/ozone/client/OzoneVolume.java | 9 +- .../hadoop/ozone/client/io/KeyInputStream.java | 2 +- .../hadoop/ozone/client/io/KeyOutputStream.java | 1 + .../hadoop/ozone/client/TestHddsClientUtils.java | 46 ++ hadoop-ozone/common/pom.xml | 8 - .../main/java/org/apache/hadoop/ozone/OmUtils.java | 61 +-- .../org/apache/hadoop/ozone/om/OMConfigKeys.java | 9 - .../apache/hadoop/ozone/om/OMMetadataManager.java | 3 +- .../OMLeaderNotReadyException.java} | 23 +- ...derException.java => OMNotLeaderException.java} | 32 +- .../ozone/om/ha/OMFailoverProxyProvider.java | 2 +- .../hadoop/ozone/om/helpers/OMRatisHelper.java | 43 -- .../apache/hadoop/ozone/om/helpers/OmKeyArgs.java | 2 - .../ozone/om/protocol/OzoneManagerHAProtocol.java | 6 +- ...OzoneManagerProtocolClientSideTranslatorPB.java | 51 +- .../java/org/apache/hadoop/ozone/TestOmUtils.java | 65 +-- hadoop-ozone/csi/pom.xml | 11 +- hadoop-ozone/dev-support/checks/findbugs.sh | 8 +- .../dist/dev-support/bin/dist-tar-stitching | 6 +- hadoop-ozone/dist/pom.xml | 39 ++ .../dist/src/main/compose/ozone-mr/common-config | 1 + .../compose/ozone-mr/hadoop27/docker-compose.yaml | 1 + .../compose/ozone-mr/hadoop31/docker-compose.yaml | 1 + .../compose/ozone-mr/hadoop32/docker-compose.yaml | 1 + .../compose/ozone-om-ha-s3/docker-compose.yaml | 1 + .../dist/src/main/compose/ozone-recon/.env | 18 - .../main/compose/ozone-recon/docker-compose.yaml | 65 --- .../src/main/compose/ozone-recon/docker-config | 35 -- .../compose/ozone-topology/docker-compose.yaml | 1 + .../src/main/compose/ozone-topology/docker-config | 1 + hadoop-ozone/dist/src/main/compose/ozone/README.md | 99 ++++ .../src/main/compose/ozone/docker-compose.yaml | 91 +-- .../dist/src/main/compose/ozone/docker-config | 11 +- .../compose/{ozoneperf => ozone}/freon-ockg.yaml | 19 +- .../compose/{ozoneperf => ozone}/freon-rk.yaml | 19 +- .../docker-config => ozone/monitoring.conf} | 15 +- .../dist/src/main/compose/ozone/monitoring.yaml | 56 ++ .../{ozoneperf/test.sh => ozone/profiling.yaml} | 30 +- .../compose/{ozoneperf => ozone}/prometheus.yml | 20 +- .../compose/{ozone-recon/test.sh => ozone/run.sh} | 15 +- hadoop-ozone/dist/src/main/compose/ozone/test.sh | 9 +- .../src/main/compose/ozoneblockade/docker-config | 2 + hadoop-ozone/dist/src/main/compose/ozoneperf/.env | 18 - .../dist/src/main/compose/ozoneperf/README.md | 74 --- .../src/main/compose/ozoneperf/docker-compose.yaml | 84 --- .../dist/src/main/compose/ozoneperf/docker-config | 36 -- .../compose/ozones3-haproxy/docker-compose.yaml | 1 + hadoop-ozone/dist/src/main/compose/ozones3/.env | 18 - .../src/main/compose/ozones3/docker-compose.yaml | 58 -- hadoop-ozone/dist/src/main/compose/ozones3/test.sh | 34 -- .../src/main/compose/ozonescripts/docker-config | 3 +- .../compose/ozonesecure-mr/docker-compose.yaml | 1 + .../main/compose/ozonesecure/docker-compose.yaml | 1 + hadoop-ozone/dist/src/main/compose/testlib.sh | 52 +- .../dist/src/main/dockerlibexec/entrypoint.sh | 8 + .../dist/src/main/smoketest/__init__.robot | 3 +- .../dist/src/main/smoketest/basic/basic.robot | 4 +- .../dist/src/main/smoketest/omha/testOMHA.robot | 28 +- hadoop-ozone/dist/src/shell/conf/log4j.properties | 10 +- .../{ => mini-chaos-tests}/pom.xml | 38 +- .../mini-chaos-tests/src/test/bin/start-chaos.sh | 46 ++ .../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 6 +- .../hadoop/ozone/MiniOzoneLoadGenerator.java | 107 +--- .../hadoop/ozone/TestMiniChaosOzoneCluster.java | 12 +- .../org/apache/hadoop/ozone/utils/LoadBucket.java | 99 ++++ .../hadoop/ozone/utils}/TestProbability.java | 2 +- .../services/org.apache.hadoop.fs.FileSystem} | 21 +- .../src/test/resources/log4j.properties | 31 ++ hadoop-ozone/fault-injection-test/pom.xml | 1 + .../apache/hadoop/ozone/insight/LogSubcommand.java | 31 +- .../TestContainerStateManagerIntegration.java | 6 +- .../metrics/TestSCMContainerManagerMetrics.java | 3 + .../hdds/scm/pipeline/TestPipelineClose.java | 4 +- .../TestRatisPipelineCreateAndDestroy.java | 29 + .../scm/pipeline/TestRatisPipelineProvider.java | 210 +++++++ .../org/apache/hadoop/ozone/MiniOzoneCluster.java | 2 +- .../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 53 +- .../hadoop/ozone/MiniOzoneHAClusterImpl.java | 24 +- .../hadoop/ozone/TestContainerOperations.java | 2 +- .../TestContainerStateMachineIdempotency.java | 2 +- .../apache/hadoop/ozone/TestMiniOzoneCluster.java | 6 +- .../hadoop/ozone/TestMiniOzoneHACluster.java | 112 ++++ .../hadoop/ozone/TestStorageContainerManager.java | 18 +- .../apache/hadoop/ozone/client/rpc/TestBCSID.java | 3 + .../ozone/client/rpc/TestBlockOutputStream.java | 5 +- .../rpc/TestBlockOutputStreamWithFailures.java | 13 +- .../rpc/TestCloseContainerHandlingByClient.java | 7 +- .../hadoop/ozone/client/rpc/TestCommitWatcher.java | 4 +- .../rpc/TestContainerReplicationEndToEnd.java | 4 +- .../client/rpc/TestContainerStateMachine.java | 10 +- .../rpc/TestContainerStateMachineFailures.java | 18 +- .../client/rpc/TestFailureHandlingByClient.java | 7 +- .../ozone/client/rpc/TestKeyInputStream.java | 7 +- .../rpc/TestMultiBlockWritesWithDnFailures.java | 5 +- .../client/rpc/TestOzoneAtRestEncryption.java | 14 +- .../rpc/TestOzoneClientRetriesOnException.java | 9 +- .../client/rpc/TestOzoneRpcClientAbstract.java | 90 ++- .../ozone/client/rpc/TestSecureOzoneRpcClient.java | 8 +- .../ozone/client/rpc/TestWatchForCommit.java | 7 +- .../ozone/container/TestContainerReplication.java | 2 +- .../apache/hadoop/ozone/container/TestHelper.java | 305 +++++++++++ .../commandhandler/TestBlockDeletion.java | 6 +- .../commandhandler/TestCloseContainerHandler.java | 2 + .../commandhandler/TestDeleteContainerHandler.java | 2 + .../hadoop/ozone/dn/scrubber/TestDataScrubber.java | 10 +- .../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 235 ++++++++ .../org/apache/hadoop/ozone/om/TestKeyPurging.java | 5 +- .../hadoop/ozone/om/TestOMRatisSnapshots.java | 20 +- .../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 34 +- .../apache/hadoop/ozone/om/TestScmSafeMode.java | 2 +- .../hadoop/ozone/ozShell/TestOzoneShellHA.java | 20 +- .../hadoop/ozone/scm/TestContainerSmallFile.java | 2 +- .../scm/TestGetCommittedBlockLengthAndPutKey.java | 2 +- .../org/apache/hadoop/ozone/scm/TestSCMMXBean.java | 2 +- .../hadoop/ozone/scm/node/TestSCMNodeMetrics.java | 4 +- hadoop-ozone/ozone-manager/pom.xml | 5 + .../apache/hadoop/ozone/om/KeyDeletingService.java | 2 +- .../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 176 ++++-- .../hadoop/ozone/om/OMDBCheckpointServlet.java | 19 +- .../java/org/apache/hadoop/ozone/om/OMStorage.java | 17 +- .../hadoop/ozone/om/OmMetadataManagerImpl.java | 35 +- .../org/apache/hadoop/ozone/om/OzoneManager.java | 103 +--- .../hadoop/ozone/om/ratis/OMRatisSnapshotInfo.java | 15 +- .../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 16 +- .../ozone/om/ratis/OzoneManagerRatisClient.java | 210 ------- .../ozone/om/ratis/OzoneManagerRatisServer.java | 83 ++- .../ozone/om/ratis/OzoneManagerStateMachine.java | 85 ++- .../hadoop/ozone/om/request/OMClientRequest.java | 6 +- .../hadoop/ozone/om/request/RequestAuditor.java | 3 - .../ozone/om/request/file/OMFileCreateRequest.java | 6 +- .../om/snapshot/OzoneManagerSnapshotProvider.java | 10 + .../protocolPB/OzoneManagerHARequestHandler.java | 39 -- .../OzoneManagerHARequestHandlerImpl.java | 56 -- ...OzoneManagerProtocolServerSideTranslatorPB.java | 27 +- .../protocolPB/OzoneManagerRequestHandler.java | 607 +-------------------- .../hadoop/ozone/protocolPB/RequestHandler.java | 28 +- .../hadoop/ozone/web/ozShell/OzoneAddress.java | 3 +- .../ozone/web/ozShell/keys/GetKeyHandler.java | 4 +- .../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 256 ++++++++- .../org/apache/hadoop/ozone/om/TestOMStorage.java | 91 +++ .../ozone/om/ratis}/TestOMRatisSnapshotInfo.java | 8 +- ...TestOzoneManagerDoubleBufferWithOMResponse.java | 1 + .../om/ratis/TestOzoneManagerRatisServer.java | 29 +- .../ozone/om/request/TestOMRequestUtils.java | 17 + .../security/acl/TestOzoneNativeAuthorizer.java | 20 +- .../ozone/web/ozShell/TestObjectPrinter.java | 6 +- hadoop-ozone/ozonefs-lib-current/pom.xml | 15 - .../fs/ozone/BasicOzoneClientAdapterImpl.java | 2 +- .../hadoop/fs/ozone/TestOzoneFileSystem.java | 12 + .../fs/ozone/TestOzoneFileSystemWithMocks.java | 2 + .../apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java | 48 +- .../hadoop/fs/ozone/TestOzoneFsRenameDir.java | 2 +- .../hadoop/ozone/recon/ReconControllerModule.java | 4 - .../hadoop/ozone/recon/ReconSchemaManager.java | 55 ++ .../org/apache/hadoop/ozone/recon/ReconServer.java | 26 +- .../ozone/recon/api/ContainerKeyService.java | 112 ++-- ...lizationService.java => TaskStatusService.java} | 39 +- .../hadoop/ozone/recon/api/UtilizationService.java | 5 - .../recon/recovery/ReconOMMetadataManager.java | 6 + .../recon/recovery/ReconOmMetadataManagerImpl.java | 11 + .../spi/impl/OzoneManagerServiceProviderImpl.java | 32 +- .../ozone/recon/tasks/ReconTaskController.java | 3 +- .../ozone/recon/tasks/ReconTaskControllerImpl.java | 70 ++- .../ozone/recon/api/TestTaskStatusService.java | 86 +++ .../recon/tasks/TestReconTaskControllerImpl.java | 38 +- hadoop-ozone/s3gateway/pom.xml | 4 + .../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 4 +- .../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 8 +- .../hadoop/ozone/s3/endpoint/RootEndpoint.java | 4 +- .../hadoop/ozone/freon/BaseFreonGenerator.java | 116 ++-- .../hadoop/ozone/freon/RandomKeyGenerator.java | 2 +- .../freon/TestFreonWithDatanodeFastRestart.java | 4 +- .../hadoop/ozone/fsck/TestContainerMapper.java | 2 +- pom.xml | 64 +-- 300 files changed, 6202 insertions(+), 4503 deletions(-) diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java index 51257c2,a2ee50a..99b2f03 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java @@@ -22,24 -22,19 +22,19 @@@ import org.apache.hadoop.conf.Configura import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState; import org.apache.hadoop.hdds.scm.ScmConfigKeys; - import org.apache.hadoop.hdds.scm.client.HddsClientUtils; - import org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementPolicy; - import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementRandom; + import org.apache.hadoop.hdds.scm.container.placement.algorithms + .ContainerPlacementPolicy; + import org.apache.hadoop.hdds.scm.container.placement.algorithms + .SCMContainerPlacementRandom; + import org.apache.hadoop.hdds.scm.events.SCMEvents; import org.apache.hadoop.hdds.scm.node.NodeManager; +import org.apache.hadoop.hdds.scm.node.NodeStatus; import org.apache.hadoop.hdds.scm.pipeline.Pipeline.PipelineState; - import org.apache.hadoop.io.MultipleIOException; - import org.apache.hadoop.hdds.ratis.RatisHelper; - import org.apache.ratis.client.RaftClient; - import org.apache.ratis.grpc.GrpcTlsConfig; - import org.apache.ratis.protocol.RaftClientReply; - import org.apache.ratis.protocol.RaftGroup; - import org.apache.ratis.protocol.RaftPeer; - import org.apache.ratis.retry.RetryPolicy; - import org.apache.ratis.rpc.SupportedRpcType; - import org.apache.ratis.util.TimeDuration; - import org.apache.ratis.util.function.CheckedBiConsumer; + import org.apache.hadoop.hdds.server.events.EventPublisher; + import org.apache.hadoop.ozone.protocol.commands.ClosePipelineCommand; + import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode; + import org.apache.hadoop.ozone.protocol.commands.CreatePipelineCommand; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --cc hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java index 28f9e96,aa190f4..af66e00 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java @@@ -29,9 -30,10 +30,11 @@@ import java.util.concurrent.TimeoutExce import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; + import org.apache.hadoop.hdds.protocol.proto + .StorageContainerDatanodeProtocolProtos.SCMCommandProto; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.TestUtils; +import org.apache.hadoop.hdds.scm.node.NodeStatus; import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager.SafeModeStatus; import org.apache.hadoop.hdds.scm.container.CloseContainerEventHandler; import org.apache.hadoop.hdds.scm.container.ContainerID; @@@ -302,9 -314,11 +315,10 @@@ public class TestBlockManager // create pipelines for (int i = 0; - i < nodeManager.getNodes(HddsProtos.NodeState.HEALTHY).size() / factor - .getNumber(); i++) { + i < nodeManager.getNodes(NodeStatus.inServiceHealthy()).size(); i++) { pipelineManager.createPipeline(type, factor); } + TestUtils.openAllRatisPipelines(pipelineManager); // wait till each pipeline has the configured number of containers. // After this each pipeline has numContainerPerOwnerInPipeline containers --------------------------------------------------------------------- To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: ozone-commits-help@hadoop.apache.org