[hadoop-ozone] branch master updated: HDDS-2622. Add config parameter for setting limit on total bytes of pending requests in Ratis. (#264)
This is an automated email from the ASF dual-hosted git repository. ljain pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git The following commit(s) were added to refs/heads/master by this push: new a9990f6 HDDS-2622. Add config parameter for setting limit on total bytes of pending requests in Ratis. (#264) a9990f6 is described below commit a9990f60d0a0d7e6fd493cb58e4a4fa85455e8bc Author: Hanisha Koneru AuthorDate: Wed Dec 4 21:21:47 2019 -0800 HDDS-2622. Add config parameter for setting limit on total bytes of pending requests in Ratis. (#264) --- .../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 5 + .../org/apache/hadoop/ozone/OzoneConfigKeys.java | 5 + .../common/src/main/resources/ozone-default.xml| 8 .../transport/server/ratis/XceiverServerRatis.java | 24 +- 4 files changed, 37 insertions(+), 5 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java index 02aba5c..00d326e 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java @@ -110,6 +110,11 @@ public final class ScmConfigKeys { "dfs.container.ratis.leader.num.pending.requests"; public static final int DFS_CONTAINER_RATIS_LEADER_NUM_PENDING_REQUESTS_DEFAULT = 4096; + public static final String DFS_CONTAINER_RATIS_LEADER_PENDING_BYTES_LIMIT = + "dfs.container.ratis.leader.pending.bytes.limit"; + public static final String + DFS_CONTAINER_RATIS_LEADER_PENDING_BYTES_LIMIT_DEFAULT = "1GB"; + // expiry interval stateMachineData cache entry inside containerStateMachine public static final String DFS_CONTAINER_RATIS_STATEMACHINEDATA_CACHE_EXPIRY_INTERVAL = diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java index 2705dfa..4b18d12 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java @@ -331,6 +331,11 @@ public final class OzoneConfigKeys { public static final int DFS_CONTAINER_RATIS_LEADER_NUM_PENDING_REQUESTS_DEFAULT = ScmConfigKeys.DFS_CONTAINER_RATIS_LEADER_NUM_PENDING_REQUESTS_DEFAULT; + public static final String DFS_CONTAINER_RATIS_LEADER_PENDING_BYTES_LIMIT = + ScmConfigKeys.DFS_CONTAINER_RATIS_LEADER_PENDING_BYTES_LIMIT; + public static final String + DFS_CONTAINER_RATIS_LEADER_PENDING_BYTES_LIMIT_DEFAULT = + ScmConfigKeys.DFS_CONTAINER_RATIS_LEADER_PENDING_BYTES_LIMIT_DEFAULT; public static final String DFS_RATIS_SERVER_REQUEST_TIMEOUT_DURATION_KEY = ScmConfigKeys.DFS_RATIS_SERVER_REQUEST_TIMEOUT_DURATION_KEY; public static final TimeDuration diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index 0dfd179..22d94f6 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -212,6 +212,14 @@ +dfs.container.ratis.leader.pending.bytes.limit +1GB +OZONE, RATIS, PERFORMANCE +Limit on the total bytes of pending requests after which + leader starts rejecting requests from client. + + + dfs.container.ratis.replication.level MAJORITY OZONE, RATIS diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java index a76944b..961df2c 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java @@ -231,11 +231,10 @@ public final class XceiverServerRatis implements XceiverServerSpi { setAutoTriggerEnabled(properties, true); RaftServerConfigKeys.Snapshot. setAutoTriggerThreshold(properties, snapshotThreshold); -int maxPendingRequets = conf.getInt( -OzoneConfigKeys.DFS_CONTAINER_RATIS_LEADER_NUM_PENDING_REQUESTS, -OzoneConfigKeys.DFS_CONTAINER_RATIS_LEADER_NUM_PENDING_REQUESTS_DEFAULT -); -RaftServerConfigKeys.Write.setElementLimit(properties, maxPendingRequets); + +// Set the limit on num/ bytes of pending requests a Ratis leader can hold +setPendingRequestsLimits(properties); + int logQueueNumElements =
[hadoop-ozone] branch master updated (f686239 -> 0ff53ef)
This is an automated email from the ASF dual-hosted git repository. aengineer pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git. from f686239 HDDS-2608. Provide command to wait until SCM is out from the safe-mode (#250) add 0ff53ef HDDS-2646. Start acceptance tests only if at least one THREE pipeline is available (#282) No new revisions were added by this update. Summary of changes: .../dist/src/main/compose/ozone-mr/common-config | 1 + .../compose/ozone-mr/hadoop27/docker-compose.yaml | 1 + .../compose/ozone-mr/hadoop31/docker-compose.yaml | 1 + .../compose/ozone-mr/hadoop32/docker-compose.yaml | 1 + .../compose/ozone-om-ha-s3/docker-compose.yaml | 1 + .../main/compose/ozone-recon/docker-compose.yaml | 1 + .../compose/ozone-topology/docker-compose.yaml | 1 + .../src/main/compose/ozone-topology/docker-config | 1 + .../src/main/compose/ozone/docker-compose.yaml | 1 + .../src/main/compose/ozoneblockade/docker-config | 2 + .../src/main/compose/ozoneperf/docker-compose.yaml | 1 + .../compose/ozones3-haproxy/docker-compose.yaml| 1 + .../src/main/compose/ozones3/docker-compose.yaml | 1 + .../src/main/compose/ozonescripts/docker-config| 3 +- .../compose/ozonesecure-mr/docker-compose.yaml | 1 + .../main/compose/ozonesecure/docker-compose.yaml | 1 + hadoop-ozone/dist/src/main/compose/testlib.sh | 51 +- 17 files changed, 19 insertions(+), 51 deletions(-) - To unsubscribe, e-mail: ozone-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: ozone-commits-h...@hadoop.apache.org
[hadoop-ozone] branch master updated (5aa7383 -> f686239)
This is an automated email from the ASF dual-hosted git repository. aengineer pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git. from 5aa7383 HDDS-2653. Improve executor memory usage in new Freon tests (#284) add f686239 HDDS-2608. Provide command to wait until SCM is out from the safe-mode (#250) No new revisions were added by this update. Summary of changes: .../scm/safemode/HealthyPipelineSafeModeRule.java | 2 +- .../hadoop/hdds/scm/cli/SafeModeCommands.java | 1 + .../hdds/scm/cli/SafeModeWaitSubcommand.java | 93 ++ 3 files changed, 95 insertions(+), 1 deletion(-) create mode 100644 hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SafeModeWaitSubcommand.java - To unsubscribe, e-mail: ozone-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: ozone-commits-h...@hadoop.apache.org
[hadoop-ozone] branch master updated (7b71678 -> 5aa7383)
This is an automated email from the ASF dual-hosted git repository. xyao pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git. from 7b71678 HDDS-2475. Unregister ContainerMetadataScrubberMetrics on thread exit (#286) add 5aa7383 HDDS-2653. Improve executor memory usage in new Freon tests (#284) No new revisions were added by this update. Summary of changes: .../hadoop/ozone/freon/BaseFreonGenerator.java | 116 ++--- 1 file changed, 80 insertions(+), 36 deletions(-) - To unsubscribe, e-mail: ozone-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: ozone-commits-h...@hadoop.apache.org
[hadoop-ozone] branch master updated (c9f2b02 -> 7b71678)
This is an automated email from the ASF dual-hosted git repository. aengineer pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git. from c9f2b02 HDDS-2656. Prefer execute() over submit() if the returned Future is ignored (#302) add 7b71678 HDDS-2475. Unregister ContainerMetadataScrubberMetrics on thread exit (#286) No new revisions were added by this update. Summary of changes: .../server/ratis/ContainerStateMachine.java| 1 + .../ozoneimpl/ContainerDataScrubberMetrics.java| 8 - .../ozoneimpl/ContainerMetadataScanner.java| 18 +++ .../ContainerMetadataScrubberMetrics.java | 8 - .../ozoneimpl/TestContainerScrubberMetrics.java| 35 +- .../hdds/scm/server/StorageContainerManager.java | 7 +++-- 6 files changed, 67 insertions(+), 10 deletions(-) - To unsubscribe, e-mail: ozone-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: ozone-commits-h...@hadoop.apache.org
[hadoop-ozone] branch master updated (a3a2bfd -> c9f2b02)
This is an automated email from the ASF dual-hosted git repository. aengineer pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git. from a3a2bfd HDDS-2571. Handle InterruptedException in SCMPipelineManager (#295) add c9f2b02 HDDS-2656. Prefer execute() over submit() if the returned Future is ignored (#302) No new revisions were added by this update. Summary of changes: .../java/org/apache/hadoop/hdds/utils/db/cache/TableCacheImpl.java| 2 +- .../hadoop/ozone/container/replication/ReplicationSupervisor.java | 2 +- .../hadoop/ozone/container/keyvalue/helpers/TestChunkUtils.java | 4 ++-- .../main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) - To unsubscribe, e-mail: ozone-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: ozone-commits-h...@hadoop.apache.org
[hadoop-ozone] branch master updated (ea6c2c9 -> a3a2bfd)
This is an automated email from the ASF dual-hosted git repository. aengineer pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git. from ea6c2c9 HDDS-1722. Use the bindings in ReconSchemaGenerationModule to create Recon SQL tables on startup. (#294) add a3a2bfd HDDS-2571. Handle InterruptedException in SCMPipelineManager (#295) No new revisions were added by this update. Summary of changes: .../java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java | 1 + 1 file changed, 1 insertion(+) - To unsubscribe, e-mail: ozone-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: ozone-commits-h...@hadoop.apache.org
[hadoop-ozone] branch master updated (dbd05b0 -> ea6c2c9)
This is an automated email from the ASF dual-hosted git repository. aengineer pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git. from dbd05b0 HDDS-2620. Fix listMultipartupload API. (#263) add ea6c2c9 HDDS-1722. Use the bindings in ReconSchemaGenerationModule to create Recon SQL tables on startup. (#294) No new revisions were added by this update. Summary of changes: .../hadoop/ozone/recon/ReconControllerModule.java | 4 -- .../hadoop/ozone/recon/ReconSchemaManager.java | 55 ++ .../org/apache/hadoop/ozone/recon/ReconServer.java | 26 +++--- 3 files changed, 61 insertions(+), 24 deletions(-) create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaManager.java - To unsubscribe, e-mail: ozone-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: ozone-commits-h...@hadoop.apache.org
[hadoop-ozone] branch master updated (2fb8747 -> dbd05b0)
This is an automated email from the ASF dual-hosted git repository. bharat pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git. from 2fb8747 HDDS-2386. Implement incremental ChunkBuffer. (#205) add dbd05b0 HDDS-2620. Fix listMultipartupload API. (#263) No new revisions were added by this update. Summary of changes: .../apache/hadoop/ozone/om/OMMetadataManager.java | 3 +- .../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 10 +- .../hadoop/ozone/om/OmMetadataManagerImpl.java | 32 +- .../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 122 + 4 files changed, 155 insertions(+), 12 deletions(-) - To unsubscribe, e-mail: ozone-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: ozone-commits-h...@hadoop.apache.org
[hadoop-ozone] branch master updated (9381dd0 -> 2fb8747)
This is an automated email from the ASF dual-hosted git repository. xyao pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git. from 9381dd0 HDDS-2637. Handle LeaderNot ready exception in OzoneManager StateMachine and upgrade ratis to latest version. (#276) add 2fb8747 HDDS-2386. Implement incremental ChunkBuffer. (#205) No new revisions were added by this update. Summary of changes: .../hadoop/hdds/scm/storage/BlockOutputStream.java | 3 +- .../apache/hadoop/hdds/scm/storage/BufferPool.java | 4 +- .../apache/hadoop/ozone/common/ChunkBuffer.java| 49 +++- .../common/ChunkBufferImplWithByteBuffer.java | 10 +- .../ozone/common/IncrementalChunkBuffer.java | 267 + .../hadoop/ozone/common/TestChunkBuffer.java | 18 ++ .../hadoop/ozone/client/rpc/TestCommitWatcher.java | 4 +- 7 files changed, 341 insertions(+), 14 deletions(-) create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/IncrementalChunkBuffer.java - To unsubscribe, e-mail: ozone-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: ozone-commits-h...@hadoop.apache.org
[hadoop-ozone] branch debug-acceptance-timeout updated: retrigger build
This is an automated email from the ASF dual-hosted git repository. elek pushed a commit to branch debug-acceptance-timeout in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git The following commit(s) were added to refs/heads/debug-acceptance-timeout by this push: new c38b311 retrigger build c38b311 is described below commit c38b311a4b489f7a82989fa4da8ce99909eb5cb8 Author: Márton Elek AuthorDate: Wed Dec 4 19:28:22 2019 +0100 retrigger build - To unsubscribe, e-mail: ozone-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: ozone-commits-h...@hadoop.apache.org
[hadoop-ozone] branch debug-acceptance-timeout updated: retrigger build
This is an automated email from the ASF dual-hosted git repository. elek pushed a commit to branch debug-acceptance-timeout in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git The following commit(s) were added to refs/heads/debug-acceptance-timeout by this push: new 4e9c4dc retrigger build 4e9c4dc is described below commit 4e9c4dc53c598ebbc2db2ef2d82a8a008582e3b3 Author: Márton Elek AuthorDate: Wed Dec 4 18:23:51 2019 +0100 retrigger build - To unsubscribe, e-mail: ozone-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: ozone-commits-h...@hadoop.apache.org
[hadoop-ozone] branch integration-test-cleanup updated: retrigger build
This is an automated email from the ASF dual-hosted git repository. elek pushed a commit to branch integration-test-cleanup in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git The following commit(s) were added to refs/heads/integration-test-cleanup by this push: new 4fff586 retrigger build 4fff586 is described below commit 4fff586d92397a8b143d6cde69af0c354f5e6e04 Author: Márton Elek AuthorDate: Wed Dec 4 18:24:16 2019 +0100 retrigger build - To unsubscribe, e-mail: ozone-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: ozone-commits-h...@hadoop.apache.org
[hadoop-ozone] branch integration-test-cleanup updated: ignore test
This is an automated email from the ASF dual-hosted git repository. elek pushed a commit to branch integration-test-cleanup in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git The following commit(s) were added to refs/heads/integration-test-cleanup by this push: new 357afd2 ignore test 357afd2 is described below commit 357afd22b79acc52bc040ff59f9dbbf58b46f8f7 Author: Márton Elek AuthorDate: Wed Dec 4 15:24:00 2019 +0100 ignore test --- .../hadoop/hdds/scm/pipeline/TestRatisPipelineCreateAndDestroy.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineCreateAndDestroy.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineCreateAndDestroy.java index 2f852f2..3590e43 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineCreateAndDestroy.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineCreateAndDestroy.java @@ -26,6 +26,7 @@ import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.test.GenericTestUtils; import org.junit.After; import org.junit.Assert; +import org.junit.Ignore; import org.junit.Test; import java.io.IOException; @@ -40,6 +41,7 @@ import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTER /** * Tests for RatisPipelineUtils. */ +@Ignore public class TestRatisPipelineCreateAndDestroy { private static MiniOzoneCluster cluster; - To unsubscribe, e-mail: ozone-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: ozone-commits-h...@hadoop.apache.org
[hadoop-ozone] branch master updated (87e02a3 -> 9381dd0)
This is an automated email from the ASF dual-hosted git repository. ljain pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git. from 87e02a3 HDDS-2573. Handle InterruptedException in KeyOutputStream. (#257) add 9381dd0 HDDS-2637. Handle LeaderNot ready exception in OzoneManager StateMachine and upgrade ratis to latest version. (#276) No new revisions were added by this update. Summary of changes: .../apache/hadoop/hdds/utils/db/DBCheckpoint.java | 12 .../hadoop/hdds/utils/db/RocksDBCheckpoint.java| 11 .../java/org/apache/hadoop/ozone/OzoneConsts.java | 2 + .../server/ratis/ContainerStateMachine.java| 6 ++ .../om/exceptions/OMLeaderNotReadyException.java | 22 +++ ...derException.java => OMNotLeaderException.java} | 32 -- .../ozone/om/ha/OMFailoverProxyProvider.java | 2 +- .../ozone/om/protocol/OzoneManagerHAProtocol.java | 6 +- ...OzoneManagerProtocolClientSideTranslatorPB.java | 51 .../java/org/apache/hadoop/ozone/TestOmUtils.java | 9 +++ .../hadoop/ozone/om/TestOMRatisSnapshots.java | 20 --- .../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 26 .../hadoop/ozone/om/OMDBCheckpointServlet.java | 19 +++--- .../org/apache/hadoop/ozone/om/OzoneManager.java | 21 --- .../hadoop/ozone/om/ratis/OMRatisSnapshotInfo.java | 15 +++-- .../ozone/om/ratis/OzoneManagerRatisServer.java| 59 +++--- .../ozone/om/ratis/OzoneManagerStateMachine.java | 69 -- .../om/snapshot/OzoneManagerSnapshotProvider.java | 10 ...OzoneManagerProtocolServerSideTranslatorPB.java | 8 +-- .../ozone/om/ratis/TestOMRatisSnapshotInfo.java| 5 +- .../om/ratis/TestOzoneManagerRatisServer.java | 17 -- pom.xml| 3 +- 22 files changed, 303 insertions(+), 122 deletions(-) copy hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigurationException.java => hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMLeaderNotReadyException.java (65%) rename hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/{NotLeaderException.java => OMNotLeaderException.java} (71%) - To unsubscribe, e-mail: ozone-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: ozone-commits-h...@hadoop.apache.org