This is an automated email from the ASF dual-hosted git repository.
jiajunwang pushed a change to branch wagedRebalancer
in repository https://gitbox.apache.org/repos/asf/helix.git.
omit 7c6f1be Add latency metric components for WAGED rebalancer (#490)
omit be1f041 Load soft constraint weight from resources/properties file
(#492)
omit 3ac3b22 Adjust the topology processing logic for instance to ensure
backward compatibility.
omit aae2e93 Enable maintenance mode for the WAGED rebalancer.
omit 1b4457d Add delayed rebalance and user-defined preference list
features to the WAGED rebalancer. (#456)
omit 88b1f80 Separate AssignableNode properties by Immutable and Mutable
(#485)
omit 935360e Integrate the WAGED rebalancer with all the related
components. (#466)
omit 718ea9c Add the remaining implementation of
ConstraintBasedAlgorithmFactory (#478)
omit e13ab51 PartitionMovementSoftConstraint Implementation (#474)
omit 951d3bf Adjust the expected replica count according to fault zone
count. (#476)
omit 7be94fe Improve ResourceTopStateAntiAffinityConstraint (#475)
omit e99e358 Add soft constraint: ResourcePartitionAntiAffinityConstraint
(#464)
omit 81fd8ce Implement MaxCapacityUsageInstanceConstraint soft constraint
(#463)
omit 8d2f752 Add soft constraint: ResourcetopStateAntiAffinityConstraint
(#465)
omit cbd295a Implement one of the soft constraints (#450)
omit 7cb8083 Fix TestWagedRebalancer and add constructor in
AssignmentMetadataStore
omit 884dc1d Implement AssignmentMetadataStore (#453)
omit c40f15b HardConstraints Implementation and unit tests (#433)
omit e654f6b Implement the WAGED rebalancer with the limited
functionality. (#443)
omit 0fa347d Validate the instance capacity/partition weight configuration
while constructing the assignable instances (#451)
omit fae0eb1 Implement the basic constraint based algorithm (#381)
omit 31a1875 Add BucketDataAccessor for large writes
omit fd18212 Record the replica objects in the AssignableNode in addition
to the partition name (#440)
omit 7938b73 Add special treatment for ClusterConfig
omit 28e6fdd Modify the expected change type from CONFIG to CLUSTER_CONFIG
in the WAGED rebalancer. (#438)
omit e6edcd9 Resubmit the change: Refine the WAGED rebalancer related
interfaces for integration (#431)
omit 6db8387 Revert "Refine the WAGED rebalancer related interfaces for
integration (#431)" (#437)
omit 08a2015 Refine the WAGED rebalancer related interfaces for
integration (#431)
omit 672c9cb Redefine the hard/soft constraints (#422)
omit 55e21b8 Add ChangeDetector interface and ResourceChangeDetector
implementation (#388)
omit 4c96562 Add cluster level default instance config. (#413)
omit b552731 Implement Cluster Model Provider. (#392)
omit 05e360d Convert all the internal assignment state objects to be
ResourceAssignment. (#399)
omit 19a6ff4 Change the rebalancer assignment record to be
ResourceAssignment instead of IdealState. (#398)
omit d2a801a Implement the WAGED rebalancer cluster model (#362)
omit 65d26aa Adding the configuration items of the WAGED rebalancer. (#348)
omit 5d1d770 Define the WAGED rebalancer interfaces.
add 22e4ec7 Fix the execution delay for the jobs
add 9434cf1 Use the system property value as the sliding window length.
#382
add cc73afd Use the system property value as the sliding window length.
#382
add d30fefb Use the system property value as the sliding window length.
#382
add 51abb53 Add a unit test case.. #382
add 6fb58a3 Fix a typo. #382
add b6aee9f Fix a typo. #382
add ab5ec69 Make the reservoir sliding window length used in Helix
monintor metrics configurable. #382
add 0f16687 Make the reservoir sliding window length used in Helix
monintor metrics configurable. #382
add 955e566 Make the reservoir sliding window length used in Helix
monintor metrics configurable. #382
add ec8ba98 Make the reservoir sliding window length used in Helix
monintor metrics configurable. #382
add d0422b7 TASK: Fix incorrect counting of numAttempts for tasks (#432)
add 33aa50c Add integration test for workflow ForceDelete
add cb8db7b remove all unused imports
add c08f9d3 TASK: Fix forceDelete for jobs in JobQueue
add cfd270f Fix the order of workflow context update
add 58a8ff8 Fix the issue where JobContext is not updated properly (#435)
add 61f7380 Make State Transition Throttling respect MIN_ACTIVE_REPLICA
add 8e5b0ec Add field for MIN_ACTIVE_REPLICA_NOT_SET
add 5c8c4af Fix helix-front build failure by downgrading types/lodash
version. (#470)
add 71becb3 Fix missed callbacks in CurrentStates based
RoutingTableProvider. (#458)
add 6197c71 Fix CustomRebalancer's assignment computation (#477)
add bf85e79 [helix-rest] Delete unused default namespace (api
"/namespaces/default") (#449)
add afcfe80 Filter instances of weight = 0 for any partition assignment
(#369)
add e4f3da4 Change the way Helix triggers rebalance (#472)
add 690e630 Add Intellij code style XML file for Helix code style. (#481)
add 3d1729b Make the Java Doc for API more clear
add bb198fc #493 Set jersey servlet application name with namespace name.
add cdbdd4f Add unit test for setting application name.
new fd20c43 Define the WAGED rebalancer interfaces.
new 39f7adf Adding the configuration items of the WAGED rebalancer. (#348)
new 730054b Implement the WAGED rebalancer cluster model (#362)
new 60dc54a Change the rebalancer assignment record to be
ResourceAssignment instead of IdealState. (#398)
new 912ed34 Convert all the internal assignment state objects to be
ResourceAssignment. (#399)
new dc828b8 Implement Cluster Model Provider. (#392)
new 0d67764 Add cluster level default instance config. (#413)
new 84ac35b Add ChangeDetector interface and ResourceChangeDetector
implementation (#388)
new ae282a8 Redefine the hard/soft constraints (#422)
new c3d8501 Refine the WAGED rebalancer related interfaces for
integration (#431)
new c65476d Revert "Refine the WAGED rebalancer related interfaces for
integration (#431)" (#437)
new 1ed9bbf Resubmit the change: Refine the WAGED rebalancer related
interfaces for integration (#431)
new 2780fd9 Modify the expected change type from CONFIG to CLUSTER_CONFIG
in the WAGED rebalancer. (#438)
new eb7a010 Add special treatment for ClusterConfig
new 9ee65a0 Record the replica objects in the AssignableNode in addition
to the partition name (#440)
new 34f8260 Add BucketDataAccessor for large writes
new 9f73d8d Implement the basic constraint based algorithm (#381)
new a4b61ee Validate the instance capacity/partition weight configuration
while constructing the assignable instances (#451)
new 702c547 Implement the WAGED rebalancer with the limited
functionality. (#443)
new 43d3476 HardConstraints Implementation and unit tests (#433)
new 3d2da32 Implement AssignmentMetadataStore (#453)
new c719273 Fix TestWagedRebalancer and add constructor in
AssignmentMetadataStore
new 06cb692 Implement one of the soft constraints (#450)
new 153d385 Add soft constraint: ResourcetopStateAntiAffinityConstraint
(#465)
new 52e1ba5 Implement MaxCapacityUsageInstanceConstraint soft constraint
(#463)
new e3cb1a6 Add soft constraint: ResourcePartitionAntiAffinityConstraint
(#464)
new eb95a19 Improve ResourceTopStateAntiAffinityConstraint (#475)
new 679547f Adjust the expected replica count according to fault zone
count. (#476)
new 50e8cec PartitionMovementSoftConstraint Implementation (#474)
new 86f1a00 Add the remaining implementation of
ConstraintBasedAlgorithmFactory (#478)
new 5c6f1e9 Integrate the WAGED rebalancer with all the related
components. (#466)
new 0b87426 Separate AssignableNode properties by Immutable and Mutable
(#485)
new 88ff705 Add delayed rebalance and user-defined preference list
features to the WAGED rebalancer. (#456)
new 1712a88 Enable maintenance mode for the WAGED rebalancer.
new 6f812d7 Adjust the topology processing logic for instance to ensure
backward compatibility.
new 75b60d1 Load soft constraint weight from resources/properties file
(#492)
new 4353f67 Add latency metric components for WAGED rebalancer (#490)
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (7c6f1be)
\
N -- N -- N refs/heads/wagedRebalancer (4353f67)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 37 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../helix/webapp/resources/ConfigResource.java | 1 -
.../webapp/resources/SchedulerTasksResource.java | 1 -
.../org/apache/helix/webapp/AdminTestHelper.java | 4 -
.../helix/webapp/TestClusterManagementWebapp.java | 1 -
.../helix/webapp/resources/TestJsonParameters.java | 2 -
.../helix/webapp/resources/TestResourceUtil.java | 2 -
.../apache/helix/agent/ProcessMonitorThread.java | 1 -
.../main/java/org/apache/helix/HelixProperty.java | 1 -
.../helix/common/caches/BasicClusterDataCache.java | 116 ++++--
.../helix/common/caches/InstanceMessagesCache.java | 12 +-
.../helix/controller/GenericHelixController.java | 43 +-
.../ResourceControllerDataProvider.java | 1 -
.../WorkflowControllerDataProvider.java | 1 -
.../controller/pipeline/AbstractBaseStage.java | 1 -
.../controller/rebalancer/CustomRebalancer.java | 17 +-
.../AbstractEvenDistributionRebalanceStrategy.java | 26 +-
.../CardDealingAdjustmentAlgorithmV2.java | 25 +-
.../rebalancer/util/RebalanceScheduler.java | 9 +-
.../stages/BestPossibleStateCalcStage.java | 2 +-
.../stages/IntermediateStateCalcStage.java | 7 +-
.../apache/helix/examples/BootstrapProcess.java | 1 -
.../helix/manager/zk/ControllerManagerHelper.java | 1 -
.../zk/DefaultControllerMessageHandlerFactory.java | 1 -
...faultParticipantErrorMessageHandlerFactory.java | 1 -
.../org/apache/helix/manager/zk/ZKHelixAdmin.java | 1 -
.../manager/zk/ZNRecordStreamingSerializer.java | 3 -
.../zk/serializer/JacksonPayloadSerializer.java | 1 -
.../org/apache/helix/messaging/AsyncCallback.java | 1 -
.../HelixStateTransitionCancellationHandler.java | 1 -
.../main/java/org/apache/helix/model/TaskSMD.java | 1 -
.../monitoring/mbeans/ClusterEventMonitor.java | 2 +-
.../monitoring/mbeans/ClusterMBeanObserver.java | 1 -
.../monitoring/mbeans/HelixCallbackMonitor.java | 2 +-
.../apache/helix/monitoring/mbeans/JobMonitor.java | 20 +-
.../monitoring/mbeans/MessageLatencyMonitor.java | 2 +-
.../helix/monitoring/mbeans/ResourceMonitor.java | 6 +-
.../mbeans/RoutingTableProviderMonitor.java | 7 +-
.../mbeans/StateTransitionStatMonitor.java | 7 +-
.../mbeans/ThreadPoolExecutorMonitor.java | 1 -
.../monitoring/mbeans/ZkClientPathMonitor.java | 10 +-
.../helix/participant/StateMachineEngine.java | 1 -
.../apache/helix/spectator/RoutingDataCache.java | 1 -
.../helix/spectator/RoutingTableProvider.java | 17 +-
.../apache/helix/task/AbstractTaskDispatcher.java | 37 +-
.../helix/task/AssignableInstanceManager.java | 1 -
.../task/FixedTargetTaskAssignmentCalculator.java | 1 -
.../main/java/org/apache/helix/task/JobDag.java | 1 -
.../java/org/apache/helix/task/JobDispatcher.java | 32 +-
.../java/org/apache/helix/task/TaskDriver.java | 84 ++--
.../main/java/org/apache/helix/task/TaskUtil.java | 4 +-
.../ThreadCountBasedTaskAssignmentCalculator.java | 1 -
.../org/apache/helix/task/WorkflowDispatcher.java | 75 ++--
.../org/apache/helix/task/WorkflowRebalancer.java | 1 -
.../java/org/apache/helix/tools/ClusterSetup.java | 1 -
.../apache/helix/tools/ClusterStateVerifier.java | 1 -
.../ClusterVerifiers/ZkHelixClusterVerifier.java | 4 -
.../tools/commandtools/CurrentStateCleanUp.java | 4 -
.../tools/commandtools/ExampleParticipant.java | 3 -
.../tools/commandtools/IntegrationTestUtil.java | 1 -
.../main/java/org/apache/helix/util/HelixUtil.java | 4 +-
.../java/org/apache/helix/util/RebalanceUtil.java | 23 ++
.../helix/common/caches/TestPropertyCache.java | 2 -
.../TestWorkflowControllerDataProvider.java | 1 -
.../helix/controller/stages/BaseStageTest.java | 17 +-
.../controller/stages/TestRecoveryLoadBalance.java | 34 +-
.../helix/controller/strategy/TestTopology.java | 1 +
.../TestCardDealingAdjustmentAlgorithmV2.java | 455 +++++++++++++++++++++
.../TestAddNodeAfterControllerStart.java | 2 -
.../helix/integration/TestDisablePartition.java | 1 -
.../helix/integration/TestEnableCompression.java | 1 -
.../helix/integration/TestErrorReplicaPersist.java | 3 -
.../integration/TestPreferenceListAsQueue.java | 1 -
.../integration/TestSyncSessionToController.java | 1 -
.../integration/common/ZkStandAloneCMTestBase.java | 2 -
.../integration/manager/TestZkHelixAdmin.java | 1 -
.../rebalancer/TestCustomRebalancer.java | 81 ++++
.../rebalancer/TestFullAutoNodeTagging.java | 1 -
.../TestRoutingTableProviderFromCurrentStates.java | 4 -
.../task/TestDeleteJobFromJobQueue.java | 83 ++++
.../helix/integration/task/TestExecutionDelay.java | 132 ++++++
.../integration/task/TestForceDeleteWorkflow.java | 371 +++++++++++++++++
.../task/TestStopWorkflowWithExecutionDelay.java | 93 +++++
.../integration/task/TestTaskConditionalRetry.java | 4 +-
.../integration/task/TestTaskNumAttempts.java | 79 ++++
.../task/TestTaskRebalancerRetryLimit.java | 5 +-
.../helix/integration/task/TestTaskRetryDelay.java | 4 +-
.../helix/integration/task/TestTaskStopQueue.java | 113 +++++
.../apache/helix/manager/zk/TestRawZkClient.java | 1 -
.../helix/manager/zk/TestZKLiveInstanceData.java | 1 -
.../helix/manager/zk/TestZNRecordSerializer.java | 1 -
.../helix/mock/controller/MockController.java | 1 -
.../org/apache/helix/model/TestConstraint.java | 2 -
.../mbeans/TestDropResourceMetricsReset.java | 1 -
.../monitoring/mbeans/TestZkClientMonitor.java | 27 ++
.../participant/TestDistControllerElection.java | 2 -
.../assigner/TestThreadCountBasedTaskAssigner.java | 2 -
.../apache/helix/util/TestRebalanceScheduler.java | 2 -
.../TestRecoveryLoadBalance.MasterSlave.json | 112 ++++-
helix-front/package-lock.json | 6 +-
helix-front/package.json | 2 +-
.../apache/helix/rest/server/HelixRestServer.java | 10 +-
.../rest/server/resources/helix/JobAccessor.java | 2 +-
.../server/resources/helix/WorkflowAccessor.java | 1 -
.../helix/rest/server/TestClusterAccessor.java | 3 -
.../rest/server/TestDefaultMonitoringMbeans.java | 32 ++
.../helix/rest/server/TestHelixRestServer.java | 2 +-
.../helix/rest/server/TestNamespacedAPIAccess.java | 50 +--
helix-style-intellij.xml | 387 ++++++++++++++++++
.../apache/helix/recipes/rabbitmq/Consumer.java | 1 -
.../apache/helix/filestore/ChangeLogGenerator.java | 2 -
.../apache/helix/filestore/ChangeLogProcessor.java | 6 -
.../apache/helix/filestore/ChangeLogReader.java | 2 -
.../java/org/apache/helix/filestore/FileStore.java | 5 -
.../main/java/org/apache/helix/filestore/Test.java | 3 -
.../helix/servicediscovery/ServiceDiscovery.java | 1 -
.../apache/helix/taskexecution/TaskStateModel.java | 2 -
116 files changed, 2451 insertions(+), 350 deletions(-)
create mode 100644
helix-core/src/test/java/org/apache/helix/controller/strategy/crushMapping/TestCardDealingAdjustmentAlgorithmV2.java
create mode 100644
helix-core/src/test/java/org/apache/helix/integration/rebalancer/TestCustomRebalancer.java
create mode 100644
helix-core/src/test/java/org/apache/helix/integration/task/TestDeleteJobFromJobQueue.java
create mode 100644
helix-core/src/test/java/org/apache/helix/integration/task/TestExecutionDelay.java
create mode 100644
helix-core/src/test/java/org/apache/helix/integration/task/TestForceDeleteWorkflow.java
create mode 100644
helix-core/src/test/java/org/apache/helix/integration/task/TestStopWorkflowWithExecutionDelay.java
create mode 100644
helix-core/src/test/java/org/apache/helix/integration/task/TestTaskNumAttempts.java
create mode 100644
helix-core/src/test/java/org/apache/helix/integration/task/TestTaskStopQueue.java
create mode 100644 helix-style-intellij.xml
