[GitHub] [helix] zhangmeng916 commented on a change in pull request #912: Add integration test to customized view aggregation
zhangmeng916 commented on a change in pull request #912: Add integration test
to customized view aggregation
URL: https://github.com/apache/helix/pull/912#discussion_r398324732
##
File path:
helix-core/src/test/java/org/apache/helix/integration/TestCustomizedViewAggregation.java
##
@@ -0,0 +1,404 @@
+package org.apache.helix.integration;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+import com.google.common.collect.Maps;
+import org.apache.helix.HelixDataAccessor;
+import org.apache.helix.HelixManager;
+import org.apache.helix.HelixManagerFactory;
+import org.apache.helix.InstanceType;
+import org.apache.helix.PropertyType;
+import org.apache.helix.TestHelper;
+import org.apache.helix.ZNRecord;
+import org.apache.helix.ZkUnitTestBase;
+import org.apache.helix.customizedstate.CustomizedStateProvider;
+import org.apache.helix.customizedstate.CustomizedStateProviderFactory;
+import org.apache.helix.integration.manager.ClusterControllerManager;
+import org.apache.helix.integration.manager.MockParticipantManager;
+import org.apache.helix.model.CustomizedStateConfig;
+import org.apache.helix.model.CustomizedView;
+import org.apache.helix.spectator.RoutingTableProvider;
+import org.apache.helix.spectator.RoutingTableSnapshot;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+
+public class TestCustomizedViewAggregation extends ZkUnitTestBase {
+
+ private static CustomizedStateProvider _customizedStateProvider_participant0;
+ private static CustomizedStateProvider _customizedStateProvider_participant1;
+ private static RoutingTableProvider _routingTableProvider;
+ private static HelixManager _spectator;
+ private static HelixManager _manager;
+ // 1st key: customized state type, 2nd key: resource name, 3rd key:
partition name, 4th key: instance name, value: state value
+ // This map contains all the customized state information that is enabled
for aggregation in config, including those are not listened by routing table
provider
+ private static Map>>>
_localCustomizedView;
+ // The set contains customized state types that are listened by routing
table provider
+ private static Set _localVisibleCustomizedStateType;
+ private String INSTANCE_0;
+ private String INSTANCE_1;
+ private final String RESOURCE_A = "TestDB0";
+ private final String RESOURCE_B = "TestDB1";
+ private final String PARTITION_A1 = "TestDB0_0";
+ private final String PARTITION_A2 = "TestDB0_1";
+ private final String PARTITION_B1 = "TestDB1_0";
+ private final String PARTITION_B2 = "TestDB1_1";
+
+ // Customized state values used for test, StatusA1 - StatusA3 are values for
Customized state TypeA, etc.
+ private enum CurrentStateValues {
+StatusA1, StatusA2, StatusA3, StatusB1, StatusB2, StatusB3, StatusC1,
StatusC2, StatusC3
+ }
+
+ private enum CustomizedStateType {
+TYPE_A, TYPE_B, TYPE_C
+ }
+
+ @BeforeClass
+ public void beforeClass() throws Exception {
+super.beforeClass();
+
+String className = TestHelper.getTestClassName();
+String methodName = TestHelper.getTestMethodName();
+String clusterName = className + "_" + methodName;
+int n = 2;
+
+System.out.println("START " + clusterName + " at " + new
Date(System.currentTimeMillis()));
+
+TestHelper.setupCluster(clusterName, ZK_ADDR, 12918, // participant port
+"localhost", // participant name prefix
+"TestDB", // resource name prefix
+2, // resources
+2, // partitions per resource
+n, // number of nodes
+2, // replicas
+"MasterSlave", true); // do rebalance
+
+ClusterControllerManager controller =
+new ClusterControllerManager(ZK_ADDR, clusterName, "controller_0");
+controller.syncStart();
+
+// start participants
+MockParticipantManager[] participants = new MockParticipantManager[n];
+for (int i = 0; i < n; i++) {
+ String instanceName = "localhost_" + (12918 + i);
+
+ participants[i] = new MockParticipantManager(ZK_ADDR, clusterName,
instanceName);
+ participants[i].syncStart();
+}
+
+INSTANCE_0 = participants[0].getInstanceName();
+INSTANCE_1 = participants[1].getInstanceName();
+
+_manager = HelixManagerFactory
+.getZKHelixManager(clusterName, "admin", InstanceType.ADMINISTRATOR,
ZK_ADDR);
+_manager.connect();
+
+_spectator = HelixManagerFactory
+.getZKHelixManager(clusterName, "spectator", InstanceType.SPECTATOR,
ZK_ADDR);
+_spectator.connect();
+HelixDataAccessor dataAccessor = _manager.getHelixDataAccessor();
+
+// Initialize customized state provider
+_customizedStateProvider_participant0 =
[GitHub] [helix] zhangmeng916 commented on a change in pull request #912: Add integration test to customized view aggregation
zhangmeng916 commented on a change in pull request #912: Add integration test
to customized view aggregation
URL: https://github.com/apache/helix/pull/912#discussion_r398325124
##
File path:
helix-core/src/test/java/org/apache/helix/integration/TestCustomizedViewAggregation.java
##
@@ -0,0 +1,404 @@
+package org.apache.helix.integration;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+import com.google.common.collect.Maps;
+import org.apache.helix.HelixDataAccessor;
+import org.apache.helix.HelixManager;
+import org.apache.helix.HelixManagerFactory;
+import org.apache.helix.InstanceType;
+import org.apache.helix.PropertyType;
+import org.apache.helix.TestHelper;
+import org.apache.helix.ZNRecord;
+import org.apache.helix.ZkUnitTestBase;
+import org.apache.helix.customizedstate.CustomizedStateProvider;
+import org.apache.helix.customizedstate.CustomizedStateProviderFactory;
+import org.apache.helix.integration.manager.ClusterControllerManager;
+import org.apache.helix.integration.manager.MockParticipantManager;
+import org.apache.helix.model.CustomizedStateConfig;
+import org.apache.helix.model.CustomizedView;
+import org.apache.helix.spectator.RoutingTableProvider;
+import org.apache.helix.spectator.RoutingTableSnapshot;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+
+public class TestCustomizedViewAggregation extends ZkUnitTestBase {
+
+ private static CustomizedStateProvider _customizedStateProvider_participant0;
+ private static CustomizedStateProvider _customizedStateProvider_participant1;
+ private static RoutingTableProvider _routingTableProvider;
+ private static HelixManager _spectator;
+ private static HelixManager _manager;
+ // 1st key: customized state type, 2nd key: resource name, 3rd key:
partition name, 4th key: instance name, value: state value
+ // This map contains all the customized state information that is enabled
for aggregation in config, including those are not listened by routing table
provider
+ private static Map>>>
_localCustomizedView;
+ // The set contains customized state types that are listened by routing
table provider
+ private static Set _localVisibleCustomizedStateType;
+ private String INSTANCE_0;
+ private String INSTANCE_1;
+ private final String RESOURCE_A = "TestDB0";
+ private final String RESOURCE_B = "TestDB1";
+ private final String PARTITION_A1 = "TestDB0_0";
+ private final String PARTITION_A2 = "TestDB0_1";
+ private final String PARTITION_B1 = "TestDB1_0";
+ private final String PARTITION_B2 = "TestDB1_1";
+
+ // Customized state values used for test, StatusA1 - StatusA3 are values for
Customized state TypeA, etc.
+ private enum CurrentStateValues {
+StatusA1, StatusA2, StatusA3, StatusB1, StatusB2, StatusB3, StatusC1,
StatusC2, StatusC3
+ }
+
+ private enum CustomizedStateType {
+TYPE_A, TYPE_B, TYPE_C
+ }
+
+ @BeforeClass
+ public void beforeClass() throws Exception {
+super.beforeClass();
+
+String className = TestHelper.getTestClassName();
+String methodName = TestHelper.getTestMethodName();
+String clusterName = className + "_" + methodName;
+int n = 2;
+
+System.out.println("START " + clusterName + " at " + new
Date(System.currentTimeMillis()));
+
+TestHelper.setupCluster(clusterName, ZK_ADDR, 12918, // participant port
+"localhost", // participant name prefix
+"TestDB", // resource name prefix
+2, // resources
+2, // partitions per resource
+n, // number of nodes
+2, // replicas
+"MasterSlave", true); // do rebalance
+
+ClusterControllerManager controller =
+new ClusterControllerManager(ZK_ADDR, clusterName, "controller_0");
+controller.syncStart();
+
+// start participants
+MockParticipantManager[] participants = new MockParticipantManager[n];
+for (int i = 0; i < n; i++) {
+ String instanceName = "localhost_" + (12918 + i);
+
+ participants[i] = new MockParticipantManager(ZK_ADDR, clusterName,
instanceName);
+ participants[i].syncStart();
+}
+
+INSTANCE_0 = participants[0].getInstanceName();
+INSTANCE_1 = participants[1].getInstanceName();
+
+_manager = HelixManagerFactory
+.getZKHelixManager(clusterName, "admin", InstanceType.ADMINISTRATOR,
ZK_ADDR);
+_manager.connect();
+
+_spectator = HelixManagerFactory
+.getZKHelixManager(clusterName, "spectator", InstanceType.SPECTATOR,
ZK_ADDR);
+_spectator.connect();
+HelixDataAccessor dataAccessor = _manager.getHelixDataAccessor();
+
+// Initialize customized state provider
+_customizedStateProvider_participant0 =
[GitHub] [helix] zhangmeng916 commented on a change in pull request #912: Add integration test to customized view aggregation
zhangmeng916 commented on a change in pull request #912: Add integration test
to customized view aggregation
URL: https://github.com/apache/helix/pull/912#discussion_r398326958
##
File path:
helix-core/src/test/java/org/apache/helix/integration/TestCustomizedViewAggregation.java
##
@@ -0,0 +1,404 @@
+package org.apache.helix.integration;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+import com.google.common.collect.Maps;
+import org.apache.helix.HelixDataAccessor;
+import org.apache.helix.HelixManager;
+import org.apache.helix.HelixManagerFactory;
+import org.apache.helix.InstanceType;
+import org.apache.helix.PropertyType;
+import org.apache.helix.TestHelper;
+import org.apache.helix.ZNRecord;
+import org.apache.helix.ZkUnitTestBase;
+import org.apache.helix.customizedstate.CustomizedStateProvider;
+import org.apache.helix.customizedstate.CustomizedStateProviderFactory;
+import org.apache.helix.integration.manager.ClusterControllerManager;
+import org.apache.helix.integration.manager.MockParticipantManager;
+import org.apache.helix.model.CustomizedStateConfig;
+import org.apache.helix.model.CustomizedView;
+import org.apache.helix.spectator.RoutingTableProvider;
+import org.apache.helix.spectator.RoutingTableSnapshot;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+
+public class TestCustomizedViewAggregation extends ZkUnitTestBase {
+
+ private static CustomizedStateProvider _customizedStateProvider_participant0;
+ private static CustomizedStateProvider _customizedStateProvider_participant1;
+ private static RoutingTableProvider _routingTableProvider;
+ private static HelixManager _spectator;
+ private static HelixManager _manager;
+ // 1st key: customized state type, 2nd key: resource name, 3rd key:
partition name, 4th key: instance name, value: state value
+ // This map contains all the customized state information that is enabled
for aggregation in config, including those are not listened by routing table
provider
+ private static Map>>>
_localCustomizedView;
+ // The set contains customized state types that are listened by routing
table provider
+ private static Set _localVisibleCustomizedStateType;
+ private String INSTANCE_0;
+ private String INSTANCE_1;
+ private final String RESOURCE_A = "TestDB0";
+ private final String RESOURCE_B = "TestDB1";
+ private final String PARTITION_A1 = "TestDB0_0";
+ private final String PARTITION_A2 = "TestDB0_1";
+ private final String PARTITION_B1 = "TestDB1_0";
+ private final String PARTITION_B2 = "TestDB1_1";
+
+ // Customized state values used for test, StatusA1 - StatusA3 are values for
Customized state TypeA, etc.
+ private enum CurrentStateValues {
+StatusA1, StatusA2, StatusA3, StatusB1, StatusB2, StatusB3, StatusC1,
StatusC2, StatusC3
+ }
+
+ private enum CustomizedStateType {
+TYPE_A, TYPE_B, TYPE_C
+ }
+
+ @BeforeClass
+ public void beforeClass() throws Exception {
+super.beforeClass();
+
+String className = TestHelper.getTestClassName();
+String methodName = TestHelper.getTestMethodName();
+String clusterName = className + "_" + methodName;
+int n = 2;
+
+System.out.println("START " + clusterName + " at " + new
Date(System.currentTimeMillis()));
+
+TestHelper.setupCluster(clusterName, ZK_ADDR, 12918, // participant port
+"localhost", // participant name prefix
+"TestDB", // resource name prefix
+2, // resources
+2, // partitions per resource
+n, // number of nodes
+2, // replicas
+"MasterSlave", true); // do rebalance
+
+ClusterControllerManager controller =
+new ClusterControllerManager(ZK_ADDR, clusterName, "controller_0");
+controller.syncStart();
+
+// start participants
+MockParticipantManager[] participants = new MockParticipantManager[n];
+for (int i = 0; i < n; i++) {
+ String instanceName = "localhost_" + (12918 + i);
+
+ participants[i] = new MockParticipantManager(ZK_ADDR, clusterName,
instanceName);
+ participants[i].syncStart();
+}
+
+INSTANCE_0 = participants[0].getInstanceName();
+INSTANCE_1 = participants[1].getInstanceName();
+
+_manager = HelixManagerFactory
+.getZKHelixManager(clusterName, "admin", InstanceType.ADMINISTRATOR,
ZK_ADDR);
+_manager.connect();
+
+_spectator = HelixManagerFactory
+.getZKHelixManager(clusterName, "spectator", InstanceType.SPECTATOR,
ZK_ADDR);
+_spectator.connect();
+HelixDataAccessor dataAccessor = _manager.getHelixDataAccessor();
+
+// Initialize customized state provider
+_customizedStateProvider_participant0 =
[GitHub] [helix] zhangmeng916 commented on a change in pull request #912: Add integration test to customized view aggregation
zhangmeng916 commented on a change in pull request #912: Add integration test
to customized view aggregation
URL: https://github.com/apache/helix/pull/912#discussion_r398324255
##
File path:
helix-core/src/test/java/org/apache/helix/integration/TestCustomizedViewAggregation.java
##
@@ -0,0 +1,404 @@
+package org.apache.helix.integration;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+import com.google.common.collect.Maps;
+import org.apache.helix.HelixDataAccessor;
+import org.apache.helix.HelixManager;
+import org.apache.helix.HelixManagerFactory;
+import org.apache.helix.InstanceType;
+import org.apache.helix.PropertyType;
+import org.apache.helix.TestHelper;
+import org.apache.helix.ZNRecord;
+import org.apache.helix.ZkUnitTestBase;
+import org.apache.helix.customizedstate.CustomizedStateProvider;
+import org.apache.helix.customizedstate.CustomizedStateProviderFactory;
+import org.apache.helix.integration.manager.ClusterControllerManager;
+import org.apache.helix.integration.manager.MockParticipantManager;
+import org.apache.helix.model.CustomizedStateConfig;
+import org.apache.helix.model.CustomizedView;
+import org.apache.helix.spectator.RoutingTableProvider;
+import org.apache.helix.spectator.RoutingTableSnapshot;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+
+public class TestCustomizedViewAggregation extends ZkUnitTestBase {
+
+ private static CustomizedStateProvider _customizedStateProvider_participant0;
+ private static CustomizedStateProvider _customizedStateProvider_participant1;
+ private static RoutingTableProvider _routingTableProvider;
+ private static HelixManager _spectator;
+ private static HelixManager _manager;
+ // 1st key: customized state type, 2nd key: resource name, 3rd key:
partition name, 4th key: instance name, value: state value
+ // This map contains all the customized state information that is enabled
for aggregation in config, including those are not listened by routing table
provider
+ private static Map>>>
_localCustomizedView;
+ // The set contains customized state types that are listened by routing
table provider
+ private static Set _localVisibleCustomizedStateType;
+ private String INSTANCE_0;
+ private String INSTANCE_1;
+ private final String RESOURCE_A = "TestDB0";
+ private final String RESOURCE_B = "TestDB1";
+ private final String PARTITION_A1 = "TestDB0_0";
+ private final String PARTITION_A2 = "TestDB0_1";
+ private final String PARTITION_B1 = "TestDB1_0";
+ private final String PARTITION_B2 = "TestDB1_1";
+
+ // Customized state values used for test, StatusA1 - StatusA3 are values for
Customized state TypeA, etc.
+ private enum CurrentStateValues {
+StatusA1, StatusA2, StatusA3, StatusB1, StatusB2, StatusB3, StatusC1,
StatusC2, StatusC3
+ }
+
+ private enum CustomizedStateType {
+TYPE_A, TYPE_B, TYPE_C
+ }
+
+ @BeforeClass
+ public void beforeClass() throws Exception {
+super.beforeClass();
+
+String className = TestHelper.getTestClassName();
+String methodName = TestHelper.getTestMethodName();
+String clusterName = className + "_" + methodName;
+int n = 2;
+
+System.out.println("START " + clusterName + " at " + new
Date(System.currentTimeMillis()));
+
+TestHelper.setupCluster(clusterName, ZK_ADDR, 12918, // participant port
+"localhost", // participant name prefix
+"TestDB", // resource name prefix
+2, // resources
+2, // partitions per resource
+n, // number of nodes
+2, // replicas
+"MasterSlave", true); // do rebalance
+
+ClusterControllerManager controller =
+new ClusterControllerManager(ZK_ADDR, clusterName, "controller_0");
+controller.syncStart();
+
+// start participants
+MockParticipantManager[] participants = new MockParticipantManager[n];
+for (int i = 0; i < n; i++) {
+ String instanceName = "localhost_" + (12918 + i);
+
+ participants[i] = new MockParticipantManager(ZK_ADDR, clusterName,
instanceName);
+ participants[i].syncStart();
+}
+
+INSTANCE_0 = participants[0].getInstanceName();
+INSTANCE_1 = participants[1].getInstanceName();
+
+_manager = HelixManagerFactory
+.getZKHelixManager(clusterName, "admin", InstanceType.ADMINISTRATOR,
ZK_ADDR);
+_manager.connect();
+
+_spectator = HelixManagerFactory
+.getZKHelixManager(clusterName, "spectator", InstanceType.SPECTATOR,
ZK_ADDR);
+_spectator.connect();
+HelixDataAccessor dataAccessor = _manager.getHelixDataAccessor();
+
+// Initialize customized state provider
+_customizedStateProvider_participant0 =
[GitHub] [helix] zhangmeng916 commented on a change in pull request #912: Add integration test to customized view aggregation
zhangmeng916 commented on a change in pull request #912: Add integration test
to customized view aggregation
URL: https://github.com/apache/helix/pull/912#discussion_r39832
##
File path:
helix-core/src/test/java/org/apache/helix/integration/TestCustomizedViewAggregation.java
##
@@ -0,0 +1,404 @@
+package org.apache.helix.integration;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+import com.google.common.collect.Maps;
+import org.apache.helix.HelixDataAccessor;
+import org.apache.helix.HelixManager;
+import org.apache.helix.HelixManagerFactory;
+import org.apache.helix.InstanceType;
+import org.apache.helix.PropertyType;
+import org.apache.helix.TestHelper;
+import org.apache.helix.ZNRecord;
+import org.apache.helix.ZkUnitTestBase;
+import org.apache.helix.customizedstate.CustomizedStateProvider;
+import org.apache.helix.customizedstate.CustomizedStateProviderFactory;
+import org.apache.helix.integration.manager.ClusterControllerManager;
+import org.apache.helix.integration.manager.MockParticipantManager;
+import org.apache.helix.model.CustomizedStateConfig;
+import org.apache.helix.model.CustomizedView;
+import org.apache.helix.spectator.RoutingTableProvider;
+import org.apache.helix.spectator.RoutingTableSnapshot;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+
+public class TestCustomizedViewAggregation extends ZkUnitTestBase {
+
+ private static CustomizedStateProvider _customizedStateProvider_participant0;
+ private static CustomizedStateProvider _customizedStateProvider_participant1;
+ private static RoutingTableProvider _routingTableProvider;
+ private static HelixManager _spectator;
+ private static HelixManager _manager;
+ // 1st key: customized state type, 2nd key: resource name, 3rd key:
partition name, 4th key: instance name, value: state value
+ // This map contains all the customized state information that is enabled
for aggregation in config, including those are not listened by routing table
provider
Review comment:
This is not the right description. Aggregation config controls which
customized states are aggregated into customized view, and routing table
provider listens to that customized view. Routing table provider does not have
the ability to make choice, and it'll listen to all changes in customized view.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
-
To unsubscribe, e-mail: reviews-unsubscr...@helix.apache.org
For additional commands, e-mail: reviews-h...@helix.apache.org
[GitHub] [helix] zhangmeng916 commented on a change in pull request #912: Add integration test to customized view aggregation
zhangmeng916 commented on a change in pull request #912: Add integration test
to customized view aggregation
URL: https://github.com/apache/helix/pull/912#discussion_r398322393
##
File path:
helix-core/src/test/java/org/apache/helix/integration/TestCustomizedViewAggregation.java
##
@@ -0,0 +1,404 @@
+package org.apache.helix.integration;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+import com.google.common.collect.Maps;
+import org.apache.helix.HelixDataAccessor;
+import org.apache.helix.HelixManager;
+import org.apache.helix.HelixManagerFactory;
+import org.apache.helix.InstanceType;
+import org.apache.helix.PropertyType;
+import org.apache.helix.TestHelper;
+import org.apache.helix.ZNRecord;
+import org.apache.helix.ZkUnitTestBase;
+import org.apache.helix.customizedstate.CustomizedStateProvider;
+import org.apache.helix.customizedstate.CustomizedStateProviderFactory;
+import org.apache.helix.integration.manager.ClusterControllerManager;
+import org.apache.helix.integration.manager.MockParticipantManager;
+import org.apache.helix.model.CustomizedStateConfig;
+import org.apache.helix.model.CustomizedView;
+import org.apache.helix.spectator.RoutingTableProvider;
+import org.apache.helix.spectator.RoutingTableSnapshot;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+
+public class TestCustomizedViewAggregation extends ZkUnitTestBase {
+
+ private static CustomizedStateProvider _customizedStateProvider_participant0;
+ private static CustomizedStateProvider _customizedStateProvider_participant1;
+ private static RoutingTableProvider _routingTableProvider;
+ private static HelixManager _spectator;
+ private static HelixManager _manager;
+ // 1st key: customized state type, 2nd key: resource name, 3rd key:
partition name, 4th key: instance name, value: state value
+ // This map contains all the customized state information that is enabled
for aggregation in config, including those are not listened by routing table
provider
+ private static Map>>>
_localCustomizedView;
+ // The set contains customized state types that are listened by routing
table provider
+ private static Set _localVisibleCustomizedStateType;
+ private String INSTANCE_0;
Review comment:
Let's try to make the naming convention more standard. It should have the
type + number, and the number should start from 0 instead of 1.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
-
To unsubscribe, e-mail: reviews-unsubscr...@helix.apache.org
For additional commands, e-mail: reviews-h...@helix.apache.org
[GitHub] [helix] zhangmeng916 commented on a change in pull request #912: Add integration test to customized view aggregation
zhangmeng916 commented on a change in pull request #912: Add integration test
to customized view aggregation
URL: https://github.com/apache/helix/pull/912#discussion_r398325974
##
File path:
helix-core/src/test/java/org/apache/helix/integration/TestCustomizedViewAggregation.java
##
@@ -0,0 +1,404 @@
+package org.apache.helix.integration;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+import com.google.common.collect.Maps;
+import org.apache.helix.HelixDataAccessor;
+import org.apache.helix.HelixManager;
+import org.apache.helix.HelixManagerFactory;
+import org.apache.helix.InstanceType;
+import org.apache.helix.PropertyType;
+import org.apache.helix.TestHelper;
+import org.apache.helix.ZNRecord;
+import org.apache.helix.ZkUnitTestBase;
+import org.apache.helix.customizedstate.CustomizedStateProvider;
+import org.apache.helix.customizedstate.CustomizedStateProviderFactory;
+import org.apache.helix.integration.manager.ClusterControllerManager;
+import org.apache.helix.integration.manager.MockParticipantManager;
+import org.apache.helix.model.CustomizedStateConfig;
+import org.apache.helix.model.CustomizedView;
+import org.apache.helix.spectator.RoutingTableProvider;
+import org.apache.helix.spectator.RoutingTableSnapshot;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+
+public class TestCustomizedViewAggregation extends ZkUnitTestBase {
+
+ private static CustomizedStateProvider _customizedStateProvider_participant0;
+ private static CustomizedStateProvider _customizedStateProvider_participant1;
+ private static RoutingTableProvider _routingTableProvider;
+ private static HelixManager _spectator;
+ private static HelixManager _manager;
+ // 1st key: customized state type, 2nd key: resource name, 3rd key:
partition name, 4th key: instance name, value: state value
+ // This map contains all the customized state information that is enabled
for aggregation in config, including those are not listened by routing table
provider
+ private static Map>>>
_localCustomizedView;
+ // The set contains customized state types that are listened by routing
table provider
+ private static Set _localVisibleCustomizedStateType;
+ private String INSTANCE_0;
+ private String INSTANCE_1;
+ private final String RESOURCE_A = "TestDB0";
+ private final String RESOURCE_B = "TestDB1";
+ private final String PARTITION_A1 = "TestDB0_0";
+ private final String PARTITION_A2 = "TestDB0_1";
+ private final String PARTITION_B1 = "TestDB1_0";
+ private final String PARTITION_B2 = "TestDB1_1";
+
+ // Customized state values used for test, StatusA1 - StatusA3 are values for
Customized state TypeA, etc.
+ private enum CurrentStateValues {
+StatusA1, StatusA2, StatusA3, StatusB1, StatusB2, StatusB3, StatusC1,
StatusC2, StatusC3
+ }
+
+ private enum CustomizedStateType {
+TYPE_A, TYPE_B, TYPE_C
+ }
+
+ @BeforeClass
+ public void beforeClass() throws Exception {
+super.beforeClass();
+
+String className = TestHelper.getTestClassName();
+String methodName = TestHelper.getTestMethodName();
+String clusterName = className + "_" + methodName;
+int n = 2;
+
+System.out.println("START " + clusterName + " at " + new
Date(System.currentTimeMillis()));
+
+TestHelper.setupCluster(clusterName, ZK_ADDR, 12918, // participant port
+"localhost", // participant name prefix
+"TestDB", // resource name prefix
+2, // resources
+2, // partitions per resource
+n, // number of nodes
+2, // replicas
+"MasterSlave", true); // do rebalance
+
+ClusterControllerManager controller =
+new ClusterControllerManager(ZK_ADDR, clusterName, "controller_0");
+controller.syncStart();
+
+// start participants
+MockParticipantManager[] participants = new MockParticipantManager[n];
+for (int i = 0; i < n; i++) {
+ String instanceName = "localhost_" + (12918 + i);
+
+ participants[i] = new MockParticipantManager(ZK_ADDR, clusterName,
instanceName);
+ participants[i].syncStart();
+}
+
+INSTANCE_0 = participants[0].getInstanceName();
+INSTANCE_1 = participants[1].getInstanceName();
+
+_manager = HelixManagerFactory
+.getZKHelixManager(clusterName, "admin", InstanceType.ADMINISTRATOR,
ZK_ADDR);
+_manager.connect();
+
+_spectator = HelixManagerFactory
+.getZKHelixManager(clusterName, "spectator", InstanceType.SPECTATOR,
ZK_ADDR);
+_spectator.connect();
+HelixDataAccessor dataAccessor = _manager.getHelixDataAccessor();
+
+// Initialize customized state provider
+_customizedStateProvider_participant0 =
[GitHub] [helix] pkuwm commented on issue #911: Fix TestZNRecordSerializeWriteSizeLimit
pkuwm commented on issue #911: Fix TestZNRecordSerializeWriteSizeLimit URL: https://github.com/apache/helix/pull/911#issuecomment-604235542 This PR is ready to be merged, approved by @narendly ``` There was a exception type change in znrecord serializers: ZkClientException -> ZkMarshallingError. So the exception types don't match in the unit test. This commit fixes the issue by changing the expected exception type in the unit test. ``` This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services - To unsubscribe, e-mail: reviews-unsubscr...@helix.apache.org For additional commands, e-mail: reviews-h...@helix.apache.org
[GitHub] [helix] narendly merged pull request #908: Make Helix REST realm-aware
narendly merged pull request #908: Make Helix REST realm-aware URL: https://github.com/apache/helix/pull/908 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services - To unsubscribe, e-mail: reviews-unsubscr...@helix.apache.org For additional commands, e-mail: reviews-h...@helix.apache.org
[GitHub] [helix] narendly commented on issue #908: Make Helix REST realm-aware
narendly commented on issue #908: Make Helix REST realm-aware URL: https://github.com/apache/helix/pull/908#issuecomment-604235186 Helix REST needs to start using a realm-aware ZkClient on multi-zk mode. Also it needs to become a listener on routing data because we don't want to restart the HelixRestServer every time we update the routing data. Changelist: Make ServerContext listen on routing data paths if run on multi-zk mode Make HelixRestServer use RealmAwareZkClient (FederatedZkClient) on multi-zk mode This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services - To unsubscribe, e-mail: reviews-unsubscr...@helix.apache.org For additional commands, e-mail: reviews-h...@helix.apache.org
[GitHub] [helix] narendly commented on issue #908: Make Helix REST realm-aware
narendly commented on issue #908: Make Helix REST realm-aware URL: https://github.com/apache/helix/pull/908#issuecomment-604234992 This PR is ready to be merged, approved by @dasahcc This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services - To unsubscribe, e-mail: reviews-unsubscr...@helix.apache.org For additional commands, e-mail: reviews-h...@helix.apache.org
[GitHub] [helix] mgao0 opened a new pull request #912: Add integration test to customized view aggregation
mgao0 opened a new pull request #912: Add integration test to customized view
aggregation
URL: https://github.com/apache/helix/pull/912
### Issues
- [ ] My PR addresses the following Helix issues and references them in the
PR description:
This PR added an integration test to customized view aggregation. The
integration test involves components: update customized state using customized
view provider, and use routing table provider to listen to changes in
customized view which is aggregated in controller.
### Description
- [ ] Here are some details about my PR, including screenshots of any UI
changes:
This integration test involves the following steps:
1. update/delete customized state to ZK
2. update/delete the same customized state in a local recorded map
3. configure the customized state types to be listened by routing table
provider
4. update the same customized state types to a local recorded set which
represents the customize state types that are currently being listened
4. get customized views from routing table provider
5. compare customized state values in the customized views fetched by
routing table provider, and in local recorded map; also compare the customized
state types available from ZK and local set
### Tests
- [ ] The following is the result of the "mvn test" command on the
appropriate module:
[INFO] Tests run: 1, Failures: 0, Errors: 0, Skipped: 0, Time elapsed:
85.425 s - in org.apache.helix.integration.TestCustomizedViewAggregation
[INFO]
[INFO] Results:
[INFO]
[INFO] Tests run: 1, Failures: 0, Errors: 0, Skipped: 0
[INFO]
[INFO]
[INFO] BUILD SUCCESS
[INFO]
[INFO] Total time: 01:29 min
[INFO] Finished at: 2020-03-25T21:05:05-07:00
[INFO]
### Commits
- [ ] My commits all reference appropriate Apache Helix GitHub issues in
their subject lines. In addition, my commits follow the guidelines from "[How
to write a good git commit message](http://chris.beams.io/posts/git-commit/)":
1. Subject is separated from body by a blank line
1. Subject is limited to 50 characters (not including Jira issue reference)
1. Subject does not end with a period
1. Subject uses the imperative mood ("add", not "adding")
1. Body wraps at 72 characters
1. Body explains "what" and "why", not "how"
### Code Quality
- [ ] My diff has been formatted using helix-style.xml
(helix-style-intellij.xml if IntelliJ IDE is used)
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
-
To unsubscribe, e-mail: reviews-unsubscr...@helix.apache.org
For additional commands, e-mail: reviews-h...@helix.apache.org
[GitHub] [helix] pkuwm opened a new pull request #911: Fix TestZNRecordSerializeWriteSizeLimit
pkuwm opened a new pull request #911: Fix TestZNRecordSerializeWriteSizeLimit
URL: https://github.com/apache/helix/pull/911
### Issues
- [ ] My PR addresses the following Helix issues and references them in the
PR description:
Fixes #910
### Description
- [ ] Here are some details about my PR, including screenshots of any UI
changes:
There was a exception type change in znrecord serializers: ZkClientException
-> ZkMarshallingError. So the exception types don't match in the unit test.
This PR fixes this issue by changing the expected exception type in the unit
test.
### Tests
- [ ] The following tests are written for this issue:
(List the names of added unit/integration tests)
- [ ] The following is the result of the "mvn test" command on the
appropriate module:
Running...
### Commits
- [ ] My commits all reference appropriate Apache Helix GitHub issues in
their subject lines. In addition, my commits follow the guidelines from "[How
to write a good git commit message](http://chris.beams.io/posts/git-commit/)":
1. Subject is separated from body by a blank line
1. Subject is limited to 50 characters (not including Jira issue reference)
1. Subject does not end with a period
1. Subject uses the imperative mood ("add", not "adding")
1. Body wraps at 72 characters
1. Body explains "what" and "why", not "how"
### Documentation (Optional)
- [ ] In case of new functionality, my PR adds documentation in the
following wiki page:
(Link the GitHub wiki you added)
### Code Quality
- [ ] My diff has been formatted using helix-style.xml
(helix-style-intellij.xml if IntelliJ IDE is used)
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
-
To unsubscribe, e-mail: reviews-unsubscr...@helix.apache.org
For additional commands, e-mail: reviews-h...@helix.apache.org
[GitHub] [helix] pkuwm opened a new issue #910: Unit test TestZNRecordSerializeWriteSizeLimit fails
pkuwm opened a new issue #910: Unit test TestZNRecordSerializeWriteSizeLimit fails URL: https://github.com/apache/helix/issues/910 Test log: ``` testZNRecordSerializerWriteSizeLimit(org.apache.helix.zookeeper.datamodel.serializer.TestZNRecordSerializeWriteSizeLimit) Time elapsed: 0.123 sec <<< FAILURE! org.apache.helix.zookeeper.zkclient.exception.ZkMarshallingError: Data size: 5445 is greater than 2000 bytes, is compressed: true, ZNRecord.id: record at org.apache.helix.zookeeper.datamodel.serializer.ZNRecordSerializer.serialize(ZNRecordSerializer.java:110) at org.apache.helix.zookeeper.datamodel.serializer.TestZNRecordSerializeWriteSizeLimit.verifyAutoCompression(TestZNRecordSerializeWriteSizeLimit.java:151) at org.apache.helix.zookeeper.datamodel.serializer.TestZNRecordSerializeWriteSizeLimit.testZNRecordSerializerWriteSizeLimit(TestZNRecordSerializeWriteSizeLimit.java:125) Results : Failed tests: TestZNRecordSerializeWriteSizeLimit.testAutoCompressionDisabled:64->verifyAutoCompression:151 » ZkMarshalling TestZNRecordSerializeWriteSizeLimit.testZNRecordSerializerWriteSizeLimit:125->verifyAutoCompression:151 » ZkMarshalling Tests run: 2, Failures: 2, Errors: 0, Skipped: 0 ``` Reason: There was a exception type change in znrecord serializers: ZkClientException -> ZkMarshallingError. So the exception types don't match in the unit test. Solution: Change the exception type in the unit test. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services - To unsubscribe, e-mail: reviews-unsubscr...@helix.apache.org For additional commands, e-mail: reviews-h...@helix.apache.org
[GitHub] [helix] zhangmeng916 closed pull request #780: Add customized state aggregation to Helix controller (WIP)
zhangmeng916 closed pull request #780: Add customized state aggregation to Helix controller (WIP) URL: https://github.com/apache/helix/pull/780 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services - To unsubscribe, e-mail: reviews-unsubscr...@helix.apache.org For additional commands, e-mail: reviews-h...@helix.apache.org
[GitHub] [helix] jiajunwang merged pull request #851: Modify Helix generic controller to include new stage for customized view aggregation
jiajunwang merged pull request #851: Modify Helix generic controller to include new stage for customized view aggregation URL: https://github.com/apache/helix/pull/851 This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services - To unsubscribe, e-mail: reviews-unsubscr...@helix.apache.org For additional commands, e-mail: reviews-h...@helix.apache.org
[GitHub] [helix] zhangmeng916 commented on issue #851: Modify Helix generic controller to include new stage for customized view aggregation
zhangmeng916 commented on issue #851: Modify Helix generic controller to include new stage for customized view aggregation URL: https://github.com/apache/helix/pull/851#issuecomment-604083600 This PR is ready to merge, approved by @jiajunwang Final commit message: Add new stages in Helix generic controller for customized view aggregation. - Add extra stages and pipelines in controller for customized state computation and customized view aggregation. - Add refresh logic in resource data provider for customized view related data refresh. - Add customized state event handling in CallbackHandler. - Add integration test for customized view aggregation. - Modify existing tests to verify new logic. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services - To unsubscribe, e-mail: reviews-unsubscr...@helix.apache.org For additional commands, e-mail: reviews-h...@helix.apache.org
[GitHub] [helix] zhangmeng916 commented on a change in pull request #869: Replace customized view cache with property cache
zhangmeng916 commented on a change in pull request #869: Replace customized
view cache with property cache
URL: https://github.com/apache/helix/pull/869#discussion_r397458124
##
File path:
helix-core/src/test/java/org/apache/helix/integration/spectator/TestRoutingTableProvider.java
##
@@ -308,19 +327,53 @@ public void testGetRoutingTableSnapshot() {
routingTableProvider.getRoutingTableSnapshot(PropertyType.CUSTOMIZEDVIEW,
"typeA");
Assert.assertEquals(routingTableSnapshot.getPropertyType(),
PropertyType.CUSTOMIZEDVIEW);
Assert.assertEquals(routingTableSnapshot.getCustomizedStateType(),
"typeA");
+
routingTableSnapshot =
routingTableProvider.getRoutingTableSnapshot(PropertyType.CUSTOMIZEDVIEW,
"typeB");
Assert.assertEquals(routingTableSnapshot.getPropertyType(),
PropertyType.CUSTOMIZEDVIEW);
Assert.assertEquals(routingTableSnapshot.getCustomizedStateType(),
"typeB");
-Map> routingTableSnapshots =
-routingTableProvider.getRoutingTableSnapshots();
-Assert.assertEquals(routingTableSnapshots.size(), 2);
-
Assert.assertEquals(routingTableSnapshots.get(PropertyType.CUSTOMIZEDVIEW.name()).size(),
2);
+// Make sure snapshot information is correct
+// Check resources are in a correct state
+boolean isRoutingTableUpdatedProperly = TestHelper.verify(() -> {
+ Map> routingTableSnapshots =
+ routingTableProvider.getRoutingTableSnapshots();
+ RoutingTableSnapshot routingTableSnapshotTypeA =
+
routingTableSnapshots.get(PropertyType.CUSTOMIZEDVIEW.name()).get("typeA");
+ RoutingTableSnapshot routingTableSnapshotTypeB =
+
routingTableSnapshots.get(PropertyType.CUSTOMIZEDVIEW.name()).get("typeB");
+ String typeAp1h1 = "noState";
+ String typeAp1h2 = "noState";
+ String typeAp2h1 = "noState";
+ String typeAp3h2 = "noState";
+ String typeBp1h2 = "noState";
+ String typeBp1h4 = "noState";
+ try {
+typeAp1h1 =
routingTableSnapshotTypeA.getCustomizeViews().iterator().next()
+.getStateMap("p1").get("h1");
+typeAp1h2 =
routingTableSnapshotTypeA.getCustomizeViews().iterator().next()
+.getStateMap("p1").get("h2");
+typeAp2h1 =
routingTableSnapshotTypeA.getCustomizeViews().iterator().next()
+.getStateMap("p2").get("h1");
+typeAp3h2 =
routingTableSnapshotTypeA.getCustomizeViews().iterator().next()
+.getStateMap("p3").get("h2");
+typeBp1h2 =
routingTableSnapshotTypeB.getCustomizeViews().iterator().next()
+.getStateMap("p1").get("h3");
+typeBp1h4 =
routingTableSnapshotTypeB.getCustomizeViews().iterator().next()
+.getStateMap("p1").get("h4");
+ } catch (Exception e) {
+// ok because RoutingTable has not been updated yet
+ }
+ return (routingTableSnapshots.size() == 2
+ &&
routingTableSnapshots.get(PropertyType.CUSTOMIZEDVIEW.name()).size() == 2
+ && typeAp1h1.equals("testState1") && typeAp1h2.equals("testState1")
+ && typeAp2h1.equals("testState2") && typeAp3h2.equals("testState3")
+ && typeBp1h2.equals("testState3") && typeBp1h4.equals("testState2"));
+}, TestHelper.WAIT_DURATION);
+Assert.assertTrue(isRoutingTableUpdatedProperly);
Review comment:
Please also add message in the assertion for easier understanding.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
-
To unsubscribe, e-mail: reviews-unsubscr...@helix.apache.org
For additional commands, e-mail: reviews-h...@helix.apache.org
[GitHub] [helix] zhangmeng916 commented on a change in pull request #851: Modify Helix generic controller to include new stage for customized view aggregation
zhangmeng916 commented on a change in pull request #851: Modify Helix generic
controller to include new stage for customized view aggregation
URL: https://github.com/apache/helix/pull/851#discussion_r398159643
##
File path:
helix-core/src/main/java/org/apache/helix/controller/stages/CustomizedViewAggregateStage.java
##
@@ -0,0 +1,171 @@
+package org.apache.helix.controller.stages;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.helix.HelixDataAccessor;
+import org.apache.helix.HelixException;
+import org.apache.helix.HelixManager;
+import org.apache.helix.PropertyKey;
+import org.apache.helix.common.caches.CustomizedViewCache;
+import org.apache.helix.controller.LogUtil;
+import
org.apache.helix.controller.dataproviders.ResourceControllerDataProvider;
+import org.apache.helix.controller.pipeline.AbstractAsyncBaseStage;
+import org.apache.helix.controller.pipeline.AsyncWorkerType;
+import org.apache.helix.controller.pipeline.StageException;
+import org.apache.helix.model.CustomizedView;
+import org.apache.helix.model.Partition;
+import org.apache.helix.model.Resource;
+import org.apache.helix.monitoring.mbeans.ClusterStatusMonitor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public class CustomizedViewAggregateStage extends AbstractAsyncBaseStage {
+ private static Logger LOG =
LoggerFactory.getLogger(CustomizedViewAggregateStage.class);
+
+ @Override
+ public AsyncWorkerType getAsyncWorkerType() {
+return AsyncWorkerType.CustomizedStateViewComputeWorker;
+ }
+
+ @Override
+ public void execute(final ClusterEvent event) throws Exception {
Review comment:
If you take a look at it now, it's simpler than before. Right now it only
has remove stale state types, and update customized view logic.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
-
To unsubscribe, e-mail: reviews-unsubscr...@helix.apache.org
For additional commands, e-mail: reviews-h...@helix.apache.org
[GitHub] [helix] zhangmeng916 commented on a change in pull request #851: Modify Helix generic controller to include new stage for customized view aggregation
zhangmeng916 commented on a change in pull request #851: Modify Helix generic controller to include new stage for customized view aggregation URL: https://github.com/apache/helix/pull/851#discussion_r398157359 ## File path: helix-core/src/main/java/org/apache/helix/manager/zk/CallbackHandler.java ## @@ -122,7 +124,7 @@ // TODO: make this be per _manager or per _listener instaed of per callbackHandler -- Lei private CallbackProcessor _batchCallbackProcessor; private boolean _watchChild = true; // Whether we should subscribe to the child znode's data - // change. + // change. Review comment: This is because the previous code was not correctly formatted in the switch section. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services - To unsubscribe, e-mail: reviews-unsubscr...@helix.apache.org For additional commands, e-mail: reviews-h...@helix.apache.org
[GitHub] [helix] jiajunwang commented on a change in pull request #851: Modify Helix generic controller to include new stage for customized view aggregation
jiajunwang commented on a change in pull request #851: Modify Helix generic
controller to include new stage for customized view aggregation
URL: https://github.com/apache/helix/pull/851#discussion_r398144033
##
File path:
helix-core/src/main/java/org/apache/helix/controller/GenericHelixController.java
##
@@ -813,23 +817,36 @@ public void onStateChange(String instanceName,
List statesInfo,
@Override
@PreFetch(enabled = false)
- public void onCustomizedStateRootChange(String instanceName,
NotificationContext changeContext) {
+ public void onCustomizedStateRootChange(String instanceName, List
customizedStateTypes,
+ NotificationContext changeContext) {
logger.info("START:
GenericClusterController.onCustomizedStateRootChange()");
-notifyCaches(changeContext, ChangeType.CUSTOMIZED_STATE_ROOT);
HelixManager manager = changeContext.getManager();
-List customizedStateTypes =
-manager.getHelixDataAccessor().getChildNames(
-
manager.getHelixDataAccessor().keyBuilder().customizedStatesRoot(instanceName));
+Builder keyBuilder = new Builder(manager.getClusterName());
-for (String customizedState : customizedStateTypes) {
- try {
-manager.addCustomizedStateChangeListener(this, instanceName,
customizedState);
-logger.info(
-manager.getInstanceName() + " added customized state listener for
" + instanceName
-+ ", listener: " + this);
- } catch (Exception e) {
-logger.error("Fail to add customized state listener for instance: " +
instanceName, e);
+synchronized (_lastSeenCustomizedStateTypes) {
+ Set lastSeenCustomizedStateTypes =
_lastSeenCustomizedStateTypes.get();
Review comment:
Then please also carry over the TODO part.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
-
To unsubscribe, e-mail: reviews-unsubscr...@helix.apache.org
For additional commands, e-mail: reviews-h...@helix.apache.org
[GitHub] [helix] jiajunwang commented on a change in pull request #851: Modify Helix generic controller to include new stage for customized view aggregation
jiajunwang commented on a change in pull request #851: Modify Helix generic
controller to include new stage for customized view aggregation
URL: https://github.com/apache/helix/pull/851#discussion_r398143820
##
File path:
helix-core/src/main/java/org/apache/helix/controller/GenericHelixController.java
##
@@ -822,6 +822,11 @@ public void onCustomizedStateRootChange(String
instanceName, List custom
logger.info("START:
GenericClusterController.onCustomizedStateRootChange()");
HelixManager manager = changeContext.getManager();
Builder keyBuilder = new Builder(manager.getClusterName());
+ if (customizedStateTypes.isEmpty()) {
+ customizedStateTypes =
+ manager.getHelixDataAccessor().getChildNames(
+
manager.getHelixDataAccessor().keyBuilder().customizedStatesRoot(instanceName));
Review comment:
nit, you have the keyBuilder above.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
-
To unsubscribe, e-mail: reviews-unsubscr...@helix.apache.org
For additional commands, e-mail: reviews-h...@helix.apache.org
[GitHub] [helix] zhangmeng916 commented on a change in pull request #851: Modify Helix generic controller to include new stage for customized view aggregation
zhangmeng916 commented on a change in pull request #851: Modify Helix generic
controller to include new stage for customized view aggregation
URL: https://github.com/apache/helix/pull/851#discussion_r398115507
##
File path:
helix-core/src/main/java/org/apache/helix/controller/GenericHelixController.java
##
@@ -813,23 +817,36 @@ public void onStateChange(String instanceName,
List statesInfo,
@Override
@PreFetch(enabled = false)
- public void onCustomizedStateRootChange(String instanceName,
NotificationContext changeContext) {
+ public void onCustomizedStateRootChange(String instanceName, List
customizedStateTypes,
+ NotificationContext changeContext) {
logger.info("START:
GenericClusterController.onCustomizedStateRootChange()");
-notifyCaches(changeContext, ChangeType.CUSTOMIZED_STATE_ROOT);
HelixManager manager = changeContext.getManager();
-List customizedStateTypes =
-manager.getHelixDataAccessor().getChildNames(
-
manager.getHelixDataAccessor().keyBuilder().customizedStatesRoot(instanceName));
+Builder keyBuilder = new Builder(manager.getClusterName());
-for (String customizedState : customizedStateTypes) {
- try {
-manager.addCustomizedStateChangeListener(this, instanceName,
customizedState);
-logger.info(
-manager.getInstanceName() + " added customized state listener for
" + instanceName
-+ ", listener: " + this);
- } catch (Exception e) {
-logger.error("Fail to add customized state listener for instance: " +
instanceName, e);
+synchronized (_lastSeenCustomizedStateTypes) {
+ Set lastSeenCustomizedStateTypes =
_lastSeenCustomizedStateTypes.get();
Review comment:
I saw a todo here:
// TODO: remove the synchronization here once we move this update into
dataCache.
synchronized (_lastSeenInstances) {
I'm not sure whether there's any pending task for this that could also apply
here, so I kept it.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
-
To unsubscribe, e-mail: reviews-unsubscr...@helix.apache.org
For additional commands, e-mail: reviews-h...@helix.apache.org
[GitHub] [helix] zhangmeng916 commented on a change in pull request #851: Modify Helix generic controller to include new stage for customized view aggregation
zhangmeng916 commented on a change in pull request #851: Modify Helix generic
controller to include new stage for customized view aggregation
URL: https://github.com/apache/helix/pull/851#discussion_r398114728
##
File path:
helix-core/src/main/java/org/apache/helix/controller/GenericHelixController.java
##
@@ -813,23 +817,36 @@ public void onStateChange(String instanceName,
List statesInfo,
@Override
@PreFetch(enabled = false)
- public void onCustomizedStateRootChange(String instanceName,
NotificationContext changeContext) {
+ public void onCustomizedStateRootChange(String instanceName, List
customizedStateTypes,
+ NotificationContext changeContext) {
logger.info("START:
GenericClusterController.onCustomizedStateRootChange()");
-notifyCaches(changeContext, ChangeType.CUSTOMIZED_STATE_ROOT);
HelixManager manager = changeContext.getManager();
-List customizedStateTypes =
-manager.getHelixDataAccessor().getChildNames(
-
manager.getHelixDataAccessor().keyBuilder().customizedStatesRoot(instanceName));
+Builder keyBuilder = new Builder(manager.getClusterName());
-for (String customizedState : customizedStateTypes) {
- try {
-manager.addCustomizedStateChangeListener(this, instanceName,
customizedState);
-logger.info(
-manager.getInstanceName() + " added customized state listener for
" + instanceName
-+ ", listener: " + this);
- } catch (Exception e) {
-logger.error("Fail to add customized state listener for instance: " +
instanceName, e);
+synchronized (_lastSeenCustomizedStateTypes) {
+ Set lastSeenCustomizedStateTypes =
_lastSeenCustomizedStateTypes.get();
+ for (String customizedState : customizedStateTypes) {
Review comment:
I made the change. However, I feel we actually do not need the prefetch
logic for this event. It's redundant now. Please let me know.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
-
To unsubscribe, e-mail: reviews-unsubscr...@helix.apache.org
For additional commands, e-mail: reviews-h...@helix.apache.org
[GitHub] [helix] narendly commented on a change in pull request #908: Make Helix REST realm-aware
narendly commented on a change in pull request #908: Make Helix REST realm-aware
URL: https://github.com/apache/helix/pull/908#discussion_r398066168
##
File path:
helix-rest/src/main/java/org/apache/helix/rest/server/ServerContext.java
##
@@ -20,42 +20,67 @@
* under the License.
*/
-import java.util.HashMap;
+import java.io.IOException;
+import java.util.List;
import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
import org.apache.helix.ConfigAccessor;
import org.apache.helix.HelixAdmin;
import org.apache.helix.HelixDataAccessor;
+import org.apache.helix.HelixException;
import org.apache.helix.InstanceType;
-import org.apache.helix.rest.metadatastore.ZkMetadataStoreDirectory;
-import org.apache.helix.zookeeper.datamodel.ZNRecord;
+import org.apache.helix.SystemPropertyKeys;
import org.apache.helix.manager.zk.ZKHelixAdmin;
import org.apache.helix.manager.zk.ZKHelixDataAccessor;
-import org.apache.helix.manager.zk.ZNRecordSerializer;
import org.apache.helix.manager.zk.ZkBaseDataAccessor;
-import org.apache.helix.zookeeper.impl.client.ZkClient;
-import org.apache.helix.zookeeper.api.client.HelixZkClient;
-import org.apache.helix.zookeeper.impl.factory.SharedZkClientFactory;
+import org.apache.helix.msdcommon.exception.InvalidRoutingDataException;
+import org.apache.helix.rest.metadatastore.ZkMetadataStoreDirectory;
+import org.apache.helix.rest.metadatastore.accessor.ZkRoutingDataReader;
import org.apache.helix.task.TaskDriver;
import org.apache.helix.tools.ClusterSetup;
+import org.apache.helix.zookeeper.api.client.HelixZkClient;
+import org.apache.helix.zookeeper.api.client.RealmAwareZkClient;
+import org.apache.helix.zookeeper.datamodel.ZNRecord;
+import org.apache.helix.zookeeper.datamodel.serializer.ZNRecordSerializer;
+import org.apache.helix.zookeeper.impl.client.FederatedZkClient;
+import org.apache.helix.zookeeper.impl.client.ZkClient;
+import org.apache.helix.zookeeper.impl.factory.DedicatedZkClientFactory;
+import org.apache.helix.zookeeper.impl.factory.SharedZkClientFactory;
+import org.apache.helix.zookeeper.util.HttpRoutingDataReader;
+import org.apache.helix.zookeeper.zkclient.IZkChildListener;
+import org.apache.helix.zookeeper.zkclient.IZkDataListener;
+import org.apache.helix.zookeeper.zkclient.IZkStateListener;
import org.apache.helix.zookeeper.zkclient.exception.ZkMarshallingError;
import org.apache.helix.zookeeper.zkclient.serialize.ZkSerializer;
+import org.apache.zookeeper.Watcher;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-public class ServerContext {
+public class ServerContext implements IZkDataListener, IZkChildListener,
IZkStateListener {
+ private static final Logger LOG =
LoggerFactory.getLogger(ServerContext.class);
+
private final String _zkAddr;
- private HelixZkClient _zkClient;
- private ZKHelixAdmin _zkHelixAdmin;
- private ClusterSetup _clusterSetup;
- private ConfigAccessor _configAccessor;
+ private volatile RealmAwareZkClient _zkClient;
+
+ private volatile ZKHelixAdmin _zkHelixAdmin;
+ private volatile ClusterSetup _clusterSetup;
+ private volatile ConfigAccessor _configAccessor;
// A lazily-initialized base data accessor that reads/writes byte array to ZK
// TODO: Only read (deserialize) is supported at this time. This
baseDataAccessor should support write (serialize) as needs arise
private volatile ZkBaseDataAccessor _byteArrayZkBaseDataAccessor;
// 1 Cluster name will correspond to 1 helix data accessor
private final Map _helixDataAccessorPool;
// 1 Cluster name will correspond to 1 task driver
private final Map _taskDriverPool;
Review comment:
I understand your concern and where you're coming from. But it's still
difficult. TaskDriver and ConfigAccessor are different APIs by design.
- TaskDriver/HelixManager/HelixDataAccessor -> only serve 1 single Helix
cluster
- ConfigAccessor/ClusterSetup/etc.. -> can serve many Helix clusters
So we need to keep a pool regardless.
To help you understand:
```
public TaskDriver(HelixManager manager) {
}
@Deprecated
public TaskDriver(RealmAwareZkClient client, String clusterName) {
}
@Deprecated
public TaskDriver(RealmAwareZkClient client, ZkBaseDataAccessor
baseAccessor,
String clusterName) {}
@Deprecated
public TaskDriver(HelixAdmin admin, HelixDataAccessor accessor,
ConfigAccessor cfgAccessor,
HelixPropertyStore propertyStore, String clusterName) {
this(admin, accessor, propertyStore, clusterName);
}
```
You see that every constructor of TaskDriver takes in a cluster name.
Although TaskDriver does not support any ephemeral operations, by design
it's meant to be a single cluster API.
I think what you wanted to see was to pass in some FederatedZkClient into
TaskDriver and use one TaskDriver for all clusters, but 1) that does not work,
and 2) if we wanted to make that work, that's going
[GitHub] [helix] dasahcc commented on a change in pull request #908: Make Helix REST realm-aware
dasahcc commented on a change in pull request #908: Make Helix REST realm-aware
URL: https://github.com/apache/helix/pull/908#discussion_r39805
##
File path:
helix-rest/src/main/java/org/apache/helix/rest/server/ServerContext.java
##
@@ -20,42 +20,67 @@
* under the License.
*/
-import java.util.HashMap;
+import java.io.IOException;
+import java.util.List;
import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
import org.apache.helix.ConfigAccessor;
import org.apache.helix.HelixAdmin;
import org.apache.helix.HelixDataAccessor;
+import org.apache.helix.HelixException;
import org.apache.helix.InstanceType;
-import org.apache.helix.rest.metadatastore.ZkMetadataStoreDirectory;
-import org.apache.helix.zookeeper.datamodel.ZNRecord;
+import org.apache.helix.SystemPropertyKeys;
import org.apache.helix.manager.zk.ZKHelixAdmin;
import org.apache.helix.manager.zk.ZKHelixDataAccessor;
-import org.apache.helix.manager.zk.ZNRecordSerializer;
import org.apache.helix.manager.zk.ZkBaseDataAccessor;
-import org.apache.helix.zookeeper.impl.client.ZkClient;
-import org.apache.helix.zookeeper.api.client.HelixZkClient;
-import org.apache.helix.zookeeper.impl.factory.SharedZkClientFactory;
+import org.apache.helix.msdcommon.exception.InvalidRoutingDataException;
+import org.apache.helix.rest.metadatastore.ZkMetadataStoreDirectory;
+import org.apache.helix.rest.metadatastore.accessor.ZkRoutingDataReader;
import org.apache.helix.task.TaskDriver;
import org.apache.helix.tools.ClusterSetup;
+import org.apache.helix.zookeeper.api.client.HelixZkClient;
+import org.apache.helix.zookeeper.api.client.RealmAwareZkClient;
+import org.apache.helix.zookeeper.datamodel.ZNRecord;
+import org.apache.helix.zookeeper.datamodel.serializer.ZNRecordSerializer;
+import org.apache.helix.zookeeper.impl.client.FederatedZkClient;
+import org.apache.helix.zookeeper.impl.client.ZkClient;
+import org.apache.helix.zookeeper.impl.factory.DedicatedZkClientFactory;
+import org.apache.helix.zookeeper.impl.factory.SharedZkClientFactory;
+import org.apache.helix.zookeeper.util.HttpRoutingDataReader;
+import org.apache.helix.zookeeper.zkclient.IZkChildListener;
+import org.apache.helix.zookeeper.zkclient.IZkDataListener;
+import org.apache.helix.zookeeper.zkclient.IZkStateListener;
import org.apache.helix.zookeeper.zkclient.exception.ZkMarshallingError;
import org.apache.helix.zookeeper.zkclient.serialize.ZkSerializer;
+import org.apache.zookeeper.Watcher;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-public class ServerContext {
+public class ServerContext implements IZkDataListener, IZkChildListener,
IZkStateListener {
+ private static final Logger LOG =
LoggerFactory.getLogger(ServerContext.class);
+
private final String _zkAddr;
- private HelixZkClient _zkClient;
- private ZKHelixAdmin _zkHelixAdmin;
- private ClusterSetup _clusterSetup;
- private ConfigAccessor _configAccessor;
+ private volatile RealmAwareZkClient _zkClient;
+
+ private volatile ZKHelixAdmin _zkHelixAdmin;
+ private volatile ClusterSetup _clusterSetup;
+ private volatile ConfigAccessor _configAccessor;
// A lazily-initialized base data accessor that reads/writes byte array to ZK
// TODO: Only read (deserialize) is supported at this time. This
baseDataAccessor should support write (serialize) as needs arise
private volatile ZkBaseDataAccessor _byteArrayZkBaseDataAccessor;
// 1 Cluster name will correspond to 1 helix data accessor
private final Map _helixDataAccessorPool;
// 1 Cluster name will correspond to 1 task driver
private final Map _taskDriverPool;
Review comment:
I agree with that. But the routing logic should not be at service layer.
Otherwise, each service using TaskDriver or ConfigAccessor will have their own
routing logic.
TaskDriver / Config Accessor is instantiated from RealmAwareZKClient. Then
can we make some smart things underneath for sharding key routing?
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
-
To unsubscribe, e-mail: reviews-unsubscr...@helix.apache.org
For additional commands, e-mail: reviews-h...@helix.apache.org
