[ https://issues.apache.org/jira/browse/SCB-506?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16451800#comment-16451800 ]
ASF GitHub Bot commented on SCB-506: ------------------------------------ xuyiyun0929 closed pull request #666: [SCB-506]服务治理相关的需要事件上报 URL: https://github.com/apache/incubator-servicecomb-java-chassis/pull/666 This is a PR merged from a forked repository. As GitHub hides the original diff on merge, it is displayed below for the sake of provenance: As this is a foreign pull request (from a fork), the diff is supplied below (as it won't show otherwise due to GitHub magic): diff --git a/foundations/foundation-common/src/main/java/org/apache/servicecomb/foundation/common/event/AlarmEvent.java b/foundations/foundation-common/src/main/java/org/apache/servicecomb/foundation/common/event/AlarmEvent.java new file mode 100644 index 000000000..ecbb5045c --- /dev/null +++ b/foundations/foundation-common/src/main/java/org/apache/servicecomb/foundation/common/event/AlarmEvent.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.servicecomb.foundation.common.event; + +import java.util.HashMap; + +public class AlarmEvent { + + Type type; + + private int id; + + private HashMap<String, Object> msg; + + public AlarmEvent(Type type, int id) { + this.type = type; + this.id = id; + } + + public Type getType() { + return this.type; + } + + public int getId() { + return id; + } + + public HashMap<String, Object> getMsg() { + return msg; + } + + public void setMsg(HashMap<String, Object> msg2) { + this.msg = msg2; + } + + public enum Type { + OPEN, + CLOSE, + SUCCED, + FAILD + }; +} diff --git a/handlers/handler-bizkeeper/src/main/java/org/apache/servicecomb/bizkeeper/BizkeeperHandler.java b/handlers/handler-bizkeeper/src/main/java/org/apache/servicecomb/bizkeeper/BizkeeperHandler.java index bf6f91140..5c3743a29 100644 --- a/handlers/handler-bizkeeper/src/main/java/org/apache/servicecomb/bizkeeper/BizkeeperHandler.java +++ b/handlers/handler-bizkeeper/src/main/java/org/apache/servicecomb/bizkeeper/BizkeeperHandler.java @@ -17,13 +17,17 @@ package org.apache.servicecomb.bizkeeper; +import org.apache.servicecomb.bizkeeper.event.CircutBreakerEvent; import org.apache.servicecomb.core.Handler; import org.apache.servicecomb.core.Invocation; +import org.apache.servicecomb.foundation.common.event.AlarmEvent.Type; +import org.apache.servicecomb.foundation.common.event.EventManager; import org.apache.servicecomb.swagger.invocation.AsyncResponse; import org.apache.servicecomb.swagger.invocation.Response; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.netflix.hystrix.HystrixCircuitBreaker; import com.netflix.hystrix.HystrixCommandProperties; import com.netflix.hystrix.HystrixInvokable; import com.netflix.hystrix.HystrixObservable; @@ -72,8 +76,14 @@ public BizkeeperHandler(String groupname) { @Override public void handle(Invocation invocation, AsyncResponse asyncResp) { HystrixObservable<Response> command = delegate.createBizkeeperCommand(invocation); - Observable<Response> observable = command.toObservable(); + HystrixCircuitBreaker circuitBreaker = + HystrixCircuitBreaker.Factory.getInstance(CommandKey.toHystrixCommandKey(groupname, invocation)); + if (circuitBreaker != null && circuitBreaker.isOpen()) { + EventManager.post(new CircutBreakerEvent(invocation, this.groupname, Type.OPEN)); + }else { + EventManager.post(new CircutBreakerEvent(invocation, this.groupname, Type.CLOSE)); + } observable.subscribe(asyncResp::complete, error -> { LOG.warn("catch error in bizkeeper:" + error.getMessage()); asyncResp.fail(invocation.getInvocationType(), error); diff --git a/handlers/handler-bizkeeper/src/main/java/org/apache/servicecomb/bizkeeper/FallbackPolicyManager.java b/handlers/handler-bizkeeper/src/main/java/org/apache/servicecomb/bizkeeper/FallbackPolicyManager.java index cf0bad2c8..5ce07c243 100644 --- a/handlers/handler-bizkeeper/src/main/java/org/apache/servicecomb/bizkeeper/FallbackPolicyManager.java +++ b/handlers/handler-bizkeeper/src/main/java/org/apache/servicecomb/bizkeeper/FallbackPolicyManager.java @@ -19,7 +19,10 @@ import java.util.HashMap; import java.util.Map; +import org.apache.servicecomb.bizkeeper.event.FallbackEvent; import org.apache.servicecomb.core.Invocation; +import org.apache.servicecomb.foundation.common.event.AlarmEvent.Type; +import org.apache.servicecomb.foundation.common.event.EventManager; import org.apache.servicecomb.swagger.invocation.Response; public class FallbackPolicyManager { @@ -39,6 +42,7 @@ public static void record(String type, Invocation invocation, Response response, public static Response getFallbackResponse(String type, Throwable error, Invocation invocation) { FallbackPolicy policy = getPolicy(type, invocation); if (policy != null) { + EventManager.post(new FallbackEvent(policy, invocation, Type.OPEN)); return policy.getFallbackResponse(invocation); } else { return Response.failResp(invocation.getInvocationType(), diff --git a/handlers/handler-bizkeeper/src/main/java/org/apache/servicecomb/bizkeeper/event/CircutBreakerEvent.java b/handlers/handler-bizkeeper/src/main/java/org/apache/servicecomb/bizkeeper/event/CircutBreakerEvent.java new file mode 100644 index 000000000..81b429df9 --- /dev/null +++ b/handlers/handler-bizkeeper/src/main/java/org/apache/servicecomb/bizkeeper/event/CircutBreakerEvent.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.servicecomb.bizkeeper.event; + +import java.util.HashMap; + +import org.apache.servicecomb.bizkeeper.CommandKey; +import org.apache.servicecomb.bizkeeper.Configuration; +import org.apache.servicecomb.core.Invocation; +import org.apache.servicecomb.foundation.common.event.AlarmEvent; +import com.netflix.hystrix.HystrixCommandMetrics; + +public class CircutBreakerEvent extends AlarmEvent { + + private static int id = 1001; + + private HashMap<String, Object> msg = new HashMap<>(); + + /** + * msg部分字段说明: + * invocationQualifiedName:当前调用的接口 + * currentTotalRequest:当前总请求数 + * currentErrorCount:当前请求出错计数 + * currentErrorPercentage:当前请求出错百分比 + */ + public CircutBreakerEvent(Invocation invocation, String groupname, Type type) { + super(type, id); + HystrixCommandMetrics hystrixCommandMetrics = + HystrixCommandMetrics.getInstance(CommandKey.toHystrixCommandKey(groupname, invocation)); + String microserviceName = invocation.getMicroserviceName(); + String invocationQualifiedName = invocation.getInvocationQualifiedName(); + msg.put("microserviceName", microserviceName); + msg.put("invocationQualifiedName", invocationQualifiedName); + if (hystrixCommandMetrics != null) { + msg.put("currentTotalRequest", hystrixCommandMetrics.getHealthCounts().getTotalRequests()); + msg.put("currentErrorCount", hystrixCommandMetrics.getHealthCounts().getErrorCount()); + msg.put("currentErrorPercentage", hystrixCommandMetrics.getHealthCounts().getErrorPercentage()); + } + msg.put("requestVolumeThreshold", + Configuration.INSTANCE.getCircuitBreakerRequestVolumeThreshold(groupname, + microserviceName, + invocationQualifiedName)); + msg.put("sleepWindowInMilliseconds", + Configuration.INSTANCE.getCircuitBreakerSleepWindowInMilliseconds(groupname, + microserviceName, + invocationQualifiedName)); + msg.put("errorThresholdPercentage", + Configuration.INSTANCE + .getCircuitBreakerErrorThresholdPercentage(groupname, microserviceName, invocationQualifiedName)); + super.setMsg(msg); + } +} diff --git a/handlers/handler-bizkeeper/src/main/java/org/apache/servicecomb/bizkeeper/event/FallbackEvent.java b/handlers/handler-bizkeeper/src/main/java/org/apache/servicecomb/bizkeeper/event/FallbackEvent.java new file mode 100644 index 000000000..2cbbfb234 --- /dev/null +++ b/handlers/handler-bizkeeper/src/main/java/org/apache/servicecomb/bizkeeper/event/FallbackEvent.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.servicecomb.bizkeeper.event; + +import java.util.HashMap; + +import org.apache.servicecomb.bizkeeper.FallbackPolicy; +import org.apache.servicecomb.core.Invocation; +import org.apache.servicecomb.foundation.common.event.AlarmEvent; + +public class FallbackEvent extends AlarmEvent { + + private static int id = 1002; + + /** + * msg部分字段说明: + * invocationQualifiedName:当前调用的接口 + * policy:当前容错策略 + */ + public FallbackEvent(FallbackPolicy policy, Invocation invocation, Type type) { + super(type, id); + HashMap<String, Object> msg = new HashMap<>(); + msg.put("microserviceName", invocation.getMicroserviceName()); + msg.put("invocationQualifiedName", invocation.getInvocationQualifiedName()); + msg.put("policy", policy.name()); + super.setMsg(msg); + } + +} diff --git a/handlers/handler-bizkeeper/src/test/java/org/apache/servicecomb/bizkeeper/TestBizkeeperHandler.java b/handlers/handler-bizkeeper/src/test/java/org/apache/servicecomb/bizkeeper/TestBizkeeperHandler.java index 4c0cb644a..efaa2e935 100644 --- a/handlers/handler-bizkeeper/src/test/java/org/apache/servicecomb/bizkeeper/TestBizkeeperHandler.java +++ b/handlers/handler-bizkeeper/src/test/java/org/apache/servicecomb/bizkeeper/TestBizkeeperHandler.java @@ -17,8 +17,13 @@ package org.apache.servicecomb.bizkeeper; +import java.util.ArrayList; +import java.util.List; + import org.apache.servicecomb.core.Invocation; import org.apache.servicecomb.core.definition.OperationMeta; +import org.apache.servicecomb.foundation.common.event.AlarmEvent; +import org.apache.servicecomb.foundation.common.event.EventManager; import org.apache.servicecomb.swagger.invocation.AsyncResponse; import org.apache.servicecomb.swagger.invocation.InvocationType; import org.junit.After; @@ -29,6 +34,7 @@ import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; +import com.google.common.eventbus.Subscribe; import com.netflix.hystrix.HystrixCommandProperties; import com.netflix.hystrix.HystrixObservableCommand; import com.netflix.hystrix.strategy.HystrixPlugins; @@ -39,6 +45,8 @@ private static final String GROUP_NAME = "Group_Name"; + private List<AlarmEvent> taskList; + Invocation invocation = null; AsyncResponse asyncResp = null; @@ -49,6 +57,7 @@ public TestBizkeeperHandler() { @Before public void setUp() throws Exception { + taskList = new ArrayList<>(); bizkeeperHandler = new TestBizkeeperHandler(); invocation = Mockito.mock(Invocation.class); asyncResp = Mockito.mock(AsyncResponse.class); @@ -56,6 +65,13 @@ public void setUp() throws Exception { FallbackPolicyManager.addPolicy(new ReturnNullFallbackPolicy()); FallbackPolicyManager.addPolicy(new ThrowExceptionFallbackPolicy()); FallbackPolicyManager.addPolicy(new FromCacheFallbackPolicy()); + + EventManager.register(new Object() { + @Subscribe + public void onEvent(AlarmEvent circutBreakerEvent) { + taskList.add(circutBreakerEvent); + } + }); } @After @@ -94,6 +110,7 @@ public void testHandle() { validAssert = false; } Assert.assertTrue(validAssert); + Assert.assertEquals(1, taskList.size()); } @Test diff --git a/handlers/handler-bizkeeper/src/test/java/org/apache/servicecomb/bizkeeper/TestFallbackPolicyManager.java b/handlers/handler-bizkeeper/src/test/java/org/apache/servicecomb/bizkeeper/TestFallbackPolicyManager.java index 04672eac1..88c27159d 100644 --- a/handlers/handler-bizkeeper/src/test/java/org/apache/servicecomb/bizkeeper/TestFallbackPolicyManager.java +++ b/handlers/handler-bizkeeper/src/test/java/org/apache/servicecomb/bizkeeper/TestFallbackPolicyManager.java @@ -16,17 +16,39 @@ */ package org.apache.servicecomb.bizkeeper; +import java.util.ArrayList; +import java.util.List; + import org.apache.servicecomb.core.Invocation; import org.apache.servicecomb.core.definition.OperationMeta; import org.apache.servicecomb.core.exception.CseException; +import org.apache.servicecomb.foundation.common.event.AlarmEvent; +import org.apache.servicecomb.foundation.common.event.EventManager; import org.apache.servicecomb.swagger.invocation.Response; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; +import com.google.common.eventbus.Subscribe; + import mockit.Expectations; import mockit.Mocked; public class TestFallbackPolicyManager { + + private List<AlarmEvent> taskList; + + @Before + public void setUp() throws Exception { + taskList = new ArrayList<>(); + EventManager.register(new Object() { + @Subscribe + public void onEvent(AlarmEvent fallbackPolicyEvent) { + taskList.add(fallbackPolicyEvent); + } + }); + } + @Test public void testFallbackPolicyManager(final @Mocked Configuration config, final @Mocked Invocation invocation, final @Mocked OperationMeta operation) { @@ -48,7 +70,7 @@ public void testFallbackPolicyManager(final @Mocked Configuration config, final }; Assert.assertEquals((String) null, FallbackPolicyManager.getFallbackResponse("Consumer", null, invocation).getResult()); - + Assert.assertEquals(1, taskList.size()); new Expectations() { { invocation.getMicroserviceName(); diff --git a/handlers/handler-loadbalance/src/main/java/org/apache/servicecomb/loadbalance/event/IsolationServerEvent.java b/handlers/handler-loadbalance/src/main/java/org/apache/servicecomb/loadbalance/event/IsolationServerEvent.java new file mode 100644 index 000000000..83a34abb1 --- /dev/null +++ b/handlers/handler-loadbalance/src/main/java/org/apache/servicecomb/loadbalance/event/IsolationServerEvent.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.servicecomb.loadbalance.event; + +import java.util.HashMap; + +import org.apache.servicecomb.foundation.common.event.AlarmEvent; + +public class IsolationServerEvent extends AlarmEvent { + + private static int id = 1003; + + /** + * msg部分字段说明: + * currentTotalRequest:当前实例总请求数 + * currentCountinuousFailureCount:当前实例连续出错次数 + * currentErrorPercentage:当前实例出错百分比 + */ + public IsolationServerEvent(String microserviceName, long totalRequest, int currentCountinuousFailureCount, + double currentErrorPercentage, int continuousFailureThreshold, + int errorThresholdPercentage, long enableRequestThreshold, long singleTestTime, Type type) { + super(type, id); + HashMap<String, Object> msg = new HashMap<>(); + msg.put("microserviceName", microserviceName); + msg.put("currentTotalRequest", totalRequest); + msg.put("currentCountinuousFailureCount", currentCountinuousFailureCount); + msg.put("currentErrorPercentage", currentErrorPercentage); + msg.put("continuousFailureThreshold", continuousFailureThreshold); + msg.put("errorThresholdPercentage", errorThresholdPercentage); + msg.put("enableRequestThreshold", enableRequestThreshold); + msg.put("singleTestTime", singleTestTime); + super.setMsg(msg); + } +} diff --git a/handlers/handler-loadbalance/src/main/java/org/apache/servicecomb/loadbalance/filter/IsolationServerListFilter.java b/handlers/handler-loadbalance/src/main/java/org/apache/servicecomb/loadbalance/filter/IsolationServerListFilter.java index 5d263e059..ec0df018a 100644 --- a/handlers/handler-loadbalance/src/main/java/org/apache/servicecomb/loadbalance/filter/IsolationServerListFilter.java +++ b/handlers/handler-loadbalance/src/main/java/org/apache/servicecomb/loadbalance/filter/IsolationServerListFilter.java @@ -20,9 +20,13 @@ import java.util.ArrayList; import java.util.List; + +import org.apache.servicecomb.foundation.common.event.AlarmEvent.Type; +import org.apache.servicecomb.foundation.common.event.EventManager; import org.apache.servicecomb.loadbalance.Configuration; import org.apache.servicecomb.loadbalance.CseServer; import org.apache.servicecomb.loadbalance.ServerListFilterExt; +import org.apache.servicecomb.loadbalance.event.IsolationServerEvent; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -91,19 +95,20 @@ private boolean allowVisit(Server server) { ServerStats serverStats = stats.getSingleServerStat(server); long totalRequest = serverStats.getTotalRequestsCount(); long failureRequest = serverStats.getSuccessiveConnectionFailureCount(); - + int currentCountinuousFailureCount = ((CseServer) server).getCountinuousFailureCount(); + double currentErrorThresholdPercentage = (failureRequest / (double) totalRequest) * PERCENT; if (totalRequest < enableRequestThreshold) { return true; } if (continuousFailureThreshold > 0) { // continuousFailureThreshold has higher priority to decide the result - if (((CseServer) server).getCountinuousFailureCount() < continuousFailureThreshold) { + if (currentCountinuousFailureCount < continuousFailureThreshold) { return true; } } else { // if continuousFailureThreshold, then check error percentage - if ((failureRequest / (double) totalRequest) * PERCENT < errorThresholdPercentage) { + if (currentErrorThresholdPercentage < errorThresholdPercentage) { return true; } } @@ -112,10 +117,18 @@ private boolean allowVisit(Server server) { LOGGER.info("The Service {}'s instance {} has been break, will give a single test opportunity.", microserviceName, server); + EventManager.post(new IsolationServerEvent(microserviceName, totalRequest, currentCountinuousFailureCount, + currentErrorThresholdPercentage, + continuousFailureThreshold, errorThresholdPercentage, enableRequestThreshold, + singleTestTime, Type.CLOSE)); return true; } LOGGER.warn("The Service {}'s instance {} has been break!", microserviceName, server); + EventManager.post(new IsolationServerEvent(microserviceName, totalRequest, currentCountinuousFailureCount, + currentErrorThresholdPercentage, + continuousFailureThreshold, errorThresholdPercentage, enableRequestThreshold, + singleTestTime, Type.OPEN)); return false; } } diff --git a/handlers/handler-loadbalance/src/test/java/org/apache/servicecomb/loadbalance/filter/TestIsolationServerListFilter.java b/handlers/handler-loadbalance/src/test/java/org/apache/servicecomb/loadbalance/filter/TestIsolationServerListFilter.java index e7a25ec34..56a41f8a3 100644 --- a/handlers/handler-loadbalance/src/test/java/org/apache/servicecomb/loadbalance/filter/TestIsolationServerListFilter.java +++ b/handlers/handler-loadbalance/src/test/java/org/apache/servicecomb/loadbalance/filter/TestIsolationServerListFilter.java @@ -23,6 +23,8 @@ import org.apache.commons.configuration.AbstractConfiguration; import org.apache.servicecomb.config.ConfigUtil; import org.apache.servicecomb.core.Invocation; +import org.apache.servicecomb.foundation.common.event.AlarmEvent; +import org.apache.servicecomb.foundation.common.event.EventManager; import org.apache.servicecomb.loadbalance.CseServer; import org.junit.After; import org.junit.AfterClass; @@ -32,6 +34,7 @@ import org.junit.Test; import org.mockito.Mockito; +import com.google.common.eventbus.Subscribe; import com.netflix.config.ConfigurationManager; import com.netflix.config.DynamicPropertyFactory; import com.netflix.loadbalancer.LoadBalancerStats; @@ -45,6 +48,8 @@ LoadBalancerStats loadBalancerStats = null; + private List<AlarmEvent> taskList; + @BeforeClass public static void initConfig() throws Exception { ConfigUtil.installDynamicConfig(); @@ -70,6 +75,14 @@ public void setUp() throws Exception { configuration.clearProperty("cse.loadbalance.isolation.enableRequestThreshold"); configuration.addProperty("cse.loadbalance.isolation.enableRequestThreshold", "3"); + + taskList = new ArrayList<>(); + EventManager.register(new Object() { + @Subscribe + public void onEvent(AlarmEvent isolationServerEvent) { + taskList.add(isolationServerEvent); + } + }); } @After @@ -139,6 +152,7 @@ public void testGetFilteredListOfServersOnContinuousFailureReachesThreshold() { IsolationServerListFilter.setInvocation(invocation); List<Server> returnedServerList = IsolationServerListFilter.getFilteredListOfServers(serverList); Assert.assertEquals(0, returnedServerList.size()); + Assert.assertEquals(1, taskList.size()); } @Test ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org > 服务治理相关的需要事件上报 > ------------- > > Key: SCB-506 > URL: https://issues.apache.org/jira/browse/SCB-506 > Project: Apache ServiceComb > Issue Type: Task > Components: Java-Chassis > Reporter: xuyiyun > Assignee: xuyiyun > Priority: Major > > 如果发生服务降级 实例隔离 熔断等操作,业务监控系统需要能获取到对应的通知,方便集成 -- This message was sent by Atlassian JIRA (v7.6.3#76005)