UNOMI-93 Add rule execution statistics to rule service endpoint - Initial commit
Signed-off-by: Serge Huber <shu...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/df6acb3f Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/df6acb3f Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/df6acb3f Branch: refs/heads/feature-UNOMI-5-KARAF4 Commit: df6acb3f1ca5c526cebddc3b66c5fd32d88b6974 Parents: 28b63e6 Author: Serge Huber <shu...@apache.org> Authored: Mon May 1 21:08:54 2017 +0200 Committer: Serge Huber <shu...@apache.org> Committed: Mon May 1 21:08:54 2017 +0200 ---------------------------------------------------------------------- .../apache/unomi/api/rules/RuleStatistics.java | 107 +++++++++++++++++ .../apache/unomi/api/services/RulesService.java | 8 ++ itests/pom.xml | 4 +- .../apache/unomi/rest/RulesServiceEndPoint.java | 13 +++ .../services/services/RulesServiceImpl.java | 117 ++++++++++++++++++- 5 files changed, 244 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/df6acb3f/api/src/main/java/org/apache/unomi/api/rules/RuleStatistics.java ---------------------------------------------------------------------- diff --git a/api/src/main/java/org/apache/unomi/api/rules/RuleStatistics.java b/api/src/main/java/org/apache/unomi/api/rules/RuleStatistics.java new file mode 100644 index 0000000..2172461 --- /dev/null +++ b/api/src/main/java/org/apache/unomi/api/rules/RuleStatistics.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.unomi.api.rules; + +import org.apache.unomi.api.Item; + +import java.util.Date; + +/** + * A separate item to track rule statistics, because we will manage the persistence and updating of these seperately + * from the rules themselves. This object contains all the relevant statistics concerning the execution of a rule. + */ +public class RuleStatistics extends Item { + + /** + * The Rule ITEM_TYPE. + * + * @see Item for a discussion of ITEM_TYPE + */ + public static final String ITEM_TYPE = "rulestats"; + private static final long serialVersionUID = 1L; + + private long executionCount = 0; + private long localExecutionCount = 0; + private long conditionsTime = 0; + private long localConditionsTime = 0; + private long actionsTime = 0; + private long localActionsTime = 0; + private Date lastSyncDate; + + public RuleStatistics() { + } + + public RuleStatistics(String itemId) { + super(itemId); + } + + public long getExecutionCount() { + return executionCount; + } + + public void setExecutionCount(long executionCount) { + this.executionCount = executionCount; + } + + public long getLocalExecutionCount() { + return localExecutionCount; + } + + public void setLocalExecutionCount(long localExecutionCount) { + this.localExecutionCount = localExecutionCount; + } + + public long getConditionsTime() { + return conditionsTime; + } + + public void setConditionsTime(long conditionsTime) { + this.conditionsTime = conditionsTime; + } + + public long getLocalConditionsTime() { + return localConditionsTime; + } + + public void setLocalConditionsTime(long localConditionsTime) { + this.localConditionsTime = localConditionsTime; + } + + public long getActionsTime() { + return actionsTime; + } + + public void setActionsTime(long actionsTime) { + this.actionsTime = actionsTime; + } + + public long getLocalActionsTime() { + return localActionsTime; + } + + public void setLocalActionsTime(long localActionsTime) { + this.localActionsTime = localActionsTime; + } + + public Date getLastSyncDate() { + return lastSyncDate; + } + + public void setLastSyncDate(Date lastSyncDate) { + this.lastSyncDate = lastSyncDate; + } +} http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/df6acb3f/api/src/main/java/org/apache/unomi/api/services/RulesService.java ---------------------------------------------------------------------- diff --git a/api/src/main/java/org/apache/unomi/api/services/RulesService.java b/api/src/main/java/org/apache/unomi/api/services/RulesService.java index 75d1f07..63f6dda 100644 --- a/api/src/main/java/org/apache/unomi/api/services/RulesService.java +++ b/api/src/main/java/org/apache/unomi/api/services/RulesService.java @@ -23,6 +23,7 @@ import org.apache.unomi.api.PartialList; import org.apache.unomi.api.conditions.Condition; import org.apache.unomi.api.query.Query; import org.apache.unomi.api.rules.Rule; +import org.apache.unomi.api.rules.RuleStatistics; import java.util.Set; @@ -55,6 +56,13 @@ public interface RulesService { Rule getRule(String ruleId); /** + * Retrieves the statistics for a rule + * @param ruleId the identifier of the rule + * @return a long representing the number of times the rule was matched and executed. + */ + RuleStatistics getRuleStatistics(String ruleId); + + /** * Persists the specified rule to the context server. * * @param rule the rule to be persisted http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/df6acb3f/itests/pom.xml ---------------------------------------------------------------------- diff --git a/itests/pom.xml b/itests/pom.xml index 3b02af1..868fb1f 100644 --- a/itests/pom.xml +++ b/itests/pom.xml @@ -144,12 +144,12 @@ <groupId>com.github.alexcojocaru</groupId> <artifactId>elasticsearch-maven-plugin</artifactId> <!-- REPLACE THE FOLLOWING WITH THE PLUGIN VERSION YOU NEED --> - <version>5.0</version> + <version>5.7</version> <configuration> <clusterName>contextElasticSearch</clusterName> <transportPort>9300</transportPort> <httpPort>9200</httpPort> - <version>5.0.2</version> + <version>${elasticsearch.version}</version> </configuration> <executions> <!-- http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/df6acb3f/rest/src/main/java/org/apache/unomi/rest/RulesServiceEndPoint.java ---------------------------------------------------------------------- diff --git a/rest/src/main/java/org/apache/unomi/rest/RulesServiceEndPoint.java b/rest/src/main/java/org/apache/unomi/rest/RulesServiceEndPoint.java index 8559ed4..d619a7f 100644 --- a/rest/src/main/java/org/apache/unomi/rest/RulesServiceEndPoint.java +++ b/rest/src/main/java/org/apache/unomi/rest/RulesServiceEndPoint.java @@ -22,6 +22,7 @@ import org.apache.unomi.api.Metadata; import org.apache.unomi.api.PartialList; import org.apache.unomi.api.query.Query; import org.apache.unomi.api.rules.Rule; +import org.apache.unomi.api.rules.RuleStatistics; import org.apache.unomi.api.services.RulesService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -104,6 +105,18 @@ public class RulesServiceEndPoint { } /** + * Retrieves the statistics for the rule with the specified identifier + * + * @param ruleId the identifier of the rule we want to retrieve + * @return the statistics for the specified rule or {@code null} if no such rule exists. + */ + @GET + @Path("/{ruleId}/statistics") + public RuleStatistics getRuleStatistics(@PathParam("ruleId") String ruleId) { + return rulesService.getRuleStatistics(ruleId); + } + + /** * Deletes the rule identified by the specified identifier. * * @param ruleId the identifier of the rule we want to delete http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/df6acb3f/services/src/main/java/org/apache/unomi/services/services/RulesServiceImpl.java ---------------------------------------------------------------------- diff --git a/services/src/main/java/org/apache/unomi/services/services/RulesServiceImpl.java b/services/src/main/java/org/apache/unomi/services/services/RulesServiceImpl.java index 2df2fff..c289891 100644 --- a/services/src/main/java/org/apache/unomi/services/services/RulesServiceImpl.java +++ b/services/src/main/java/org/apache/unomi/services/services/RulesServiceImpl.java @@ -26,6 +26,7 @@ import org.apache.unomi.api.actions.ActionExecutor; import org.apache.unomi.api.conditions.Condition; import org.apache.unomi.api.query.Query; import org.apache.unomi.api.rules.Rule; +import org.apache.unomi.api.rules.RuleStatistics; import org.apache.unomi.api.services.DefinitionsService; import org.apache.unomi.api.services.EventListenerService; import org.apache.unomi.api.services.EventService; @@ -57,6 +58,9 @@ public class RulesServiceImpl implements RulesService, EventListenerService, Syn private List<Rule> allRules; private Timer rulesTimer; + private Timer ruleStatisticsTimer; + + private Map<String,RuleStatistics> allRuleStatistics = new HashMap<String,RuleStatistics>(); public void setBundleContext(BundleContext bundleContext) { this.bundleContext = bundleContext; @@ -102,7 +106,7 @@ public class RulesServiceImpl implements RulesService, EventListenerService, Syn bundleContext.addBundleListener(this); - initializeTimer(); + initializeTimers(); logger.info("Rule service initialized."); } @@ -116,6 +120,9 @@ public class RulesServiceImpl implements RulesService, EventListenerService, Syn if(rulesTimer != null) { rulesTimer.cancel(); } + if (ruleStatisticsTimer != null) { + ruleStatisticsTimer.cancel(); + } logger.info("Rule purge: Purge unscheduled"); } @@ -161,41 +168,50 @@ public class RulesServiceImpl implements RulesService, EventListenerService, Syn List<Rule> allItems = allRules; for (Rule rule : allItems) { + RuleStatistics ruleStatistics = getLocalRuleStatistics(rule); + long ruleConditionStartTime = System.currentTimeMillis(); String scope = rule.getMetadata().getScope(); if (scope.equals(Metadata.SYSTEM_SCOPE) || scope.equals(event.getScope())) { Condition eventCondition = definitionsService.extractConditionByTag(rule.getCondition(), "eventCondition"); if (eventCondition == null) { + updateRuleStatistics(ruleStatistics, ruleConditionStartTime); continue; } if (!persistenceService.testMatch(eventCondition, event)) { + updateRuleStatistics(ruleStatistics, ruleConditionStartTime); continue; } Condition sourceCondition = definitionsService.extractConditionByTag(rule.getCondition(), "sourceEventCondition"); if (sourceCondition != null && !persistenceService.testMatch(sourceCondition, event.getSource())) { + updateRuleStatistics(ruleStatistics, ruleConditionStartTime); continue; } if (rule.isRaiseEventOnlyOnceForProfile()) { hasEventAlreadyBeenRaisedForProfile = hasEventAlreadyBeenRaisedForProfile != null ? hasEventAlreadyBeenRaisedForProfile : eventService.hasEventAlreadyBeenRaised(event, false); if (hasEventAlreadyBeenRaisedForProfile) { + updateRuleStatistics(ruleStatistics, ruleConditionStartTime); continue; } } else if (rule.isRaiseEventOnlyOnceForSession()) { hasEventAlreadyBeenRaisedForSession = hasEventAlreadyBeenRaisedForSession != null ? hasEventAlreadyBeenRaisedForSession : eventService.hasEventAlreadyBeenRaised(event, true); if (hasEventAlreadyBeenRaisedForSession) { + updateRuleStatistics(ruleStatistics, ruleConditionStartTime); continue; } } Condition profileCondition = definitionsService.extractConditionByTag(rule.getCondition(), "profileCondition"); if (profileCondition != null && !persistenceService.testMatch(profileCondition, event.getProfile())) { + updateRuleStatistics(ruleStatistics, ruleConditionStartTime); continue; } Condition sessionCondition = definitionsService.extractConditionByTag(rule.getCondition(), "sessionCondition"); if (sessionCondition != null && !persistenceService.testMatch(sessionCondition, event.getSession())) { + updateRuleStatistics(ruleStatistics, ruleConditionStartTime); continue; } matchedRules.add(rule); @@ -205,6 +221,20 @@ public class RulesServiceImpl implements RulesService, EventListenerService, Syn return matchedRules; } + private RuleStatistics getLocalRuleStatistics(Rule rule) { + RuleStatistics ruleStatistics = this.allRuleStatistics.get(rule.getItemId()); + if (ruleStatistics == null) { + ruleStatistics = new RuleStatistics(rule.getItemId()); + } + return ruleStatistics; + } + + private void updateRuleStatistics(RuleStatistics ruleStatistics, long ruleConditionStartTime) { + long totalRuleConditionTime = System.currentTimeMillis() - ruleConditionStartTime; + ruleStatistics.setLocalConditionsTime(ruleStatistics.getLocalConditionsTime() + totalRuleConditionTime); + allRuleStatistics.put(ruleStatistics.getItemId(), ruleStatistics); + } + private List<Rule> getAllRules() { List<Rule> allItems = persistenceService.getAllItems(Rule.class, 0, -1, "priority").getList(); for (Rule rule : allItems) { @@ -225,18 +255,32 @@ public class RulesServiceImpl implements RulesService, EventListenerService, Syn int changes = EventService.NO_CHANGE; for (Rule rule : rules) { logger.debug("Fired rule " + rule.getMetadata().getId() + " for " + event.getEventType() + " - " + event.getItemId()); + long actionsStartTime = System.currentTimeMillis(); for (Action action : rule.getActions()) { changes |= actionExecutorDispatcher.execute(action, event); } - + long totalActionsTime = System.currentTimeMillis() - actionsStartTime; Event ruleFired = new Event("ruleFired", event.getSession(), event.getProfile(), event.getScope(), event, rule, event.getTimeStamp()); ruleFired.getAttributes().putAll(event.getAttributes()); ruleFired.setPersistent(false); changes |= eventService.send(ruleFired); + + RuleStatistics ruleStatistics = getLocalRuleStatistics(rule); + ruleStatistics.setLocalExecutionCount(ruleStatistics.getLocalExecutionCount()+1); + ruleStatistics.setLocalActionsTime(ruleStatistics.getLocalActionsTime() + totalActionsTime); + this.allRuleStatistics.put(rule.getItemId(), ruleStatistics); } return changes; } + @Override + public RuleStatistics getRuleStatistics(String ruleId) { + if (allRuleStatistics.containsKey(ruleId)) { + return allRuleStatistics.get(ruleId); + } + return persistenceService.load(ruleId, RuleStatistics.class); + } + public Set<Metadata> getRuleMetadatas() { Set<Metadata> metadatas = new HashSet<Metadata>(); for (Rule rule : persistenceService.getAllItems(Rule.class, 0, 50, null).getList()) { @@ -309,7 +353,7 @@ public class RulesServiceImpl implements RulesService, EventListenerService, Syn persistenceService.remove(ruleId, Rule.class); } - private void initializeTimer() { + private void initializeTimers() { rulesTimer = new Timer(); TimerTask task = new TimerTask() { @Override @@ -318,6 +362,14 @@ public class RulesServiceImpl implements RulesService, EventListenerService, Syn } }; rulesTimer.schedule(task, 0, 1000); + ruleStatisticsTimer = new Timer(); + TimerTask statisticsTask = new TimerTask() { + @Override + public void run() { + syncRuleStatistics(); + } + }; + ruleStatisticsTimer.schedule(statisticsTask, 0, 10000); } public void bundleChanged(BundleEvent event) { @@ -330,4 +382,63 @@ public class RulesServiceImpl implements RulesService, EventListenerService, Syn break; } } + + private void syncRuleStatistics() { + List<RuleStatistics> allPersistedRuleStatisticsList = persistenceService.getAllItems(RuleStatistics.class); + Map<String,RuleStatistics> allPersistedRuleStatistics = new HashMap<>(); + for (RuleStatistics ruleStatistics : allPersistedRuleStatisticsList) { + allPersistedRuleStatistics.put(ruleStatistics.getItemId(), ruleStatistics); + } + // first we iterate over the rules we have in memory + for (RuleStatistics ruleStatistics : allRuleStatistics.values()) { + boolean mustPersist = false; + if (allPersistedRuleStatistics.containsKey(ruleStatistics.getItemId())) { + // we must sync with the data coming from the persistence service. + RuleStatistics persistedRuleStatistics = allPersistedRuleStatistics.get(ruleStatistics.getItemId()); + ruleStatistics.setExecutionCount(persistedRuleStatistics.getExecutionCount() + ruleStatistics.getLocalExecutionCount()); + if (ruleStatistics.getLocalExecutionCount() > 0) { + ruleStatistics.setLocalExecutionCount(0); + mustPersist = true; + } + ruleStatistics.setConditionsTime(persistedRuleStatistics.getConditionsTime() + ruleStatistics.getLocalConditionsTime()); + if (ruleStatistics.getLocalConditionsTime() > 0) { + ruleStatistics.setLocalConditionsTime(0); + mustPersist = true; + } + ruleStatistics.setActionsTime(persistedRuleStatistics.getActionsTime() + ruleStatistics.getLocalActionsTime()); + if (ruleStatistics.getLocalActionsTime() > 0) { + ruleStatistics.setLocalActionsTime(0); + mustPersist = true; + } + ruleStatistics.setLastSyncDate(new Date()); + } else { + ruleStatistics.setExecutionCount(ruleStatistics.getExecutionCount() + ruleStatistics.getLocalExecutionCount()); + if (ruleStatistics.getLocalExecutionCount() > 0) { + ruleStatistics.setLocalExecutionCount(0); + mustPersist = true; + } + ruleStatistics.setConditionsTime(ruleStatistics.getConditionsTime() + ruleStatistics.getLocalConditionsTime()); + if (ruleStatistics.getLocalConditionsTime() > 0) { + ruleStatistics.setLocalConditionsTime(0); + mustPersist = true; + } + ruleStatistics.setActionsTime(ruleStatistics.getActionsTime() + ruleStatistics.getLocalActionsTime()); + if (ruleStatistics.getLocalActionsTime() > 0) { + ruleStatistics.setLocalActionsTime(0); + mustPersist = true; + } + ruleStatistics.setLastSyncDate(new Date()); + } + allRuleStatistics.put(ruleStatistics.getItemId(), ruleStatistics); + if (mustPersist) { + persistenceService.save(ruleStatistics); + } + } + // now let's iterate over the rules coming from the persistence service, as we may have new ones. + for (RuleStatistics ruleStatistics : allPersistedRuleStatistics.values()) { + if (!allRuleStatistics.containsKey(ruleStatistics.getItemId())) { + allRuleStatistics.put(ruleStatistics.getItemId(), ruleStatistics); + } + } + } }