AMBARI-19847 Improve Log Feeder simulation to help scale testing (mgergely)
Change-Id: I8a53b3285b919523a929656c10d4d1215169fbab Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/61f8f6d5 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/61f8f6d5 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/61f8f6d5 Branch: refs/heads/branch-dev-patch-upgrade Commit: 61f8f6d56937996ea7ca2f64b8961e83b72422fd Parents: 14cd837 Author: Miklos Gergely <[email protected]> Authored: Fri Feb 3 09:56:10 2017 +0100 Committer: Miklos Gergely <[email protected]> Committed: Fri Feb 3 09:56:10 2017 +0100 ---------------------------------------------------------------------- .../ambari/logfeeder/input/InputSimulate.java | 52 ++++++++++++-------- .../ambari/logfeeder/util/LogFeederUtil.java | 12 +++++ 2 files changed, 44 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/61f8f6d5/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputSimulate.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputSimulate.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputSimulate.java index 743be69..be97a52 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputSimulate.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputSimulate.java @@ -21,24 +21,25 @@ package org.apache.ambari.logfeeder.input; import java.net.InetAddress; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Random; +import java.util.Set; +import java.util.TreeSet; import org.apache.ambari.logfeeder.filter.Filter; import org.apache.ambari.logfeeder.filter.FilterJSON; import org.apache.ambari.logfeeder.util.LogFeederUtil; -import org.apache.commons.lang3.StringUtils; import org.apache.solr.common.util.Base64; +import com.google.common.base.Joiner; + public class InputSimulate extends Input { - private static final String LOG_MESSAGE_PREFIX = "Simulated log message for testing, line"; - - private static final String LOG_TEXT_PATTERN = - "{ logtime=\"%d\", level=\"%s\", log_message=\"<LOG_MESSAGE_PATTERN>\"}"; + private static final String LOG_TEXT_PATTERN = "{ logtime=\"%d\", level=\"%s\", log_message=\"%s\"}"; private static final Map<String, String> typeToFilePath = new HashMap<>(); public static void loadTypeToFilePath(List<Map<String, Object>> inputList) { @@ -55,16 +56,21 @@ public class InputSimulate extends Input { private final List<String> types; private final String level; - private final String logText; + private final int numberOfWords; + private final int minLogWords; + private final int maxLogWords; private final long sleepMillis; public InputSimulate() throws Exception { this.types = getSimulatedLogTypes(); this.level = LogFeederUtil.getStringProperty("logfeeder.simulate.log_level", "WARN"); - this.logText = getLogText(); + this.numberOfWords = LogFeederUtil.getIntProperty("logfeeder.simulate.number_of_words", 1000, 50, 1000000); + this.minLogWords = LogFeederUtil.getIntProperty("logfeeder.simulate.min_log_words", 5, 1, 10); + this.maxLogWords = LogFeederUtil.getIntProperty("logfeeder.simulate.max_log_words", 10, 10, 20); this.sleepMillis = LogFeederUtil.getIntProperty("logfeeder.simulate.sleep_milliseconds", 10000); Filter filter = new FilterJSON(); + filter.loadConfig(Collections.<String, Object> emptyMap()); filter.setInput(this); addFilter(filter); } @@ -80,15 +86,6 @@ public class InputSimulate extends Input { } } - private String getLogText() { - int logTextSize = LogFeederUtil.getIntProperty("logfeeder.simulate.log_message_size", 100); - int fillerSize = Math.max(logTextSize - LOG_MESSAGE_PREFIX.length() - 10, 0); - String filler = StringUtils.repeat("X", fillerSize); - String logMessagePattern = LOG_MESSAGE_PREFIX + " %08d " + filler; - - return LOG_TEXT_PATTERN.replaceAll("<LOG_MESSAGE_PATTERN>", logMessagePattern); - } - @Override public boolean isReady() { return true; @@ -103,12 +100,12 @@ public class InputSimulate extends Input { while (true) { String type = imitateRandomLogFile(); + String line = getLine(); InputMarker marker = getInputMarker(type); - String line = getLine(marker); outputLine(line, marker); - try { Thread.sleep(sleepMillis); } catch(Exception e) {} + try { Thread.sleep(sleepMillis); } catch(Exception e) { /* Ignore */ } } } @@ -143,9 +140,24 @@ public class InputSimulate extends Input { return Base64.byteArrayToBase64(fileKey.getBytes()); } - private String getLine(InputMarker marker) { + private String getLine() { Date d = new Date(); - return String.format(logText, d.getTime(), level, marker.lineNumber); + String logMessage = createLogMessage(); + return String.format(LOG_TEXT_PATTERN, d.getTime(), level, logMessage); + } + + private String createLogMessage() { + int logMessageLength = minLogWords + random.nextInt(maxLogWords - minLogWords + 1); + Set<Integer> words = new TreeSet<>(); + List<String> logMessage = new ArrayList<>(); + while (words.size() < logMessageLength) { + int word = random.nextInt(numberOfWords); + if (words.add(word)) { + logMessage.add(String.format("Word%06d", word)); + } + } + + return Joiner.on(' ').join(logMessage); } @Override http://git-wip-us.apache.org/repos/asf/ambari/blob/61f8f6d5/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java index 5bf600e..73cf449 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java @@ -171,8 +171,20 @@ public class LogFeederUtil { } public static int getIntProperty(String key, int defaultValue) { + return getIntProperty(key, defaultValue, null, null); + } + + public static int getIntProperty(String key, int defaultValue, Integer minValue, Integer maxValue) { String value = getStringProperty(key); int retValue = objectToInt(value, defaultValue, ", key=" + key); + if (minValue != null && retValue < minValue) { + LOG.info("Minimum rule was applied for " + key + ": " + retValue + " < " + minValue); + retValue = minValue; + } + if (maxValue != null && retValue > maxValue) { + LOG.info("Maximum rule was applied for " + key + ": " + retValue + " > " + maxValue); + retValue = maxValue; + } return retValue; }
