[ https://issues.apache.org/jira/browse/CASSANDRA-16120?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
David Capwell updated CASSANDRA-16120: -------------------------------------- Description: One of the main gaps between python dtest and jvm dtest is python dtest supports the ability to grep the logs of an instance; we need this capability as some tests require validating logs were triggered. Pydocs for common log methods {code} | grep_log(self, expr, filename='system.log', from_mark=None) | Returns a list of lines matching the regular expression in parameter | in the Cassandra log of this node | | grep_log_for_errors(self, filename='system.log') | Returns a list of errors with stack traces | in the Cassandra log of this node | | grep_log_for_errors_from(self, filename='system.log', seek_start=0) {code} {code} | watch_log_for(self, exprs, from_mark=None, timeout=600, process=None, verbose=False, filename='system.log') | Watch the log until one or more (regular) expression are found. | This methods when all the expressions have been found or the method | timeouts (a TimeoutError is then raised). On successful completion, | a list of pair (line matched, match object) is returned. {code} Below is a POC showing a way to do such logic {code} package org.apache.cassandra.distributed.test; import java.io.BufferedReader; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStreamReader; import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; import java.util.Iterator; import java.util.Spliterator; import java.util.Spliterators; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Stream; import java.util.stream.StreamSupport; import com.google.common.io.Closeables; import org.junit.Test; import org.apache.cassandra.distributed.Cluster; import org.apache.cassandra.utils.AbstractIterator; public class AllTheLogs extends TestBaseImpl { @Test public void test() throws IOException { try (final Cluster cluster = init(Cluster.build(1).start())) { String tag = System.getProperty("cassandra.testtag", "cassandra.testtag_IS_UNDEFINED"); String suite = System.getProperty("suitename", "suitename_IS_UNDEFINED"); String log = String.format("build/test/logs/%s/TEST-%s.log", tag, suite); grep(log, "Enqueuing flush of tables").forEach(l -> System.out.println("I found the thing: " + l)); } } private static Stream<String> grep(String file, String regex) throws IOException { return grep(file, Pattern.compile(regex)); } private static Stream<String> grep(String file, Pattern regex) throws IOException { BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), StandardCharsets.UTF_8)); Iterator<String> it = new AbstractIterator<String>() { protected String computeNext() { try { String s; while ((s = reader.readLine()) != null) { Matcher m = regex.matcher(s); if (m.find()) return s; } reader.close(); return endOfData(); } catch (IOException e) { Closeables.closeQuietly(reader); throw new UncheckedIOException(e); } } }; return StreamSupport.stream(Spliterators.spliteratorUnknownSize(it, Spliterator.ORDERED), false); } } {code} And {code} @Test public void test() throws IOException { try (final Cluster cluster = init(Cluster.build(1).start())) { String tag = System.getProperty("cassandra.testtag", "cassandra.testtag_IS_UNDEFINED"); String suite = System.getProperty("suitename", "suitename_IS_UNDEFINED"); //TODO missing way to get node id // cluster.get(1); String log = String.format("build/test/logs/%s/TEST-%s-node%d.log", tag, suite, 1); grep(log, "Enqueuing flush of tables").forEach(l -> System.out.println("I found the thing: " + l)); } } {code} was: One of the main gaps between python dtest and jvm dtest is python dtest supports the ability to grep the logs of an instance; we need this capability as some tests require validating logs were triggered. Pydocs for common log methods | grep_log(self, expr, filename='system.log', from_mark=None) | Returns a list of lines matching the regular expression in parameter | in the Cassandra log of this node | | grep_log_for_errors(self, filename='system.log') | Returns a list of errors with stack traces | in the Cassandra log of this node | | grep_log_for_errors_from(self, filename='system.log', seek_start=0) | watch_log_for(self, exprs, from_mark=None, timeout=600, process=None, verbose=False, filename='system.log') | Watch the log until one or more (regular) expression are found. | This methods when all the expressions have been found or the method | timeouts (a TimeoutError is then raised). On successful completion, | a list of pair (line matched, match object) is returned. Below is a POC showing a way to do such logic {code} package org.apache.cassandra.distributed.test; import java.io.BufferedReader; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStreamReader; import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; import java.util.Iterator; import java.util.Spliterator; import java.util.Spliterators; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Stream; import java.util.stream.StreamSupport; import com.google.common.io.Closeables; import org.junit.Test; import org.apache.cassandra.distributed.Cluster; import org.apache.cassandra.utils.AbstractIterator; public class AllTheLogs extends TestBaseImpl { @Test public void test() throws IOException { try (final Cluster cluster = init(Cluster.build(1).start())) { String tag = System.getProperty("cassandra.testtag", "cassandra.testtag_IS_UNDEFINED"); String suite = System.getProperty("suitename", "suitename_IS_UNDEFINED"); String log = String.format("build/test/logs/%s/TEST-%s.log", tag, suite); grep(log, "Enqueuing flush of tables").forEach(l -> System.out.println("I found the thing: " + l)); } } private static Stream<String> grep(String file, String regex) throws IOException { return grep(file, Pattern.compile(regex)); } private static Stream<String> grep(String file, Pattern regex) throws IOException { BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), StandardCharsets.UTF_8)); Iterator<String> it = new AbstractIterator<String>() { protected String computeNext() { try { String s; while ((s = reader.readLine()) != null) { Matcher m = regex.matcher(s); if (m.find()) return s; } reader.close(); return endOfData(); } catch (IOException e) { Closeables.closeQuietly(reader); throw new UncheckedIOException(e); } } }; return StreamSupport.stream(Spliterators.spliteratorUnknownSize(it, Spliterator.ORDERED), false); } } {code} And {code} @Test public void test() throws IOException { try (final Cluster cluster = init(Cluster.build(1).start())) { String tag = System.getProperty("cassandra.testtag", "cassandra.testtag_IS_UNDEFINED"); String suite = System.getProperty("suitename", "suitename_IS_UNDEFINED"); //TODO missing way to get node id // cluster.get(1); String log = String.format("build/test/logs/%s/TEST-%s-node%d.log", tag, suite, 1); grep(log, "Enqueuing flush of tables").forEach(l -> System.out.println("I found the thing: " + l)); } } {code} > Add ability for jvm-dtest to grep instance logs > ----------------------------------------------- > > Key: CASSANDRA-16120 > URL: https://issues.apache.org/jira/browse/CASSANDRA-16120 > Project: Cassandra > Issue Type: Improvement > Components: Test/dtest/java > Reporter: David Capwell > Priority: Normal > > One of the main gaps between python dtest and jvm dtest is python dtest > supports the ability to grep the logs of an instance; we need this capability > as some tests require validating logs were triggered. > Pydocs for common log methods > {code} > | grep_log(self, expr, filename='system.log', from_mark=None) > | Returns a list of lines matching the regular expression in parameter > | in the Cassandra log of this node > | > | grep_log_for_errors(self, filename='system.log') > | Returns a list of errors with stack traces > | in the Cassandra log of this node > | > | grep_log_for_errors_from(self, filename='system.log', seek_start=0) > {code} > {code} > | watch_log_for(self, exprs, from_mark=None, timeout=600, process=None, > verbose=False, filename='system.log') > | Watch the log until one or more (regular) expression are found. > | This methods when all the expressions have been found or the method > | timeouts (a TimeoutError is then raised). On successful completion, > | a list of pair (line matched, match object) is returned. > {code} > Below is a POC showing a way to do such logic > {code} > package org.apache.cassandra.distributed.test; > import java.io.BufferedReader; > import java.io.FileInputStream; > import java.io.IOException; > import java.io.InputStreamReader; > import java.io.UncheckedIOException; > import java.nio.charset.StandardCharsets; > import java.util.Iterator; > import java.util.Spliterator; > import java.util.Spliterators; > import java.util.regex.Matcher; > import java.util.regex.Pattern; > import java.util.stream.Stream; > import java.util.stream.StreamSupport; > import com.google.common.io.Closeables; > import org.junit.Test; > import org.apache.cassandra.distributed.Cluster; > import org.apache.cassandra.utils.AbstractIterator; > public class AllTheLogs extends TestBaseImpl > { > @Test > public void test() throws IOException > { > try (final Cluster cluster = init(Cluster.build(1).start())) > { > String tag = System.getProperty("cassandra.testtag", > "cassandra.testtag_IS_UNDEFINED"); > String suite = System.getProperty("suitename", > "suitename_IS_UNDEFINED"); > String log = String.format("build/test/logs/%s/TEST-%s.log", tag, > suite); > grep(log, "Enqueuing flush of tables").forEach(l -> > System.out.println("I found the thing: " + l)); > } > } > private static Stream<String> grep(String file, String regex) throws > IOException > { > return grep(file, Pattern.compile(regex)); > } > private static Stream<String> grep(String file, Pattern regex) throws > IOException > { > BufferedReader reader = new BufferedReader(new InputStreamReader(new > FileInputStream(file), StandardCharsets.UTF_8)); > Iterator<String> it = new AbstractIterator<String>() > { > protected String computeNext() > { > try > { > String s; > while ((s = reader.readLine()) != null) > { > Matcher m = regex.matcher(s); > if (m.find()) > return s; > } > reader.close(); > return endOfData(); > } > catch (IOException e) > { > Closeables.closeQuietly(reader); > throw new UncheckedIOException(e); > } > } > }; > return StreamSupport.stream(Spliterators.spliteratorUnknownSize(it, > Spliterator.ORDERED), false); > } > } > {code} > And > {code} > @Test > public void test() throws IOException > { > try (final Cluster cluster = init(Cluster.build(1).start())) > { > String tag = System.getProperty("cassandra.testtag", > "cassandra.testtag_IS_UNDEFINED"); > String suite = System.getProperty("suitename", > "suitename_IS_UNDEFINED"); > //TODO missing way to get node id > // cluster.get(1); > String log = > String.format("build/test/logs/%s/TEST-%s-node%d.log", tag, suite, 1); > grep(log, "Enqueuing flush of tables").forEach(l -> > System.out.println("I found the thing: " + l)); > } > } > {code} -- This message was sent by Atlassian Jira (v8.3.4#803005) --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org