[ https://issues.apache.org/jira/browse/CASSANDRA-16120?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17193384#comment-17193384 ]
David Capwell commented on CASSANDRA-16120: ------------------------------------------- Sent out PR for dtest API, each Cassandra branch would have a FileLogAction which looks like the following {code} package org.apache.cassandra.distributed.impl; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.io.RandomAccessFile; import java.io.UncheckedIOException; import java.util.Objects; import java.util.function.Predicate; import com.google.common.io.Closeables; import org.apache.cassandra.utils.AbstractIterator; public class FileLogAction implements LogAction { private final File file; public FileLogAction(File file) { this.file = Objects.requireNonNull(file); } @Override public long mark() { return file.length(); } @Override public LineIterator matching(long startPosition, Predicate<String> fn) { RandomAccessFile reader; try { reader = new RandomAccessFile(file, "r"); } catch (FileNotFoundException e) { // if file isn't present, don't return an empty stream as it looks the same as no log lines matched throw new UncheckedIOException(e); } if (startPosition > 0) // -1 used to disable, so ignore any negative values or 0 (default offset) { try { reader.seek(startPosition); } catch (IOException e) { throw new UncheckedIOException("Unable to seek to " + startPosition, e); } } return new FileLineIterator(reader, fn); } private static final class FileLineIterator extends AbstractIterator<String> implements LineIterator { private final RandomAccessFile reader; private final Predicate<String> fn; private FileLineIterator(RandomAccessFile reader, Predicate<String> fn) { this.reader = reader; this.fn = fn; } @Override public long mark() { try { return reader.getFilePointer(); } catch (IOException e) { throw new UncheckedIOException(e); } } @Override protected String computeNext() { try { String s; while ((s = reader.readLine()) != null) { if (fn.test(s)) return s; } close(); return endOfData(); } catch (IOException e) { close(); throw new UncheckedIOException(e); } } @Override public void close() { try { Closeables.close(reader, true); } catch (IOException impossible) { throw new AssertionError(impossible); } } } } {code} The main reason this lives in cassandra is org.apache.cassandra.utils.AbstractIterator is used, if I fork it I can move this class into the api. In my testing I updated JVMDTestTest with the following test {code} @Test public void instanceLogs() throws IOException, TimeoutException { try (Cluster cluster = init(Cluster.build(2).withConfig(c -> c.with(Feature.values())).start())) { logs(cluster.get(1)).grep("^DEBUG").forEach(s -> System.out.println("######## " + s)); LogAction logs = logs(cluster.get(2)); long mark = logs.mark(); cluster.get(2).runOnInstance(() -> { CassandraDaemon.uncaughtException(Thread.currentThread(), new RuntimeException("fail without fail")); }); List<String> errors = logs.watchFor(mark, "^ERROR"); Assertions.assertThat(errors).isNotEmpty(); } } {code} I will send cassandra branch tomorrow morning... > Add ability for jvm-dtest to grep instance logs > ----------------------------------------------- > > Key: CASSANDRA-16120 > URL: https://issues.apache.org/jira/browse/CASSANDRA-16120 > Project: Cassandra > Issue Type: Improvement > Components: Test/dtest/java > Reporter: David Capwell > Assignee: David Capwell > Priority: Normal > Labels: pull-request-available > Fix For: 4.0-beta > > > One of the main gaps between python dtest and jvm dtest is python dtest > supports the ability to grep the logs of an instance; we need this capability > as some tests require validating logs were triggered. > Pydocs for common log methods > {code} > | grep_log(self, expr, filename='system.log', from_mark=None) > | Returns a list of lines matching the regular expression in parameter > | in the Cassandra log of this node > | > | grep_log_for_errors(self, filename='system.log') > | Returns a list of errors with stack traces > | in the Cassandra log of this node > | > | grep_log_for_errors_from(self, filename='system.log', seek_start=0) > {code} > {code} > | watch_log_for(self, exprs, from_mark=None, timeout=600, process=None, > verbose=False, filename='system.log') > | Watch the log until one or more (regular) expression are found. > | This methods when all the expressions have been found or the method > | timeouts (a TimeoutError is then raised). On successful completion, > | a list of pair (line matched, match object) is returned. > {code} > Below is a POC showing a way to do such logic > {code} > package org.apache.cassandra.distributed.test; > import java.io.BufferedReader; > import java.io.FileInputStream; > import java.io.IOException; > import java.io.InputStreamReader; > import java.io.UncheckedIOException; > import java.nio.charset.StandardCharsets; > import java.util.Iterator; > import java.util.Spliterator; > import java.util.Spliterators; > import java.util.regex.Matcher; > import java.util.regex.Pattern; > import java.util.stream.Stream; > import java.util.stream.StreamSupport; > import com.google.common.io.Closeables; > import org.junit.Test; > import org.apache.cassandra.distributed.Cluster; > import org.apache.cassandra.utils.AbstractIterator; > public class AllTheLogs extends TestBaseImpl > { > @Test > public void test() throws IOException > { > try (final Cluster cluster = init(Cluster.build(1).start())) > { > String tag = System.getProperty("cassandra.testtag", > "cassandra.testtag_IS_UNDEFINED"); > String suite = System.getProperty("suitename", > "suitename_IS_UNDEFINED"); > String log = String.format("build/test/logs/%s/TEST-%s.log", tag, > suite); > grep(log, "Enqueuing flush of tables").forEach(l -> > System.out.println("I found the thing: " + l)); > } > } > private static Stream<String> grep(String file, String regex) throws > IOException > { > return grep(file, Pattern.compile(regex)); > } > private static Stream<String> grep(String file, Pattern regex) throws > IOException > { > BufferedReader reader = new BufferedReader(new InputStreamReader(new > FileInputStream(file), StandardCharsets.UTF_8)); > Iterator<String> it = new AbstractIterator<String>() > { > protected String computeNext() > { > try > { > String s; > while ((s = reader.readLine()) != null) > { > Matcher m = regex.matcher(s); > if (m.find()) > return s; > } > reader.close(); > return endOfData(); > } > catch (IOException e) > { > Closeables.closeQuietly(reader); > throw new UncheckedIOException(e); > } > } > }; > return StreamSupport.stream(Spliterators.spliteratorUnknownSize(it, > Spliterator.ORDERED), false); > } > } > {code} > And > {code} > @Test > public void test() throws IOException > { > try (final Cluster cluster = init(Cluster.build(1).start())) > { > String tag = System.getProperty("cassandra.testtag", > "cassandra.testtag_IS_UNDEFINED"); > String suite = System.getProperty("suitename", > "suitename_IS_UNDEFINED"); > //TODO missing way to get node id > // cluster.get(1); > String log = > String.format("build/test/logs/%s/TEST-%s-node%d.log", tag, suite, 1); > grep(log, "Enqueuing flush of tables").forEach(l -> > System.out.println("I found the thing: " + l)); > } > } > {code} -- This message was sent by Atlassian Jira (v8.3.4#803005) --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org