This is an automated email from the ASF dual-hosted git repository. marcuse pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/trunk by this push: new 7d138e2 Add a new tool to dump audit logs 7d138e2 is described below commit 7d138e20ea987d44fffbc47de4674b253b7431ff Author: Vinay Chella <vinaykumar...@gmail.com> AuthorDate: Mon Dec 17 23:42:14 2018 -0800 Add a new tool to dump audit logs Patch by Vinay Chella; reviewed by marcuse for CASSANDRA-14885 --- CHANGES.txt | 1 + doc/source/operating/audit_logging.rst | 27 ++- .../org/apache/cassandra/audit/BinAuditLogger.java | 14 +- .../org/apache/cassandra/tools/AuditLogViewer.java | 212 +++++++++++++++++++++ .../apache/cassandra/tools/AuditLogViewerTest.java | 84 ++++++++ tools/bin/auditlogviewer | 49 +++++ tools/bin/auditlogviewer.bat | 41 ++++ 7 files changed, 423 insertions(+), 5 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index ec59fa6..a0c51f0 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 4.0 + * Add a new tool to dump audit logs (CASSANDRA-14885) * Fix generating javadoc with Java11 (CASSANDRA-14988) * Only cancel conflicting compactions when starting anticompactions and sub range compactions (CASSANDRA-14935) * Use a stub IndexRegistry for non-daemon use cases (CASSANDRA-14938) diff --git a/doc/source/operating/audit_logging.rst b/doc/source/operating/audit_logging.rst index 6cfd141..068209e 100644 --- a/doc/source/operating/audit_logging.rst +++ b/doc/source/operating/audit_logging.rst @@ -143,19 +143,44 @@ NodeTool command to reload AuditLog filters ``enableauditlog``: NodeTool enableauditlog command can be used to reload auditlog filters when called with default or previous ``loggername`` and updated filters E.g., + :: nodetool enableauditlog --loggername <Default/ existing loggerName> --included-keyspaces <New Filter values> +View the contents of AuditLog Files +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +``auditlogviewer`` is the new tool introduced to help view the contents of binlog file in human readable text format. + +:: + auditlogviewer <path1> [<path2>...<pathN>] [options] + +Options +"""""""" +``-f,--follow`` + Upon reacahing the end of the log continue indefinitely + waiting for more records +``-r,--roll_cycle`` + How often to roll the log file was rolled. May be + necessary for Chronicle to correctly parse file names. (MINUTELY, HOURLY, + DAILY). Default HOURLY. +``-h,--help`` + display this help message +For example, to dump the contents of audit log files on the console + +:: + + auditlogviewer /logs/cassandra/audit Sample output -^^^^^^^^^^^^^^^^ +""""""""""""" + :: LogMessage: user:anonymous|host:localhost/X.X.X.X|source:/X.X.X.X|port:60878|timestamp:1521158923615|type:USE_KS|category:DDL|ks:dev1|operation:USE "dev1" diff --git a/src/java/org/apache/cassandra/audit/BinAuditLogger.java b/src/java/org/apache/cassandra/audit/BinAuditLogger.java index bd3a158..23b9977 100644 --- a/src/java/org/apache/cassandra/audit/BinAuditLogger.java +++ b/src/java/org/apache/cassandra/audit/BinAuditLogger.java @@ -19,6 +19,7 @@ package org.apache.cassandra.audit; import java.nio.file.Paths; +import com.google.common.annotations.VisibleForTesting; import com.google.common.primitives.Ints; import net.openhft.chronicle.wire.WireOut; @@ -29,6 +30,10 @@ import org.apache.cassandra.utils.concurrent.WeightedQueue; public class BinAuditLogger extends BinLogAuditLogger implements IAuditLogger { + public static final String TYPE = "type"; + public static final String AUDITLOG_TYPE = "AuditLog"; + public static final String AUDITLOG_MESSAGE = "message"; + public BinAuditLogger() { // due to the way that IAuditLogger instance are created in AuditLogManager, via reflection, we can't assume @@ -56,11 +61,12 @@ public class BinAuditLogger extends BinLogAuditLogger implements IAuditLogger super.logRecord(new Message(auditLogEntry.getLogString()), binLog); } - static class Message extends BinLog.ReleaseableWriteMarshallable implements WeightedQueue.Weighable + @VisibleForTesting + public static class Message extends BinLog.ReleaseableWriteMarshallable implements WeightedQueue.Weighable { private final String message; - Message(String message) + public Message(String message) { this.message = message; } @@ -68,8 +74,8 @@ public class BinAuditLogger extends BinLogAuditLogger implements IAuditLogger @Override public void writeMarshallable(WireOut wire) { - wire.write("type").text("AuditLog"); - wire.write("message").text(message); + wire.write(TYPE).text(AUDITLOG_TYPE); + wire.write(AUDITLOG_MESSAGE).text(message); } @Override diff --git a/src/java/org/apache/cassandra/tools/AuditLogViewer.java b/src/java/org/apache/cassandra/tools/AuditLogViewer.java new file mode 100644 index 0000000..01ea7b3 --- /dev/null +++ b/src/java/org/apache/cassandra/tools/AuditLogViewer.java @@ -0,0 +1,212 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.tools; + +import java.io.File; +import java.util.Arrays; +import java.util.List; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.GnuParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; + +import net.openhft.chronicle.core.io.IORuntimeException; +import net.openhft.chronicle.queue.ChronicleQueueBuilder; +import net.openhft.chronicle.queue.ExcerptTailer; +import net.openhft.chronicle.queue.RollCycles; +import net.openhft.chronicle.queue.impl.single.SingleChronicleQueue; +import net.openhft.chronicle.threads.Pauser; +import net.openhft.chronicle.wire.ReadMarshallable; +import net.openhft.chronicle.wire.WireIn; +import org.apache.cassandra.audit.BinAuditLogger; + +/** + * Tool to view the contenst of AuditLog files in human readable format. Default implementation for AuditLog files + * logs audit messages in {@link org.apache.cassandra.utils.binlog.BinLog} format, this tool prints the contens of + * binary audit log files in text format. + */ +public class AuditLogViewer +{ + private static final String TOOL_NAME = "auditlogviewer"; + private static final String ROLL_CYCLE = "roll_cycle"; + private static final String FOLLOW = "follow"; + private static final String HELP_OPTION = "help"; + + public static void main(String[] args) + { + AuditLogViewerOptions options = AuditLogViewerOptions.parseArgs(args); + + try + { + dump(options.pathList, options.rollCycle, options.follow, System.out::print); + } + catch (Exception e) + { + System.err.println(e.getMessage()); + System.exit(1); + } + } + + static void dump(List<String> pathList, String rollCycle, boolean follow, Consumer<String> displayFun) + { + //Backoff strategy for spinning on the queue, not aggressive at all as this doesn't need to be low latency + Pauser pauser = Pauser.millis(100); + List<ExcerptTailer> tailers = pathList.stream() + .distinct() + .map(path -> ChronicleQueueBuilder.single(new File(path)).readOnly(true).rollCycle(RollCycles.valueOf(rollCycle)).build()) + .map(SingleChronicleQueue::createTailer) + .collect(Collectors.toList()); + boolean hadWork = true; + while (hadWork) + { + hadWork = false; + for (ExcerptTailer tailer : tailers) + { + while (tailer.readDocument(new DisplayRecord(displayFun))) + { + hadWork = true; + } + } + + if (follow) + { + if (!hadWork) + { + //Chronicle queue doesn't support blocking so use this backoff strategy + pauser.pause(); + } + //Don't terminate the loop even if there wasn't work + hadWork = true; + } + } + } + + private static class DisplayRecord implements ReadMarshallable + { + private final Consumer<String> displayFun; + + DisplayRecord(Consumer<String> displayFun) + { + this.displayFun = displayFun; + } + + public void readMarshallable(WireIn wireIn) throws IORuntimeException + { + StringBuilder sb = new StringBuilder(); + + String type = wireIn.read(BinAuditLogger.TYPE).text(); + sb.append("Type: ") + .append(type) + .append(System.lineSeparator()); + + if (null != type && type.equals(BinAuditLogger.AUDITLOG_TYPE)) + { + sb.append("LogMessage: ").append(wireIn.read(BinAuditLogger.AUDITLOG_MESSAGE).text()).append(System.lineSeparator()); + } + + displayFun.accept(sb.toString()); + } + } + + private static class AuditLogViewerOptions + { + private final List<String> pathList; + private String rollCycle = "HOURLY"; + private boolean follow; + + private AuditLogViewerOptions(String[] pathList) + { + this.pathList = Arrays.asList(pathList); + } + + static AuditLogViewerOptions parseArgs(String cmdArgs[]) + { + CommandLineParser parser = new GnuParser(); + Options options = getCmdLineOptions(); + try + { + CommandLine cmd = parser.parse(options, cmdArgs, false); + + if (cmd.hasOption(HELP_OPTION)) + { + printUsage(options); + System.exit(0); + } + + String[] args = cmd.getArgs(); + if (args.length <= 0) + { + System.err.println("Audit log files directory path is a required argument."); + printUsage(options); + System.exit(1); + } + + AuditLogViewerOptions opts = new AuditLogViewerOptions(args); + + opts.follow = cmd.hasOption(FOLLOW); + + if (cmd.hasOption(ROLL_CYCLE)) + { + opts.rollCycle = cmd.getOptionValue(ROLL_CYCLE); + } + + return opts; + } + catch (ParseException e) + { + errorMsg(e.getMessage(), options); + return null; + } + } + + static void errorMsg(String msg, Options options) + { + System.err.println(msg); + printUsage(options); + System.exit(1); + } + + static Options getCmdLineOptions() + { + Options options = new Options(); + + options.addOption(new Option("r", ROLL_CYCLE, false, "How often to roll the log file was rolled. May be necessary for Chronicle to correctly parse file names. (MINUTELY, HOURLY, DAILY). Default HOURLY.")); + options.addOption(new Option("f", FOLLOW, false, "Upon reacahing the end of the log continue indefinitely waiting for more records")); + options.addOption(new Option("h", HELP_OPTION, false, "display this help message")); + + return options; + } + + static void printUsage(Options options) + { + String usage = String.format("%s <path1> [<path2>...<pathN>] [options]", TOOL_NAME); + StringBuilder header = new StringBuilder(); + header.append("--\n"); + header.append("View the audit log contents in human readable format"); + header.append("\n--\n"); + header.append("Options are:"); + new HelpFormatter().printHelp(usage, header.toString(), options, ""); + } + } +} diff --git a/test/unit/org/apache/cassandra/tools/AuditLogViewerTest.java b/test/unit/org/apache/cassandra/tools/AuditLogViewerTest.java new file mode 100644 index 0000000..078f899 --- /dev/null +++ b/test/unit/org/apache/cassandra/tools/AuditLogViewerTest.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.tools; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +import com.google.common.collect.ImmutableList; +import org.apache.commons.io.FileUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import net.openhft.chronicle.queue.ChronicleQueue; +import net.openhft.chronicle.queue.ChronicleQueueBuilder; +import net.openhft.chronicle.queue.ExcerptAppender; +import net.openhft.chronicle.queue.RollCycles; +import org.apache.cassandra.audit.BinAuditLogger; + +public class AuditLogViewerTest +{ + private Path path; + + @Before + public void setUp() throws IOException + { + path = Files.createTempDirectory("foo"); + } + + @After + public void tearDown() throws IOException + { + if (path.toFile().exists() && path.toFile().isDirectory()) + { + //Deletes directory and all of it's contents + FileUtils.deleteDirectory(path.toFile()); + } + } + + @Test + public void testDisplayRecord() + { + List<String> records = new ArrayList<>(); + records.add("Test foo bar 1"); + records.add("Test foo bar 2"); + + try (ChronicleQueue queue = ChronicleQueueBuilder.single(path.toFile()).rollCycle(RollCycles.TEST_SECONDLY).build()) + { + ExcerptAppender appender = queue.acquireAppender(); + + //Write bunch of records + records.forEach(s -> appender.writeDocument(new BinAuditLogger.Message(s))); + + //Read those written records + List<String> actualRecords = new ArrayList<>(); + AuditLogViewer.dump(ImmutableList.of(path.toString()), RollCycles.TEST_SECONDLY.toString(), false, actualRecords::add); + + for (int i = 0; i < records.size(); i++) + { + Assert.assertTrue(actualRecords.get(i).contains(records.get(i))); + } + } + } +} \ No newline at end of file diff --git a/tools/bin/auditlogviewer b/tools/bin/auditlogviewer new file mode 100755 index 0000000..a6a7375 --- /dev/null +++ b/tools/bin/auditlogviewer @@ -0,0 +1,49 @@ +#!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ "x$CASSANDRA_INCLUDE" = "x" ]; then + # Locations (in order) to use when searching for an include file. + for include in "`dirname "$0"`/cassandra.in.sh" \ + "$HOME/.cassandra.in.sh" \ + /usr/share/cassandra/cassandra.in.sh \ + /usr/local/share/cassandra/cassandra.in.sh \ + /opt/cassandra/cassandra.in.sh; do + if [ -r "$include" ]; then + . "$include" + break + fi + done +elif [ -r "$CASSANDRA_INCLUDE" ]; then + . "$CASSANDRA_INCLUDE" +fi + +if [ -z "$CLASSPATH" ]; then + echo "You must set the CLASSPATH var" >&2 + exit 1 +fi + +if [ "x$MAX_HEAP_SIZE" = "x" ]; then + MAX_HEAP_SIZE="256M" +fi + +"$JAVA" $JAVA_AGENT -ea -cp "$CLASSPATH" $JVM_OPTS -Xmx$MAX_HEAP_SIZE \ + -Dcassandra.storagedir="$cassandra_storagedir" \ + -Dlogback.configurationFile=logback-tools.xml \ + org.apache.cassandra.tools.AuditLogViewer "$@" + +# vi:ai sw=4 ts=4 tw=0 et diff --git a/tools/bin/auditlogviewer.bat b/tools/bin/auditlogviewer.bat new file mode 100644 index 0000000..3b6bd81 --- /dev/null +++ b/tools/bin/auditlogviewer.bat @@ -0,0 +1,41 @@ +@REM +@REM Licensed to the Apache Software Foundation (ASF) under one or more +@REM contributor license agreements. See the NOTICE file distributed with +@REM this work for additional information regarding copyright ownership. +@REM The ASF licenses this file to You under the Apache License, Version 2.0 +@REM (the "License"); you may not use this file except in compliance with +@REM the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, software +@REM distributed under the License is distributed on an "AS IS" BASIS, +@REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@REM See the License for the specific language governing permissions and +@REM limitations under the License. + +@echo off +if "%OS%" == "Windows_NT" setlocal + +pushd "%~dp0" +call cassandra.in.bat + +if NOT DEFINED CASSANDRA_MAIN set CASSANDRA_MAIN=org.apache.cassandra.tools.AuditLogViewer +if NOT DEFINED JAVA_HOME goto :err + +REM ***** JAVA options ***** +set JAVA_OPTS=^ + -Dlogback.configurationFile=logback-tools.xml + +set TOOLS_PARAMS= + +"%JAVA_HOME%\bin\java" %JAVA_OPTS% %CASSANDRA_PARAMS% -cp %CASSANDRA_CLASSPATH% "%CASSANDRA_MAIN%" %* +goto finally + +:err +echo JAVA_HOME environment variable must be set! +pause + +:finally + +ENDLOCAL --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org