The branch, master has been updated via ca5cc05b22b s3:script: Replace --merge by --merge-by-timestamp in samba-log-parser via 16386bfd4cd docs-xml:manpages: Fix tabs in samba-log-parser.1.xml from 6539f1e4cd6 s3:winbindd: Change the TALLOC_CTX to fix the tevent call depth tracking
https://git.samba.org/?p=samba.git;a=shortlog;h=master - Log ----------------------------------------------------------------- commit ca5cc05b22bf92dd7236d2c0d4a424340a0048f3 Author: Pavel Filipenský <pfilipen...@samba.org> Date: Wed Jun 28 14:59:29 2023 +0200 s3:script: Replace --merge by --merge-by-timestamp in samba-log-parser For --merge-by-timestamp the traces do not need to contain the traceid header field. Signed-off-by: Pavel Filipenský <pfilipen...@samba.org> Reviewed-by: Douglas Bagnall <douglas.bagn...@catalyst.net.nz> Autobuild-User(master): Pavel Filipensky <pfilipen...@samba.org> Autobuild-Date(master): Thu Jul 20 19:14:05 UTC 2023 on atb-devel-224 commit 16386bfd4cd10bbb4a5588973220212287a2f868 Author: Pavel Filipenský <pfilipen...@samba.org> Date: Wed Jun 28 16:51:35 2023 +0200 docs-xml:manpages: Fix tabs in samba-log-parser.1.xml Signed-off-by: Pavel Filipenský <pfilipen...@samba.org> Reviewed-by: Douglas Bagnall <douglas.bagn...@catalyst.net.nz> ----------------------------------------------------------------------- Summary of changes: docs-xml/manpages/samba-log-parser.1.xml | 43 ++++++++----- source3/script/samba-log-parser | 103 ++++++++++++++++++++++++------- 2 files changed, 106 insertions(+), 40 deletions(-) Changeset truncated at 500 lines: diff --git a/docs-xml/manpages/samba-log-parser.1.xml b/docs-xml/manpages/samba-log-parser.1.xml index ea6fd9150df..7a62cd10a47 100644 --- a/docs-xml/manpages/samba-log-parser.1.xml +++ b/docs-xml/manpages/samba-log-parser.1.xml @@ -20,7 +20,7 @@ options: --traceid ID specify the traceid of the trace records --pid PID specify the pid of winbind client --breakdown breakdown the traces into per traceid files - --merge merge logs by timestamp + --merge-by-timestamp merge logs by timestamp --flow show the request/sub-request flow traces --flow-compact show the request/sub-request flow traces without dcerpc details @@ -29,9 +29,9 @@ options: <command>samba-log-parser</command> <arg choice="req">path</arg> <arg choice="opt">--pid=PID</arg> - <arg choice="opt">--traceid=ID</arg> - <arg choice="opt">--breakdown</arg> - <arg choice="opt">--merge</arg> + <arg choice="opt">--traceid=ID</arg> + <arg choice="opt">--breakdown</arg> + <arg choice="opt">--merge-by-timestamp</arg> <arg choice="opt">--flow</arg> <arg choice="opt">--flow-compact</arg> <arg choice="opt">-h|--help</arg> @@ -70,31 +70,32 @@ options: <varlistentry> <term>--breakdown</term> - <listitem><para>Break down all traces to separate files in the current - working directory. For each traceid, three files are created: - traceid.full - traceid.flow - traceid.flowcompact + <listitem><para>Break down all traces to separate files in the current + working directory. For each traceid, three files are created: + traceid.full + traceid.flow + traceid.flowcompact </para></listitem> </varlistentry> <varlistentry> - <term>--merge</term> - <listitem><para>Sort the trace lines according to the timestamp. - </para></listitem> + <term>--merge-by-timestamp</term> + <listitem><para>Sort the trace lines according to the timestamp. + Works with log files without traceid header field as well. + </para></listitem> </varlistentry> <varlistentry> <term>--flow</term> - <listitem><para>Display the request/sub-request flow. + <listitem><para>Display the request/sub-request flow. </para></listitem> </varlistentry> <varlistentry> <term>--flow-compact</term> - <listitem><para>Display the request/sub-request flow without dcerpc - call details. + <listitem><para>Display the request/sub-request flow without dcerpc + call details. </para></listitem> </varlistentry> @@ -118,7 +119,7 @@ options: </para> <programlisting> - # samba-log-parser --pid 999999 --merge /var/log/samba + # samba-log-parser --pid 999999 --merge-by-timestamp /var/log/samba </programlisting> <para>Break down the traces into separate files according to traceid sorted @@ -126,7 +127,15 @@ options: </para> <programlisting> - # samba-log-parser --breakdown --merge /var/log/samba + # samba-log-parser --breakdown --merge-by-timestamp /var/log/samba + </programlisting> + + <para>Sort traces using the timestamp for log files found in the samba log + directory. Traces do not need to contain the traceid header field. + </para> + + <programlisting> + # samba-log-parser --merge-by-timestamp /var/log/samba </programlisting> </refsect1> diff --git a/source3/script/samba-log-parser b/source3/script/samba-log-parser index 1c1d2ced33d..a07dfdb4a6c 100755 --- a/source3/script/samba-log-parser +++ b/source3/script/samba-log-parser @@ -5,11 +5,12 @@ # A script to parse samba (especially winbind) logfiles. # Trace files should be in a non-syslog format (debug syslog format = no). # -# --traceid ... Specify the traceid of the request to parse -# --pid ... Specify the pid -# --merge ... Merge logs by timestamp -# --flow ... Show the request/sub-request call flow -# --flow-compact ... Show the request/sub-request call flow without dcerpc +# --traceid ... Specify the traceid of the request to parse +# --pid ... Specify the pid +# --breakdown ... Break to separate files per each traceid +# --merge-by-timestamp ... Merge logs by timestamp +# --flow ... Show the request/sub-request call flow +# --flow-compact ... Show the request/sub-request call flow without dcerpc # # # Copyright (c) 2023 Andreas Schneider <a...@samba.org> @@ -39,6 +40,16 @@ from argparse import ArgumentParser from collections import defaultdict # Trace record consists of a trace header followed by one or more text lines. +# +# This tool expects trace header format based on these smb.conf parameters: +# +# debug syslog format = no +# debug hires timestamp = yes +# winbind debug traceid = yes +# +# If 'winbind debug traceid = no' is set, then the option --merge-by-timestamp +# still can be used. +# # Each trace header contains a traceid, which is the main identifier for this # tool. A single traceid is either provided via command line option --traceid # or a list of traceids is derived from the PID specified via option --pid. @@ -56,6 +67,26 @@ from collections import defaultdict # memory if files are big. +def process_file_no_traceid(record_list, fname): + with open(fname, "r") as infile: + data = infile.readlines() + date = "" + record_lines = [] + + RE_HEADER_NO_TRACEID = re.compile( + r"^\[(\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}\.\d{6}).*") + for line in data: + header = RE_HEADER_NO_TRACEID.search(line) + if header: + # Append all previous trace lines of a record + if record_lines: + record_list.append((date, None, record_lines, fname)) + record_lines = [] + # Remember the new date + date = header.group(1) + record_lines.append(line) + + def process_file(record_list, traceid_set, fname, opid, otraceid): with open(fname, "r") as infile: data = infile.readlines() @@ -204,7 +235,7 @@ def setup_parser(): help="breakdown the traces into per traceid files" ) parser.add_argument( - "--merge", + "--merge-by-timestamp", action="store_true", dest="merge", default=False, @@ -235,11 +266,16 @@ def main(): # noqa parser = setup_parser() options = parser.parse_args() - if not options.traceid and not options.pid and not options.breakdown: - print("One of --traceid or --pid is needed or --breakdown.") + if (not options.traceid and not options.pid and not options.breakdown + and not options.merge): + print("One of --traceid or --pid is needed" + " or --breakdown or --merge-by-timestamp.") sys.exit(1) elif options.traceid and options.pid: - print("Only one of --traceid or --pid or --breakdown is allowed.") + print("Only one of --traceid or --pid is allowed.") + sys.exit(1) + elif options.breakdown and (options.traceid or options.pid): + print("--breakdown cannot be combined with --traceid and --pid.") sys.exit(1) if options.flow and not options.traceid: @@ -259,29 +295,50 @@ def main(): # noqa print("Path to logfile or directory with logs is needed.") sys.exit(1) + merge_with_no_traceid = (not options.traceid and not options.pid + and not options.breakdown) and options.merge + path = options.path if os.path.isdir(path): for root, dirs, files in os.walk(path): for name in files: - process_file( - record_list, - traceid_set, - os.path.join(root, name), - options.pid, - options.traceid, - ) + if merge_with_no_traceid: + process_file_no_traceid( + record_list, + os.path.join(root, name) + ) + else: + process_file( + record_list, + traceid_set, + os.path.join(root, name), + options.pid, + options.traceid, + ) elif os.path.isfile(path): - process_file( - record_list, - traceid_set, - path, - options.pid, - options.traceid - ) + if merge_with_no_traceid: + process_file_no_traceid( + record_list, + path + ) + else: + process_file( + record_list, + traceid_set, + path, + options.pid, + options.traceid + ) else: print(path, "Path is neither file or directory.") sys.exit(1) + # Sort only using timestamps, no use of traceid + if merge_with_no_traceid: + record_list.sort() + print_record_list(record_list, sys.stdout) + sys.exit(0) + # Keep only records with matching traceids if not options.breakdown: record_list = filter_traceids(record_list, traceid_set) -- Samba Shared Repository