Github user nickwallen commented on a diff in the pull request: https://github.com/apache/metron/pull/1245#discussion_r237651771 --- Diff: metron-platform/metron-parsers/src/test/resources/config/RegularExpressionsInvalidParserConfig.json --- @@ -0,0 +1,208 @@ +{ + "convertCamelCaseToUnderScore": true, + "messageHeaderRegex": "(?<syslogpriority>(?<=^<)\\d{1,4}(?=>)).*?(?<timestampDeviceOriginal>(?<=>)[A-Za-z]{3}\\s{1,2}\\d{1,2}\\s\\d{1,2}:\\d{1,2}:\\d{1,2}(?=\\s)).*?(?<deviceName>(?<=\\s).*?(?=\\s))", + "recordTypeRegex": "(?<dstProcessName>(?<=\\s)\\b(tch-replicant|audispd|syslog|ntpd|sendmail|pure-ftpd|usermod|useradd|anacron|unix_chkpwd|sudo|dovecot|postfix\\/smtpd|postfix\\/smtp|postfix\\/qmgr|klnagent|systemd|(?i)crond(?-i)|clamd|kesl|sshd|run-parts|automount|suexec|freshclam|kernel|vsftpd|ftpd|su)\\b(?=\\[|:))", + "fields": [ + { + "recordType": "syslog", + "regex": ".*(?<dstProcessId>(?<=PID\\s=\\s).*?(?=\\sLine)).*(?<filePath>(?<=64\\s)\/([A-Za-z0-9_-]+\/)+(?=\\w))(?<fileName>.*?(?=\")).*(?<eventInfo>(?<=\").*?(?=$))" + }, + { + "recordType": "pure-ftpd", + "regex": ".*(?<srcUserId>(?<=\\:\\s\\().*?(?=\\)\\s)).*?(?<messageLevel>(?<=\\s\\[).*?(?=\\]\\s)).*?(?<eventInfo>(?<=\\]\\s).*?(?=$))" + }, + { + "recordType": "systemd", + "regex": [ + ".*(?<eventInfo>(?<=\\ssystemd\\:\\s).*?(?=\\d+)).*?(?<sessionName>(?<=\\sSession\\s).*?(?=\\sof)).*?(?<srcUserId>(?<=\\suser\\s).*?(?=\\.)).*$", + ".*(?<eventInfo>(?<=\\ssystemd\\:\\s).*?(?=\\sof)).*?(?<srcUserId>(?<=\\sof\\s).*?(?=\\.)).*$" + ] + }, + { + "recordType": "kesl", + "regex": ".*(?<eventInfo>(?<=\\:).*?(?=$))" + }, + { + "recordType": "dovecot", + "regex": [ + ".*(?<subprocess>(?<=\\sdovecot:\\s).*?(?=\\:)).*?(?<eventInfo>(?<=\\:).*?(?=\\:\\suser)).*?(?<srcUserId>(?<=user\\=\\<).*?(?=\\>)).*?(?<rip>(?<=rip\\=).*?(?=,)).*?(?<lip>(?<=lip\\=).*?(?=,)).*?(?<connectionType>(?<=,\\s).*?(?=,)).*?(?<sessionName>(?<=session\\=\\<).*?(?=\\>)).*$", + ".*(?<subprocess>(?<=\\sdovecot:\\s).*?(?=\\:)).*?(?<eventInfo>(?<=\\:).*?(?=\\:\\srip)).*?(?<rip>(?<=rip\\=).*?(?=,)).*?(?<lip>(?<=lip\\=).*?(?=,)).*?(?<connectionType>(?<=,\\s).*?(?=$))", + ".*(?<subprocess>(?<=\\sdovecot:\\s).*?(?=\\:)).*?(?<eventInfo>(?<=\\:).*?(?=$))" + ] + }, + { + "recordType": "postfix/smtpd", + "regex": [ + ".*(?<dstProcessId>(?<=\\[).*?(?=\\])).*?(?<eventInfo>(?<=\\:).*?(?=$))", + ".*(?<dstProcessId>(?<=\\[).*?(?=\\]:)).*?(?<eventInfo>(?<=\\:\\s)disconnect(?=\\sfrom)).*?(?<srcHost>(?<=from).*(?=\\[)).*?(?<ipSrcAddr>(?<=\\[).*(?=\\])).*$" + ] + }, + { + "recordType": "postfix/smtp", + "regex": [ + ".*(?<dstProcessId>(?<=smtp\\[).*?(?=\\]:)).*(?<toEmail>(?<=to=#\\<).*?(?=>,)).*(?<relay>(?<=relay=).*?(?=,)).*(?<delay>(?<=delay=).*?(?=,)).*(?<delays>(?<=delays=).*?(?=,)).*(?<dsn>(?<=dsn=).*?(?=,)).*(?<status>(?<=status=).*?(?=\\()).*?(?<dstHost>(?<=connect to).*?(?=\\[)).*?(?<ipDstAddr>(?<=\\[).*?(?=\\])).*?(?<ipDstPort>(?<=\\]:).*?(?=:\\s)).*?(?<eventInfo>(?<=:\\s).*?(?=$))", + ".*(?<dstProcessId>(?<=smtp\\[).*?(?=\\]:)).*?(?<dstHost>(?<=connect to).*?(?=\\[)).*?(?<ipDstAddr>(?<=\\[).*?(?=\\])).*(?<ipDstPort>(?<=:).*?(?=\\s)).*(?<eventInfo>(?<=\\s).*?(?=$))", + ".*(?<dstProcessId>(?<=\\[).*?(?=\\])).*?(?<eventInfo>(?<=\\:).*?(?=$))" + ] + }, + { + "recordType": "crond", + "regex": [ + ".*(?<dstProcessId>(?<=\\[).*?(?=\\])).*?(?<srcUserId>(?<=\\]:\\s\\().*?(?=\\)\\s)).*?(?<commandLine>(?<=CMD\\s\\().*?(?=\\))).*$", + ".*(?<dstProcessId>(?<=\\[).*?(?=\\])).*?(?<srcUserId>(?<=\\]:\\s\\().*?(?=\\)\\s)).*?(?<eventInfo>(?<=\\().*?(?=\\))).*$", + ".*(?<dstProcessId>(?<=\\[).*?(?=\\])).*?(?<srcUserId>(?<=\\]:\\s\\().*?(?=\\)\\s)).*?(?<commandLine>(?<=CMD\\s\\().*?(?=\\))).*$", + ".*(?<dstProcessId>(?<=\\[).*?(?=\\])).*?(?<eventInfo>(?<=\\:).*?(?=$))" + ] + }, + { + "recordType": "clamd", + "regex": [ + ".*(?<dstProcessId>(?<=\\[).*?(?=\\])).*?(?<subProcess>(?<=\\:\\s).*?(?=\\:)).*?(?<eventInfo>(?<=\\:).*?(?=$))", + ".*(?<subProcess>(?<=\\:\\s).*?(?=\\:)).*?(?<eventInfo>(?<=\\:).*?(?=$))" + ] + }, + { + "recordType": "run-parts", + "regex": ".*(?<eventInfo>(?<=\\sparts).*?(?=$))" + }, + { + "recordType": "sshd", + "regex": [ + ".*(?<dstProcessId>(?<=\\[).*?(?=\\])).*?(?<event_Info>(?<=\\]:\\s).*?(?=\\sfor)).*?(?<dstUserId>(?<=\\sfor\\s).*?(?=\\sfrom)).*?(?<ipSrcAddr>(?<=\\sfrom\\s).*?(?=\\sport)).*?(?<ipSrcPort>(?<=\\sport\\s).*?(?=\\s)).*?(?<appProtocol>(?<=port\\s\\d{1,5}\\s).*(?=:\\s)).*?(?<encryptionAlgorithm>(?<=:\\s).+?(?=\\s)).*(?<correlationId>(?<=\\s).+?(?=$))", + ".*(?<dstProcessId>(?<=\\[).*?(?=\\])).*?(?<eventInfo>(?<=\\]:\\s).*?(?=\\sfor)).*?(?<dstUserId>(?<=\\sfor\\s).*?(?=\\sfrom)).*?(?<ipSrcAddr>(?<=\\sfrom\\s).*?(?=\\sport)).*?(?<ipSrcPort>(?<=\\sport\\s).*?(?=\\s)).*?(?<appProtocol>(?<=port\\s\\d{1,5}\\s).*?(?=$))", + ".*(?<dstProcessId>(?<=\\[).*?(?=\\])).*?(?<ipDstAddr>(?<=Remote:).*?(?=\\-)).*?(?<ipDstPort>(?<=\\-).*?(?=;)).*?(?<appProtocol>(?<=Protocol:).*?(?=;)).*?(?<sshClient>(?<=Client:).*?(?=$))", + ".*(?<dstProcessId>(?<=\\[).*?(?=\\])).*?(?<appProtocol>(?<=\\]:).*?(?=:)).*?(?<ipDstAddr>(?<=Remote:).*?(?=\\-)).*?(?<ipDstPort>(?<=\\-).*?(?=;)).*?(?<encryptionAlgorithm>(?<=Enc:\\s).*?(?=$))", + ".*(?<dstProcessId>(?<=\\[).*?(?=\\])).*?(?<ipDstAddr>(?<=Remote:).*?(?=\\-)).*?(?<ipDstPort>(?<=\\-).*?(?=;)).*?(?<encryptionAlgorithm>(?<=Enc:\\s).*?(?=$))", + ".*(?<dstProcessId>(?<=\\[).*?(?=\\])).*?(?<eventInfo>(?<=:).*?(?=for)).*?(?<dstUserId>(?<=for).*?(?=from)).*?(?<ipSrcAddr>(?<=from).*?(?=port)).*?(?<ipSrcPort>(?<=port).*?(?=\\s)).*?(?<appProtocol>(?<=\\s).*?(?=$))", + ".*(?<dstProcessId>(?<=\\[).*?(?=\\]))]:\\s.*?(?<eventInfo>subsystem.*?(?=by\\suser)).*?(?<srcUserId>(?<=user).*?(?=$))", + ".*(?<dstProcessId>(?<=\\[).*?(?=\\])).*?(?<action>(?<=Received).*?(?=from)).*?(?<ipSrcAddr>(?<=from).*?(?=:)).*?(?<eventInfo>(?<=11:).*?(?=$))", + ".*(?<dstProcessId>(?<=\\[).*?(?=\\])).*?(?<eventInfo>(?<=\\]:\\s)Server\\slistening(?=\\s)).*?(?<ipSrcAddr>(?<=\\son\\s).*?(?=port)).*?(?<ipSrcPort>(?<=port\\s)\\d{1,6}(?=\\.)).*$", + ".*(?<dstProcessId>(?<=\\[).*?(?=\\])).*?(?<eventInfo>(?<=\\]:\\s)Invalid user(?=\\s)).*?(?<dstUserId>(?<=\\s).*?(?=from)).*?(?<ipSrcAddr>(?<=from\\s).*(?=$))", + ".*(?<dstProcessId>(?<=\\[).*?(?=]:\\s)).*(?<subProcess>(?<=]:\\s).*\\)(?=:)).*(?<eventInfo>(?<=:\\s).*(?=;)).*(?<logname>(?<=logname=).*?(?=\\s)).*(?<dstUserId>(?<=uid=).*?(?=\\s)).*(?<effectiveUserId>(?<=euid=).*?(?=\\s)).*(?<sessionName>(?<=tty=).*?(?=\\s)).*(?<srcUserId>(?<=ruser=).*?(?=\\s)).*(?<ipSrcAddr>(?<=rhost=).*?(?=\\s)).*(?<userId>(?<=user=).*?(?=$))", + ".*(?<dstProcessId>(?<=\\[).*?(?=]:\\s)).*(?<eventInfo>(?<=:\\s).*(?=;)).*(?<logname>(?<=logname=).*?(?=\\s)).*(?<dstUserId>(?<=uid=).*?(?=\\s)).*(?<effectiveUserId>(?<=euid=).*?(?=\\s)).*(?<sessionName>(?<=tty=).*?(?=\\s)).*(?<srcUserId>(?<=ruser=).*?(?=\\s)).*(?<ipSrcAddr>(?<=rhost=).*?(?=\\s)).*(?<userId>(?<=user=).*?(?=$))", + ".*(?<dstProcessId>(?<=\\[).*?(?=\\])).*?(?<eventInfo>(?<=\\]:\\s).*?(?=for)).*?(?<dstUserId>(?<=\\sfor).*?(?=\\[)).*?(?<subProcess>(?<=\\[).*?(?=\\])).*$", + ".*(?<dstProcessId>(?<=\\[).*?(?=\\])).*?(?<eventInfo>(?<=:\\s)Excess permission or bad ownership on file(?=\\s\\/)).*?(?<filePath>(?<=\\s).*(?=\\/)).*?(?<fileName>(?<=\\/).*(?=$))", + ".*(?<dstProcessId>(?<=\\[).*?(?=\\])).*?(?<eventInfo>(?<=:).*?(?=;)).*$", + ".*(?<dstProcessId>(?<=\\[).*?(?=\\])).*?(?<eventInfo>(?<=:).*?(?=\\d)).*$", + ".*(?<dstProcessId>(?<=\\[).*?(?=\\])).*?(?<eventInfo>(?<=:).*?(?=$))" --- End diff -- Help me understand why you need 17 different regular expressions to parse SSHD records? Is it just that you see it in 17 different forms?
---