2015-03-21 14:50 GMT+01:00 chenlin rao <rao.chen...@gmail.com>: > $MaxMessageSize 32m > module( load="imtcp" ) > module( load="imuxsock" ) > module( load="imklog" ) > module( load="mmfields" ) > module( load="mmjsonparse" ) > module( load="omelasticsearch" ) > $ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat > $RepeatedMsgReduction off > $WorkDirectory /data0/rsyslog > $MainMsgQueueFilename mainQ > $MainMsgQueueType LinkedList > $MainMsgQueueMaxFileSize 512M > $MainMsgQueueMaxDiskSpace 20G > $PreserveFQDN on > input(type="imtcp" port="514") > template(name="local6IndexName" type="list") { > constant(value="logstash-mweibo-") > property(name="timereported" dateFormat="rfc3339" position.from="1" > position.to="4") > constant(value=".") > property(name="timereported" dateFormat="rfc3339" position.from="6" > position.to="7") > constant(value=".") > property(name="timereported" dateFormat="rfc3339" position.from="9" > position.to="10") > } > template( name="local6TypeName" type="string" string="%programname%" ) > template( name="local6JsonArray" type="list" ) { > constant(value="{\"@timestamp\":\"") property(name="timereported" > dateFormat="rfc3339") > constant(value="\",\"host\":\"") property(name="hostname") > constant(value="\",") property(name="$.line" position.from="2") > } > Ruleset( name="forwardRuleSetJsonArray" ) > { > action ( type="mmjsonparse" > name="action_jsonarray-parse" > queue.size="3000" > queue.dequeuebatchsize="300" > queue.maxdiskspace="15G" > queue.checkpointinterval="10" > queue.type="linkedlist" > queue.workerthreads="30" > queue.workerthreadminimummessages="100" > queue.maxfilesize="500M" > queue.saveonshutdown="on" > )
running mmjsonparse on a queue (asynchronously!) does not make any sense. When running async, the json properties will never be seen by the rest of the ruleset. Maybe this even triggers the fault... Rainer > # maybe recieved a too large data being truncated > if ( re_match($msg, ']}$') ) then { > foreach ($.line in $!msg) do { > action ( > # test for other output modules > # type="omfile" file="/data/rsyslog-debug.log" > # type="omfwd" Target="127.0.0.1" Port="5140" Protocol="tcp" > type="omelasticsearch" server="es.domain.com" > dynSearchIndex="on" searchIndex="local6IndexName" dynSearchType="on" > searchType="local6TypeName" asyncrepl="on" bulkmode="on" > template="local6JsonArray" > name="action_json2log-es1003" > queue.filename="action_json2log-es1003" > queue.size="10000" > queue.dequeuebatchsize="2000" > queue.maxdiskspace="15G" > queue.discardseverity="3" > queue.checkpointinterval="10" > queue.type="linkedlist" > queue.workerthreads="5" > queue.workerthreadminimummessages="2000" > queue.maxfilesize="500M" > queue.saveonshutdown="on" > ) > } > stop > } > } > > if ( $programname == 'mobile_client_net_fatal_error' ) then > { > call forwardRuleSetJsonArray > stop > } > *.info;mail.none;authpriv.none;cron.none;local6.none;local7.none;user.none > /var/log/messages > authpriv.* /var/log/secure > mail.* /var/log/maillog > cron.* /var/log/cron > uucp,news.crit /var/log/spooler > > 2015-03-21 13:14 GMT+08:00 singh.janmejay <singh.janme...@gmail.com>: > >> Can you please share your config as well? >> >> Also, I'll likely be able to look at it only after 27th Mar. As of now I >> don't have access to a computer and github doesn't seem to show line >> numbers while browsing code using mobile phone. >> >> -- >> Regards, >> Janmejay >> >> PS: Please blame the typos in this mail on my phone's uncivilized soft >> keyboard sporting it's not-so-smart-assist technology. >> >> On Mar 21, 2015 8:49 AM, "chenlin rao" <rao.chen...@gmail.com> wrote: >> >> > I try to build rsyslogd from github master with "./configure >> --enable-debug >> > --enable-valgrind --enable-memcheck --enable-elasticsearch >> > --enable-mmjsonparse --enable-mmsequence --enable-mmfields >> > --disable-liblogging-stdlog --enable-omruleset" >> > >> > then process exit with: >> > >> > 7778.301430428:main Q[DA]:Reg/w0: ACTION 16 >> > [builtin:omfwd:action(type="builtin:omfwd" ...)] >> > 7778.301676909:main Q[DA]:Reg/w0: executing action 16 >> > 7778.301791807:main Q[DA]:Reg/w0: Called action, logging to builtin:omfwd >> > ==20833== Thread 2: >> > ==20833== Invalid read of size 4 >> > ==20833== at 0x3A044093A0: pthread_mutex_lock (in /lib64/ >> > libpthread-2.12.so) >> > ==20833== by 0x438053: qqueueEnqMsg (queue.c:2856) >> > ==20833== by 0x4421AB: doSubmitToActionQ (action.c:1403) >> > ==20833== by 0x439D6C: scriptExec (ruleset.c:197) >> > ==20833== by 0x439ECC: scriptExec (ruleset.c:284) >> > ==20833== by 0x439CD9: scriptExec (ruleset.c:416) >> > ==20833== by 0x43A40B: processBatch (ruleset.c:503) >> > ==20833== by 0x44A403: msgConsumer (rsyslogd.c:575) >> > ==20833== by 0x438D12: ConsumerReg (queue.c:1897) >> > ==20833== by 0x43314B: wtiWorker (wti.c:334) >> > ==20833== by 0x431BA6: wtpWorker (wtp.c:389) >> > ==20833== by 0x3A044079D0: start_thread (in /lib64/libpthread-2.12.so >> ) >> > ==20833== Address 0x10 is not stack'd, malloc'd or (recently) free'd >> > ==20833== >> > ==20833== >> > ==20833== Process terminating with default action of signal 11 (SIGSEGV) >> > ==20833== Access not within mapped region at address 0x10 >> > ==20833== at 0x3A044093A0: pthread_mutex_lock (in /lib64/ >> > libpthread-2.12.so) >> > ==20833== by 0x438053: qqueueEnqMsg (queue.c:2856) >> > ==20833== by 0x4421AB: doSubmitToActionQ (action.c:1403) >> > ==20833== by 0x439D6C: scriptExec (ruleset.c:197) >> > ==20833== by 0x439ECC: scriptExec (ruleset.c:284) >> > ==20833== by 0x439CD9: scriptExec (ruleset.c:416) >> > ==20833== by 0x43A40B: processBatch (ruleset.c:503) >> > ==20833== by 0x44A403: msgConsumer (rsyslogd.c:575) >> > ==20833== by 0x438D12: ConsumerReg (queue.c:1897) >> > ==20833== by 0x43314B: wtiWorker (wti.c:334) >> > ==20833== by 0x431BA6: wtpWorker (wtp.c:389) >> > ==20833== by 0x3A044079D0: start_thread (in /lib64/libpthread-2.12.so >> ) >> > ==20833== If you believe this happened as a result of a stack >> > ==20833== overflow in your program's main thread (unlikely but >> > ==20833== possible), you can try to increase the size of the >> > ==20833== main thread stack using the --main-stacksize= flag. >> > ==20833== The main thread stack size used in this run was 10485760. >> > ==20833== >> > ==20833== HEAP SUMMARY: >> > ==20833== in use at exit: 44,605,326 bytes in 5,019 blocks >> > ==20833== total heap usage: 10,310 allocs, 5,291 frees, 78,621,963 >> bytes >> > allocated >> > ==20833== >> > ==20833== LEAK SUMMARY: >> > ==20833== definitely lost: 516 bytes in 12 blocks >> > ==20833== indirectly lost: 0 bytes in 0 blocks >> > ==20833== possibly lost: 33,559,553 bytes in 17 blocks >> > ==20833== still reachable: 11,045,257 bytes in 4,990 blocks >> > ==20833== suppressed: 0 bytes in 0 blocks >> > ==20833== Rerun with --leak-check=full to see details of leaked memory >> > ==20833== >> > ==20833== For counts of detected and suppressed errors, rerun with: -v >> > ==20833== Use --track-origins=yes to see where uninitialised values come >> > from >> > ==20833== ERROR SUMMARY: 11 errors from 3 contexts (suppressed: 100 from >> 9) >> > 已杀死 >> > >> > 2015-03-20 23:29 GMT+08:00 singh.janmejay <singh.janme...@gmail.com>: >> > >> > > Can you please build with debug symbols and repeat the valgrind run? >> > > >> > > -- >> > > Regards, >> > > Janmejay >> > > >> > > PS: Please blame the typos in this mail on my phone's uncivilized soft >> > > keyboard sporting it's not-so-smart-assist technology. >> > > >> > > On Mar 20, 2015 6:03 PM, "chenlin rao" <rao.chen...@gmail.com> wrote: >> > > >> > > > btw: if I change omelasticsearch/omfwd to omfile, rsyslogd would be >> > > fine... >> > > > >> > > > 2015-03-20 20:13 GMT+08:00 chenlin rao <rao.chen...@gmail.com>: >> > > > >> > > > > 3498.767218405:main Q[DA]:Reg/w0: rainerscript: var 200:!msg: '[ { >> > > "uid": >> > > > > "1941604034", "request_header": >> > > "{\"Accept-Encoding\":\"gzip,deflate\"}", >> > > > > "network_type": "wifi", "end_time": "1426836307406", "dns_ip": >> > > > > "218.15.203.34,192.168.1.1", "response_code": "200", >> "response_data": >> > > > "{}", >> > > > > "start_time": "1426836307328", "act": "net_fatal_error", "type": >> > > > > "net_fatal_error", "request_url": "https:\/\/api.weibo.cn >> > > > >> > > >> > >> \/2\/client\/url_list?accuracy_level=0&c=android&i=9eef7ba&s=0088f6a4&ua=Xiaomi-MI%202S__weibo__5.0.0__android__android4.4.4&wm=5311_4002&v_f=2&from=1050095010&gsid=4uzH00573m6YZU5oVhYra896c0y&lang=zh_CN&skin=default&oldwm=5311_4002", >> > > > > "response_status_line": "HTTP\/1.1 200 OK" }, { "uid": >> "1941604034", >> > > > > "request_header": "{\"Accept-Encoding\":\"gzip,deflate\"}", >> > > > "network_type": >> > > > > "wifi", "end_time": "1426836320563", "dns_ip": >> > > > "218.15.203.34,192.168.1.1", >> > > > > "response_code": "200", "response_data": "{}", "start_time": >> > > > > "1426836320505", "act": "net_fatal_error", "type": >> "net_fatal_error", >> > > > > "request_url": "https:\/\/api.weibo.cn >> > > > >> > > >> > >> \/2\/client\/url_list?accuracy_level=0&c=android&i=9eef7ba&s=0088f6a4&ua=Xiaomi-MI%202S__weibo__5.0.0__android__android4.4.4&wm=5311_4002&v_f=2&from=1050095010&gsid=4uzH00573m6YZU5oVhYra896c0y&lang=zh_CN&skin=default&oldwm=5311_4002", >> > > > > "response_status_line": "HTTP\/1.1 200 OK" } ]' >> > > > > 3498.767338611:main Q[DA]:Reg/w0: eval expr 0x62e63a0, return >> > datatype >> > > > 'J' >> > > > > 3498.770732236:main Q[DA]:Reg/w0: ACTION 17 >> > > > > [builtin:omfwd:action(type="builtin:omfwd" ...)] >> > > > > 3498.770963455:main Q[DA]:Reg/w0: executing action 17 >> > > > > 3498.771052511:main Q[DA]:Reg/w0: Called action, logging to >> > > builtin:omfwd >> > > > > ==30138== Thread 3: >> > > > > ==30138== Invalid read of size 4 >> > > > > ==30138== at 0x4E3E3A0: pthread_mutex_lock (in /lib64/ >> > > > > libpthread-2.12.so) >> > > > > ==30138== by 0x14228D: qqueueEnqMsg (in /sbin/rsyslogd) >> > > > > ==30138== by 0x14BB5B: ??? (in /sbin/rsyslogd) >> > > > > ==30138== by 0x143C4C: ??? (in /sbin/rsyslogd) >> > > > > ==30138== by 0x143DAC: ??? (in /sbin/rsyslogd) >> > > > > ==30138== by 0x143BB9: ??? (in /sbin/rsyslogd) >> > > > > ==30138== by 0x1442CB: ??? (in /sbin/rsyslogd) >> > > > > ==30138== by 0x15336B: ??? (in /sbin/rsyslogd) >> > > > > ==30138== by 0x143112: ??? (in /sbin/rsyslogd) >> > > > > ==30138== by 0x13D671: wtiWorker (in /sbin/rsyslogd) >> > > > > ==30138== by 0x13D1C1: ??? (in /sbin/rsyslogd) >> > > > > ==30138== by 0x4E3C9D0: start_thread (in /lib64/ >> > libpthread-2.12.so) >> > > > > ==30138== Address 0x10 is not stack'd, malloc'd or (recently) >> free'd >> > > > > ==30138== >> > > > > ==30138== >> > > > > ==30138== Process terminating with default action of signal 11 >> > > (SIGSEGV) >> > > > > ==30138== Access not within mapped region at address 0x10 >> > > > > ==30138== at 0x4E3E3A0: pthread_mutex_lock (in /lib64/ >> > > > > libpthread-2.12.so) >> > > > > ==30138== by 0x14228D: qqueueEnqMsg (in /sbin/rsyslogd) >> > > > > ==30138== by 0x14BB5B: ??? (in /sbin/rsyslogd) >> > > > > ==30138== by 0x143C4C: ??? (in /sbin/rsyslogd) >> > > > > ==30138== by 0x143DAC: ??? (in /sbin/rsyslogd) >> > > > > ==30138== by 0x143BB9: ??? (in /sbin/rsyslogd) >> > > > > ==30138== by 0x1442CB: ??? (in /sbin/rsyslogd) >> > > > > ==30138== by 0x15336B: ??? (in /sbin/rsyslogd) >> > > > > ==30138== by 0x143112: ??? (in /sbin/rsyslogd) >> > > > > ==30138== by 0x13D671: wtiWorker (in /sbin/rsyslogd) >> > > > > ==30138== by 0x13D1C1: ??? (in /sbin/rsyslogd) >> > > > > ==30138== by 0x4E3C9D0: start_thread (in /lib64/ >> > libpthread-2.12.so) >> > > > > ==30138== If you believe this happened as a result of a stack >> > > > > ==30138== overflow in your program's main thread (unlikely but >> > > > > ==30138== possible), you can try to increase the size of the >> > > > > ==30138== main thread stack using the --main-stacksize= flag. >> > > > > ==30138== The main thread stack size used in this run was >> 10485760. >> > > > > ==30138== >> > > > > ==30138== HEAP SUMMARY: >> > > > > ==30138== in use at exit: 1,078,127,378 bytes in 291,997 blocks >> > > > > ==30138== total heap usage: 614,872 allocs, 322,875 frees, >> > > > 1,586,744,205 >> > > > > bytes allocated >> > > > > ==30138== >> > > > > ==30138== LEAK SUMMARY: >> > > > > ==30138== definitely lost: 33,555,027 bytes in 13 blocks >> > > > > ==30138== indirectly lost: 0 bytes in 0 blocks >> > > > > ==30138== possibly lost: 5,760 bytes in 18 blocks >> > > > > ==30138== still reachable: 1,044,566,591 bytes in 291,966 blocks >> > > > > ==30138== suppressed: 0 bytes in 0 blocks >> > > > > ==30138== Rerun with --leak-check=full to see details of leaked >> > memory >> > > > > ==30138== >> > > > > ==30138== For counts of detected and suppressed errors, rerun with: >> > -v >> > > > > ==30138== Use --track-origins=yes to see where uninitialised values >> > > come >> > > > > from >> > > > > ==30138== ERROR SUMMARY: 1809 errors from 5 contexts (suppressed: >> 131 >> > > > from >> > > > > 9) >> > > > > Killed >> > > > > >> > > > > >> > > > > 2015-03-20 8:50 GMT+08:00 singh.janmejay <singh.janme...@gmail.com >> >: >> > > > > >> > > > >> Can you please run this with valgrind and share its output on >> crash? >> > > > >> >> > > > >> -- >> > > > >> Regards, >> > > > >> Janmejay >> > > > >> >> > > > >> PS: Please blame the typos in this mail on my phone's uncivilized >> > soft >> > > > >> keyboard sporting it's not-so-smart-assist technology. >> > > > >> >> > > > >> On Mar 19, 2015 11:10 PM, "chenlin rao" <rao.chen...@gmail.com> >> > > wrote: >> > > > >> >> > > > >> > Hello everyone. >> > > > >> > >> > > > >> > I just learnt a foreach syntax from >> > > `src/tests/json_array_looping.sh`, >> > > > >> so I >> > > > >> > try to parse my logdata(yes, long json array in msg) as follow: >> > > > >> > >> > > > >> > ``` >> > > > >> > >> > > > >> > $MaxMessageSize 256k >> > > > >> > >> > > > >> > template( name="local6JsonArray" type="string" >> > string="%$.line%\n" ) >> > > > >> > >> > > > >> > Ruleset( name="forwardRuleSetJsonArray" ) >> > > > >> > >> > > > >> > { >> > > > >> > >> > > > >> > action( type="mmjsonparse" ) >> > > > >> > >> > > > >> > foreach ($.line in $!msg) do { >> > > > >> > >> > > > >> > action ( type="omfile" file="/data0/logfile" >> > > > >> > template="local6JsonArray") >> > > > >> > } >> > > > >> > } >> > > > >> > ``` >> > > > >> > >> > > > >> > But I always got segment fault after restarted few minutes. >> > > > >> > >> > > > >> > Before the rsyslogd died , I can watch some correct lines in the >> > > > >> > "/data0/logfile". >> > > > >> > >> > > > >> > While running `rsyslogd -dn`, I watch the last five lines were: >> > > > >> > >> > > > >> > ``` >> > > > >> > >> > > > >> > 3295.820218609:main Q:Reg/w0 : FOREACH .line IN >> > > > >> > >> > > > >> > 3295.820227632:main Q:Reg/w0 : var '!msg' >> > > > >> > >> > > > >> > 3295.820240109:main Q:Reg/w0 : eval expr 0x7fe2044fefa0, type >> > > 'V[86]' >> > > > >> > >> > > > >> > 3295.821747011:main Q:Reg/w0 : rainerscript: var 200:!msg: >> > > '{"msg":[{ >> > > > >> > "end_time":"142...longmsghere....\"verieval expr 0x7fe2044fefa0, >> > > > return >> > > > >> > datatype 'J' >> > > > >> > ``` >> > > > >> > >> > > > >> > The length of such line is about 20k~30k, far away below my >> > > > >> > $MaxMessageSize. >> > > > >> > >> > > > >> > So, why rsyslogd segment fault? >> > > > >> > _______________________________________________ >> > > > >> > rsyslog mailing list >> > > > >> > http://lists.adiscon.net/mailman/listinfo/rsyslog >> > > > >> > http://www.rsyslog.com/professional-services/ >> > > > >> > What's up with rsyslog? Follow https://twitter.com/rgerhards >> > > > >> > NOTE WELL: This is a PUBLIC mailing list, posts are ARCHIVED by >> a >> > > > myriad >> > > > >> > of sites beyond our control. PLEASE UNSUBSCRIBE and DO NOT POST >> if >> > > you >> > > > >> > DON'T LIKE THAT. >> > > > >> > >> > > > >> _______________________________________________ >> > > > >> rsyslog mailing list >> > > > >> http://lists.adiscon.net/mailman/listinfo/rsyslog >> > > > >> http://www.rsyslog.com/professional-services/ >> > > > >> What's up with rsyslog? Follow https://twitter.com/rgerhards >> > > > >> NOTE WELL: This is a PUBLIC mailing list, posts are ARCHIVED by a >> > > myriad >> > > > >> of sites beyond our control. PLEASE UNSUBSCRIBE and DO NOT POST if >> > you >> > > > >> DON'T LIKE THAT. >> > > > >> >> > > > > >> > > > > >> > > > _______________________________________________ >> > > > rsyslog mailing list >> > > > http://lists.adiscon.net/mailman/listinfo/rsyslog >> > > > http://www.rsyslog.com/professional-services/ >> > > > What's up with rsyslog? Follow https://twitter.com/rgerhards >> > > > NOTE WELL: This is a PUBLIC mailing list, posts are ARCHIVED by a >> > myriad >> > > > of sites beyond our control. PLEASE UNSUBSCRIBE and DO NOT POST if >> you >> > > > DON'T LIKE THAT. >> > > > >> > > _______________________________________________ >> > > rsyslog mailing list >> > > http://lists.adiscon.net/mailman/listinfo/rsyslog >> > > http://www.rsyslog.com/professional-services/ >> > > What's up with rsyslog? Follow https://twitter.com/rgerhards >> > > NOTE WELL: This is a PUBLIC mailing list, posts are ARCHIVED by a >> myriad >> > > of sites beyond our control. PLEASE UNSUBSCRIBE and DO NOT POST if you >> > > DON'T LIKE THAT. >> > > >> > _______________________________________________ >> > rsyslog mailing list >> > http://lists.adiscon.net/mailman/listinfo/rsyslog >> > http://www.rsyslog.com/professional-services/ >> > What's up with rsyslog? Follow https://twitter.com/rgerhards >> > NOTE WELL: This is a PUBLIC mailing list, posts are ARCHIVED by a myriad >> > of sites beyond our control. PLEASE UNSUBSCRIBE and DO NOT POST if you >> > DON'T LIKE THAT. >> _______________________________________________ >> rsyslog mailing list >> http://lists.adiscon.net/mailman/listinfo/rsyslog >> http://www.rsyslog.com/professional-services/ >> What's up with rsyslog? Follow https://twitter.com/rgerhards >> NOTE WELL: This is a PUBLIC mailing list, posts are ARCHIVED by a myriad >> of sites beyond our control. PLEASE UNSUBSCRIBE and DO NOT POST if you >> DON'T LIKE THAT. >> > _______________________________________________ > rsyslog mailing list > http://lists.adiscon.net/mailman/listinfo/rsyslog > http://www.rsyslog.com/professional-services/ > What's up with rsyslog? Follow https://twitter.com/rgerhards > NOTE WELL: This is a PUBLIC mailing list, posts are ARCHIVED by a myriad of > sites beyond our control. PLEASE UNSUBSCRIBE and DO NOT POST if you DON'T > LIKE THAT. _______________________________________________ rsyslog mailing list http://lists.adiscon.net/mailman/listinfo/rsyslog http://www.rsyslog.com/professional-services/ What's up with rsyslog? Follow https://twitter.com/rgerhards NOTE WELL: This is a PUBLIC mailing list, posts are ARCHIVED by a myriad of sites beyond our control. PLEASE UNSUBSCRIBE and DO NOT POST if you DON'T LIKE THAT.