Hi, I have two graylog servers clustered. One is configured as the master with full configuration the other as bakend-server. I upgraded both virtual appliances from 2.1.2 to 2.2.0. Before the upgrade everything was working fine. Now i have multiple errors and no logs show when searching.
Every 20 seconds the page will reload and for a second and "server currently unavailable" page will show. The pages are blank under /system/indices, or streams, or alerts. But some like dashboards, or sources, or input work fine <https://lh3.googleusercontent.com/-cEiCJZcIXsw/WKdaVAoFtKI/AAAAAAAAM-E/eB8dmHFn7ew-gEw-00AQ59c3PLWyFkmpACLcB/s1600/server_currently_unavailble.png> I get the following errors in the logs. I'm not sure what changed with the new version, but it used to parse this logs with no problem. 2017-02-17_19:58:39.81255 [3053]: index [graylog_447], type [message], id [fa52e365-f54a-11e6-8af1-005056a7396f], message [MapperParsingException[failed to parse [EventDate]]; nested: IllegalArgumentException[Invalid format: "2017/02/17" is malformed at "/02/17"];] payloadSize=156, timestamp=2017-02-17T20:08:41.486Z, remoteAddress=/1.1.1.1:1030} on input <57239495e765a00aa151081e>. 2017-02-17_20:31:14.33021 2017-02-17 12:31:14,329 ERROR: org.graylog2.shared.buffers.processors.DecodingProcessor - Error processing message RawMessage{id=e08a52e1-f54c-11e6-9231-005056a7396f, journalOffset=9857804159, codec=syslog, payloadSize=156, timestamp=2017-02-17T20:08:41.486Z, remoteAddress=/10.4.1.110:1030} 2017-02-17_20:31:14.33105 java.lang.IllegalArgumentException: Invalid format: "19293274:" is malformed at ":" 2017-02-17_20:31:14.33584 at org.joda.time.format.DateTimeFormatter.parseDateTime(DateTimeFormatter.java:945) ~[graylog.jar:?] 2017-02-17_20:31:14.33727 at org.joda.time.DateTime.parse(DateTime.java:160) ~[graylog.jar:?] 2017-02-17_20:31:14.33762 at org.joda.time.DateTime.parse(DateTime.java:149) ~[graylog.jar:?] 2017-02-17_20:31:14.33811 at org.graylog2.syslog4j.server.impl.event.SyslogServerEvent.parseDate(SyslogServerEvent.java:108) ~[graylog.jar:?] 2017-02-17_20:31:14.33955 at org.graylog2.syslog4j.server.impl.event.SyslogServerEvent.parsePriority(SyslogServerEvent.java:136) ~[graylog.jar:?] 2017-02-17_20:31:14.34209 at org.graylog2.syslog4j.server.impl.event.SyslogServerEvent.parse(SyslogServerEvent.java:152) ~[graylog.jar:?] 2017-02-17_20:31:14.34211 at org.graylog2.syslog4j.server.impl.event.SyslogServerEvent.<init>(SyslogServerEvent.java:50) ~[graylog.jar:?] 2017-02-17_20:31:14.34212 at org.graylog2.inputs.codecs.SyslogCodec.parse(SyslogCodec.java:123) ~[graylog.jar:?] 2017-02-17_20:31:14.34398 at org.graylog2.inputs.codecs.SyslogCodec.decode(SyslogCodec.java:91) ~[graylog.jar:?] 2017-02-17_20:31:14.34595 at org.graylog2.shared.buffers.processors.DecodingProcessor.processMessage(DecodingProcessor.java:146) ~[graylog.jar:?] 2017-02-17_20:31:14.34625 at org.graylog2.shared.buffers.processors.DecodingProcessor.onEvent(DecodingProcessor.java:87) [graylog.jar:?] 2017-02-17_20:31:14.34929 at org.graylog2.shared.buffers.processors.ProcessBufferProcessor.onEvent(ProcessBufferProcessor.java:79) [graylog.jar:?] 2017-02-17_20:31:14.34963 at org.graylog2.shared.buffers.processors.ProcessBufferProcessor.onEvent(ProcessBufferProcessor.java:45) [graylog.jar:?] 2017-02-17_20:31:14.35012 at com.lmax.disruptor.WorkProcessor.run(WorkProcessor.java:143) [graylog.jar:?] 2017-02-17_20:31:14.35134 at com.codahale.metrics.InstrumentedThreadFactory$InstrumentedRunnable.run(InstrumentedThreadFactory.java:66) [graylog.jar:?] 2017-02-17_20:31:14.35179 at java.lang.Thread.run(Thread.java:745) [?:1.8.0_101] /elasticsearch/current 17_20:33:58.10920 [2017-02-17 12:33:57,437][DEBUG][action.bulk ] [Morg] [graylog_447][2] failed to execute bulk item (index) index {[graylog_deflector][message][79384092-f54f-11e6-969d-005056a71aa5], source[{"RepeatCount":"1","EventDate":"2017/02/17","gl2_remote_ip":"2.3.60.12","gl2_remote_port":43149,"IngressInterface":"ethernet1/3","source":"source","gl2_source_input":"57eafbb1e765a0322da6254e","DestinationPort":"161","Bytes":187,"SessionEndReason":"aged-out","SourceZone":"untrust","PktsSent":1,"YEAR":"2017","gl2_source_node":"33a8a3ac-4bd2-4295-889b-eea9ced9c321","MINUTE":"55","NATSourceIP":"0.0.0.0","DestinationLocation":"10.0.0.0-10.255.255.255","NATDestinationPort":"0","PktsReceived":1,"RuleName":"GL-VW-Rule-Inbound","MONTHNUM":"02","level":6,"ConfigVersion":"1","IPV4":"0.0.0.0","streams":["000000000000000000000001"],"Sequence":"6216921628","LogForwardingProfile":"Log Profile","SerialNumber":"001801032530","EventTime":"11:55:25","LoggedTime":"11:55:25","BytesSent":93,"ActionFlags":"0x0","DestinationZone":"trust","Domain":"1","Application":"snmpv2","SessionID":"177745","Subtype":"end","MONTHDAY":"17","NATSourcePort":"0","SourceLocation":"some city","VirtualSystem":"vsys1","Action":"allow","Category":"any","HOUR":"11","ElapsedTime":"0","SourcePort":"35988","SourceIP":"2.3.100.4","EgressInterface":"ethernet1/4","Packets":2,"Padding":"0","Protocol":"udp","timestamp":"2017-02-17 19:55:26.000","LoggedDate":"2017/02/17","NATDestinationIP":"0.0.0.0","Flags":"0x19","message":"source 1,2017/02/17 11:55:25,001801032530,TRAFFIC,end,1,2017/02/17 11:55:25,2.3.100.4,10.4.1.94,0.0.0.0,0.0.0.0,bl-VW-Rule-Inbound,,,snmpv2,vsys1,untrust,trust,ethernet1/3,ethernet1/1,Log Profile,2017/02/17 11:55:25,177745,1,35988,161,0,0,0x19,udp,allow,187,93,94,2,2017/02/17 11:54:55,0,any,0,6216921628,0x0,some palce,10.0.0.0-10.255.255.255,0,1,1,aged-out,12,0,0,0,vsys1,source,from-policy","BytesReceived":94,"DestinationIP":"10.4.1.94","Type":"TRAFFIC","HOSTNAME":"source","SECOND":"25","facility":"local4"}]} 2017-02-17_20:33:58.11106 MapperParsingException[failed to parse [EventDate]]; nested: IllegalArgumentException[Invalid format: "2017/02/17" is malformed at "/02/17"]; 2017-02-17_20:33:58.11142 at org.elasticsearch.index.mapper.FieldMapper.parse(FieldMapper.java:329) 2017-02-17_20:33:58.11201 at org.elasticsearch.index.mapper.DocumentParser.parseObjectOrField(DocumentParser.java:311) 2017-02-17_20:33:58.11295 at org.elasticsearch.index.mapper.DocumentParser.parseAndMergeUpdate(DocumentParser.java:740) 2017-02-17_20:33:58.12648 at org.elasticsearch.index.mapper.DocumentParser.parseDynamicValue(DocumentParser.java:627) 2017-02-17_20:33:58.12682 at org.elasticsearch.index.mapper.DocumentParser.parseValue(DocumentParser.java:444) 2017-02-17_20:33:58.12758 at org.elasticsearch.index.mapper.DocumentParser.parseObject(DocumentParser.java:264) 2017-02-17_20:33:58.13058 at org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:124) 2017-02-17_20:33:58.13127 at org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:309) 2017-02-17_20:33:58.13458 at org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:584) 2017-02-17_20:33:58.13581 at org.elasticsearch.index.shard.IndexShard.prepareIndexOnPrimary(IndexShard.java:563) 2017-02-17_20:33:58.13616 at org.elasticsearch.action.index.TransportIndexAction.prepareIndexOperationOnPrimary(TransportIndexAction.java:211) 2017-02-17_20:33:58.13674 at org.elasticsearch.action.index.TransportIndexAction.executeIndexRequestOnPrimary(TransportIndexAction.java:223) 2017-02-17_20:33:58.13777 at org.elasticsearch.action.bulk.TransportShardBulkAction.shardIndexOperation(TransportShardBulkAction.java:327) 2017-02-17_20:33:58.13889 at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:120) 2017-02-17_20:33:58.13936 at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:68) 2017-02-17_20:33:58.14061 at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryPhase.doRun(TransportReplicationAction.java:657) 2017-02-17_20:33:58.14095 at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) 2017-02-17_20:33:58.14165 at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:287) 2017-02-17_20:33:58.14258 at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:279) 2017-02-17_20:33:58.14318 at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77) 2017-02-17_20:33:58.14556 at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:376) 2017-02-17_20:33:58.14607 at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) 2017-02-17_20:33:58.14738 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) 2017-02-17_20:33:58.14791 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) 2017-02-17_20:33:58.15194 at java.lang.Thread.run(Thread.java:745) 2017-02-17_20:33:58.15250 Caused by: java.lang.IllegalArgumentException: Invalid format: "2017/02/17" is malformed at "/02/17" 2017-02-17_20:33:58.15372 at org.joda.time.format.DateTimeParserBucket.doParseMillis(DateTimeParserBucket.java:187) 2017-02-17_20:33:58.15438 at org.joda.time.format.DateTimeFormatter.parseMillis(DateTimeFormatter.java:826) 2017-02-17_20:33:58.16123 at org.elasticsearch.index.mapper.core.DateFieldMapper$DateFieldType.parseStringValue(DateFieldMapper.java:366) 2017-02-17_20:33:58.16173 at org.elasticsearch.index.mapper.core.DateFieldMapper.innerParseCreateField(DateFieldMapper.java:534) 2017-02-17_20:33:58.16472 at org.elasticsearch.index.mapper.core.NumberFieldMapper.parseCreateField(NumberFieldMapper.java:241) 2017-02-17_20:33:58.16538 at org.elasticsearch.index.mapper.FieldMapper.parse(FieldMapper.java:321) 2017-02-17_20:33:58.16713 ... 24 more -- You received this message because you are subscribed to the Google Groups "Graylog Users" group. To unsubscribe from this group and stop receiving emails from it, send an email to graylog2+unsubscr...@googlegroups.com. To view this discussion on the web visit https://groups.google.com/d/msgid/graylog2/8bdbd89e-03bb-4a70-8393-233bf06543e7%40googlegroups.com. For more options, visit https://groups.google.com/d/optout.
