plainolneesh commented on a change in pull request #338: URL: https://github.com/apache/fluo-muchos/pull/338#discussion_r465877411
########## File path: ansible/roles/logstash/templates/logstash-simple-2.conf ########## @@ -0,0 +1,150 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS,e +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +input { + beats { + port => 5044 + } + + file { + path => "/media/ephemeral0/logs/accumulo/*.log" + type => "accumulo" + start_position => "beginning" + } + + file { + path => "/media/ephemeral0/logs/hadoop/*.log" + type => "hadoop" + start_position => "beginning" + } + + file { + path => "/media/ephemeral0/logs/zookeeper/*.out" + type => "zookeeper" + start_position => "beginning" + } +} + +filter { + if [type] == "accumulo" { + + grok { + + #2020-03-30 14:01:06,322 [gc.GarbageCollectWriteAheadLogs] DEBUG: New tablet servers noticed: [worker3:9997[1000005c5390005], worker1:9997[1000005c5390004], worker2:9997[1000005c5390006]] + match => { "message" => "%{TIMESTAMP_ISO8601:TIMESTAMP} \[%{GREEDYDATA:module}\.%{GREEDYDATA:className\] %{LOGLEVEL:level}: %{GREEDYDATA:message}\[%{GREEDYDATA:tservername}:%{GREEDYDATA:tservername}\:%{GREEDYDATA:tservername}\:%{GREEDYDATA:hostname}\]"} + + #2020-06-18 14:27:15,793 [impl.MetricsConfig] INFO : Loaded properties from hadoop-metrics2-accumulo.properties + match => { "message" => "%{TIMESTAMP_ISO8601:TIMESTAMP} \[%{GREEDYDATA:module}\.%{GREEDYDATA:className}\] %{LOGLEVEL:level}: %{GREEDYDATA:message}\[%{GREEDYDATA:module}\]"} + + #2020-03-31 04:43:48,553 [gc.GarbageCollectWriteAheadLogs] DEBUG: New tablet servers noticed: [] + #2020-04-01 01:56:04,271 [gc.GarbageCollectWriteAheadLogs] DEBUG: Tablet servers removed: [] + #2020-03-30 14:00:35,898 [gc.SimpleGarbageCollector] DEBUG: Got GC ZooKeeper lock + #2020-03-30 14:03:39,048 [master.Master] DEBUG: [Root Table]: scan time 0.00 seconds + #2020-03-30 14:03:29,331 [master.Master] DEBUG: Finished gathering information from 3 of 3 servers in 0.02 seconds + match => { "message" => "%{TIMESTAMP_ISO8601:TIMESTAMP} \[%{GREEDYDATA:module}\.%{GREEDYDATA:className}\] %{LOGLEVEL:level}: %{GREEDYDATA:message}"} + + #2020-04-01 12:56:22,176 [state.ZooTabletStateStore] DEBUG: Returning root tablet state: +r<<@(null,worker1:9997[1000005c5390004],worker1:9997[1000005c5390004]) + #2020-03-30 14:03:39,048 [master.Master] DEBUG: Root Table location State: +r<<@(null,worker1:9997[1000005c5390004],worker1:9997[1000005c5390004]) + match => { "message" => "%{TIMESTAMP_ISO8601:TIMESTAMP} \[%{GREEDYDATA:module}\.%{GREEDYDATA:className}\] %{LOGLEVEL:level}: %{GREEDYDATA:message}\(%{GREEDYDATA:tservers}\)"} + + #2020-06-17 14:06:03,898 [sasl.SaslDataTransferClient] INFO : SASL encryption trust check: localHostTrusted = false, remoteHostTrusted = false + #2020-06-17 13:40:55,503 [compress.CodecPool] INFO : Got brand-new decompressor [.deflate] + #2020-06-17 13:40:19,188 [bcfile.Compression] INFO : Trying to load Lzo codec class: org.apache.hadoop.io.compress.LzoCodec + #2020-06-17 13:40:11,052 [lru.LruBlockCacheManager] INFO : Creating INDEX cache with configuration org.apache.accumulo.core.file.blockfile.cache.lru.LruBlockCacheConfiguration@22e49d8, acceptableFactor: 0.85, minFactor: 0.75, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, mapLoadFactor: 0.75, mapConcurrencyLevel: 16, useEvictionThread: true + #2020-03-30 14:01:06,223 [gc.SimpleGarbageCollector] INFO : Number of successfully deleted data files: 0 + #2020-06-17 13:40:11,212 [tserver.TabletServer] INFO : Tablet server starting on 0.0.0.0 + #2020-06-17 13:40:10,742 [server.ServerUtil] INFO : tserver.memory.manager = org.apache.accumulo.server.tabletserver.LargestFirstMemoryManager + #2020-06-17 13:40:09,222 [conf.SiteConfiguration] INFO : Found Accumulo configuration on classpath at /home/centos/install/accumulo-2.0.0/conf/accumulo.properties + #2020-06-24 14:15:58,727 [compact.CompactRange] INFO : No iterators or compaction strategy + match => {"message" => "%{TIMESTAMP_ISO8601:TIMESTAMP} \[%{GREEDYDATA:module}\.%{GREEDYDATA:className}] %{LOGLEVEL:level} : %{GREEDYDATA:message}"} + + #2020-06-24 11:10:00,094 [tracer.TraceServer] INFO : Instance a7f82873-7c32-4510-8560-665523d9b57f + match => {"message" => "%{TIMESTAMP_ISO8601:TIMESTAMP} \[%{GREEDYDATA:module}\.%{GREEDYDATA:className}\] %{LOGLEVEL:level} : %{GREEDYDATA:instance} %{GREEDYDATA:instanceID}"} + + #2020-06-24 11:09:59,929 [server.ServerUtil] INFO : instance.volumes = hdfs://leader1:8020/accumulo + match => {"message" => "%{TIMESTAMP_ISO8601:TIMESTAMP} \[%{GREEDYDATA:module}\.%{GREEDYDATA:className}\] %{LOGLEVEL:level} : %{GREEDYDATA:instance} %{URI:path}"} + + #2020-06-24 11:09:55,624 [conf.SiteConfiguration] INFO : Found Accumulo configuration on classpath at /home/centos/install/accumulo-2.0.0/conf/accumulo.properties + match => {"message" => "%{TIMESTAMP_ISO8601:TIMESTAMP} \[%{GREEDYDATA:module}\.%{GREEDYDATA:className}\] %{LOGLEVEL:level} : %{GREEDYDATA:message} %{PATH:path}"} + + #2020-06-24 15:06:05,196 [zookeeper.DistributedReadWriteLock] INFO : Added lock entry 0 userData 620e830c39101580 lockTpye READ + #2020-06-24 14:10:58,054 [tableOps.Utils] INFO : table +r (79bfe74ef98dc6c) locked for read operation: COMPACT + #2020-06-24 15:11:06,486 [tableOps.Utils] INFO : namespace +accumulo (24c7ba560088d55b) locked for read operation: COMPACT + match => {"message" => "%{TIMESTAMP_ISO8601:TIMESTAMP} \[%{GREEDYDATA:module}\.%{GREEDYDATA:className}\] %{LOGLEVEL:level} : %{GREEDYDATA:message} %{WORD:operation}"} + + + #2020-06-24 11:10:00,900 [security.SecurityOperation] INFO : Granted table permission READ for user root on the table 1 at the request of user !SYSTEM + #2020-06-24 11:10:00,926 [security.SecurityOperation] INFO : Granted table permission WRITE for user root on the table 1 at the request of user !SYSTEM + #2020-06-24 11:10:00,935 [security.SecurityOperation] INFO : Granted table permission BULK_IMPORT for user root on the table 1 at the request of user !SYSTEM + #2020-06-24 11:10:00,939 [security.SecurityOperation] INFO : Granted table permission ALTER_TABLE for user root on the table 1 at the request of user !SYSTEM + #2020-06-24 11:10:00,955 [security.SecurityOperation] INFO : Granted table permission GRANT for user root on the table 1 at the request of user !SYSTEM + #2020-06-24 11:10:00,962 [security.SecurityOperation] INFO : Granted table permission DROP_TABLE for user root on the table 1 at the request of user !SYSTEM + #2020-06-24 11:10:00,965 [security.SecurityOperation] INFO : Granted table permission GET_SUMMARIES for user root on the table 1 at the request of user !SYSTEM + #2020-06-24 11:10:00,989 [zookeeper.DistributedReadWriteLock] INFO : Added lock entry 0 userData 0665dc60a64ce2cd lockType WRITE + match => {"message" => "%{TIMESTAMP_ISO8601:TIMESTAMP} \[%{GREEDYDATA:module}\.%{GREEDYDATA:className}\] %{LOGLEVEL:level} : %{GREEDYDATA:message}%{WORD:operation}"} + + + + + + } + } + else if [type] == "hadoop"{ + grok { + #2020-03-30 15:01:17,801 INFO org.apache.hadoop.hdfs.StateChange: DIR* completeFile: /accumulo/tables/+r/root_tablet/F000003o.rf_tmp is closed by DFSClient_NONMAPREDUCE_69506301_11 + match => { "message" => "%{TIMESTAMP_ISO8601}%{SPACE}%{WORD:log level}%{GREEDYDATA:hostname}%{GREEDYDATA:LINK}:%{GREEDYDATA:state}%{LOGLEVEL:level}: %{GREEDYDATA:message}"} + match => { "message" => "%{TIMESTAMP_ISO8601}%{SPACE}%{WORD:log level}%{GREEDYDATA:ip}%{GREEDYDATA:LINK}:%{GREEDYDATA:state}%{LOGLEVEL:level}: %{GREEDYDATA:message}"} + + #2020-06-17 13:39:10,434 INFO org.apache.hadoop.yarn.logaggregation.AggregatedLogDeletionService: aggregated log deletion finished. + #2020-06-17 13:39:06,483 INFO org.apache.hadoop.mapreduce.v2.hs.JobHistoryServer: registered UNIX signal handlers for [TERM, HUP, INT] + #2020-06-17 13:39:07,064 INFO org.apache.hadoop.metrics2.impl.MetricsConfig: Loaded properties from hadoop-metrics2.properties + #2020-06-17 13:39:07,147 INFO org.apache.hadoop.metrics2.impl.MetricsSystemImpl: Scheduled Metric snapshot period at 10 second(s). + #2020-06-17 13:39:07,703 INFO org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils: Default file system [hdfs://leader1:8020] + match => {"message" => "%{TIMESTAMP_ISO8601:TIMESTAMP} %{LOGLEVEL:level} %{GREEDYDATA:className}: %{GREEDYDATA:message}"} + } +} + + else if [type] == "zookeeper" { + + grok { + #2020-06-15 17:15:15,275 [myid:] - INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181:NIOServerCnxnFactory@222] - Accepted socket connection from /172.31.66.109:55402 + match => { "message" => "%{TIMESTAMP_ISO8601:TIMESTAMP} \[%{GREEDYDATA:ID}:]\ - %{LOGLEVEL:level} \[%{GREEDYDATA:location}\]\ - %{GREEDYDATA:message}" } + + #2020-06-16 14:34:20,639 [myid:] - INFO [main:NIOServerCnxnFactory@89] - binding to port 0.0.0.0/0.0.0.0:2181 + #2020-06-16 14:34:20,623 [myid:] - INFO [main:Environment@100] - Server environment:java.compiler=<NA> + match => {"message" => "%{TIMESTAMP_ISO8601:TIMESTAMP} \[%{GREEDYDATA:ID}:]\ - %{LOGLEVEL:level} \[%{GREEDYDATA:method}:%{GREEDYDATA:className}\]\ \- %{GREEDYDATA:message}"} + + #2020-06-16 18:41:28,734 [myid:] - INFO [ProcessThread(sid:0 cport:2181)::PrepRequestProcessor@653] - Got user-level KeeperException when processing sessionid:0x1000007cddd000b type:create cxid:0xb7ef zxid:0x164a txntype:-1 reqpath:n/a Error Path:/accumulo/4d65522e-62cd-4540-9270-2272a3db2f85/masters/tick Error:KeeperErrorCode = NodeExists for /accumulo/4d65522e-62cd-4540-9270-2272a3db2f85/masters/tick + #2020-06-16 14:34:20,622 [myid:] - INFO [main:Environment@100] - Server environment:java.class.path=/home/centos/install/zookeeper-3.4.14/bin/../zookeeper-server/target/classes:/home/centos/install/zookeeper-3.4.14/bin/../build/classes:/home/centos/install/zookeeper-3.4.14/bin/../zookeeper-server/target/lib/*.jar:/home/centos/install/zookeeper-3.4.14/bin/../build/lib/*.jar:/home/centos/install/zookeeper-3.4.14/bin/../lib/slf4j-log4j12-1.7.25.jar:/home/centos/install/zookeeper-3.4.14/bin/../lib/slf4j-api-1.7.25.jar:/home/centos/install/zookeeper-3.4.14/bin/../lib/netty-3.10.6.Final.jar:/home/centos/install/zookeeper-3.4.14/bin/../lib/log4j-1.2.17.jar:/home/centos/install/zookeeper-3.4.14/bin/../lib/jline-0.9.94.jar:/home/centos/install/zookeeper-3.4.14/bin/../lib/audience-annotations-0.5.0.jar:/home/centos/install/zookeeper-3.4.14/bin/../zookeeper-3.4.14.jar:/home/centos/install/zookeeper-3.4.14/bin/../zookeeper-server/src/main/resources/lib/*.jar:/home/centos/install/zookee per-3.4.14/bin/../conf: + match => {"message" => "%{TIMESTAMP_ISO8601:TIMESTAMP} \[%{GREEDYDATA:ID}:]\ - %{LOGLEVEL:level} \[%{GREEDYDATA:method}:%{GREEDYDATA:className}\]\ \- %{GREEDYDATA:message}\:%{PATH}"} + + #2020-06-17 13:37:17,041 [myid:] - INFO [main:ZooKeeperServerMain@98] - Starting server + match => {"message" => "%{TIMESTAMP_ISO8601:TIMESTAMP} \[%{GREEDYDATA:ID}:]\ - %{LOGLEVEL:level} \[%{GREEDYDATA:method}:%{GREEDYDATA:className}\] - %{GREEDYDATA:message}"} + + match => {"message" => "%{TIMESTAMP_ISO8601:TIMESTAMP} \[%{GREEDYDATA:ID}:]\ - %{LOGLEVEL:level} \[%{GREEDYDATA:method}:%{GREEDYDATA:className}\]\ \- %{GREEDYDATA:message}\ /%{IPORHOST}"} + + + + } + + } + } +output { + elasticsearch { + hosts => ["http://leader1:9200";] Review comment: I hard-coded the hostname here `http://leader1:9200` because I use this section to set up the hostnames for Elasticsearch. But if you have any suggestions for change here, please don't hesitate to let me know. ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
