Repository: ambari Updated Branches: refs/heads/trunk 5845d5463 -> f0d970b5b
AMBARI-21144 Create .md files to describe Log Search input configurations (mgergely) Change-Id: I632de5e66607119548ba083fdfed31d39a4e085f Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f0d970b5 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f0d970b5 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f0d970b5 Branch: refs/heads/trunk Commit: f0d970b5b93a40c2e0830d5eaf3039c81ada2290 Parents: 5845d54 Author: Miklos Gergely <mgerg...@hortonworks.com> Authored: Wed May 31 09:10:00 2017 +0200 Committer: Miklos Gergely <mgerg...@hortonworks.com> Committed: Wed May 31 09:10:00 2017 +0200 ---------------------------------------------------------------------- .../api/model/inputconfig/InputDescriptor.java | 2 - .../inputconfig/impl/InputDescriptorImpl.java | 12 - .../inputconfig/impl/MapDateDescriptorImpl.java | 2 +- .../ambari-logsearch-logfeeder/README.md | 17 +- .../ambari-logsearch-logfeeder/docs/filter.md | 59 ++++ .../ambari-logsearch-logfeeder/docs/input.md | 63 ++++ .../docs/inputConfig.md | 29 ++ .../docs/postMapValues.md | 61 ++++ .../logfeeder/input/AbstractInputFile.java | 9 +- .../ambari/logfeeder/mapper/MapperDate.java | 2 +- .../ambari/logfeeder/input/InputFileTest.java | 1 - .../resources/samples/config/config_audit.json | 339 +++++++++---------- .../samples/config/config_service.json | 79 +++-- .../logsearch/model/common/LSServerInput.java | 8 - .../logsearch/model/common/LSServerMapDate.java | 2 +- .../logfeeder/shipper-conf/global.config.json | 5 +- .../package/templates/global.config.json.j2 | 3 +- 17 files changed, 442 insertions(+), 251 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java index c41da93..82e9504 100644 --- a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java +++ b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java @@ -38,8 +38,6 @@ public interface InputDescriptor { Boolean isUseEventMd5AsId(); - String getStartPosition(); - Boolean isCacheEnabled(); String getCacheKeyField(); http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputDescriptorImpl.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputDescriptorImpl.java index 94dcc2a..54b4b9b 100644 --- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputDescriptorImpl.java +++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputDescriptorImpl.java @@ -53,10 +53,6 @@ public abstract class InputDescriptorImpl implements InputDescriptor { @Expose @SerializedName("use_event_md5_as_id") private Boolean useEventMd5AsId; - - @Expose - @SerializedName("start_position") - private String startPosition; @Expose @SerializedName("cache_enabled") @@ -146,14 +142,6 @@ public abstract class InputDescriptorImpl implements InputDescriptor { this.useEventMd5AsId = useEventMd5AsId; } - public String getStartPosition() { - return startPosition; - } - - public void setStartPosition(String startPosition) { - this.startPosition = startPosition; - } - public Boolean isCacheEnabled() { return cacheEnabled; } http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapDateDescriptorImpl.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapDateDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapDateDescriptorImpl.java index 9daad2b..2e54e7a 100644 --- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapDateDescriptorImpl.java +++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapDateDescriptorImpl.java @@ -31,7 +31,7 @@ public class MapDateDescriptorImpl implements MapDateDescriptor { } @Expose - @SerializedName("source_date_pattern") + @SerializedName("src_date_pattern") private String sourceDatePattern; @Expose http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-logsearch/ambari-logsearch-logfeeder/README.md ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/README.md b/ambari-logsearch/ambari-logsearch-logfeeder/README.md index d00eb57..c5c6848 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/README.md +++ b/ambari-logsearch/ambari-logsearch-logfeeder/README.md @@ -17,9 +17,10 @@ limitations under the License. {% endcomment %} --> -#LogSearch - LogFeeder: +#Log Search - Log Feeder: -Logfeeder is a tool that reads log, parses it and stores it in Apache Solr for analyising purpose. +Log Feeder is a component of the Log Search service that reads logs, parses them and stores them in Apache Solr for the purpose +of later analysis. #Compilation mvn clean compile package @@ -34,3 +35,15 @@ mkdir /opt/logfeeder cd /opt/logfeeder tar xfz ~/logsearch-logfeeder.tar.gz ``` + +#Input Configuration + +The configuration for the log feeder contains +* description of the log files +* description of the filters that parse the data of the log entries +* description of the mappers that modify the parsed fields + +The element description can be found [here](docs/inputConfig.md) + +All these data are stored in json files, which should be named in the directory /etc/ambari-logsearch-logfeeder/conf, and the +name of the files should be input.config-<service\_name>.json \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-logsearch/ambari-logsearch-logfeeder/docs/filter.md ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/docs/filter.md b/ambari-logsearch/ambari-logsearch-logfeeder/docs/filter.md new file mode 100644 index 0000000..42c3271 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-logfeeder/docs/filter.md @@ -0,0 +1,59 @@ +<!-- +{% comment %} +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to you under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +{% endcomment %} +--> + +#Filter + +The filter element in the [input configuration](inputConfig.md) contains a list of filter descriptions, each describing one filter +applied on an input. + +The general elements in the json are the following: + +| Field | Description | Default | +|-----------------------|-------------------------------------------------------------------------------------------------|-------------| +| filter | The type of the filter, currently grok, keyvalue and json are supported | - | +| conditions | The conditions of which input to filter | - | +| sort\_order | Describes the order in which the filters should be applied | - | +| source\_field | The source of the filter, must be set for keyvalue filters | log_message | +| remove\_source\_field | Remove the source field after the filter is applied | false | +| post\_map\_values | Mappings done after the filtering provided it's result, see [post map values](postMapValues.md) | - | +| is\_enabled | A flag to show if the filter should be used | true | + + +##Grok Filter + +Grok filters have the following additional parameters: + +| Field | Description | Default | +|--------------------|------------------------------------------------------------------------------------------------------------|---------| +| log4j\_format | The log4j pattern of the log, not used, it is only there for documentation | - | +| multiline\_pattern | The grok pattern that shows that the line is not a log line on it's own but the part of a multi line entry | - | +| message\_pattern | The grok pattern to use to parse the log entry | - | + + +##Key-value Filter + +Key-value filters have the following additional parameters: + +| Field | Description | Default | +|----------------|-------------------------------------------------------------------------------------------|---------| +| field\_split | The string that splits the key-value pairs | "\t" | +| value\_split | The string that separates keys from values | "=" | +| value\_borders | The borders around the value, must be 2 characters long, first before it, second after it | - | + +If value\_borders is only used if it is specified, and value\_split is not. http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-logsearch/ambari-logsearch-logfeeder/docs/input.md ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/docs/input.md b/ambari-logsearch/ambari-logsearch-logfeeder/docs/input.md new file mode 100644 index 0000000..e95401a --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-logfeeder/docs/input.md @@ -0,0 +1,63 @@ +<!-- +{% comment %} +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to you under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +{% endcomment %} +--> + +#Input + +The input element in the [input configuration](inputConfig.md) contains a list of input descriptions, each describing one source +of input. + +The general elements in the json are the following: + +| Field | Description | Default | +|-----------------------------|-------------------------------------------------------------------------------------------------------|--------------| +| type | The type of the input source, currently file and s3_file are supported | - | +| rowtype | The type of the row, can be service / audit | - | +| path | The path of the source, may contain '*' characters too | - | +| add\_fields | The element contains field\_name: field\_value pairs which will be added to each rows data | - | +| tail | The input should check for only the latest file matching the pattern, not all of them | true | +| gen\_event\_md5 | Generate an event\_md5 field for each row by creating a hash of the row data | true | +| use\_event\_md5\_as\_id | Generate an id for each row by creating a hash of the row data | false | +| start\_position | Should the parsing start from the beginning | beginning | +| cache\_enabled | Allows the input to use a cache to filter out duplications | true | +| cache\_key\_field | Specifies the field for which to use the cache to find duplications of | log\_message | +| cache\_last\_dedup\_enabled | Allow to filter out entries which are same as the most recent one irrelevant of it's time | false | +| cache\_size | The number of entries to store in the cache | 100 | +| cache\_dedup\_interval | The maximum interval in ms which may pass between two identical log messages to filter the latter out | 1000 | +| is\_enabled | A flag to show if the input should be used | true | + + +##File Input + +File inputs have the following parameters too: + +| Field | Description | Default | +|--------------------------|--------------------------------------------------------------------|---------| +| checkpoint\_interval\_ms | The time interval in ms when the checkpoint file should be updated | 5000 | +| process\_file | Should the file be processed | true | +| copy\_file | Should the file be copied \(only if not processed\) | false | + + +##S3 File Input + +S3 file inputs have the following parameters in addition to the general file parameters: + +| Field | Description | Default | +|-----------------|-----------------------------------------|---------| +| s3\_access\_key | The access key used for AWS credentials | - | +| s3\_secret\_key | The secret key used for AWS credentials | - | http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-logsearch/ambari-logsearch-logfeeder/docs/inputConfig.md ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/docs/inputConfig.md b/ambari-logsearch/ambari-logsearch-logfeeder/docs/inputConfig.md new file mode 100644 index 0000000..ae7ac05 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-logfeeder/docs/inputConfig.md @@ -0,0 +1,29 @@ +<!-- +{% comment %} +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to you under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +{% endcomment %} +--> + +#Input Configuration + +The input configurations are stored in json files. Each of them are describing the processing of the log files of a service. + +The json contains two elements: + +| Field | Description | +|--------|--------------------------------------------| +| input | A list of [input descriptions](input.md) | +| filter | A list of [filter descriptions](filter.md) | \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-logsearch/ambari-logsearch-logfeeder/docs/postMapValues.md ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/docs/postMapValues.md b/ambari-logsearch/ambari-logsearch-logfeeder/docs/postMapValues.md new file mode 100644 index 0000000..aa381cd --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-logfeeder/docs/postMapValues.md @@ -0,0 +1,61 @@ +<!-- +{% comment %} +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to you under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +{% endcomment %} +--> + +#Post Map Values + +The Post Map Values element in the [filter](filter.md) field names as keys, the values are lists of sets of post map values, each +describing one mapping done on a field named before obtained after filtering. + +Currently there are four kind of mappings are supported: + +##Map Date + +The name of the mapping element should be map\_date. The value json element may contain the following parameters: + +| Field | Description | +|-----------------------|--------------------------------------------------------------------------------------------------------| +| src\_date\_pattern | If it is specified than the mapper converts from this format to the target, and also adds missing year | +| target\_date\_pattern | If 'epoch' then the field is parsed as seconds from 1970, otherwise the content used as pattern | + + +##Map Copy + +The name of the mapping element should be map\_copy. The value json element should contain the following parameter: + +| Field | Description | +|------------|-------------------------------| +| copy\_name | The name of the copied field | + + +##Map Field Name + +The name of the mapping element should be map\_fieldname. The value json element should contain the following parameter: + +| Field | Description | +|------------------|-------------------------------| +| new\_field\_name | The name of the renamed field | + +##Map Field Value + +The name of the mapping element should be map\_fieldvalue. The value json element should contain the following parameter: + +| Field | Description | +|-------------|--------------------------------------------------------------------| +| pre\_value | The value that the field must match \(ignoring case\) to be mapped | +| post\_value | The value to which the field is modified to | http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/AbstractInputFile.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/AbstractInputFile.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/AbstractInputFile.java index cfa1903..2bc4e68 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/AbstractInputFile.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/AbstractInputFile.java @@ -45,7 +45,6 @@ public abstract class AbstractInputFile extends Input { protected String base64FileKey; protected boolean isReady; - private boolean isStartFromBegining = true; private String checkPointExtension; private File checkPointFile; @@ -82,12 +81,6 @@ public abstract class AbstractInputFile extends Input { return; } - String startPosition = inputDescriptor.getStartPosition(); - if (StringUtils.isEmpty(startPosition) || startPosition.equalsIgnoreCase("beginning") || - startPosition.equalsIgnoreCase("begining") || !tail) { - isStartFromBegining = true; - } - setFilePath(logPath); boolean isFileReady = isReady(); @@ -109,7 +102,7 @@ public abstract class AbstractInputFile extends Input { br = openLogFile(logPathFile); - boolean resume = isStartFromBegining; + boolean resume = true; int resumeFromLineNumber = getResumeFromLineNumber(); if (resumeFromLineNumber > 0) { resume = false; http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperDate.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperDate.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperDate.java index 5d34c06..305688b 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperDate.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperDate.java @@ -38,7 +38,7 @@ public class MapperDate extends Mapper { private SimpleDateFormat targetDateFormatter = null; private boolean isEpoch = false; - private SimpleDateFormat srcDateFormatter=null; + private SimpleDateFormat srcDateFormatter = null; @Override public boolean init(String inputDesc, String fieldName, String mapClassCode, MapFieldDescriptor mapFieldDescriptor) { http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputFileTest.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputFileTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputFileTest.java index 3a5f31e..efebc08 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputFileTest.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputFileTest.java @@ -81,7 +81,6 @@ public class InputFileTest { inputFileDescriptor.setSource("file"); inputFileDescriptor.setTail(true); inputFileDescriptor.setGenEventMd5(true); - inputFileDescriptor.setStartPosition("beginning"); inputFileDescriptor.setType("hdfs_datanode"); inputFileDescriptor.setRowtype("service"); inputFileDescriptor.setPath(path); http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-logsearch/ambari-logsearch-logfeeder/src/test/resources/samples/config/config_audit.json ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/resources/samples/config/config_audit.json b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/resources/samples/config/config_audit.json index b7d0a6e..05f3bff 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/resources/samples/config/config_audit.json +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/resources/samples/config/config_audit.json @@ -1,173 +1,172 @@ { - "global": { - "add_fields": { - "cluster": "audit" - }, - "source": "file", - "tail": "true", - "gen_event_md5": "true", - "start_position": "beginning" - }, - "input": [{ - "type": "hdfs_audit", - "rowtype": "hdfs_audit", - "path": "{path}/src/test/resources/samples/jsonlogs/audit_log.json" - }], - "filter": [{ - "filter": "json", - "conditions": { - "fields": { - "type": [ - "hdfs_audit" - ] - - } - } - }, { - "filter": "keyvalue", - "sort_order": 1, - "conditions": { - "fields": { - "type": [ - "hdfs_audit" - ] - - } - - }, - "source_field": "log_message", - "value_split": "=", - "field_split": "\t", - "post_map_values": { - "src": { - "map_fieldname": { - "new_fieldname": "resource" - } - - }, - "ip": { - "map_fieldname": { - "new_fieldname": "cliIP" - } - - }, - "allowed": [{ - "map_fieldvalue": { - "pre_value": "true", - "post_value": "1" - } - - }, { - "map_fieldvalue": { - "pre_value": "false", - "post_value": "0" - } - - }, { - "map_fieldname": { - "new_fieldname": "result" - } - - } - - ], - "cmd": { - "map_fieldname": { - "new_fieldname": "action" - } - - }, - "proto": { - "map_fieldname": { - "new_fieldname": "cliType" - } - - }, - "callerContext": { - "map_fieldname": { - "new_fieldname": "req_caller_id" - } - - } - - } - - }, { - "filter": "grok", - "sort_order": 2, - "source_field": "ugi", - "remove_source_field": "false", - "conditions": { - "fields": { - "type": [ - "hdfs_audit" - ] - - } - - }, - "message_pattern": "%{USERNAME:p_user}.+auth:%{USERNAME:p_authType}.+via %{USERNAME:k_user}.+auth:%{USERNAME:k_authType}|%{USERNAME:user}.+auth:%{USERNAME:authType}|%{USERNAME:x_user}", - "post_map_values": { - "user": { - "map_fieldname": { - "new_fieldname": "reqUser" - } - - }, - "x_user": { - "map_fieldname": { - "new_fieldname": "reqUser" - } - - }, - "p_user": { - "map_fieldname": { - "new_fieldname": "reqUser" - } - - }, - "k_user": { - "map_fieldname": { - "new_fieldname": "proxyUsers" - } - - }, - "p_authType": { - "map_fieldname": { - "new_fieldname": "authType" - } - - }, - "k_authType": { - "map_fieldname": { - "new_fieldname": "proxyAuthType" - } - - } - - } - - } - - ], - - "output": [{ - "is_enabled": "true", - "comment": "Output to file for audit logs", - "destination": "solr", - "url": "http://localhost:8983/solr/audit_logs";, - "collection": "audit_logs", - "number_of_shards": "1", - "splits_interval_mins": "100000", - "conditions": { - "fields": { - "rowtype": [ - "hdfs_audit" - ] - } - } - } - - ] + "global": { + "add_fields": { + "cluster": "audit" + }, + "source": "file", + "tail": "true", + "gen_event_md5": "true" + }, + "input": [{ + "type": "hdfs_audit", + "rowtype": "hdfs_audit", + "path": "{path}/src/test/resources/samples/jsonlogs/audit_log.json" + }], + "filter": [{ + "filter": "json", + "conditions": { + "fields": { + "type": [ + "hdfs_audit" + ] + + } + } + }, { + "filter": "keyvalue", + "sort_order": 1, + "conditions": { + "fields": { + "type": [ + "hdfs_audit" + ] + + } + + }, + "source_field": "log_message", + "value_split": "=", + "field_split": "\t", + "post_map_values": { + "src": { + "map_fieldname": { + "new_fieldname": "resource" + } + + }, + "ip": { + "map_fieldname": { + "new_fieldname": "cliIP" + } + + }, + "allowed": [{ + "map_fieldvalue": { + "pre_value": "true", + "post_value": "1" + } + + }, { + "map_fieldvalue": { + "pre_value": "false", + "post_value": "0" + } + + }, { + "map_fieldname": { + "new_fieldname": "result" + } + + } + + ], + "cmd": { + "map_fieldname": { + "new_fieldname": "action" + } + + }, + "proto": { + "map_fieldname": { + "new_fieldname": "cliType" + } + + }, + "callerContext": { + "map_fieldname": { + "new_fieldname": "req_caller_id" + } + + } + + } + + }, { + "filter": "grok", + "sort_order": 2, + "source_field": "ugi", + "remove_source_field": "false", + "conditions": { + "fields": { + "type": [ + "hdfs_audit" + ] + + } + + }, + "message_pattern": "%{USERNAME:p_user}.+auth:%{USERNAME:p_authType}.+via %{USERNAME:k_user}.+auth:%{USERNAME:k_authType}|%{USERNAME:user}.+auth:%{USERNAME:authType}|%{USERNAME:x_user}", + "post_map_values": { + "user": { + "map_fieldname": { + "new_fieldname": "reqUser" + } + + }, + "x_user": { + "map_fieldname": { + "new_fieldname": "reqUser" + } + + }, + "p_user": { + "map_fieldname": { + "new_fieldname": "reqUser" + } + + }, + "k_user": { + "map_fieldname": { + "new_fieldname": "proxyUsers" + } + + }, + "p_authType": { + "map_fieldname": { + "new_fieldname": "authType" + } + + }, + "k_authType": { + "map_fieldname": { + "new_fieldname": "proxyAuthType" + } + + } + + } + + } + + ], + + "output": [{ + "is_enabled": "true", + "comment": "Output to file for audit logs", + "destination": "solr", + "url": "http://localhost:8983/solr/audit_logs";, + "collection": "audit_logs", + "number_of_shards": "1", + "splits_interval_mins": "100000", + "conditions": { + "fields": { + "rowtype": [ + "hdfs_audit" + ] + } + } + } + + ] } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-logsearch/ambari-logsearch-logfeeder/src/test/resources/samples/config/config_service.json ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/resources/samples/config/config_service.json b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/resources/samples/config/config_service.json index 2e96fd7..0fff050 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/resources/samples/config/config_service.json +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/resources/samples/config/config_service.json @@ -1,43 +1,42 @@ { - "global": { - "add_fields": { - "cluster": "cluster_name" - }, - "source": "file", - "tail": "true", - "gen_event_md5": "true", - "start_position": "beginning" - }, - "input": [{ - "type": "logsearch", - "rowtype": "service", - "path": "{path}/src/test/resources/samples/jsonlogs/service_log.json" - }], - "filter": [{ - "filter": "json", - "conditions": { - "fields": { - "type": [ - "logsearch" - ] - } - } - }], - "output": [{ - "comment": "Output to solr for service records", - "is_enabled": "true", - "destination": "solr", - "url": "http://localhost:8983/solr/hadoop_logs";, - "collection": "hadoop_logs", - "number_of_shards": "1", - "splits_interval_mins": "100000", - "conditions": { - "fields": { - "rowtype": [ - "service" - ] - } - } - }] + "global": { + "add_fields": { + "cluster": "cluster_name" + }, + "source": "file", + "tail": "true", + "gen_event_md5": "true" + }, + "input": [{ + "type": "logsearch", + "rowtype": "service", + "path": "{path}/src/test/resources/samples/jsonlogs/service_log.json" + }], + "filter": [{ + "filter": "json", + "conditions": { + "fields": { + "type": [ + "logsearch" + ] + } + } + }], + "output": [{ + "comment": "Output to solr for service records", + "is_enabled": "true", + "destination": "solr", + "url": "http://localhost:8983/solr/hadoop_logs";, + "collection": "hadoop_logs", + "number_of_shards": "1", + "splits_interval_mins": "100000", + "conditions": { + "fields": { + "rowtype": [ + "service" + ] + } + } + }] } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerInput.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerInput.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerInput.java index e0f7564..6ef3d3f 100644 --- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerInput.java +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerInput.java @@ -53,9 +53,6 @@ public abstract class LSServerInput { @JsonProperty("use_event_md5_as_id") private Boolean useEventMd5AsId; - @JsonProperty("start_position") - private String startPosition; - @JsonProperty("cache_enabled") private Boolean cacheEnabled; @@ -85,7 +82,6 @@ public abstract class LSServerInput { this.tail = inputDescriptor.isTail(); this.genEventMd5 = inputDescriptor.isGenEventMd5(); this.useEventMd5AsId = inputDescriptor.isUseEventMd5AsId(); - this.startPosition = inputDescriptor.getStartPosition(); this.cacheEnabled = inputDescriptor.isCacheEnabled(); this.cacheKeyField = inputDescriptor.getCacheKeyField(); this.cacheLastDedupEnabled = inputDescriptor.getCacheLastDedupEnabled(); @@ -126,10 +122,6 @@ public abstract class LSServerInput { return useEventMd5AsId; } - public String getStartPosition() { - return startPosition; - } - public Boolean getCacheEnabled() { return cacheEnabled; } http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerMapDate.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerMapDate.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerMapDate.java index 3147402..dcb0393 100644 --- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerMapDate.java +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerMapDate.java @@ -34,7 +34,7 @@ public class LSServerMapDate extends LSServerMapField { return "map_date"; } - @JsonProperty("source_date_pattern") + @JsonProperty("src_date_pattern") private String sourceDatePattern; @NotNull http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/global.config.json ---------------------------------------------------------------------- diff --git a/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/global.config.json b/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/global.config.json index 7ac29dd..65da726 100644 --- a/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/global.config.json +++ b/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/global.config.json @@ -5,7 +5,6 @@ }, "source":"file", "tail":"true", - "gen_event_md5":"true", - "start_position":"beginning" + "gen_event_md5":"true" } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/global.config.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/global.config.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/global.config.json.j2 index f337527..1b63e2a 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/global.config.json.j2 +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/global.config.json.j2 @@ -22,7 +22,6 @@ }, "source":"file", "tail":"true", - "gen_event_md5":"true", - "start_position":"beginning" + "gen_event_md5":"true" } } \ No newline at end of file
