AMBARI-21423 Add REST end point for the documentation of the Log Feeder shipper properties (mgergely)
Change-Id: If6d1b66c3a1f74b118ae60a7edc26624d49fb7e6 Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/15dd999f Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/15dd999f Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/15dd999f Branch: refs/heads/feature-branch-AMBARI-21307 Commit: 15dd999fff99fb80bc65ddfc94513e890a6efdef Parents: c088289 Author: Miklos Gergely <mgerg...@hortonworks.com> Authored: Mon Jul 10 14:51:23 2017 +0200 Committer: Miklos Gergely <mgerg...@hortonworks.com> Committed: Mon Jul 10 14:51:23 2017 +0200 ---------------------------------------------------------------------- .../api/ShipperConfigElementDescription.java | 59 +++++++++++ .../api/ShipperConfigTypeDescription.java | 44 ++++++++ .../model/inputconfig/impl/ConditionsImpl.java | 13 +++ .../model/inputconfig/impl/FieldsImpl.java | 14 +++ .../inputconfig/impl/FilterDescriptorImpl.java | 51 ++++++++++ .../impl/FilterGrokDescriptorImpl.java | 24 +++++ .../impl/FilterKeyValueDescriptorImpl.java | 28 +++++ .../model/inputconfig/impl/InputConfigImpl.java | 18 ++++ .../inputconfig/impl/InputDescriptorImpl.java | 101 +++++++++++++++++++ .../impl/InputFileBaseDescriptorImpl.java | 27 +++++ .../impl/InputS3FileDescriptorImpl.java | 16 +++ .../impl/MapAnonymizeDescriptorImpl.java | 21 +++- .../inputconfig/impl/MapDateDescriptorImpl.java | 20 +++- .../impl/MapFieldCopyDescriptorImpl.java | 14 ++- .../impl/MapFieldDescriptorImpl.java | 33 ++++++ .../impl/MapFieldNameDescriptorImpl.java | 14 ++- .../impl/MapFieldValueDescriptorImpl.java | 20 +++- .../inputconfig/impl/PostMapValuesAdapter.java | 2 +- .../ambari-logsearch-logfeeder/docs/filter.md | 4 +- .../ambari-logsearch-logfeeder/docs/input.md | 10 +- .../docs/postMapValues.md | 2 +- .../ambari/logfeeder/filter/FilterJSONTest.java | 12 ++- .../common/ShipperConfigDescriptionStorage.java | 67 ++++++++++++ .../ambari/logsearch/doc/DocConstants.java | 1 + .../ambari/logsearch/manager/InfoManager.java | 9 ++ .../response/ShipperConfigDescriptionData.java | 52 ++++++++++ .../ambari/logsearch/rest/InfoResource.java | 10 ++ 27 files changed, 667 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/ShipperConfigElementDescription.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/ShipperConfigElementDescription.java b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/ShipperConfigElementDescription.java new file mode 100644 index 0000000..d65bf8e --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/ShipperConfigElementDescription.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.config.api; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Marker for the shipper configuration properties. + * Can be used to generate documentation about the shipper configs. + */ +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.FIELD}) +public @interface ShipperConfigElementDescription { + + /** + * The path of the json element. + */ + String path(); + + /** + * The type of the json element. + */ + String type(); + + /** + * Describe what the json element is used for. + */ + String description(); + + /** + * An example value for the element, if applicable. + */ + String[] examples() default {}; + + /** + * Default value of the json element, if applicable. + */ + String defaultValue() default ""; + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/ShipperConfigTypeDescription.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/ShipperConfigTypeDescription.java b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/ShipperConfigTypeDescription.java new file mode 100644 index 0000000..1c112d8 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/ShipperConfigTypeDescription.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.config.api; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Marker for the shipper configuration types. + * Can be used to generate documentation about the shipper configs. + */ +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.TYPE}) +public @interface ShipperConfigTypeDescription { + + /** + * The name of the element type. + */ + String name(); + + /** + * The description of the json element. + */ + String description(); + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/ConditionsImpl.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/ConditionsImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/ConditionsImpl.java index 8bbff8f..2ba472c 100644 --- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/ConditionsImpl.java +++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/ConditionsImpl.java @@ -19,11 +19,24 @@ package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl; +import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription; +import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription; import org.apache.ambari.logsearch.config.api.model.inputconfig.Conditions; import com.google.gson.annotations.Expose; +@ShipperConfigTypeDescription( + name = "Conditions", + description = "Describes the conditions that should be met in order to match a filter to an input element.\n" + + "\n" + + "It has the following attributes:" +) public class ConditionsImpl implements Conditions { + @ShipperConfigElementDescription( + path = "/filter/[]/conditions/fields", + type = "json object", + description = "The fields in the input element of which's value should be met." + ) @Expose private FieldsImpl fields; http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FieldsImpl.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FieldsImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FieldsImpl.java index 68cd0e2..32a0348 100644 --- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FieldsImpl.java +++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FieldsImpl.java @@ -21,11 +21,25 @@ package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl; import java.util.Set; +import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription; +import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription; import org.apache.ambari.logsearch.config.api.model.inputconfig.Fields; import com.google.gson.annotations.Expose; +@ShipperConfigTypeDescription( + name = "Fields", + description = "Describes a the fields which's value should be met in order to match a filter to an input element.\n" + + "\n" + + "It has the following attributes:" + ) public class FieldsImpl implements Fields { + @ShipperConfigElementDescription( + path = "/filter/[]/conditions/fields/type", + type = "list of strings", + description = "The acceptable values for the type field in the input element.", + examples = {"ambari_server", "\"spark_jobhistory_server\", \"spark_thriftserver\", \"livy_server\""} + ) @Expose private Set<String> type; http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterDescriptorImpl.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterDescriptorImpl.java index 4e11715..eb9d38c 100644 --- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterDescriptorImpl.java +++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterDescriptorImpl.java @@ -22,35 +22,86 @@ package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl; import java.util.List; import java.util.Map; +import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription; +import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription; import org.apache.ambari.logsearch.config.api.model.inputconfig.FilterDescriptor; import org.apache.ambari.logsearch.config.api.model.inputconfig.PostMapValues; import com.google.gson.annotations.Expose; import com.google.gson.annotations.SerializedName; +@ShipperConfigTypeDescription( + name = "Filter", + description = "The filter element in the [input configuration](inputConfig.md) contains a list of filter descriptions, each describing one filter applied on an input.\n" + + "\n" + + "The general elements in the json are the following:" +) public abstract class FilterDescriptorImpl implements FilterDescriptor { + @ShipperConfigElementDescription( + path = "/filter/[]/filter", + type = "string", + description = "The type of the filter.", + examples = {"grok", "keyvalue", "json"} + ) @Expose private String filter; + @ShipperConfigElementDescription( + path = "/filter/[]/conditions", + type = "json object", + description = "The conditions of which input to filter." + ) @Expose private ConditionsImpl conditions; + @ShipperConfigElementDescription( + path = "/filter/[]/sort_order", + type = "integer", + description = "Describes the order in which the filters should be applied.", + examples = {"1", "3"} + ) @Expose @SerializedName("sort_order") private Integer sortOrder; + @ShipperConfigElementDescription( + path = "/filter/[]/source_field", + type = "integer", + description = "The source of the filter, must be set for keyvalue filters.", + examples = {"field_further_to_filter"}, + defaultValue = "log_message" + ) @Expose @SerializedName("source_field") private String sourceField; + @ShipperConfigElementDescription( + path = "/filter/[]/remove_source_field", + type = "boolean", + description = "Remove the source field after the filter is applied.", + examples = {"true", "false"}, + defaultValue = "false" + ) @Expose @SerializedName("remove_source_field") private Boolean removeSourceField; + @ShipperConfigElementDescription( + path = "/filter/[]/post_map_values", + type = "dictionary string to list of json objects", + description = "Mappings done after the filtering provided it's result." + ) @Expose @SerializedName("post_map_values") private Map<String, List<PostMapValuesImpl>> postMapValues; + @ShipperConfigElementDescription( + path = "/filter/[]/is_enabled", + type = "boolean", + description = "A flag to show if the filter should be used.", + examples = {"true", "false"}, + defaultValue = "true" + ) @Expose @SerializedName("is_enabled") private Boolean isEnabled; http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterGrokDescriptorImpl.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterGrokDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterGrokDescriptorImpl.java index 995f76b..e140df0 100644 --- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterGrokDescriptorImpl.java +++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterGrokDescriptorImpl.java @@ -19,20 +19,44 @@ package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl; +import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription; +import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription; import org.apache.ambari.logsearch.config.api.model.inputconfig.FilterGrokDescriptor; import com.google.gson.annotations.Expose; import com.google.gson.annotations.SerializedName; +@ShipperConfigTypeDescription( + name = "Grok Filter", + description = "Grok filters have the following additional parameters:" +) public class FilterGrokDescriptorImpl extends FilterDescriptorImpl implements FilterGrokDescriptor { + @ShipperConfigElementDescription( + path = "/filter/[]/log4j_format", + type = "string", + description = "The log4j pattern of the log, not used, it is only there for documentation.", + examples = {"%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n"} + ) @Expose @SerializedName("log4j_format") private String log4jFormat; + @ShipperConfigElementDescription( + path = "/filter/[]/multiline_pattern", + type = "string", + description = "The grok pattern that shows that the line is not a log line on it's own but the part of a multi line entry.", + examples = {"^(%{TIMESTAMP_ISO8601:logtime})"} + ) @Expose @SerializedName("multiline_pattern") private String multilinePattern; + @ShipperConfigElementDescription( + path = "/filter/[]/message_pattern", + type = "string", + description = "The grok pattern to use to parse the log entry.", + examples = {"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}-%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\@%{INT:line_number}\\]%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}"} + ) @Expose @SerializedName("message_pattern") private String messagePattern; http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterKeyValueDescriptorImpl.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterKeyValueDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterKeyValueDescriptorImpl.java index 8e89990..1c782c5 100644 --- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterKeyValueDescriptorImpl.java +++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterKeyValueDescriptorImpl.java @@ -19,20 +19,48 @@ package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl; +import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription; +import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription; import org.apache.ambari.logsearch.config.api.model.inputconfig.FilterKeyValueDescriptor; import com.google.gson.annotations.Expose; import com.google.gson.annotations.SerializedName; +@ShipperConfigTypeDescription( + name = "Key-value Filter", + description = "value_borders is only used if it is specified, and value_split is not.\n" + + "\n" + + "Key-value filters have the following additional parameters:" +) public class FilterKeyValueDescriptorImpl extends FilterDescriptorImpl implements FilterKeyValueDescriptor { + @ShipperConfigElementDescription( + path = "/filter/[]/field_split", + type = "string", + description = "The string that splits the key-value pairs.", + examples = {" ", ","}, + defaultValue = "\\t" + ) @Expose @SerializedName("field_split") private String fieldSplit; + @ShipperConfigElementDescription( + path = "/filter/[]/value_split", + type = "string", + description = "The string that separates keys from values.", + examples = {":", "->"}, + defaultValue = "=" + ) @Expose @SerializedName("value_split") private String valueSplit; + @ShipperConfigElementDescription( + path = "/filter/[]/value_borders", + type = "string", + description = "The borders around the value, must be 2 characters long, first before it, second after it.", + examples = {"()", "[]", "{}"} + ) @Expose @SerializedName("value_borders") private String valueBorders; http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputConfigImpl.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputConfigImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputConfigImpl.java index a4eba8e..6ce634f 100644 --- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputConfigImpl.java +++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputConfigImpl.java @@ -21,16 +21,34 @@ package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl; import java.util.List; +import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription; +import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription; import org.apache.ambari.logsearch.config.api.model.inputconfig.FilterDescriptor; import org.apache.ambari.logsearch.config.api.model.inputconfig.InputConfig; import org.apache.ambari.logsearch.config.api.model.inputconfig.InputDescriptor; import com.google.gson.annotations.Expose; +@ShipperConfigTypeDescription( + name = "Input Config", + description = "The input configurations are stored in json files. Each of them are describing the processing of the log files of a service.\n" + + "\n" + + "The json contains two elements:" +) public class InputConfigImpl implements InputConfig { + @ShipperConfigElementDescription( + path = "/input", + type = "list of json objects", + description = "A list of input descriptions" + ) @Expose private List<InputDescriptorImpl> input; + @ShipperConfigElementDescription( + path = "/filter", + type = "list of json objects", + description = "A list of filter descriptions" + ) @Expose private List<FilterDescriptorImpl> filter; http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputDescriptorImpl.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputDescriptorImpl.java index 54b4b9b..cec16c8 100644 --- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputDescriptorImpl.java +++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputDescriptorImpl.java @@ -21,59 +21,160 @@ package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl; import java.util.Map; +import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription; +import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription; import org.apache.ambari.logsearch.config.api.model.inputconfig.InputDescriptor; import com.google.gson.annotations.Expose; import com.google.gson.annotations.SerializedName; +@ShipperConfigTypeDescription( + name = "Input", + description = "The input element in the input configuration contains a list of input descriptions, each describing one source of input.\n" + + "\n" + + "The general elements in the json are the following:" +) public abstract class InputDescriptorImpl implements InputDescriptor { + @ShipperConfigElementDescription( + path = "/input/[]/type", + type = "string", + description = "The log id for this source.", + examples = {"zookeeper", "ambari_server"} + ) @Expose private String type; + @ShipperConfigElementDescription( + path = "/input/[]/rowtype", + type = "string", + description = "The type of the row.", + examples = {"service", "audit"} + ) @Expose private String rowtype; + @ShipperConfigElementDescription( + path = "/input/[]/path", + type = "string", + description = "The path of the source, may contain '*' characters too.", + examples = {"/var/log/ambari-logsearch-logfeeder/logsearch-logfeeder.json", "/var/log/zookeeper/zookeeper*.log"} + ) @Expose private String path; + @ShipperConfigElementDescription( + path = "/input/[]/add_fields", + type = "dictionary", + description = "The element contains field_name: field_value pairs which will be added to each rows data.", + examples = {"\"cluster\":\"cluster_name\""} + ) @Expose @SerializedName("add_fields") private Map<String, String> addFields; + @ShipperConfigElementDescription( + path = "/input/[]/source", + type = "dictionary", + description = "The type of the input source.", + examples = {"file", "s3_file"} + ) @Expose private String source; + @ShipperConfigElementDescription( + path = "/input/[]/tail", + type = "boolean", + description = "The input should check for only the latest file matching the pattern, not all of them.", + examples = {"true", "false"}, + defaultValue = "true" + ) @Expose private Boolean tail; + @ShipperConfigElementDescription( + path = "/input/[]/gen_event_md5", + type = "boolean", + description = "Generate an event_md5 field for each row by creating a hash of the row data.", + examples = {"true", "false"}, + defaultValue = "true" + ) @Expose @SerializedName("gen_event_md5") private Boolean genEventMd5; + @ShipperConfigElementDescription( + path = "/input/[]/use_event_md5_as_id", + type = "boolean", + description = "Generate an id for each row by creating a hash of the row data.", + examples = {"true", "false"}, + defaultValue = "false" + ) @Expose @SerializedName("use_event_md5_as_id") private Boolean useEventMd5AsId; + @ShipperConfigElementDescription( + path = "/input/[]/cache_enabled", + type = "boolean", + description = "Allows the input to use a cache to filter out duplications.", + examples = {"true", "false"}, + defaultValue = "false" + ) @Expose @SerializedName("cache_enabled") private Boolean cacheEnabled; + @ShipperConfigElementDescription( + path = "/input/[]/cache_key_field", + type = "string", + description = "Specifies the field for which to use the cache to find duplications of.", + examples = {"some_field_prone_to_repeating_value"}, + defaultValue = "log_message" + ) @Expose @SerializedName("cache_key_field") private String cacheKeyField; + @ShipperConfigElementDescription( + path = "/input/[]/cache_last_dedup_enabled", + type = "boolean", + description = "Allow to filter out entries which are same as the most recent one irrelevant of it's time.", + examples = {"true", "false"}, + defaultValue = "false" + ) @Expose @SerializedName("cache_last_dedup_enabled") private Boolean cacheLastDedupEnabled; + @ShipperConfigElementDescription( + path = "/input/[]/cache_size", + type = "integer", + description = "The number of entries to store in the cache.", + examples = {"50"}, + defaultValue = "100" + ) @Expose @SerializedName("cache_size") private Integer cacheSize; + @ShipperConfigElementDescription( + path = "/input/[]/cache_dedup_interval", + type = "integer", + description = "The maximum interval in ms which may pass between two identical log messages to filter the latter out.", + examples = {"500"}, + defaultValue = "1000" + ) @Expose @SerializedName("cache_dedup_interval") private Long cacheDedupInterval; + @ShipperConfigElementDescription( + path = "/input/[]/is_enabled", + type = "boolean", + description = "A flag to show if the input should be used.", + examples = {"true", "false"}, + defaultValue = "true" + ) @Expose @SerializedName("is_enabled") private Boolean isEnabled; http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputFileBaseDescriptorImpl.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputFileBaseDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputFileBaseDescriptorImpl.java index 51c7ec8..8281daa 100644 --- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputFileBaseDescriptorImpl.java +++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputFileBaseDescriptorImpl.java @@ -19,20 +19,47 @@ package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl; +import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription; +import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription; import org.apache.ambari.logsearch.config.api.model.inputconfig.InputFileBaseDescriptor; import com.google.gson.annotations.Expose; import com.google.gson.annotations.SerializedName; +@ShipperConfigTypeDescription( + name = "File Input", + description = "File inputs have some additional parameters:" +) public class InputFileBaseDescriptorImpl extends InputDescriptorImpl implements InputFileBaseDescriptor { + @ShipperConfigElementDescription( + path = "/input/[]/checkpoint_interval_ms", + type = "integer", + description = "The time interval in ms when the checkpoint file should be updated.", + examples = {"10000"}, + defaultValue = "5000" + ) @Expose @SerializedName("checkpoint_interval_ms") private Integer checkpointIntervalMs; + @ShipperConfigElementDescription( + path = "/input/[]/process_file", + type = "boolean", + description = "Should the file be processed.", + examples = {"true", "false"}, + defaultValue = "true" + ) @Expose @SerializedName("process_file") private Boolean processFile; + @ShipperConfigElementDescription( + path = "/input/[]/copy_file", + type = "boolean", + description = "Should the file be copied (only if not processed).", + examples = {"true", "false"}, + defaultValue = "false" + ) @Expose @SerializedName("copy_file") private Boolean copyFile; http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputS3FileDescriptorImpl.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputS3FileDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputS3FileDescriptorImpl.java index 277a57c..19f52d3 100644 --- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputS3FileDescriptorImpl.java +++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputS3FileDescriptorImpl.java @@ -19,16 +19,32 @@ package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl; +import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription; +import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription; import org.apache.ambari.logsearch.config.api.model.inputconfig.InputS3FileDescriptor; import com.google.gson.annotations.Expose; import com.google.gson.annotations.SerializedName; +@ShipperConfigTypeDescription( + name = "S3 File Input", + description = "S3 file inputs have the following parameters in addition to the general file parameters:" +) public class InputS3FileDescriptorImpl extends InputFileBaseDescriptorImpl implements InputS3FileDescriptor { + @ShipperConfigElementDescription( + path = "/input/[]/s3_access_key", + type = "string", + description = "The access key used for AWS credentials." + ) @Expose @SerializedName("s3_access_key") private String s3AccessKey; + @ShipperConfigElementDescription( + path = "/input/[]/s3_secret_key", + type = "string", + description = "The secret key used for AWS credentials." + ) @Expose @SerializedName("s3_secret_key") private String s3SecretKey; http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapAnonymizeDescriptorImpl.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapAnonymizeDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapAnonymizeDescriptorImpl.java index 5fdbbab..8c128de 100644 --- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapAnonymizeDescriptorImpl.java +++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapAnonymizeDescriptorImpl.java @@ -19,20 +19,39 @@ package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl; +import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription; +import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription; import org.apache.ambari.logsearch.config.api.model.inputconfig.MapAnonymizeDescriptor; import com.google.gson.annotations.Expose; import com.google.gson.annotations.SerializedName; -public class MapAnonymizeDescriptorImpl implements MapAnonymizeDescriptor { +@ShipperConfigTypeDescription( + name = "Map Anonymize", + description = "The name of the mapping element should be map_anonymize. The value json element should contain the following parameter:" +) +public class MapAnonymizeDescriptorImpl extends MapFieldDescriptorImpl implements MapAnonymizeDescriptor { @Override public String getJsonName() { return "map_anonymize"; } + @ShipperConfigElementDescription( + path = "/filter/[]/post_map_values/{field_name}/[]/map_anonymize/pattern", + type = "string", + description = "The pattern to use to identify parts to anonymize. The parts to hide should be marked with the \"<hide>\" string.", + examples = {"Some secret is here: <hide>, and another one is here: <hide>"} + ) @Expose private String pattern; + @ShipperConfigElementDescription( + path = "/filter/[]/post_map_values/{field_name}/[]/map_anonymize/hide_char", + type = "string", + description = "The character to hide with", + defaultValue = "*", + examples = {"X", "-"} + ) @Expose @SerializedName("hide_char") private Character hideChar; http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapDateDescriptorImpl.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapDateDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapDateDescriptorImpl.java index 2e54e7a..feec4b6 100644 --- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapDateDescriptorImpl.java +++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapDateDescriptorImpl.java @@ -19,21 +19,39 @@ package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl; +import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription; +import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription; import org.apache.ambari.logsearch.config.api.model.inputconfig.MapDateDescriptor; import com.google.gson.annotations.Expose; import com.google.gson.annotations.SerializedName; -public class MapDateDescriptorImpl implements MapDateDescriptor { +@ShipperConfigTypeDescription( + name = "Map Date", + description = "The name of the mapping element should be map_date. The value json element may contain the following parameters:" +) +public class MapDateDescriptorImpl extends MapFieldDescriptorImpl implements MapDateDescriptor { @Override public String getJsonName() { return "map_date"; } + @ShipperConfigElementDescription( + path = "/filter/[]/post_map_values/{field_name}/[]/map_date/src_date_pattern", + type = "string", + description = "If it is specified than the mapper converts from this format to the target, and also adds missing year", + examples = {"MMM dd HH:mm:ss"} + ) @Expose @SerializedName("src_date_pattern") private String sourceDatePattern; + @ShipperConfigElementDescription( + path = "/filter/[]/post_map_values/{field_name}/[]/map_date/target_date_pattern", + type = "string", + description = "If 'epoch' then the field is parsed as seconds from 1970, otherwise the content used as pattern", + examples = {"yyyy-MM-dd HH:mm:ss,SSS", "epoch"} + ) @Expose @SerializedName("target_date_pattern") private String targetDatePattern; http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldCopyDescriptorImpl.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldCopyDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldCopyDescriptorImpl.java index 4a8d746..e7b8fdf 100644 --- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldCopyDescriptorImpl.java +++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldCopyDescriptorImpl.java @@ -19,17 +19,29 @@ package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl; +import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription; +import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription; import org.apache.ambari.logsearch.config.api.model.inputconfig.MapFieldCopyDescriptor; import com.google.gson.annotations.Expose; import com.google.gson.annotations.SerializedName; -public class MapFieldCopyDescriptorImpl implements MapFieldCopyDescriptor { +@ShipperConfigTypeDescription( + name = "Map Copy", + description = "The name of the mapping element should be map_copy. The value json element should contain the following parameter:" +) +public class MapFieldCopyDescriptorImpl extends MapFieldDescriptorImpl implements MapFieldCopyDescriptor { @Override public String getJsonName() { return "map_fieldcopy"; } + @ShipperConfigElementDescription( + path = "/filter/[]/post_map_values/{field_name}/[]/map_copy/copy_name", + type = "string", + description = "The name of the copied field", + examples = {"new_name"} + ) @Expose @SerializedName("copy_name") private String copyName; http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldDescriptorImpl.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldDescriptorImpl.java new file mode 100644 index 0000000..101e0d4 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldDescriptorImpl.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl; + +import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription; +import org.apache.ambari.logsearch.config.api.model.inputconfig.MapFieldDescriptor; + +@ShipperConfigTypeDescription( + name = "Post Map Values", + description = "The Post Map Values element in the [filter](filter.md) field names as keys, the values are lists of sets of " + + "post map values, each describing one mapping done on a field named before obtained after filtering.\n" + + "\n" + + "Currently there are four kind of mappings are supported:" + ) +public abstract class MapFieldDescriptorImpl implements MapFieldDescriptor { +} http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldNameDescriptorImpl.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldNameDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldNameDescriptorImpl.java index bd32018..e1b71e6 100644 --- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldNameDescriptorImpl.java +++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldNameDescriptorImpl.java @@ -19,17 +19,29 @@ package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl; +import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription; +import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription; import org.apache.ambari.logsearch.config.api.model.inputconfig.MapFieldNameDescriptor; import com.google.gson.annotations.Expose; import com.google.gson.annotations.SerializedName; -public class MapFieldNameDescriptorImpl implements MapFieldNameDescriptor { +@ShipperConfigTypeDescription( + name = "Map Field Name", + description = "The name of the mapping element should be map_fieldname. The value json element should contain the following parameter:" +) +public class MapFieldNameDescriptorImpl extends MapFieldDescriptorImpl implements MapFieldNameDescriptor { @Override public String getJsonName() { return "map_fieldname"; } + @ShipperConfigElementDescription( + path = "/filter/[]/post_map_values/{field_name}/[]/map_fieldname/new_field_name", + type = "string", + description = "The name of the renamed field", + examples = {"new_name"} + ) @Expose @SerializedName("new_field_name") private String newFieldName; http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldValueDescriptorImpl.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldValueDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldValueDescriptorImpl.java index 599e152..a80a994 100644 --- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldValueDescriptorImpl.java +++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldValueDescriptorImpl.java @@ -19,21 +19,39 @@ package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl; +import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription; +import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription; import org.apache.ambari.logsearch.config.api.model.inputconfig.MapFieldValueDescriptor; import com.google.gson.annotations.Expose; import com.google.gson.annotations.SerializedName; -public class MapFieldValueDescriptorImpl implements MapFieldValueDescriptor { +@ShipperConfigTypeDescription( + name = "Map Field Value", + description = "The name of the mapping element should be map_fieldvalue. The value json element should contain the following parameter:" +) +public class MapFieldValueDescriptorImpl extends MapFieldDescriptorImpl implements MapFieldValueDescriptor { @Override public String getJsonName() { return "map_fieldvalue"; } + @ShipperConfigElementDescription( + path = "/filter/[]/post_map_values/{field_name}/[]/map_fieldvalue/pre_value", + type = "string", + description = "The value that the field must match (ignoring case) to be mapped", + examples = {"old_value"} + ) @Expose @SerializedName("pre_value") private String preValue; + @ShipperConfigElementDescription( + path = "/filter/[]/post_map_values/{field_name}/[]/map_fieldvalue/post_value", + type = "string", + description = "The value to which the field is modified to", + examples = {"new_value"} + ) @Expose @SerializedName("post_value") private String postValue; http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/PostMapValuesAdapter.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/PostMapValuesAdapter.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/PostMapValuesAdapter.java index 3c21fd8..e3f9886 100644 --- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/PostMapValuesAdapter.java +++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/PostMapValuesAdapter.java @@ -95,7 +95,7 @@ public class PostMapValuesAdapter implements JsonDeserializer<List<PostMapValues private JsonElement createMapperObject(PostMapValuesImpl postMapValues, JsonSerializationContext context) { JsonObject jsonObject = new JsonObject(); for (MapFieldDescriptor m : postMapValues.getMappers()) { - jsonObject.add(((MapFieldDescriptor)m).getJsonName(), context.serialize(m)); + jsonObject.add(((MapFieldDescriptorImpl)m).getJsonName(), context.serialize(m)); } return jsonObject; } http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-logfeeder/docs/filter.md ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/docs/filter.md b/ambari-logsearch/ambari-logsearch-logfeeder/docs/filter.md index 129279b..d825290 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/docs/filter.md +++ b/ambari-logsearch/ambari-logsearch-logfeeder/docs/filter.md @@ -48,6 +48,8 @@ Grok filters have the following additional parameters: ## Key-value Filter +value\_borders is only used if it is specified, and value\_split is not. + Key-value filters have the following additional parameters: | Field | Description | Default | @@ -56,4 +58,4 @@ Key-value filters have the following additional parameters: | value\_split | The string that separates keys from values | "=" | | value\_borders | The borders around the value, must be 2 characters long, first before it, second after it | - | -If value\_borders is only used if it is specified, and value\_split is not. + http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-logfeeder/docs/input.md ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/docs/input.md b/ambari-logsearch/ambari-logsearch-logfeeder/docs/input.md index 661eeb8..1a9ce8d 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/docs/input.md +++ b/ambari-logsearch/ambari-logsearch-logfeeder/docs/input.md @@ -20,20 +20,18 @@ limitations under the License. # Input The input element in the [input configuration](inputConfig.md) contains a list of input descriptions, each describing one source -of input. - -The general elements in the json are the following: +of input. The general elements in the json are the following: | Field | Description | Default | |-----------------------------|-------------------------------------------------------------------------------------------------------|--------------| -| type | The type of the input source, currently file and s3_file are supported | - | +| type | The log id for this source | - | | rowtype | The type of the row, can be service / audit | - | | path | The path of the source, may contain '*' characters too | - | | add\_fields | The element contains field\_name: field\_value pairs which will be added to each rows data | - | +| source | The type of the input source, currently file and s3_file are supported | - | | tail | The input should check for only the latest file matching the pattern, not all of them | true | | gen\_event\_md5 | Generate an event\_md5 field for each row by creating a hash of the row data | true | | use\_event\_md5\_as\_id | Generate an id for each row by creating a hash of the row data | false | -| start\_position | Should the parsing start from the beginning | beginning | | cache\_enabled | Allows the input to use a cache to filter out duplications | true | | cache\_key\_field | Specifies the field for which to use the cache to find duplications of | log\_message | | cache\_last\_dedup\_enabled | Allow to filter out entries which are same as the most recent one irrelevant of it's time | false | @@ -44,7 +42,7 @@ The general elements in the json are the following: ## File Input -File inputs have the following parameters too: +File inputs have some additional parameters: | Field | Description | Default | |--------------------------|--------------------------------------------------------------------|---------| http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-logfeeder/docs/postMapValues.md ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/docs/postMapValues.md b/ambari-logsearch/ambari-logsearch-logfeeder/docs/postMapValues.md index 7ec439a..bc219df 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/docs/postMapValues.md +++ b/ambari-logsearch/ambari-logsearch-logfeeder/docs/postMapValues.md @@ -67,4 +67,4 @@ The name of the mapping element should be map\_anonymize. The value json element | Field | Description | |------------|-----------------------------------------------------------------------------------------------------------------| | pattern | The pattern to use to identify parts to anonymize. The parts to hide should be marked with the "<hide>" string. | -| hide\_char | The character to hide with, if it is not specified then the default is 'X' | +| hide\_char | The character to hide with, if it is not specified then the default is '*' | http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java index 7abf177..acc3d4d 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java @@ -131,13 +131,15 @@ public class FilterJSONTest { @Test public void testJSONFilterCode_invalidJson() throws Exception { LOG.info("testJSONFilterCode_invalidJson()"); + init(new FilterJsonDescriptorImpl()); - String inputStr="invalid json"; + + String inputStr = "invalid json"; try{ - filterJson.apply(inputStr,new InputMarker(null, null, 0)); - fail("Expected LogFeederException was not occured"); - }catch(LogFeederException logFeederException){ - assertEquals("Json parsing failed for inputstr = "+inputStr, logFeederException.getLocalizedMessage()); + filterJson.apply(inputStr,new InputMarker(null, null, 0)); + fail("Expected LogFeederException was not occured"); + } catch(LogFeederException logFeederException) { + assertEquals("Json parsing failed for inputstr = " + inputStr, logFeederException.getLocalizedMessage()); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/common/ShipperConfigDescriptionStorage.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/common/ShipperConfigDescriptionStorage.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/common/ShipperConfigDescriptionStorage.java new file mode 100644 index 0000000..7d4bc2c --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/common/ShipperConfigDescriptionStorage.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.common; + +import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription; +import org.apache.ambari.logsearch.model.response.ShipperConfigDescriptionData; +import org.reflections.Reflections; +import org.reflections.scanners.FieldAnnotationsScanner; + +import javax.annotation.PostConstruct; +import javax.inject.Named; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +@Named +public class ShipperConfigDescriptionStorage { + + private static final String SHIPPER_CONFIG_PACKAGE = "org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl"; + + private final List<ShipperConfigDescriptionData> shipperConfigDescription = new ArrayList<>(); + + @PostConstruct + public void postConstruct() { + Thread loadShipperConfigDescriptionThread = new Thread("load_shipper_config_description") { + @Override + public void run() { + fillShipperConfigDescriptions(); + } + }; + loadShipperConfigDescriptionThread.setDaemon(true); + loadShipperConfigDescriptionThread.start(); + } + + public List<ShipperConfigDescriptionData> getShipperConfigDescription() { + return shipperConfigDescription; + } + + private void fillShipperConfigDescriptions() { + Reflections reflections = new Reflections(SHIPPER_CONFIG_PACKAGE, new FieldAnnotationsScanner()); + Set<Field> fields = reflections.getFieldsAnnotatedWith(ShipperConfigElementDescription.class); + for (Field field : fields) { + ShipperConfigElementDescription description = field.getAnnotation(ShipperConfigElementDescription.class); + shipperConfigDescription.add(new ShipperConfigDescriptionData(description.path(), description.description(), + description.examples(), description.defaultValue())); + } + + shipperConfigDescription.sort((o1, o2) -> o1.getPath().compareTo(o2.getPath())); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java index 6d1382d..da0a8bb 100644 --- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java @@ -104,6 +104,7 @@ public class DocConstants { public static final String GET_AUTH_DETAILS_OD = "Get authentication details."; public static final String GET_ALL_PROPERTIES_INFO_OD = "List all available properties for Log Search and Log Feeder"; public static final String GET_LOGSEARCH_PROPERTIES_INFO_OD = "List all available properties for Log Search property file (e.g: logsearch.properties/logfeeder.properties)"; + public static final String GET_ALL_SHIPPER_CONFIG_INFO_OD = "List all available shipper configuration element"; } public class EventHistoryDescriptions { http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/InfoManager.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/InfoManager.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/InfoManager.java index f6d0449..2f63492 100644 --- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/InfoManager.java +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/InfoManager.java @@ -25,7 +25,9 @@ import java.util.Map; import org.apache.ambari.logsearch.conf.AuthPropsConfig; import org.apache.ambari.logsearch.common.PropertyDescriptionStorage; +import org.apache.ambari.logsearch.common.ShipperConfigDescriptionStorage; import org.apache.ambari.logsearch.model.response.PropertyDescriptionData; +import org.apache.ambari.logsearch.model.response.ShipperConfigDescriptionData; import javax.inject.Inject; import javax.inject.Named; @@ -39,6 +41,9 @@ public class InfoManager extends JsonManagerBase { @Inject private PropertyDescriptionStorage propertyDescriptionStore; + @Inject + private ShipperConfigDescriptionStorage shipperConfigDescriptionStore; + public Map<String, Boolean> getAuthMap() { Map<String, Boolean> authMap = new HashMap<>(); authMap.put("external", authPropsConfig.isAuthExternalEnabled()); @@ -56,4 +61,8 @@ public class InfoManager extends JsonManagerBase { public List<PropertyDescriptionData> getLogSearchPropertyDescriptions(String propertiesFile) { return getPropertyDescriptions().get(propertiesFile); } + + public List<ShipperConfigDescriptionData> getLogSearchShipperConfigDescription() { + return shipperConfigDescriptionStore.getShipperConfigDescription(); + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/response/ShipperConfigDescriptionData.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/response/ShipperConfigDescriptionData.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/response/ShipperConfigDescriptionData.java new file mode 100644 index 0000000..91f7420 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/response/ShipperConfigDescriptionData.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.model.response; + +public class ShipperConfigDescriptionData { + private final String path; + + private final String description; + + private final String[] examples; + + private final String defaultValue; + + public ShipperConfigDescriptionData(String path, String description, String[] examples, String defaultValue) { + this.path = path; + this.description = description; + this.examples = examples; + this.defaultValue = defaultValue; + } + + public String getPath() { + return path; + } + + public String getDescription() { + return description; + } + + public String[] getExamples() { + return examples; + } + + public String getDefaultValue() { + return defaultValue; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/rest/InfoResource.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/rest/InfoResource.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/rest/InfoResource.java index 6ea0bab..e49be90 100644 --- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/rest/InfoResource.java +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/rest/InfoResource.java @@ -29,12 +29,14 @@ import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import org.apache.ambari.logsearch.manager.InfoManager; import org.apache.ambari.logsearch.model.response.PropertyDescriptionData; +import org.apache.ambari.logsearch.model.response.ShipperConfigDescriptionData; import org.springframework.context.annotation.Scope; import java.util.List; import java.util.Map; import static org.apache.ambari.logsearch.doc.DocConstants.PublicOperationDescriptions.GET_ALL_PROPERTIES_INFO_OD; +import static org.apache.ambari.logsearch.doc.DocConstants.PublicOperationDescriptions.GET_ALL_SHIPPER_CONFIG_INFO_OD; import static org.apache.ambari.logsearch.doc.DocConstants.PublicOperationDescriptions.GET_LOGSEARCH_PROPERTIES_INFO_OD; import static org.apache.ambari.logsearch.doc.DocConstants.PublicOperationDescriptions.GET_AUTH_DETAILS_OD; @@ -70,4 +72,12 @@ public class InfoResource { public List<PropertyDescriptionData> getPropertyFileDescription(@PathParam("propertyFile") String propertyFile) { return infoManager.getLogSearchPropertyDescriptions(propertyFile); } + + @GET + @Path("/shipperconfig") + @Produces({"application/json"}) + @ApiOperation(GET_ALL_SHIPPER_CONFIG_INFO_OD) + public List<ShipperConfigDescriptionData> getShipperConfigDescription() { + return infoManager.getLogSearchShipperConfigDescription(); + } }