AMBARI-21033 Log Search use POJOs for input configuration (mgergely) Change-Id: Ibf28c16309cf3ced0f0eea69d832ecd8accd2d62
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/fd4a7a46 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/fd4a7a46 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/fd4a7a46 Branch: refs/heads/trunk Commit: fd4a7a46a2db9869dca28294660ca40e693504ea Parents: cbb1e90 Author: Miklos Gergely <mgerg...@hortonworks.com> Authored: Mon May 22 12:49:50 2017 +0200 Committer: Miklos Gergely <mgerg...@hortonworks.com> Committed: Mon May 22 12:49:50 2017 +0200 ---------------------------------------------------------------------- .../ambari-logsearch-config-api/pom.xml | 14 +- .../config/api/InputConfigMonitor.java | 13 +- .../logsearch/config/api/LogSearchConfig.java | 3 +- .../config/api/LogSearchConfigFactory.java | 10 +- .../api/model/inputconfig/Conditions.java | 24 +++ .../config/api/model/inputconfig/Fields.java | 26 +++ .../api/model/inputconfig/FilterDescriptor.java | 39 ++++ .../model/inputconfig/FilterGrokDescriptor.java | 28 +++ .../model/inputconfig/FilterJsonDescriptor.java | 23 +++ .../inputconfig/FilterKeyValueDescriptor.java | 28 +++ .../api/model/inputconfig/InputConfig.java | 28 +++ .../api/model/inputconfig/InputDescriptor.java | 54 +++++ .../inputconfig/InputFileBaseDescriptor.java | 28 +++ .../model/inputconfig/InputFileDescriptor.java | 23 +++ .../inputconfig/InputS3FileDescriptor.java | 26 +++ .../model/inputconfig/MapDateDescriptor.java | 26 +++ .../inputconfig/MapFieldCopyDescriptor.java | 24 +++ .../model/inputconfig/MapFieldDescriptor.java | 24 +++ .../inputconfig/MapFieldNameDescriptor.java | 24 +++ .../inputconfig/MapFieldValueDescriptor.java | 26 +++ .../api/model/inputconfig/PostMapValues.java | 26 +++ .../config/api/LogSearchConfigClass1.java | 3 +- .../config/api/LogSearchConfigClass2.java | 3 +- .../ambari-logsearch-config-zookeeper/pom.xml | 6 + .../config/zookeeper/LogSearchConfigZK.java | 72 ++++++- .../model/inputconfig/impl/ConditionsImpl.java | 37 ++++ .../model/inputconfig/impl/FieldsImpl.java | 39 ++++ .../model/inputconfig/impl/FilterAdapter.java | 42 ++++ .../inputconfig/impl/FilterDescriptorImpl.java | 113 ++++++++++ .../impl/FilterGrokDescriptorImpl.java | 66 ++++++ .../impl/FilterJsonDescriptorImpl.java | 25 +++ .../impl/FilterKeyValueDescriptorImpl.java | 63 ++++++ .../model/inputconfig/impl/InputAdapter.java | 58 ++++++ .../model/inputconfig/impl/InputConfigGson.java | 46 +++++ .../model/inputconfig/impl/InputConfigImpl.java | 54 +++++ .../inputconfig/impl/InputDescriptorImpl.java | 204 +++++++++++++++++++ .../impl/InputFileBaseDescriptorImpl.java | 66 ++++++ .../impl/InputFileDescriptorImpl.java | 25 +++ .../impl/InputS3FileDescriptorImpl.java | 53 +++++ .../inputconfig/impl/MapDateDescriptorImpl.java | 58 ++++++ .../impl/MapFieldCopyDescriptorImpl.java | 45 ++++ .../impl/MapFieldNameDescriptorImpl.java | 45 ++++ .../impl/MapFieldValueDescriptorImpl.java | 58 ++++++ .../inputconfig/impl/PostMapValuesAdapter.java | 99 +++++++++ .../inputconfig/impl/PostMapValuesImpl.java | 40 ++++ .../org/apache/ambari/logfeeder/LogFeeder.java | 2 +- .../ambari/logfeeder/common/ConfigBlock.java | 107 +--------- .../ambari/logfeeder/common/ConfigHandler.java | 126 ++++++------ .../ambari/logfeeder/common/ConfigItem.java | 97 +++++++++ .../apache/ambari/logfeeder/filter/Filter.java | 53 +++-- .../ambari/logfeeder/filter/FilterGrok.java | 11 +- .../ambari/logfeeder/filter/FilterJSON.java | 3 - .../ambari/logfeeder/filter/FilterKeyValue.java | 12 +- .../logfeeder/input/AbstractInputFile.java | 16 +- .../apache/ambari/logfeeder/input/Input.java | 112 ++++++---- .../ambari/logfeeder/input/InputFile.java | 6 +- .../ambari/logfeeder/input/InputS3File.java | 5 +- .../ambari/logfeeder/input/InputSimulate.java | 23 +-- .../logfeeder/loglevelfilter/FilterLogData.java | 2 +- .../apache/ambari/logfeeder/mapper/Mapper.java | 4 +- .../ambari/logfeeder/mapper/MapperDate.java | 15 +- .../logfeeder/mapper/MapperFieldCopy.java | 13 +- .../logfeeder/mapper/MapperFieldName.java | 14 +- .../logfeeder/mapper/MapperFieldValue.java | 14 +- .../apache/ambari/logfeeder/output/Output.java | 3 - .../logfeeder/output/OutputLineFilter.java | 2 +- .../ambari/logfeeder/output/OutputManager.java | 8 +- .../ambari/logfeeder/output/OutputS3File.java | 96 ++++----- .../ambari/logfeeder/util/LogFeederUtil.java | 51 ----- .../ambari/logfeeder/filter/FilterGrokTest.java | 37 ++-- .../ambari/logfeeder/filter/FilterJSONTest.java | 14 +- .../logfeeder/filter/FilterKeyValueTest.java | 41 ++-- .../ambari/logfeeder/input/InputFileTest.java | 22 +- .../logconfig/LogConfigHandlerTest.java | 18 +- .../ambari/logfeeder/mapper/MapperDateTest.java | 44 ++-- .../logfeeder/mapper/MapperFieldCopyTest.java | 19 +- .../logfeeder/mapper/MapperFieldNameTest.java | 19 +- .../logfeeder/mapper/MapperFieldValueTest.java | 29 +-- .../logfeeder/output/OutputLineFilterTest.java | 22 +- .../logfeeder/output/OutputManagerTest.java | 10 +- .../logfeeder/output/OutputS3FileTest.java | 17 +- .../logsearch/manager/ShipperConfigManager.java | 7 +- .../model/common/LSServerConditions.java | 41 ++++ .../logsearch/model/common/LSServerFields.java | 43 ++++ .../logsearch/model/common/LSServerFilter.java | 130 ++++++++++++ .../model/common/LSServerFilterGrok.java | 73 +++++++ .../model/common/LSServerFilterJson.java | 31 +++ .../model/common/LSServerFilterKeyValue.java | 71 +++++++ .../logsearch/model/common/LSServerInput.java | 149 ++++++++++++++ .../model/common/LSServerInputConfig.java | 87 ++++++++ .../model/common/LSServerInputFile.java | 31 +++ .../model/common/LSServerInputFileBase.java | 72 +++++++ .../model/common/LSServerInputS3File.java | 59 ++++++ .../logsearch/model/common/LSServerMapDate.java | 61 ++++++ .../model/common/LSServerMapField.java | 30 +++ .../model/common/LSServerMapFieldCopy.java | 49 +++++ .../model/common/LSServerMapFieldName.java | 49 +++++ .../model/common/LSServerMapFieldValue.java | 61 ++++++ .../model/common/LSServerPostMapValues.java | 63 ++++++ .../common/LSServerPostMapValuesSerializer.java | 39 ++++ .../logsearch/rest/ShipperConfigResource.java | 6 +- .../test-config/logfeeder/logfeeder.properties | 1 + .../test-config/logsearch/logsearch.properties | 4 - 103 files changed, 3460 insertions(+), 621 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-config-api/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-api/pom.xml b/ambari-logsearch/ambari-logsearch-config-api/pom.xml index 72fcc80..5355906 100644 --- a/ambari-logsearch/ambari-logsearch-config-api/pom.xml +++ b/ambari-logsearch/ambari-logsearch-config-api/pom.xml @@ -41,17 +41,9 @@ <scope>test</scope> </dependency> <dependency> - <groupId>org.apache.commons</groupId> - <artifactId>commons-lang3</artifactId> - <version>3.4</version> - </dependency> - <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - </dependency> - <dependency> - <groupId>log4j</groupId> - <artifactId>log4j</artifactId> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + <version>1.7.7</version> </dependency> </dependencies> </project> http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/InputConfigMonitor.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/InputConfigMonitor.java b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/InputConfigMonitor.java index 29a82a6..746c14c 100644 --- a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/InputConfigMonitor.java +++ b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/InputConfigMonitor.java @@ -19,19 +19,28 @@ package org.apache.ambari.logsearch.config.api; +import java.util.List; + +import org.apache.ambari.logsearch.config.api.model.inputconfig.InputConfig; + /** * Monitors input configuration changes. */ public interface InputConfigMonitor { /** + * @return A list of json strings for all the global config jsons. + */ + List<String> getGlobalConfigJsons(); + + /** * Notification of a new input configuration. * * @param serviceName The name of the service for which the input configuration was created. * @param inputConfig The input configuration. * @throws Exception */ - void loadInputConfigs(String serviceName, String inputConfig) throws Exception; - + void loadInputConfigs(String serviceName, InputConfig inputConfig) throws Exception; + /** * Notification of the removal of an input configuration. * http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/LogSearchConfig.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/LogSearchConfig.java b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/LogSearchConfig.java index 07921d0..4cbf21f 100644 --- a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/LogSearchConfig.java +++ b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/LogSearchConfig.java @@ -25,6 +25,7 @@ import java.util.Map; import org.apache.ambari.logsearch.config.api.model.loglevelfilter.LogLevelFilter; import org.apache.ambari.logsearch.config.api.model.loglevelfilter.LogLevelFilterMap; +import org.apache.ambari.logsearch.config.api.model.inputconfig.InputConfig; /** * Log Search Configuration, which uploads, retrieves configurations, and monitors it's changes. @@ -71,7 +72,7 @@ public interface LogSearchConfig extends Closeable { * @param serviceName The name of the service looked for. * @return The input configuration for the service if it exists, null otherwise. */ - String getInputConfig(String clusterName, String serviceName); + InputConfig getInputConfig(String clusterName, String serviceName); /** * Uploads the input configuration for a service in a cluster. http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/LogSearchConfigFactory.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/LogSearchConfigFactory.java b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/LogSearchConfigFactory.java index 6ef4b90..947e7e7 100644 --- a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/LogSearchConfigFactory.java +++ b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/LogSearchConfigFactory.java @@ -22,14 +22,14 @@ package org.apache.ambari.logsearch.config.api; import java.util.Map; import org.apache.ambari.logsearch.config.api.LogSearchConfig.Component; -import org.apache.commons.lang3.StringUtils; -import org.apache.log4j.Logger; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Factory class for LogSearchConfig. */ public class LogSearchConfigFactory { - private static final Logger LOG = Logger.getLogger(LogSearchConfigFactory.class); + private static final Logger LOG = LoggerFactory.getLogger(LogSearchConfigFactory.class); /** * Creates a Log Search Configuration instance that implements {@link org.apache.ambari.logsearch.config.api.LogSearchConfig}. @@ -47,7 +47,7 @@ public class LogSearchConfigFactory { try { LogSearchConfig logSearchConfig = null; String configClassName = properties.get("logsearch.config.class"); - if (!StringUtils.isBlank(configClassName)) { + if (configClassName != null && !"".equals(configClassName.trim())) { Class<?> clazz = Class.forName(configClassName); if (LogSearchConfig.class.isAssignableFrom(clazz)) { logSearchConfig = (LogSearchConfig) clazz.newInstance(); @@ -61,7 +61,7 @@ public class LogSearchConfigFactory { logSearchConfig.init(component, properties); return logSearchConfig; } catch (Exception e) { - LOG.fatal("Could not initialize logsearch config.", e); + LOG.error("Could not initialize logsearch config.", e); throw e; } } http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/Conditions.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/Conditions.java b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/Conditions.java new file mode 100644 index 0000000..4da400a --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/Conditions.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ambari.logsearch.config.api.model.inputconfig; + +public interface Conditions { + Fields getFields(); +} http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/Fields.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/Fields.java b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/Fields.java new file mode 100644 index 0000000..5d34b1e --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/Fields.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ambari.logsearch.config.api.model.inputconfig; + +import java.util.Set; + +public interface Fields { + Set<String> getType(); +} http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/FilterDescriptor.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/FilterDescriptor.java b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/FilterDescriptor.java new file mode 100644 index 0000000..632c6cb --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/FilterDescriptor.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ambari.logsearch.config.api.model.inputconfig; + +import java.util.List; +import java.util.Map; + +public interface FilterDescriptor { + String getFilter(); + + Conditions getConditions(); + + Integer getSortOrder(); + + String getSourceField(); + + Boolean isRemoveSourceField(); + + Map<String, ? extends List<? extends PostMapValues>> getPostMapValues(); + + Boolean isEnabled(); +} http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/FilterGrokDescriptor.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/FilterGrokDescriptor.java b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/FilterGrokDescriptor.java new file mode 100644 index 0000000..e85ce97 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/FilterGrokDescriptor.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ambari.logsearch.config.api.model.inputconfig; + +public interface FilterGrokDescriptor extends FilterDescriptor { + String getLog4jFormat(); + + String getMultilinePattern(); + + String getMessagePattern(); +} http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/FilterJsonDescriptor.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/FilterJsonDescriptor.java b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/FilterJsonDescriptor.java new file mode 100644 index 0000000..08f1893 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/FilterJsonDescriptor.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ambari.logsearch.config.api.model.inputconfig; + +public interface FilterJsonDescriptor extends FilterDescriptor { +} http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/FilterKeyValueDescriptor.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/FilterKeyValueDescriptor.java b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/FilterKeyValueDescriptor.java new file mode 100644 index 0000000..6edd140 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/FilterKeyValueDescriptor.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ambari.logsearch.config.api.model.inputconfig; + +public interface FilterKeyValueDescriptor extends FilterDescriptor { + String getFieldSplit(); + + String getValueSplit(); + + String getValueBorders(); +} http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputConfig.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputConfig.java b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputConfig.java new file mode 100644 index 0000000..8126ac9 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputConfig.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ambari.logsearch.config.api.model.inputconfig; + +import java.util.List; + +public interface InputConfig { + List<? extends InputDescriptor> getInput(); + + List<? extends FilterDescriptor> getFilter(); +} http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java new file mode 100644 index 0000000..c41da93 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ambari.logsearch.config.api.model.inputconfig; + +import java.util.Map; + +public interface InputDescriptor { + String getType(); + + String getRowtype(); + + String getPath(); + + Map<String, String> getAddFields(); + + String getSource(); + + Boolean isTail(); + + Boolean isGenEventMd5(); + + Boolean isUseEventMd5AsId(); + + String getStartPosition(); + + Boolean isCacheEnabled(); + + String getCacheKeyField(); + + Boolean getCacheLastDedupEnabled(); + + Integer getCacheSize(); + + Long getCacheDedupInterval(); + + Boolean isEnabled(); +} http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputFileBaseDescriptor.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputFileBaseDescriptor.java b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputFileBaseDescriptor.java new file mode 100644 index 0000000..a393dc7 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputFileBaseDescriptor.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ambari.logsearch.config.api.model.inputconfig; + +public interface InputFileBaseDescriptor extends InputDescriptor { + Boolean getProcessFile(); + + Boolean getCopyFile(); + + Integer getCheckpointIntervalMs(); +} http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputFileDescriptor.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputFileDescriptor.java b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputFileDescriptor.java new file mode 100644 index 0000000..0070ad9 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputFileDescriptor.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ambari.logsearch.config.api.model.inputconfig; + +public interface InputFileDescriptor extends InputFileBaseDescriptor { +} http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputS3FileDescriptor.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputS3FileDescriptor.java b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputS3FileDescriptor.java new file mode 100644 index 0000000..b075629 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputS3FileDescriptor.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ambari.logsearch.config.api.model.inputconfig; + +public interface InputS3FileDescriptor extends InputFileBaseDescriptor { + String getS3AccessKey(); + + String getS3SecretKey(); +} http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/MapDateDescriptor.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/MapDateDescriptor.java b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/MapDateDescriptor.java new file mode 100644 index 0000000..f88435f --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/MapDateDescriptor.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ambari.logsearch.config.api.model.inputconfig; + +public interface MapDateDescriptor extends MapFieldDescriptor { + String getSourceDatePattern(); + + public String getTargetDatePattern(); +} http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/MapFieldCopyDescriptor.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/MapFieldCopyDescriptor.java b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/MapFieldCopyDescriptor.java new file mode 100644 index 0000000..596c173 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/MapFieldCopyDescriptor.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ambari.logsearch.config.api.model.inputconfig; + +public interface MapFieldCopyDescriptor extends MapFieldDescriptor { + String getCopyName(); +} http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/MapFieldDescriptor.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/MapFieldDescriptor.java b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/MapFieldDescriptor.java new file mode 100644 index 0000000..db086c5 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/MapFieldDescriptor.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ambari.logsearch.config.api.model.inputconfig; + +public interface MapFieldDescriptor { + public String getJsonName(); +} http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/MapFieldNameDescriptor.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/MapFieldNameDescriptor.java b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/MapFieldNameDescriptor.java new file mode 100644 index 0000000..da8cd0d --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/MapFieldNameDescriptor.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ambari.logsearch.config.api.model.inputconfig; + +public interface MapFieldNameDescriptor extends MapFieldDescriptor { + String getNewFieldName(); +} http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/MapFieldValueDescriptor.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/MapFieldValueDescriptor.java b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/MapFieldValueDescriptor.java new file mode 100644 index 0000000..cf37e62 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/MapFieldValueDescriptor.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ambari.logsearch.config.api.model.inputconfig; + +public interface MapFieldValueDescriptor extends MapFieldDescriptor { + String getPreValue(); + + public String getPostValue(); +} http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/PostMapValues.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/PostMapValues.java b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/PostMapValues.java new file mode 100644 index 0000000..5be7287 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/PostMapValues.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ambari.logsearch.config.api.model.inputconfig; + +import java.util.List; + +public interface PostMapValues { + List<MapFieldDescriptor> getMappers(); +} http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-config-api/src/test/java/org/apache/ambari/logsearch/config/api/LogSearchConfigClass1.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/test/java/org/apache/ambari/logsearch/config/api/LogSearchConfigClass1.java b/ambari-logsearch/ambari-logsearch-config-api/src/test/java/org/apache/ambari/logsearch/config/api/LogSearchConfigClass1.java index fc3fe5b..d7e3c0a 100644 --- a/ambari-logsearch/ambari-logsearch-config-api/src/test/java/org/apache/ambari/logsearch/config/api/LogSearchConfigClass1.java +++ b/ambari-logsearch/ambari-logsearch-config-api/src/test/java/org/apache/ambari/logsearch/config/api/LogSearchConfigClass1.java @@ -26,6 +26,7 @@ import org.apache.ambari.logsearch.config.api.InputConfigMonitor; import org.apache.ambari.logsearch.config.api.LogSearchConfig; import org.apache.ambari.logsearch.config.api.model.loglevelfilter.LogLevelFilter; import org.apache.ambari.logsearch.config.api.model.loglevelfilter.LogLevelFilterMap; +import org.apache.ambari.logsearch.config.api.model.inputconfig.InputConfig; public class LogSearchConfigClass1 implements LogSearchConfig { @Override @@ -52,7 +53,7 @@ public class LogSearchConfigClass1 implements LogSearchConfig { } @Override - public String getInputConfig(String clusterName, String serviceName) { + public InputConfig getInputConfig(String clusterName, String serviceName) { return null; } http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-config-api/src/test/java/org/apache/ambari/logsearch/config/api/LogSearchConfigClass2.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/test/java/org/apache/ambari/logsearch/config/api/LogSearchConfigClass2.java b/ambari-logsearch/ambari-logsearch-config-api/src/test/java/org/apache/ambari/logsearch/config/api/LogSearchConfigClass2.java index 346edb3..198c133 100644 --- a/ambari-logsearch/ambari-logsearch-config-api/src/test/java/org/apache/ambari/logsearch/config/api/LogSearchConfigClass2.java +++ b/ambari-logsearch/ambari-logsearch-config-api/src/test/java/org/apache/ambari/logsearch/config/api/LogSearchConfigClass2.java @@ -26,6 +26,7 @@ import org.apache.ambari.logsearch.config.api.InputConfigMonitor; import org.apache.ambari.logsearch.config.api.LogSearchConfig; import org.apache.ambari.logsearch.config.api.model.loglevelfilter.LogLevelFilter; import org.apache.ambari.logsearch.config.api.model.loglevelfilter.LogLevelFilterMap; +import org.apache.ambari.logsearch.config.api.model.inputconfig.InputConfig; public class LogSearchConfigClass2 implements LogSearchConfig { @Override @@ -52,7 +53,7 @@ public class LogSearchConfigClass2 implements LogSearchConfig { } @Override - public String getInputConfig(String clusterName, String serviceName) { + public InputConfig getInputConfig(String clusterName, String serviceName) { return null; } http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-config-zookeeper/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/pom.xml b/ambari-logsearch/ambari-logsearch-config-zookeeper/pom.xml index 2c59a4a..7ecda60 100644 --- a/ambari-logsearch/ambari-logsearch-config-zookeeper/pom.xml +++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/pom.xml @@ -71,8 +71,14 @@ <version>2.12.0</version> </dependency> <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + <version>1.7.7</version> + </dependency> + <dependency> <groupId>com.google.code.gson</groupId> <artifactId>gson</artifactId> + <version>2.6.2</version> </dependency> </dependencies> </project> http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/LogSearchConfigZK.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/LogSearchConfigZK.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/LogSearchConfigZK.java index 5e22374..4d10a5b 100644 --- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/LogSearchConfigZK.java +++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/LogSearchConfigZK.java @@ -27,6 +27,10 @@ import java.util.TreeMap; import org.apache.ambari.logsearch.config.api.LogSearchConfig; import org.apache.ambari.logsearch.config.api.model.loglevelfilter.LogLevelFilter; import org.apache.ambari.logsearch.config.api.model.loglevelfilter.LogLevelFilterMap; +import org.apache.ambari.logsearch.config.api.model.inputconfig.InputConfig; +import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.InputAdapter; +import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.InputConfigGson; +import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.InputConfigImpl; import org.apache.ambari.logsearch.config.api.InputConfigMonitor; import org.apache.ambari.logsearch.config.api.LogLevelFilterMonitor; import org.apache.commons.collections.MapUtils; @@ -40,18 +44,23 @@ import org.apache.curator.framework.recipes.cache.TreeCacheEvent.Type; import org.apache.curator.framework.recipes.cache.TreeCacheListener; import org.apache.curator.retry.ExponentialBackoffRetry; import org.apache.curator.utils.ZKPaths; -import org.apache.log4j.Logger; import org.apache.zookeeper.KeeperException.NodeExistsException; import org.apache.zookeeper.ZooDefs; import org.apache.zookeeper.data.ACL; import org.apache.zookeeper.data.Id; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.google.common.base.Splitter; import com.google.gson.Gson; import com.google.gson.GsonBuilder; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; public class LogSearchConfigZK implements LogSearchConfig { - private static final Logger LOG = Logger.getLogger(LogSearchConfigZK.class); + private static final Logger LOG = LoggerFactory.getLogger(LogSearchConfigZK.class); private static final int SESSION_TIMEOUT = 15000; private static final int CONNECTION_TIMEOUT = 30000; @@ -129,7 +138,16 @@ public class LogSearchConfigZK implements LogSearchConfig { @Override public void monitorInputConfigChanges(final InputConfigMonitor inputConfigMonitor, - final LogLevelFilterMonitor logLevelFilterMonitor ) throws Exception { + final LogLevelFilterMonitor logLevelFilterMonitor) throws Exception { + final JsonParser parser = new JsonParser(); + final JsonArray globalConfigNode = new JsonArray(); + for (String globalConfigJsonString : inputConfigMonitor.getGlobalConfigJsons()) { + JsonElement globalConfigJson = parser.parse(globalConfigJsonString); + globalConfigNode.add(globalConfigJson.getAsJsonObject().get("global")); + } + + createGlobalConfigNode(globalConfigNode); + TreeCacheListener listener = new TreeCacheListener() { public void childEvent(CuratorFramework client, TreeCacheEvent event) throws Exception { String nodeName = ZKPaths.getNodeFromPath(event.getData().getPath()); @@ -171,7 +189,16 @@ public class LogSearchConfigZK implements LogSearchConfig { private void addInputs(String serviceName, String inputConfig) { try { - inputConfigMonitor.loadInputConfigs(serviceName, inputConfig); + JsonElement inputConfigJson = parser.parse(inputConfig); + for (Map.Entry<String, JsonElement> typeEntry : inputConfigJson.getAsJsonObject().entrySet()) { + for (JsonElement e : typeEntry.getValue().getAsJsonArray()) { + for (JsonElement globalConfig : globalConfigNode) { + merge(globalConfig.getAsJsonObject(), e.getAsJsonObject()); + } + } + } + + inputConfigMonitor.loadInputConfigs(serviceName, InputConfigGson.gson.fromJson(inputConfigJson, InputConfigImpl.class)); } catch (Exception e) { LOG.error("Could not load input configuration for service " + serviceName + ":\n" + inputConfig, e); } @@ -193,11 +220,39 @@ public class LogSearchConfigZK implements LogSearchConfig { break; } } + + private void merge(JsonObject source, JsonObject target) { + for (Map.Entry<String, JsonElement> e : source.entrySet()) { + if (!target.has(e.getKey())) { + target.add(e.getKey(), e.getValue()); + } else { + if (e.getValue().isJsonObject()) { + JsonObject valueJson = (JsonObject)e.getValue(); + merge(valueJson, target.get(e.getKey()).getAsJsonObject()); + } + } + } + } }; cache.getListenable().addListener(listener); cache.start(); } + private void createGlobalConfigNode(JsonArray globalConfigNode) { + String globalConfigNodePath = String.format("%s/%s/global", root, properties.get(CLUSTER_NAME_PROPERTY)); + String data = InputConfigGson.gson.toJson(globalConfigNode); + + try { + if (cache.getCurrentData(globalConfigNodePath) != null) { + client.setData().forPath(globalConfigNodePath, data.getBytes()); + } else { + client.create().creatingParentContainersIfNeeded().withACL(getAcls()).forPath(globalConfigNodePath, data.getBytes()); + } + } catch (Exception e) { + LOG.warn("Exception during global config node creation/update", e); + } + } + @Override public List<String> getServices(String clusterName) { String parentPath = String.format("%s/%s/input", root, clusterName); @@ -206,9 +261,14 @@ public class LogSearchConfigZK implements LogSearchConfig { } @Override - public String getInputConfig(String clusterName, String serviceName) { + public InputConfig getInputConfig(String clusterName, String serviceName) { + String globalConfigNodePath = String.format("%s/%s/global", root, clusterName); + String globalConfigData = new String(cache.getCurrentData(globalConfigNodePath).getData()); + JsonArray globalConfigs = (JsonArray) new JsonParser().parse(globalConfigData); + InputAdapter.setGlobalConfigs(globalConfigs); + ChildData childData = cache.getCurrentData(String.format("%s/%s/input/%s", root, clusterName, serviceName)); - return childData == null ? null : new String(childData.getData()); + return childData == null ? null : InputConfigGson.gson.fromJson(new String(childData.getData()), InputConfigImpl.class); } @Override http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/ConditionsImpl.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/ConditionsImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/ConditionsImpl.java new file mode 100644 index 0000000..8bbff8f --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/ConditionsImpl.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl; + +import org.apache.ambari.logsearch.config.api.model.inputconfig.Conditions; + +import com.google.gson.annotations.Expose; + +public class ConditionsImpl implements Conditions { + @Expose + private FieldsImpl fields; + + public FieldsImpl getFields() { + return fields; + } + + public void setFields(FieldsImpl fields) { + this.fields = fields; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FieldsImpl.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FieldsImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FieldsImpl.java new file mode 100644 index 0000000..68cd0e2 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FieldsImpl.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl; + +import java.util.Set; + +import org.apache.ambari.logsearch.config.api.model.inputconfig.Fields; + +import com.google.gson.annotations.Expose; + +public class FieldsImpl implements Fields { + @Expose + private Set<String> type; + + public Set<String> getType() { + return type; + } + + public void setType(Set<String> type) { + this.type = type; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterAdapter.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterAdapter.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterAdapter.java new file mode 100644 index 0000000..b84403b --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterAdapter.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl; + +import java.lang.reflect.Type; + +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonElement; + +public class FilterAdapter implements JsonDeserializer<FilterDescriptorImpl> { + @Override + public FilterDescriptorImpl deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) { + switch (json.getAsJsonObject().get("filter").getAsString()) { + case "grok": + return (FilterDescriptorImpl)context.deserialize(json, FilterGrokDescriptorImpl.class); + case "keyvalue": + return (FilterDescriptorImpl)context.deserialize(json, FilterKeyValueDescriptorImpl.class); + case "json": + return (FilterDescriptorImpl)context.deserialize(json, FilterJsonDescriptorImpl.class); + default: + throw new IllegalArgumentException("Unknown filter type: " + json.getAsJsonObject().get("filter").getAsString()); + } + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterDescriptorImpl.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterDescriptorImpl.java new file mode 100644 index 0000000..4e11715 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterDescriptorImpl.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl; + +import java.util.List; +import java.util.Map; + +import org.apache.ambari.logsearch.config.api.model.inputconfig.FilterDescriptor; +import org.apache.ambari.logsearch.config.api.model.inputconfig.PostMapValues; + +import com.google.gson.annotations.Expose; +import com.google.gson.annotations.SerializedName; + +public abstract class FilterDescriptorImpl implements FilterDescriptor { + @Expose + private String filter; + + @Expose + private ConditionsImpl conditions; + + @Expose + @SerializedName("sort_order") + private Integer sortOrder; + + @Expose + @SerializedName("source_field") + private String sourceField; + + @Expose + @SerializedName("remove_source_field") + private Boolean removeSourceField; + + @Expose + @SerializedName("post_map_values") + private Map<String, List<PostMapValuesImpl>> postMapValues; + + @Expose + @SerializedName("is_enabled") + private Boolean isEnabled; + + public String getFilter() { + return filter; + } + + public void setFilter(String filter) { + this.filter = filter; + } + + public ConditionsImpl getConditions() { + return conditions; + } + + public void setConditions(ConditionsImpl conditions) { + this.conditions = conditions; + } + + public Integer getSortOrder() { + return sortOrder; + } + + public void setSortOrder(Integer sortOrder) { + this.sortOrder = sortOrder; + } + + public String getSourceField() { + return sourceField; + } + + public void setSourceField(String sourceField) { + this.sourceField = sourceField; + } + + public Boolean isRemoveSourceField() { + return removeSourceField; + } + + public void setRemoveSourceField(Boolean removeSourceField) { + this.removeSourceField = removeSourceField; + } + + public Map<String, ? extends List<? extends PostMapValues>> getPostMapValues() { + return postMapValues; + } + + public void setPostMapValues(Map<String, List<PostMapValuesImpl>> postMapValues) { + this.postMapValues = postMapValues; + } + + public Boolean isEnabled() { + return isEnabled; + } + + public void setIsEnabled(Boolean isEnabled) { + this.isEnabled = isEnabled; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterGrokDescriptorImpl.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterGrokDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterGrokDescriptorImpl.java new file mode 100644 index 0000000..7f40b7f --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterGrokDescriptorImpl.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl; + +import org.apache.ambari.logsearch.config.api.model.inputconfig.FilterGrokDescriptor; + +import com.google.gson.annotations.Expose; +import com.google.gson.annotations.SerializedName; + +public class FilterGrokDescriptorImpl extends FilterDescriptorImpl implements FilterGrokDescriptor { + @Expose + @SerializedName("log4j_format") + private String log4jFormat; + + @Expose + @SerializedName("multiline_pattern") + private String multilinePattern; + + @Expose + @SerializedName("message_pattern") + private String messagePattern; + + @Override + public String getLog4jFormat() { + return log4jFormat; + } + + public void setLog4jFormat(String log4jFormat) { + this.log4jFormat = log4jFormat; + } + + @Override + public String getMultilinePattern() { + return multilinePattern; + } + + public void setMultilinePattern(String multilinePattern) { + this.multilinePattern = multilinePattern; + } + + @Override + public String getMessagePattern() { + return messagePattern; + } + + public void setMessagePattern(String messagePattern) { + this.messagePattern = messagePattern; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterJsonDescriptorImpl.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterJsonDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterJsonDescriptorImpl.java new file mode 100644 index 0000000..9bf1a2b --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterJsonDescriptorImpl.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl; + +import org.apache.ambari.logsearch.config.api.model.inputconfig.FilterJsonDescriptor; + +public class FilterJsonDescriptorImpl extends FilterDescriptorImpl implements FilterJsonDescriptor { +} http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterKeyValueDescriptorImpl.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterKeyValueDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterKeyValueDescriptorImpl.java new file mode 100644 index 0000000..8e89990 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterKeyValueDescriptorImpl.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl; + +import org.apache.ambari.logsearch.config.api.model.inputconfig.FilterKeyValueDescriptor; + +import com.google.gson.annotations.Expose; +import com.google.gson.annotations.SerializedName; + +public class FilterKeyValueDescriptorImpl extends FilterDescriptorImpl implements FilterKeyValueDescriptor { + @Expose + @SerializedName("field_split") + private String fieldSplit; + + @Expose + @SerializedName("value_split") + private String valueSplit; + + @Expose + @SerializedName("value_borders") + private String valueBorders; + + public String getFieldSplit() { + return fieldSplit; + } + + public void setFieldSplit(String fieldSplit) { + this.fieldSplit = fieldSplit; + } + + public String getValueSplit() { + return valueSplit; + } + + public void setValueSplit(String valueSplit) { + this.valueSplit = valueSplit; + } + + public String getValueBorders() { + return valueBorders; + } + + public void setValueBorders(String valueBorders) { + this.valueBorders = valueBorders; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputAdapter.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputAdapter.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputAdapter.java new file mode 100644 index 0000000..86741c6 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputAdapter.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl; + +import java.lang.reflect.Type; + +import com.google.gson.JsonArray; +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonElement; + +public class InputAdapter implements JsonDeserializer<InputDescriptorImpl> { + private static JsonArray globalConfigs; + public static void setGlobalConfigs(JsonArray globalConfigs_) { + globalConfigs = globalConfigs_; + } + + @Override + public InputDescriptorImpl deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) { + String source = null; + if (json.getAsJsonObject().has("source")) { + source = json.getAsJsonObject().get("source").getAsString(); + } else { + for (JsonElement e : globalConfigs) { + if (e.getAsJsonObject().has("source")) { + source = e.getAsJsonObject().get("source").getAsString(); + break; + } + } + } + + switch (source) { + case "file": + return (InputDescriptorImpl)context.deserialize(json, InputFileDescriptorImpl.class); + case "s3_file": + return (InputDescriptorImpl)context.deserialize(json, InputS3FileDescriptorImpl.class); + default: + throw new IllegalArgumentException("Unknown input type: " + json.getAsJsonObject().get("source").getAsString()); + } + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputConfigGson.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputConfigGson.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputConfigGson.java new file mode 100644 index 0000000..3b78aff --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputConfigGson.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl; + +import java.lang.reflect.Type; +import java.util.List; + +import com.google.common.reflect.TypeToken; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; + +/** + * Helper class to convert betweeb json string and InputConfig class. + */ +public class InputConfigGson { + public static Gson gson; + static { + Type inputType = new TypeToken<InputDescriptorImpl>() {}.getType(); + Type filterType = new TypeToken<FilterDescriptorImpl>() {}.getType(); + Type postMapValuesType = new TypeToken<List<PostMapValuesImpl>>() {}.getType(); + gson = new GsonBuilder() + .registerTypeAdapter(inputType, new InputAdapter()) + .registerTypeAdapter(filterType, new FilterAdapter()) + .registerTypeAdapter(postMapValuesType, new PostMapValuesAdapter()) + .setPrettyPrinting() + .excludeFieldsWithoutExposeAnnotation() + .create(); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputConfigImpl.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputConfigImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputConfigImpl.java new file mode 100644 index 0000000..a4eba8e --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputConfigImpl.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl; + +import java.util.List; + +import org.apache.ambari.logsearch.config.api.model.inputconfig.FilterDescriptor; +import org.apache.ambari.logsearch.config.api.model.inputconfig.InputConfig; +import org.apache.ambari.logsearch.config.api.model.inputconfig.InputDescriptor; + +import com.google.gson.annotations.Expose; + +public class InputConfigImpl implements InputConfig { + @Expose + private List<InputDescriptorImpl> input; + + @Expose + private List<FilterDescriptorImpl> filter; + + @Override + public List<? extends InputDescriptor> getInput() { + return input; + } + + public void setInput(List<InputDescriptorImpl> input) { + this.input = input; + } + + @Override + public List<? extends FilterDescriptor> getFilter() { + return filter; + } + + public void setFilter(List<FilterDescriptorImpl> filter) { + this.filter = filter; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputDescriptorImpl.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputDescriptorImpl.java new file mode 100644 index 0000000..94dcc2a --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputDescriptorImpl.java @@ -0,0 +1,204 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl; + +import java.util.Map; + +import org.apache.ambari.logsearch.config.api.model.inputconfig.InputDescriptor; + +import com.google.gson.annotations.Expose; +import com.google.gson.annotations.SerializedName; + +public abstract class InputDescriptorImpl implements InputDescriptor { + @Expose + private String type; + + @Expose + private String rowtype; + + @Expose + private String path; + + @Expose + @SerializedName("add_fields") + private Map<String, String> addFields; + + @Expose + private String source; + + @Expose + private Boolean tail; + + @Expose + @SerializedName("gen_event_md5") + private Boolean genEventMd5; + + @Expose + @SerializedName("use_event_md5_as_id") + private Boolean useEventMd5AsId; + + @Expose + @SerializedName("start_position") + private String startPosition; + + @Expose + @SerializedName("cache_enabled") + private Boolean cacheEnabled; + + @Expose + @SerializedName("cache_key_field") + private String cacheKeyField; + + @Expose + @SerializedName("cache_last_dedup_enabled") + private Boolean cacheLastDedupEnabled; + + @Expose + @SerializedName("cache_size") + private Integer cacheSize; + + @Expose + @SerializedName("cache_dedup_interval") + private Long cacheDedupInterval; + + @Expose + @SerializedName("is_enabled") + private Boolean isEnabled; + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getRowtype() { + return rowtype; + } + + public void setRowtype(String rowType) { + this.rowtype = rowType; + } + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + + public Map<String, String> getAddFields() { + return addFields; + } + + public void setAddFields(Map<String, String> addFields) { + this.addFields = addFields; + } + + public String getSource() { + return source; + } + + public void setSource(String source) { + this.source = source; + } + + public Boolean isTail() { + return tail; + } + + public void setTail(Boolean tail) { + this.tail = tail; + } + + public Boolean isGenEventMd5() { + return genEventMd5; + } + + public void setGenEventMd5(Boolean genEventMd5) { + this.genEventMd5 = genEventMd5; + } + + public Boolean isUseEventMd5AsId() { + return useEventMd5AsId; + } + + public void setUseEventMd5AsId(Boolean useEventMd5AsId) { + this.useEventMd5AsId = useEventMd5AsId; + } + + public String getStartPosition() { + return startPosition; + } + + public void setStartPosition(String startPosition) { + this.startPosition = startPosition; + } + + public Boolean isCacheEnabled() { + return cacheEnabled; + } + + public void setCacheEnabled(Boolean cacheEnabled) { + this.cacheEnabled = cacheEnabled; + } + + public String getCacheKeyField() { + return cacheKeyField; + } + + public void setCacheKeyField(String cacheKeyField) { + this.cacheKeyField = cacheKeyField; + } + + public Boolean getCacheLastDedupEnabled() { + return cacheLastDedupEnabled; + } + + public void setCacheLastDedupEnabled(Boolean cacheLastDedupEnabled) { + this.cacheLastDedupEnabled = cacheLastDedupEnabled; + } + + public Integer getCacheSize() { + return cacheSize; + } + + public void setCacheSize(Integer cacheSize) { + this.cacheSize = cacheSize; + } + + public Long getCacheDedupInterval() { + return cacheDedupInterval; + } + + public void setCacheDedupInterval(Long cacheDedupInterval) { + this.cacheDedupInterval = cacheDedupInterval; + } + + public Boolean isEnabled() { + return isEnabled; + } + + public void setIsEnabled(Boolean isEnabled) { + this.isEnabled = isEnabled; + } +}