http://git-wip-us.apache.org/repos/asf/ambari/blob/5c9bdbfe/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java index 0fdc21f..a244a4e 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java @@ -32,6 +32,7 @@ import java.util.Properties; import org.apache.ambari.logfeeder.LogFeeder; import org.apache.ambari.logfeeder.metrics.MetricData; +import org.apache.ambari.logsearch.config.api.LogSearchPropertyDescription; import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Level; import org.apache.log4j.Logger; @@ -46,6 +47,26 @@ import com.google.gson.reflect.TypeToken; public class LogFeederUtil { private static final Logger LOG = Logger.getLogger(LogFeederUtil.class); + public static final String LOGFEEDER_PROPERTIES_FILE = "logfeeder.properties"; + + @LogSearchPropertyDescription( + name = "cluster.name", + description = "The name of the cluster the Log Feeder program runs in.", + examples = {"cl1"}, + sources = {LOGFEEDER_PROPERTIES_FILE} + ) + private static final String CLUSTER_NAME_PROPERTY = "cluster.name"; + + private static final String DEFAULT_TMP_DIR = "/tmp/$username/logfeeder/"; + @LogSearchPropertyDescription( + name = "logfeeder.tmp.dir", + description = "The tmp dir used for creating temporary files.", + examples = {"/tmp/"}, + defaultValue = DEFAULT_TMP_DIR, + sources = {LOGFEEDER_PROPERTIES_FILE} + ) + private static final String TMP_DIR_PROPERTY = "logfeeder.tmp.dir"; + private final static String GSON_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss.SSS"; private static Gson gson = new GsonBuilder().setDateFormat(GSON_DATE_FORMAT).create(); @@ -81,6 +102,10 @@ public class LogFeederUtil { return props; } + public static void loadProperties() throws Exception { + loadProperties(LOGFEEDER_PROPERTIES_FILE); + } + /** * This method will read the properties from System, followed by propFile and finally from the map */ @@ -236,16 +261,20 @@ public class LogFeederUtil { } } - private static String logfeederTempDir = null; + public static String getClusterName() { + return getStringProperty(CLUSTER_NAME_PROPERTY); + } + + private static String logFeederTempDir = null; - public synchronized static String getLogfeederTempDir() { - if (logfeederTempDir == null) { - String tempDirValue = getStringProperty("logfeeder.tmp.dir", "/tmp/$username/logfeeder/"); + public synchronized static String getLogFeederTempDir() { + if (logFeederTempDir == null) { + String tempDirValue = getStringProperty(TMP_DIR_PROPERTY, DEFAULT_TMP_DIR); HashMap<String, String> contextParam = new HashMap<String, String>(); String username = System.getProperty("user.name"); contextParam.put("username", username); - logfeederTempDir = PlaceholderUtil.replaceVariables(tempDirValue, contextParam); + logFeederTempDir = PlaceholderUtil.replaceVariables(tempDirValue, contextParam); } - return logfeederTempDir; + return logFeederTempDir; } }
http://git-wip-us.apache.org/repos/asf/ambari/blob/5c9bdbfe/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogfeederHDFSUtil.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogfeederHDFSUtil.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogfeederHDFSUtil.java deleted file mode 100644 index c975b99..0000000 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogfeederHDFSUtil.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.ambari.logfeeder.util; - -import java.io.IOException; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.log4j.Logger; - -public class LogfeederHDFSUtil { - private static final Logger LOG = Logger.getLogger(LogfeederHDFSUtil.class); - - private LogfeederHDFSUtil() { - throw new UnsupportedOperationException(); - } - - public static boolean copyFromLocal(String sourceFilepath, String destFilePath, FileSystem fileSystem, boolean overwrite, - boolean delSrc) { - Path src = new Path(sourceFilepath); - Path dst = new Path(destFilePath); - boolean isCopied = false; - try { - LOG.info("copying localfile := " + sourceFilepath + " to hdfsPath := " + destFilePath); - fileSystem.copyFromLocalFile(delSrc, overwrite, src, dst); - isCopied = true; - } catch (Exception e) { - LOG.error("Error copying local file :" + sourceFilepath + " to hdfs location : " + destFilePath, e); - } - return isCopied; - } - - public static FileSystem buildFileSystem(String hdfsHost, String hdfsPort) { - try { - Configuration configuration = buildHdfsConfiguration(hdfsHost, hdfsPort); - FileSystem fs = FileSystem.get(configuration); - return fs; - } catch (Exception e) { - LOG.error("Exception is buildFileSystem :", e); - } - return null; - } - - private static Configuration buildHdfsConfiguration(String hdfsHost, String hdfsPort) { - String url = "hdfs://" + hdfsHost + ":" + hdfsPort + "/"; - Configuration configuration = new Configuration(); - configuration.set("fs.default.name", url); - return configuration; - } - - public static void closeFileSystem(FileSystem fileSystem) { - if (fileSystem != null) { - try { - fileSystem.close(); - } catch (IOException e) { - LOG.error(e.getLocalizedMessage(), e.getCause()); - } - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/5c9bdbfe/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/SSLUtil.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/SSLUtil.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/SSLUtil.java index d963de3..573bb1c 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/SSLUtil.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/SSLUtil.java @@ -19,6 +19,7 @@ package org.apache.ambari.logfeeder.util; +import org.apache.ambari.logsearch.config.api.LogSearchPropertyDescription; import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang3.ArrayUtils; @@ -28,6 +29,8 @@ import org.apache.log4j.Logger; import java.io.File; import java.nio.charset.Charset; +import static org.apache.ambari.logfeeder.util.LogFeederUtil.LOGFEEDER_PROPERTIES_FILE; + public class SSLUtil { private static final Logger LOG = Logger.getLogger(SSLUtil.class); @@ -42,6 +45,12 @@ public class SSLUtil { private static final String KEYSTORE_PASSWORD_FILE = "ks_pass.txt"; private static final String TRUSTSTORE_PASSWORD_FILE = "ts_pass.txt"; + @LogSearchPropertyDescription( + name = "hadoop.security.credential.provider.path", + description = "The jceks file that provides passwords.", + examples = {"jceks://file/etc/ambari-logsearch-logfeeder/conf/logfeeder.jceks"}, + sources = {LOGFEEDER_PROPERTIES_FILE} + ) private static final String CREDENTIAL_STORE_PROVIDER_PATH = "hadoop.security.credential.provider.path"; private static final String LOGFEEDER_CERT_DEFAULT_FOLDER = "/etc/ambari-logsearch-portal/conf/keys"; private static final String LOGFEEDER_STORE_DEFAULT_PASSWORD = "bigdata"; http://git-wip-us.apache.org/repos/asf/ambari/blob/5c9bdbfe/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java index 8f75c3a..7abf177 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java @@ -25,7 +25,7 @@ import java.util.Map; import java.util.TimeZone; import org.apache.ambari.logfeeder.common.LogFeederConstants; -import org.apache.ambari.logfeeder.common.LogfeederException; +import org.apache.ambari.logfeeder.common.LogFeederException; import org.apache.ambari.logfeeder.input.InputMarker; import org.apache.ambari.logfeeder.output.OutputManager; import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.FilterJsonDescriptorImpl; @@ -135,9 +135,9 @@ public class FilterJSONTest { String inputStr="invalid json"; try{ filterJson.apply(inputStr,new InputMarker(null, null, 0)); - fail("Expected LogfeederException was not occured"); - }catch(LogfeederException logfeederException){ - assertEquals("Json parsing failed for inputstr = "+inputStr, logfeederException.getLocalizedMessage()); + fail("Expected LogFeederException was not occured"); + }catch(LogFeederException logFeederException){ + assertEquals("Json parsing failed for inputstr = "+inputStr, logFeederException.getLocalizedMessage()); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/5c9bdbfe/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputManagerTest.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputManagerTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputManagerTest.java index 625e362..46fbc3b 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputManagerTest.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputManagerTest.java @@ -143,32 +143,6 @@ public class InputManagerTest { } @Test - public void testInputManagr_waitOnAllInputs() throws Exception { - Input input1 = strictMock(Input.class); - Input input2 = strictMock(Input.class); - Input input3 = strictMock(Input.class); - - Thread mockThread = strictMock(Thread.class); - - expect(input1.getThread()).andReturn(null); - expect(input2.getThread()).andReturn(null); - expect(input3.getThread()).andReturn(mockThread); - - mockThread.join(); expectLastCall(); - - replay(input1, input2, input3); - - InputManager manager = new InputManager(); - manager.add("serviceName", input1); - manager.add("serviceName", input2); - manager.add("serviceName", input3); - - manager.waitOnAllInputs(); - - verify(input1, input2, input3); - } - - @Test public void testInputManager_checkInAll() throws Exception { Input input1 = strictMock(Input.class); Input input2 = strictMock(Input.class); http://git-wip-us.apache.org/repos/asf/ambari/blob/5c9bdbfe/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/metrics/MetricsManagerTest.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/metrics/MetricsManagerTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/metrics/MetricsManagerTest.java index 1461352..4b4067c 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/metrics/MetricsManagerTest.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/metrics/MetricsManagerTest.java @@ -80,7 +80,6 @@ public class MetricsManagerTest { assertEquals(metricList.size(), 1); TimelineMetric metric = metricList.get(0); - assertEquals(metric.getHostName(), "test_host_name"); assertEquals(metric.getAppId(), "logfeeder"); assertEquals(metric.getMetricName(), "metric1"); assertEquals(metric.getType(), "Long"); @@ -116,7 +115,6 @@ public class MetricsManagerTest { assertEquals(metricList.size(), 1); TimelineMetric metric = metricList.get(0); - assertEquals(metric.getHostName(), "test_host_name"); assertEquals(metric.getAppId(), "logfeeder"); assertEquals(metric.getMetricName(), "metric1"); assertEquals(metric.getType(), "Long"); http://git-wip-us.apache.org/repos/asf/ambari/blob/5c9bdbfe/ambari-logsearch/ambari-logsearch-logfeeder/src/test/resources/logfeeder.properties ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/resources/logfeeder.properties b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/resources/logfeeder.properties index 19027d1..5476c96 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/resources/logfeeder.properties +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/resources/logfeeder.properties @@ -17,5 +17,4 @@ logfeeder.log.filter.enable=true logfeeder.solr.config.interval=5 logfeeder.solr.zk_connect_string=some_connect_string logfeeder.metrics.collector.hosts=some_collector_host -node.hostname=test_host_name logfeeder.include.default.level=FATAL,ERROR,WARN \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/5c9bdbfe/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/LogSearchConfigConfigurer.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/LogSearchConfigConfigurer.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/LogSearchConfigConfigurer.java index 978e91a..c34dce6 100644 --- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/LogSearchConfigConfigurer.java +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/LogSearchConfigConfigurer.java @@ -53,7 +53,7 @@ public class LogSearchConfigConfigurer implements Configurer { while (true) { try { logSearchConfig = LogSearchConfigFactory.createLogSearchConfig(Component.SERVER, PropertiesHelper.getProperties(), - LogSearchConfigZK.class); + null, LogSearchConfigZK.class); logSearchConfigState.setLogSearchConfigAvailable(true); break; } catch (Exception e) {