Repository: incubator-eagle Updated Branches: refs/heads/master 67c915127 -> f430f521f
[EAGLE-834] Add Daily Job Summary Report https://issues.apache.org/jira/browse/EAGLE-834 Author: Zhao, Qingwen <qingwz...@apache.org> Author: Qingwen Zhao <qingwen...@gmail.com> Closes #742 from qingwen220/EAGLE-834. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/f430f521 Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/f430f521 Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/f430f521 Branch: refs/heads/master Commit: f430f521faea959491208c199b59244386cef438 Parents: 67c9151 Author: Zhao, Qingwen <qingwz...@apache.org> Authored: Thu Dec 15 11:49:13 2016 +0800 Committer: Zhao, Qingwen <qingwz...@apache.org> Committed: Thu Dec 15 11:49:13 2016 +0800 ---------------------------------------------------------------------- .../engine/coordinator/AlertDefinition.java | 12 +- .../publisher/impl/AlertEmailPublisher.java | 30 +- .../src/test/resources/application-test.conf | 18 +- .../app/service/ApplicationEmailService.java | 104 ++++++ .../eagle/common/mail/AbstractEmailService.java | 82 +++++ .../eagle/common/mail/AlertEmailConstants.java | 57 ++++ .../eagle/common/mail/AlertEmailContext.java | 68 ++++ .../eagle/common/mail/AlertEmailSender.java | 111 +++++++ .../eagle/common/mail/EagleMailClient.java | 229 ++++++++++++++ eagle-jpm/eagle-jpm-mr-history/pom.xml | 6 + .../mr/history/MRHistoryJobDailyReporter.java | 317 +++++++++++++++++++ .../src/main/resources/JobReportTemplate.vm | 141 +++++++++ .../history/MRHistoryJobDailyReporterTest.java | 109 +++++++ .../src/test/resources/JobReportTemplate.vm | 141 +++++++++ .../src/test/resources/application-test.conf | 58 ++++ .../src/test/resources/log4j.properties | 34 ++ eagle-server-assembly/src/main/conf/eagle.conf | 25 +- .../apache/eagle/server/ServerApplication.java | 11 + .../src/main/resources/application.conf | 27 +- 19 files changed, 1537 insertions(+), 43 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/f430f521/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/AlertDefinition.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/AlertDefinition.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/AlertDefinition.java index 66a9bce..e94d6fa 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/AlertDefinition.java +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/AlertDefinition.java @@ -18,6 +18,8 @@ package org.apache.eagle.alert.engine.coordinator; import org.apache.commons.lang3.builder.HashCodeBuilder; +import java.util.Objects; + public class AlertDefinition { private TemplateType templateType = TemplateType.TEXT; private String subject; @@ -92,11 +94,11 @@ public class AlertDefinition { return false; } AlertDefinition another = (AlertDefinition) that; - if (another.templateType.equals(this.templateType) - && another.body.equals(this.body) - && another.category.equals(this.category) - && another.severity.equals(this.severity) - && another.subject.equals(this.subject)) { + if (Objects.equals(another.templateType, this.templateType) + && Objects.equals(another.body, this.body) + && Objects.equals(another.category, this.category) + && Objects.equals(another.severity, this.severity) + && Objects.equals(another.subject, this.subject)) { return true; } return false; http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/f430f521/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertEmailPublisher.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertEmailPublisher.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertEmailPublisher.java index d81ec2a..d08d114 100644 --- a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertEmailPublisher.java +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertEmailPublisher.java @@ -39,6 +39,7 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import static org.apache.eagle.alert.service.MetadataServiceClientImpl.*; +import static org.apache.eagle.common.mail.AlertEmailConstants.*; public class AlertEmailPublisher extends AbstractPublishPlugin { @@ -47,14 +48,6 @@ public class AlertEmailPublisher extends AbstractPublishPlugin { private static final int DEFAULT_THREAD_POOL_MAX_SIZE = 8; private static final long DEFAULT_THREAD_POOL_SHRINK_TIME = 60000L; // 1 minute - private static final String EAGLE_CORRELATION_SMTP_SERVER = "metadataService.mailSmtpServer"; - private static final String EAGLE_CORRELATION_SMTP_PORT = "metadataService.mailSmtpPort"; - private static final String EAGLE_CORRELATION_SMTP_CONN = "metadataService.mailSmtpConn"; - private static final String EAGLE_CORRELATION_SMTP_AUTH = "metadataService.mailSmtpAuth"; - private static final String EAGLE_CORRELATION_SMTP_USERNAME = "metadataService.mailSmtpUsername"; - private static final String EAGLE_CORRELATION_SMTP_PASSWORD = "metadataService.mailSmtpPassword"; - private static final String EAGLE_CORRELATION_SMTP_DEBUG = "metadataService.mailSmtpDebug"; - private AlertEmailGenerator emailGenerator; private Map<String, Object> emailConfig; @@ -83,23 +76,28 @@ public class AlertEmailPublisher extends AbstractPublishPlugin { private Properties parseMailClientConfig(Config config) { Properties props = new Properties(); - String mailSmtpServer = config.getString(EAGLE_CORRELATION_SMTP_SERVER); - String mailSmtpPort = config.getString(EAGLE_CORRELATION_SMTP_PORT); - String mailSmtpAuth = config.getString(EAGLE_CORRELATION_SMTP_AUTH); + Config mailConfig = null; + if (config.hasPath(EAGLE_COORDINATOR_EMAIL_SERVICE)) { + mailConfig = config.getConfig(EAGLE_COORDINATOR_EMAIL_SERVICE); + } else if (config.hasPath(EAGLE_APPLICATION_EMAIL_SERVICE)) { + mailConfig = config.getConfig(EAGLE_APPLICATION_EMAIL_SERVICE); + } + String mailSmtpServer = mailConfig.getString(EAGLE_EMAIL_SMTP_SERVER); + String mailSmtpPort = mailConfig.getString(EAGLE_EMAIL_SMTP_PORT); + String mailSmtpAuth = mailConfig.getString(EAGLE_EMAIL_SMTP_AUTH); props.put(AlertEmailConstants.CONF_MAIL_HOST, mailSmtpServer); props.put(AlertEmailConstants.CONF_MAIL_PORT, mailSmtpPort); props.put(AlertEmailConstants.CONF_MAIL_AUTH, mailSmtpAuth); if (Boolean.parseBoolean(mailSmtpAuth)) { - String mailSmtpUsername = config.getString(EAGLE_CORRELATION_SMTP_USERNAME); - String mailSmtpPassword = config.getString(EAGLE_CORRELATION_SMTP_PASSWORD); + String mailSmtpUsername = mailConfig.getString(EAGLE_EMAIL_SMTP_USERNAME); + String mailSmtpPassword = mailConfig.getString(EAGLE_EMAIL_SMTP_PASSWORD); props.put(AlertEmailConstants.CONF_AUTH_USER, mailSmtpUsername); props.put(AlertEmailConstants.CONF_AUTH_PASSWORD, mailSmtpPassword); } - String mailSmtpConn = config.hasPath(EAGLE_CORRELATION_SMTP_CONN) ? config.getString(EAGLE_CORRELATION_SMTP_CONN) : AlertEmailConstants.CONN_PLAINTEXT; - String mailSmtpDebug = config.hasPath(EAGLE_CORRELATION_SMTP_DEBUG) ? config.getString(EAGLE_CORRELATION_SMTP_DEBUG) : "false"; + String mailSmtpConn = mailConfig.hasPath(EAGLE_EMAIL_SMTP_CONN) ? mailConfig.getString(EAGLE_EMAIL_SMTP_CONN) : AlertEmailConstants.CONN_PLAINTEXT; if (mailSmtpConn.equalsIgnoreCase(AlertEmailConstants.CONN_TLS)) { props.put("mail.smtp.starttls.enable", "true"); } @@ -108,6 +106,8 @@ public class AlertEmailPublisher extends AbstractPublishPlugin { props.put("mail.smtp.socketFactory.class", "javax.net.ssl.SSLSocketFactory"); } + + String mailSmtpDebug = mailConfig.hasPath(EAGLE_EMAIL_SMTP_DEBUG) ? mailConfig.getString(EAGLE_EMAIL_SMTP_DEBUG) : "false"; props.put(AlertEmailConstants.CONF_MAIL_DEBUG, mailSmtpDebug); return props; } http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/f430f521/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/test/resources/application-test.conf ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/test/resources/application-test.conf b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/test/resources/application-test.conf index 3573507..99bbee6 100755 --- a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/test/resources/application-test.conf +++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/test/resources/application-test.conf @@ -48,14 +48,7 @@ "metadataService": { "context": "/api", "host": "localhost", - "port": 8080, - mailSmtpServer = "localhost", - mailSmtpPort = 5025, - mailSmtpAuth = "false" - //mailSmtpConn = "plaintext", - //mailSmtpUsername = "" - //mailSmtpPassword = "" - //mailSmtpDebug = false + "port": 8080 }, "metric": { "sink": { @@ -73,4 +66,13 @@ } }, "connection": "mongodb://localhost:27017" + application.mailService { + mailSmtpServer = "localhost", + mailSmtpPort = 5025, + mailSmtpAuth = "false" + //mailSmtpConn = "plaintext", + //mailSmtpUsername = "" + //mailSmtpPassword = "" + //mailSmtpDebug = false + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/f430f521/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationEmailService.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationEmailService.java b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationEmailService.java new file mode 100644 index 0000000..7498748 --- /dev/null +++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationEmailService.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eagle.app.service; + +import com.typesafe.config.Config; +import org.apache.eagle.common.mail.AbstractEmailService; +import org.apache.eagle.common.mail.AlertEmailConstants; +import org.apache.eagle.common.mail.AlertEmailContext; +import org.apache.eagle.common.mail.AlertEmailSender; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Map; + +import static org.apache.eagle.common.mail.AlertEmailConstants.EAGLE_APPLICATION_EMAIL_SERVICE; + +public class ApplicationEmailService extends AbstractEmailService { + private static final Logger LOG = LoggerFactory.getLogger(ApplicationEmailService.class); + private String appConfPath; + private Config config; + + public ApplicationEmailService(Config config, String appConfPath) { + super(config.getConfig(EAGLE_APPLICATION_EMAIL_SERVICE)); + this.appConfPath = appConfPath; + this.config = config; + } + + public boolean onAlert(Map<String, Object> alertData) { + return super.onAlert(buildEmailContext(), alertData); + } + + private String buildDefaultSender() { + String hostname = ""; + try { + hostname = InetAddress.getLocalHost().getHostName(); + if (!hostname.endsWith(".com")) { + //avoid invalid host exception + hostname += ".com"; + } + } catch (UnknownHostException e) { + LOG.warn("UnknownHostException when get local hostname"); + } + return System.getProperty("user.name") + "@" + hostname; + } + + public AlertEmailContext buildEmailContext() { + return buildEmailContext(null); + } + + public AlertEmailContext buildEmailContext(String mailSubject) { + AlertEmailContext mailProps = new AlertEmailContext(); + Config appConfig = config.getConfig(appConfPath); + String tplFileName = appConfig.getString(AlertEmailConstants.TEMPLATE); + if (tplFileName == null || tplFileName.equals("")) { + tplFileName = "ALERT_INLINED_TEMPLATE.vm"; + } + String subject; + if (mailSubject != null) { + subject = mailSubject; + } else { + subject = appConfig.getString(AlertEmailConstants.SUBJECT); + } + String sender; + if (!appConfig.hasPath(AlertEmailConstants.SENDER)) { + sender = buildDefaultSender(); + } else { + sender = appConfig.getString(AlertEmailConstants.SENDER); + } + if (!appConfig.hasPath(AlertEmailConstants.RECIPIENTS)) { + throw new IllegalArgumentException("email recipients is null or unset"); + } + if (appConfig.hasPath(AlertEmailConstants.CC_RECIPIENTS)) { + mailProps.setCc(appConfig.getString(AlertEmailConstants.CC_RECIPIENTS)); + } + String recipients = appConfig.getString(AlertEmailConstants.RECIPIENTS); + mailProps.setSubject(subject); + mailProps.setRecipients(recipients); + mailProps.setSender(sender); + mailProps.setVelocityTplFile(tplFileName); + return mailProps; + } + + @Override + protected Logger getLogger() { + return LOG; + } +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/f430f521/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/mail/AbstractEmailService.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/mail/AbstractEmailService.java b/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/mail/AbstractEmailService.java new file mode 100644 index 0000000..57e5cba --- /dev/null +++ b/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/mail/AbstractEmailService.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.eagle.common.mail; + +import com.typesafe.config.Config; +import org.slf4j.Logger; + +import java.util.Map; +import java.util.Properties; + +import static org.apache.eagle.common.mail.AlertEmailConstants.*; + +public abstract class AbstractEmailService { + + private Properties serverProps; + + public AbstractEmailService(Config config) { + serverProps = parseMailClientConfig(config); + } + + protected abstract Logger getLogger(); + + private Properties parseMailClientConfig(Config config) { + Properties props = new Properties(); + String mailSmtpServer = config.getString(EAGLE_EMAIL_SMTP_SERVER); + String mailSmtpPort = config.getString(EAGLE_EMAIL_SMTP_PORT); + String mailSmtpAuth = config.getString(EAGLE_EMAIL_SMTP_AUTH); + + props.put(AlertEmailConstants.CONF_MAIL_HOST, mailSmtpServer); + props.put(AlertEmailConstants.CONF_MAIL_PORT, mailSmtpPort); + props.put(AlertEmailConstants.CONF_MAIL_AUTH, mailSmtpAuth); + + if (Boolean.parseBoolean(mailSmtpAuth)) { + String mailSmtpUsername = config.getString(EAGLE_EMAIL_SMTP_USERNAME); + String mailSmtpPassword = config.getString(EAGLE_EMAIL_SMTP_PASSWORD); + props.put(AlertEmailConstants.CONF_AUTH_USER, mailSmtpUsername); + props.put(AlertEmailConstants.CONF_AUTH_PASSWORD, mailSmtpPassword); + } + + String mailSmtpConn = config.hasPath(EAGLE_EMAIL_SMTP_CONN) ? config.getString(EAGLE_EMAIL_SMTP_CONN) : AlertEmailConstants.CONN_PLAINTEXT; + String mailSmtpDebug = config.hasPath(EAGLE_EMAIL_SMTP_DEBUG) ? config.getString(EAGLE_EMAIL_SMTP_DEBUG) : "false"; + if (mailSmtpConn.equalsIgnoreCase(AlertEmailConstants.CONN_TLS)) { + props.put("mail.smtp.starttls.enable", "true"); + } + if (mailSmtpConn.equalsIgnoreCase(AlertEmailConstants.CONN_SSL)) { + props.put("mail.smtp.socketFactory.port", "465"); + props.put("mail.smtp.socketFactory.class", + "javax.net.ssl.SSLSocketFactory"); + } + props.put(AlertEmailConstants.CONF_MAIL_DEBUG, mailSmtpDebug); + return props; + } + + public boolean onAlert(AlertEmailContext mailContext, Map<String, Object> alertData) { + /** synchronized email sending. */ + if (alertData == null || alertData.isEmpty()) { + getLogger().warn("alertData for {} is empty"); + return false; + } + AlertEmailSender mailSender = new AlertEmailSender(mailContext, serverProps); + mailSender.addAlertContext(alertData); + getLogger().info("Sending email in synchronous mode to: {} cc: {}", mailContext.getRecipients(), mailContext.getCc()); + return mailSender.send(); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/f430f521/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/mail/AlertEmailConstants.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/mail/AlertEmailConstants.java b/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/mail/AlertEmailConstants.java new file mode 100644 index 0000000..140d306 --- /dev/null +++ b/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/mail/AlertEmailConstants.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eagle.common.mail; + +public class AlertEmailConstants { + + public static final String CONN_PLAINTEXT = "plaintext"; + public static final String CONN_TLS = "tls"; + public static final String CONN_SSL = "ssl"; + + public static final String CONF_MAIL_HOST = "mail.smtp.host"; + public static final String CONF_MAIL_PORT = "mail.smtp.port"; + public static final String CONF_MAIL_AUTH = "mail.smtp.auth"; + public static final String CONF_AUTH_USER = "mail.username"; + public static final String CONF_AUTH_PASSWORD = "mail.password"; + public static final String CONF_MAIL_CONN = "mail.connection"; + public static final String CONF_MAIL_DEBUG = "mail.debug"; + + public static final String SUBJECT = "subject"; + public static final String SENDER = "sender"; + public static final String RECIPIENTS = "recipients"; + public static final String TEMPLATE = "template"; + public static final String CC_RECIPIENTS = "cc"; + + public static final String ALERT_EMAIL_TIME_PROPERTY = "timestamp"; + public static final String ALERT_EMAIL_COUNT_PROPERTY = "count"; + public static final String ALERT_EMAIL_ALERTLIST_PROPERTY = "alertList"; + public static final String ALERT_EMAIL_ORIGIN_PROPERTY = "alertEmailOrigin"; + + public static final String EAGLE_APPLICATION_EMAIL_SERVICE = "application.mailService"; + public static final String EAGLE_COORDINATOR_EMAIL_SERVICE = "coordinator.mailService"; + + public static final String EAGLE_EMAIL_SMTP_SERVER = "mailSmtpServer"; + public static final String EAGLE_EMAIL_SMTP_PORT = "mailSmtpPort"; + public static final String EAGLE_EMAIL_SMTP_CONN = "mailSmtpConn"; + public static final String EAGLE_EMAIL_SMTP_AUTH = "mailSmtpAuth"; + public static final String EAGLE_EMAIL_SMTP_USERNAME = "mailSmtpUsername"; + public static final String EAGLE_EMAIL_SMTP_PASSWORD = "mailSmtpPassword"; + public static final String EAGLE_EMAIL_SMTP_DEBUG = "mailSmtpDebug"; + + +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/f430f521/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/mail/AlertEmailContext.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/mail/AlertEmailContext.java b/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/mail/AlertEmailContext.java new file mode 100644 index 0000000..c01d51b --- /dev/null +++ b/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/mail/AlertEmailContext.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.eagle.common.mail; + +public class AlertEmailContext { + private String sender; + private String subject; + private String recipients; + private String velocityTplFile; + private String cc; + + public String getSender() { + return sender; + } + + public void setSender(String sender) { + this.sender = sender; + } + + public String getSubject() { + return subject; + } + + public void setSubject(String subject) { + this.subject = subject; + } + + public String getRecipients() { + return recipients; + } + + public void setRecipients(String recipients) { + this.recipients = recipients; + } + + public String getVelocityTplFile() { + return velocityTplFile; + } + + public void setVelocityTplFile(String velocityTplFile) { + this.velocityTplFile = velocityTplFile; + } + + public String getCc() { + return cc; + } + + public void setCc(String cc) { + this.cc = cc; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/f430f521/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/mail/AlertEmailSender.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/mail/AlertEmailSender.java b/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/mail/AlertEmailSender.java new file mode 100644 index 0000000..ffc249d --- /dev/null +++ b/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/mail/AlertEmailSender.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.eagle.common.mail; + +import org.apache.eagle.common.DateTimeUtil; +import org.apache.velocity.VelocityContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.management.ManagementFactory; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +public class AlertEmailSender { + private static final Logger LOG = LoggerFactory.getLogger(AlertEmailSender.class); + + private List<Map<String, Object>> alertContexts = new ArrayList<>(); + private String origin; + + private AlertEmailContext mailProps; + private Properties serverProps; + + private static final int MAX_RETRY_COUNT = 3; + + /** + * Derived class may have some additional context properties to add. + * @param context velocity context + */ + public void addAlertContext(Map<String, Object> context) { + alertContexts.add(context); + } + + public AlertEmailSender(AlertEmailContext mailProps, Properties serverProps) { + this.mailProps = mailProps; + this.serverProps = serverProps; + String tmp = ManagementFactory.getRuntimeMXBean().getName(); + this.origin = tmp.split("@")[1] + "(pid:" + tmp.split("@")[0] + ")"; + LOG.info("Initialized email sender: origin is :{}, recipient of the email: {}, velocity TPL file: {}", + origin, mailProps.getRecipients(), mailProps.getVelocityTplFile()); + } + + public boolean send() { + int count = 0; + boolean success = false; + while (count++ < MAX_RETRY_COUNT && !success) { + LOG.info("Sending email, tried: " + count + ", max: " + MAX_RETRY_COUNT); + try { + final EagleMailClient client; + if (serverProps != null) { + client = new EagleMailClient(serverProps); + } else { + client = new EagleMailClient(); + } + + final VelocityContext context = new VelocityContext(); + generateCommonContext(context); + LOG.info("After calling generateCommonContext..."); + + if (mailProps.getRecipients() == null || mailProps.getRecipients().equals("")) { + LOG.error("Recipients is null, skip sending emails "); + return success; + } + + success = client.send(mailProps.getSender(), + mailProps.getRecipients(), + mailProps.getCc(), + mailProps.getSubject(), + mailProps.getVelocityTplFile(), + context, + null); + LOG.info("Success of sending email: " + success); + if (!success && count < MAX_RETRY_COUNT) { + LOG.info("Sleep for a while before retrying"); + Thread.sleep(10 * 1000); + } + } catch (Exception e) { + LOG.warn("Sending mail exception", e); + } + } + if (success) { + LOG.info("Successfully send email with subject {}", mailProps.getSubject()); + } else { + LOG.warn("Fail sending email after tries {} times, subject: %s", MAX_RETRY_COUNT, mailProps.getSubject()); + } + return success; + } + + private void generateCommonContext(VelocityContext context) { + context.put(AlertEmailConstants.ALERT_EMAIL_TIME_PROPERTY, DateTimeUtil.millisecondsToHumanDateWithSeconds(System.currentTimeMillis())); + context.put(AlertEmailConstants.ALERT_EMAIL_COUNT_PROPERTY, alertContexts.size()); + context.put(AlertEmailConstants.ALERT_EMAIL_ALERTLIST_PROPERTY, alertContexts); + context.put(AlertEmailConstants.ALERT_EMAIL_ORIGIN_PROPERTY, origin); + } + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/f430f521/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/mail/EagleMailClient.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/mail/EagleMailClient.java b/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/mail/EagleMailClient.java new file mode 100644 index 0000000..81b1114 --- /dev/null +++ b/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/mail/EagleMailClient.java @@ -0,0 +1,229 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.eagle.common.mail; + +import org.apache.velocity.Template; +import org.apache.velocity.VelocityContext; +import org.apache.velocity.app.VelocityEngine; +import org.apache.velocity.exception.ResourceNotFoundException; +import org.apache.velocity.runtime.RuntimeConstants; +import org.apache.velocity.runtime.resource.loader.ClasspathResourceLoader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.activation.DataHandler; +import javax.activation.DataSource; +import javax.activation.FileDataSource; +import javax.mail.*; +import javax.mail.internet.*; +import java.io.File; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +public class EagleMailClient { + private static final Logger LOG = LoggerFactory.getLogger(EagleMailClient.class); + private static final String BASE_PATH = "templates/"; + + private VelocityEngine velocityEngine; + private Session session; + + public EagleMailClient() { + this(new Properties()); + } + + public EagleMailClient(final Properties config) { + try { + velocityEngine = new VelocityEngine(); + velocityEngine.setProperty(RuntimeConstants.RESOURCE_LOADER, "classpath"); + velocityEngine.setProperty("classpath.resource.loader.class", ClasspathResourceLoader.class.getName()); + velocityEngine.init(); + + config.put("mail.transport.protocol", "smtp"); + if (Boolean.parseBoolean(config.getProperty(AlertEmailConstants.CONF_MAIL_AUTH))) { + session = Session.getInstance(config, new Authenticator() { + protected PasswordAuthentication getPasswordAuthentication() { + return new PasswordAuthentication( + config.getProperty(AlertEmailConstants.CONF_AUTH_USER), + config.getProperty(AlertEmailConstants.CONF_AUTH_PASSWORD) + ); + } + }); + } else { + session = Session.getInstance(config, new Authenticator() { + }); + } + + final String debugMode = config.getProperty(AlertEmailConstants.CONF_MAIL_DEBUG, "false"); + final boolean debug = Boolean.parseBoolean(debugMode); + LOG.info("Set email debug mode: " + debugMode); + session.setDebug(debug); + } catch (Exception e) { + LOG.error("Failed to connect to smtp server", e); + } + } + + private boolean sendInternal(String from, String to, String cc, String title, String content) { + Message msg = new MimeMessage(session); + try { + msg.setFrom(new InternetAddress(from)); + msg.setSubject(title); + if (to != null) { + msg.setRecipients(Message.RecipientType.TO, InternetAddress.parse(to)); + } + if (cc != null) { + msg.setRecipients(Message.RecipientType.CC, InternetAddress.parse(cc)); + } + //msg.setRecipients(Message.RecipientType.BCC, InternetAddress.parse(DEFAULT_BCC_ADDRESS)); + msg.setContent(content, "text/html;charset=utf-8"); + LOG.info(String.format("Going to send mail: from[%s], to[%s], cc[%s], title[%s]", from, to, cc, title)); + Transport.send(msg); + return true; + } catch (AddressException e) { + LOG.info("Failed to send mail, got an AddressException: " + e.getMessage(), e); + return false; + } catch (MessagingException e) { + LOG.info("Failed to send mail, got an AddressException: " + e.getMessage(), e); + return false; + } + } + + private boolean sendInternal(String from, String to, String cc, String title, String content, List<MimeBodyPart> attachments) { + MimeMessage mail = new MimeMessage(session); + try { + mail.setFrom(new InternetAddress(from)); + mail.setSubject(title); + if (to != null) { + mail.setRecipients(Message.RecipientType.TO, InternetAddress.parse(to)); + } + if (cc != null) { + mail.setRecipients(Message.RecipientType.CC, InternetAddress.parse(cc)); + } + + //mail.setRecipients(Message.RecipientType.BCC, InternetAddress.parse(DEFAULT_BCC_ADDRESS)); + + MimeBodyPart mimeBodyPart = new MimeBodyPart(); + mimeBodyPart.setContent(content, "text/html;charset=utf-8"); + + Multipart multipart = new MimeMultipart(); + multipart.addBodyPart(mimeBodyPart); + + for (MimeBodyPart attachment : attachments) { + multipart.addBodyPart(attachment); + } + + mail.setContent(multipart); + // mail.setContent(content, "text/html;charset=utf-8"); + LOG.info(String.format("Going to send mail: from[%s], to[%s], cc[%s], title[%s]", from, to, cc, title)); + Transport.send(mail); + return true; + } catch (AddressException e) { + LOG.info("Failed to send mail, got an AddressException: " + e.getMessage(), e); + return false; + } catch (MessagingException e) { + LOG.info("Failed to send mail, got an AddressException: " + e.getMessage(), e); + return false; + } + } + + public boolean send(String from, String to, String cc, String title, + String content) { + return this.sendInternal(from, to, cc, title, content); + } + + public boolean send(String from, String to, String cc, String title, + String templatePath, VelocityContext context) { + Template t = null; + try { + t = velocityEngine.getTemplate(BASE_PATH + templatePath); + } catch (ResourceNotFoundException ex) { + // ignored + } + if (t == null) { + try { + t = velocityEngine.getTemplate(templatePath); + } catch (ResourceNotFoundException e) { + t = velocityEngine.getTemplate("/" + templatePath); + } + } + final StringWriter writer = new StringWriter(); + t.merge(context, writer); + if (LOG.isDebugEnabled()) { + LOG.debug(writer.toString()); + } + + return this.send(from, to, cc, title, writer.toString()); + } + + public boolean send(String from, String to, String cc, String title, + String templatePath, VelocityContext context, Map<String, File> attachments) { + if (attachments == null || attachments.isEmpty()) { + return send(from, to, cc, title, templatePath, context); + } + Template t = null; + + List<MimeBodyPart> mimeBodyParts = new ArrayList<MimeBodyPart>(); + + for (Map.Entry<String, File> entry : attachments.entrySet()) { + final String attachment = entry.getKey(); + final File attachmentFile = entry.getValue(); + final MimeBodyPart mimeBodyPart = new MimeBodyPart(); + if (attachmentFile != null && attachmentFile.exists()) { + DataSource source = new FileDataSource(attachmentFile); + try { + mimeBodyPart.setDataHandler(new DataHandler(source)); + mimeBodyPart.setFileName(attachment); + mimeBodyPart.setDisposition(MimeBodyPart.ATTACHMENT); + mimeBodyPart.setContentID(attachment); + mimeBodyParts.add(mimeBodyPart); + } catch (MessagingException e) { + LOG.error("Generate mail failed, got exception while attaching files: " + e.getMessage(), e); + } + } else { + LOG.error("Attachment: " + attachment + " is null or not exists"); + } + } + + try { + t = velocityEngine.getTemplate(BASE_PATH + templatePath); + } catch (ResourceNotFoundException ex) { + // ignored + } + + if (t == null) { + try { + t = velocityEngine.getTemplate(templatePath); + } catch (ResourceNotFoundException e) { + try { + t = velocityEngine.getTemplate("/" + templatePath); + } catch (Exception ex) { + LOG.error("Template not found:" + "/" + templatePath, ex); + } + } + } + + final StringWriter writer = new StringWriter(); + t.merge(context, writer); + if (LOG.isDebugEnabled()) { + LOG.debug(writer.toString()); + } + + return this.sendInternal(from, to, cc, title, writer.toString(), mimeBodyParts); + } +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/f430f521/eagle-jpm/eagle-jpm-mr-history/pom.xml ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-mr-history/pom.xml b/eagle-jpm/eagle-jpm-mr-history/pom.xml index dde7056..cbe8de1 100644 --- a/eagle-jpm/eagle-jpm-mr-history/pom.xml +++ b/eagle-jpm/eagle-jpm-mr-history/pom.xml @@ -74,6 +74,12 @@ </exclusion> </exclusions> </dependency> + <dependency> + <groupId>dumbster</groupId> + <artifactId>dumbster</artifactId> + <version>1.6</version> + <scope>test</scope> + </dependency> </dependencies> <build> <resources> http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/f430f521/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporter.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporter.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporter.java new file mode 100644 index 0000000..42f2b29 --- /dev/null +++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporter.java @@ -0,0 +1,317 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eagle.jpm.mr.history; + +import com.google.common.util.concurrent.AbstractScheduledService; +import com.google.inject.Inject; +import com.typesafe.config.Config; +import org.apache.commons.lang.time.StopWatch; +import org.apache.eagle.app.service.ApplicationEmailService; +import org.apache.eagle.common.DateTimeUtil; +import org.apache.eagle.common.mail.AlertEmailConstants; +import org.apache.eagle.common.mail.AlertEmailContext; +import org.apache.eagle.jpm.util.Constants; +import org.apache.eagle.log.entity.GenericServiceAPIResponseEntity; +import org.apache.eagle.metadata.model.ApplicationEntity; +import org.apache.eagle.metadata.service.ApplicationEntityService; +import org.apache.eagle.service.client.EagleServiceClientException; +import org.apache.eagle.service.client.IEagleServiceClient; +import org.apache.eagle.service.client.impl.EagleServiceClientImpl; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.*; +import java.util.concurrent.TimeUnit; + +import static org.apache.eagle.common.config.EagleConfigConstants.SERVICE_HOST; +import static org.apache.eagle.common.config.EagleConfigConstants.SERVICE_PORT; + +public class MRHistoryJobDailyReporter extends AbstractScheduledService { + private static final Logger LOG = LoggerFactory.getLogger(MRHistoryJobDailyReporter.class); + private static final String TIMEZONE_PATH = "service.timezone"; + private static final String DAILY_SENT_HOUROFDAY = "application.dailyJobReport.reportHourTime"; + private static final String DAILY_SENT_PERIOD = "application.dailyJobReport.reportPeriodInHour"; + private static final String NUM_TOP_USERS = "application.dailyJobReport.numTopUsers"; + private static final String JOB_OVERTIME_LIMIT_HOUR = "application.dailyJobReport.jobOvertimeLimitInHour"; + + public static final String SERVICE_PATH = "application.dailyJobReport"; + public static final String APP_TYPE = "MR_HISTORY_JOB_APP"; + + // alert context keys + protected static final String NUM_TOP_USERS_KEY = "numTopUsers"; + protected static final String JOB_OVERTIME_LIMIT_KEY = "jobOvertimeLimit"; + protected static final String ALERT_TITLE_KEY = "alertTitle"; + protected static final String REPORT_RANGE_KEY = "reportRange"; + protected static final String SUMMARY_INFO_KEY = "summaryInfo"; + protected static final String FAILED_JOB_USERS_KEY = "failedJobUsers"; + protected static final String SUCCEEDED_JOB_USERS_KEY = "succeededJobUsers"; + protected static final String FINISHED_JOB_USERS_KEY = "finishedJobUsers"; + protected static final String EAGLE_JOB_LINK_KEY = "eagleJobLink"; + + private Config config; + private IEagleServiceClient client; + private ApplicationEntityService applicationResource; + private ApplicationEmailService emailService; + private boolean isDailySent = false; + private long lastSentTime; + + private int dailySentHour; + private int dailySentPeriod; + private int numTopUsers = 10; + private int jobOvertimeLimit = 6; + + // scheduler + private int initialDelayMin = 10; + private int periodInMin = 60; + private TimeZone timeZone; + + @Inject + public MRHistoryJobDailyReporter(Config config, ApplicationEntityService applicationEntityService) { + this.timeZone = TimeZone.getTimeZone(config.getString(TIMEZONE_PATH)); + + if (config.hasPath(SERVICE_PATH) && config.hasPath(AlertEmailConstants.EAGLE_APPLICATION_EMAIL_SERVICE)) { + this.emailService = new ApplicationEmailService(config, SERVICE_PATH); + } + if (config.hasPath(DAILY_SENT_HOUROFDAY)) { + this.dailySentHour = config.getInt(DAILY_SENT_HOUROFDAY); + } + if (config.hasPath(DAILY_SENT_PERIOD)) { + this.dailySentPeriod = config.getInt(DAILY_SENT_PERIOD); + } + if (config.hasPath(NUM_TOP_USERS)) { + this.numTopUsers = config.getInt(NUM_TOP_USERS); + } + if (config.hasPath(JOB_OVERTIME_LIMIT_HOUR)) { + this.jobOvertimeLimit = config.getInt(JOB_OVERTIME_LIMIT_HOUR); + } + this.config = config; + this.applicationResource = applicationEntityService; + } + + private boolean isSentHour(int currentHour) { + return Math.abs(currentHour - dailySentHour) % dailySentPeriod == 0; + } + + private Collection<String> loadSites(String appType) { + Set<String> sites = new HashSet<>(); + Collection<ApplicationEntity> apps = applicationResource.findAll(); + for (ApplicationEntity app : apps) { + if (app.getDescriptor().getType().equalsIgnoreCase(appType) && app.getStatus().equals(ApplicationEntity.Status.RUNNING)) { + sites.add(app.getSite().getSiteId()); + } + } + LOG.info("Detected {} sites where MR_HISTORY_JOB_APP is Running: {}", sites.size(), sites); + return sites; + } + + @Override + protected void runOneIteration() throws Exception { + GregorianCalendar calendar = new GregorianCalendar(timeZone); + int currentHour = calendar.get(Calendar.HOUR_OF_DAY); + long currentTimestamp = calendar.getTimeInMillis(); + if (!isSentHour(currentHour)) { + isDailySent = false; + } else if (!isDailySent) { + isDailySent = true; + LOG.info("last job report time is {} %s", DateTimeUtil.millisecondsToHumanDateWithSeconds(lastSentTime), timeZone.getID()); + try { + Collection<String> sites = loadSites(APP_TYPE); + if (sites == null || sites.isEmpty()) { + LOG.warn("application MR_HISTORY_JOB_APP does not run on any sites!"); + return; + } + for (String site : sites) { + int reportHour = currentHour / dailySentPeriod * dailySentPeriod; + calendar.set(Calendar.HOUR_OF_DAY, reportHour); + String subject = String.format("%s %s", site.toUpperCase(), config.getString(SERVICE_PATH + "." + AlertEmailConstants.SUBJECT)); + Map<String, Object> alertData = buildAlertData(site, calendar.getTimeInMillis()); + sendByEmailWithSubject(alertData, subject); + } + } catch (Exception ex) { + LOG.error("Fail to get job summery info due to {}", ex.getMessage(), ex); + } + lastSentTime = currentTimestamp; + } + } + + protected void sendByEmail(Map<String, Object> alertData) { + emailService.onAlert(alertData); + } + + protected void sendByEmailWithSubject(Map<String, Object> alertData, String subject) { + AlertEmailContext alertContext = emailService.buildEmailContext(subject); + emailService.onAlert(alertContext, alertData); + } + + private Map<String, Object> buildAlertData(String site, long endTime) { + StopWatch watch = new StopWatch(); + Map<String, Object> data = new HashMap<>(); + this.client = new EagleServiceClientImpl(config); + long startTime = endTime - DateTimeUtil.ONEHOUR * dailySentPeriod; + LOG.info("Going to report job summery info for site {} from {} to {}", site, + DateTimeUtil.millisecondsToHumanDateWithSeconds(startTime), + DateTimeUtil.millisecondsToHumanDateWithSeconds(endTime)); + try { + watch.start(); + data.putAll(buildJobSummery(site, startTime, endTime)); + data.putAll(buildFailedJobInfo(site, startTime, endTime)); + data.putAll(buildSucceededJobInfo(site, startTime, endTime)); + data.putAll(buildFinishedJobInfo(site, startTime, endTime)); + data.put(NUM_TOP_USERS_KEY, numTopUsers); + data.put(JOB_OVERTIME_LIMIT_KEY, jobOvertimeLimit); + data.put(ALERT_TITLE_KEY, String.format("%s Daily Job Report", site.toUpperCase())); + data.put(REPORT_RANGE_KEY, String.format("%s ~ %s %s", + DateTimeUtil.millisecondsToHumanDateWithSeconds(startTime), + DateTimeUtil.millisecondsToHumanDateWithSeconds(endTime), + DateTimeUtil.CURRENT_TIME_ZONE.getID())); + watch.stop(); + LOG.info("Fetching DailyJobReport tasks {} seconds", watch.getTime() / DateTimeUtil.ONESECOND); + } finally { + try { + client.close(); + } catch (IOException e) { + LOG.info("fail to close eagle service client"); + } + } + return data; + } + + private Map<String, Long> parseQueryResult(List<Map<List<String>, List<Double>>> result, int limit) { + Map<String, Long> stateCount = new LinkedHashMap<>(); + for (Map<List<String>, List<Double>> map : result) { + if (stateCount.size() >= limit) { + break; + } + String key = String.valueOf(map.get("key").get(0)); + Long value = map.get("value").get(0).longValue(); + stateCount.put(key, value); + } + return stateCount; + } + + private Map<String, Long> queryGroupByMetrics(String condition, long startTime, long endTime, int limit) { + try { + GenericServiceAPIResponseEntity response = client.search() + .pageSize(Integer.MAX_VALUE) + .query(condition) + .startTime(startTime) + .endTime(endTime).send(); + if (!response.isSuccess()) { + return null; + } + List<Map<List<String>, List<Double>>> result = response.getObj(); + return parseQueryResult(result, limit); + } catch (EagleServiceClientException e) { + LOG.error(e.getMessage(), e); + return new HashMap<>(); + } + } + + private Map<String, Object> buildJobSummery(String site, long startTime, long endTime) { + Map<String, Object> data = new HashMap<>(); + List<JobSummeryInfo> statusCount = new ArrayList<>(); + String query = String.format("%s[@site=\"%s\" and @endTime<=%s]<@currentState>{count}", + Constants.JPA_JOB_EXECUTION_SERVICE_NAME, site, endTime); + Map<String, Long> jobSummery = queryGroupByMetrics(query, startTime, endTime, Integer.MAX_VALUE); + if (jobSummery == null || jobSummery.isEmpty()) { + return data; + } + Optional<Long> totalJobs = jobSummery.values().stream().reduce((a, b) -> a + b); + for (Map.Entry<String, Long> entry : jobSummery.entrySet()) { + JobSummeryInfo summeryInfo = new JobSummeryInfo(); + summeryInfo.status = entry.getKey(); + summeryInfo.numOfJobs = entry.getValue(); + summeryInfo.ratio = String.format("%.2f", entry.getValue() * 1d / totalJobs.get()); + statusCount.add(summeryInfo); + } + data.put(SUMMARY_INFO_KEY, statusCount); + return data; + } + + private Map<String, Object> buildFailedJobInfo(String site, long startTime, long endTime) { + Map<String, Object> data = new HashMap<>(); + String query = String.format("%s[@site=\"%s\" and @currentState=\"FAILED\" and @endTime<=%s]<@user>{count}.{count desc}", + Constants.JPA_JOB_EXECUTION_SERVICE_NAME, site, endTime); + + Map<String, Long> failedJobUsers = queryGroupByMetrics(query, startTime, endTime, numTopUsers); + if (failedJobUsers == null || failedJobUsers.isEmpty()) { + LOG.warn("Result set is empty for query={}", query); + return data; + } + data.put(FAILED_JOB_USERS_KEY, failedJobUsers); + data.put(EAGLE_JOB_LINK_KEY, String.format("http://%s:%d/#/site/%s/jpm/list?startTime=%s&endTime=%s";, + config.getString(SERVICE_HOST), + config.getInt(SERVICE_PORT), + site, + DateTimeUtil.millisecondsToHumanDateWithSeconds(startTime), + DateTimeUtil.millisecondsToHumanDateWithSeconds(endTime))); + return data; + } + + private Map<String, Object> buildSucceededJobInfo(String site, long startTime, long endTime) { + Map<String, Object> data = new HashMap<>(); + long overtimeLimit = jobOvertimeLimit * DateTimeUtil.ONEHOUR; + String query = String.format("%s[@site=\"%s\" and @currentState=\"SUCCEEDED\" and @durationTime>%s and @endTime<=%s]<@user>{count}.{count desc}", + Constants.JPA_JOB_EXECUTION_SERVICE_NAME, site, overtimeLimit, endTime); + Map<String, Long> succeededJobUsers = queryGroupByMetrics(query, startTime, endTime, numTopUsers); + if (succeededJobUsers == null || succeededJobUsers.isEmpty()) { + LOG.warn("Result set is empty for query={}", query); + return data; + } + data.put(SUCCEEDED_JOB_USERS_KEY, succeededJobUsers); + return data; + } + + private Map<String, Object> buildFinishedJobInfo(String site, long startTime, long endTime) { + Map<String, Object> data = new HashMap<>(); + String query = String.format("%s[@site=\"%s\" and @endTime<=%s]<@user>{count}.{count desc}", + Constants.JPA_JOB_EXECUTION_SERVICE_NAME, site, endTime); + Map<String, Long> jobUsers = queryGroupByMetrics(query, startTime, endTime, numTopUsers); + if (jobUsers == null || jobUsers.isEmpty()) { + LOG.warn("Result set is empty for query={}", query); + return data; + } + data.put(FINISHED_JOB_USERS_KEY, jobUsers); + return data; + } + + @Override + protected Scheduler scheduler() { + return Scheduler.newFixedRateSchedule(initialDelayMin, periodInMin, TimeUnit.MINUTES); + } + + public static class JobSummeryInfo { + public String status; + public long numOfJobs; + public String ratio; + + public String getStatus() { + return status; + } + + public long getNumOfJobs() { + return numOfJobs; + } + + public String getRatio() { + return ratio; + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/f430f521/eagle-jpm/eagle-jpm-mr-history/src/main/resources/JobReportTemplate.vm ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/resources/JobReportTemplate.vm b/eagle-jpm/eagle-jpm-mr-history/src/main/resources/JobReportTemplate.vm new file mode 100644 index 0000000..fef83da --- /dev/null +++ b/eagle-jpm/eagle-jpm-mr-history/src/main/resources/JobReportTemplate.vm @@ -0,0 +1,141 @@ +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one or more + ~ contributor license agreements. See the NOTICE file distributed with + ~ this work for additional information regarding copyright ownership. + ~ The ASF licenses this file to You under the Apache License, Version 2.0 + ~ (the "License"); you may not use this file except in compliance with + ~ the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd";> +<html xmlns="http://www.w3.org/1999/xhtml";> + #set ( $alert = $alertList[0] ) +<head> + <meta http-equiv="Content-Type" content="text/html; charset=utf-8"/> + <meta name="viewport" content="width=device-width"/> + <title>$alert["alertTitle"]</title> +</head> +<body> +<table class="body"> + <tr> + <td align="center" valign="top"> + <!-- Eagle Body --> + <table width="580"> + <tr> + <!-- Title --> + <td align="center"> + <h2>$alert["alertTitle"]</h2> + </td> + </tr> + + <!-- Basic Information --> + <tr> + <td style="padding: 20px 0 10px 0;"> + <p><b>Summery ($alert["reportRange"])</b></p> + </td> + </tr> + <tr> + <td> + <table class="tableBordered" width="580"> + <tr> + <th>Metrics</th> + <th>Number of Jobs</th> + <th>Ratio</th> + </tr> + #foreach($item in $alert["summaryInfo"]) + <tr> + <td>$item.status</td> + <td>$item.numOfJobs</td> + <td>$item.ratio</td> + </tr> + #end + </table> + </td> + </tr> + + <!-- Top Users for Failed Jobs --> + <tr> + <td style="padding: 20px 0 10px 0;"> + <p><b>Top $alert["numTopUsers"] Users (Order by Number of Failed Jobs)</b></p> + </td> + </tr> + <tr> + <td> + <table class="tableBordered" width="580"> + <tr> + <th>Name</th> + <th>Number of Failed Jobs</th> + </tr> + #foreach($userItem in $alert["failedJobUsers"].entrySet()) + <tr> + <td>$userItem.key</td> + <td>$userItem.value</td> + </tr> + #end + </table> + </td> + </tr> + <tr> + <td style="padding: 20px 0 10px 0;"> + <p>View more job information on <a href="$alert["eagleJobLink"]">Eagle</a></p> + </td> + </tr> + + <!-- Top Users for Failed Jobs --> + <tr> + <td style="padding: 20px 0 10px 0;"> + <p><b>Top $alert["numTopUsers"] Users (Order by Number of Succeeded Jobs Running over $alert["jobOvertimeLimit"] Hours)</b></p> + </td> + </tr> + <tr> + <td> + <table class="tableBordered" width="580"> + <tr> + <th>Name</th> + <th>Number of Jobs Runing over $alert["jobOvertimeLimit"] hrs</th> + </tr> + #foreach($userItem in $alert["succeededJobUsers"].entrySet()) + <tr> + <td>$userItem.key</td> + <td>$userItem.value</td> + </tr> + #end + </table> + </td> + </tr> + + <!-- Top Users for All Jobs --> + <tr> + <td style="padding: 20px 0 10px 0;"> + <p><b>Top $alert["numTopUsers"] Users (Order by Number of Finished Jobs)</b></p> + </td> + </tr> + <tr> + <td> + <table class="tableBordered" width="580"> + <tr> + <th>Name</th> + <th>Number of Finished Jobs</th> + </tr> + #foreach($userItem in $alert["finishedJobUsers"].entrySet()) + <tr> + <td>$userItem.key</td> + <td>$userItem.value</td> + </tr> + #end + </table> + </td> + </tr> + </table> + </td> + </tr> +</table> +</body> +</html> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/f430f521/eagle-jpm/eagle-jpm-mr-history/src/test/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporterTest.java ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-mr-history/src/test/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporterTest.java b/eagle-jpm/eagle-jpm-mr-history/src/test/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporterTest.java new file mode 100644 index 0000000..13c3350 --- /dev/null +++ b/eagle-jpm/eagle-jpm-mr-history/src/test/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporterTest.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eagle.jpm.mr.history; + +import com.dumbster.smtp.SimpleSmtpServer; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; +import org.apache.eagle.common.DateTimeUtil; +import org.junit.After; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; + +import static org.apache.eagle.jpm.mr.history.MRHistoryJobDailyReporter.*; + +public class MRHistoryJobDailyReporterTest { + + private static final Logger LOG = LoggerFactory.getLogger(MRHistoryJobDailyReporterTest.class); + private static final int SMTP_PORT = 5025; + private Config config; + private SimpleSmtpServer server; + + @Before + public void setUp(){ + config = ConfigFactory.load("application-test.conf"); + server = SimpleSmtpServer.start(SMTP_PORT); + } + + @After + public void clear(){ + if(server!=null) { + server.stop(); + } + } + @Test + public void test() throws Exception { + MRHistoryJobDailyReporter reporter = new MRHistoryJobDailyReporter(config, null); + reporter.sendByEmail(mockAlertData()); + Iterator it = server.getReceivedEmail(); + Assert.assertTrue(server.getReceivedEmailSize() == 1); + while (it.hasNext()) { + LOG.info(it.next().toString()); + } + } + + private Map<String, Object> mockAlertData() { + Map<String, Object> alertData = new HashMap<>(); + List<MRHistoryJobDailyReporter.JobSummeryInfo> summeryInfos = new ArrayList<>(); + MRHistoryJobDailyReporter.JobSummeryInfo summeryInfo1 = new MRHistoryJobDailyReporter.JobSummeryInfo(); + summeryInfo1.status = "failed"; + summeryInfo1.numOfJobs = 10; + summeryInfo1.ratio = "0.1"; + MRHistoryJobDailyReporter.JobSummeryInfo summeryInfo2 = new MRHistoryJobDailyReporter.JobSummeryInfo(); + summeryInfo2.status = "succeeded"; + summeryInfo2.numOfJobs = 90; + summeryInfo2.ratio = "0.9"; + summeryInfos.add(summeryInfo1); + summeryInfos.add(summeryInfo2); + alertData.put(SUMMARY_INFO_KEY, summeryInfos); + + Map<String,Double> failedJobUsers = new TreeMap<>(); + failedJobUsers.put("alice", 100d); + failedJobUsers.put("bob", 97d); + alertData.put(FAILED_JOB_USERS_KEY, failedJobUsers); + + + Map<String,Double> succeededJobUsers = new TreeMap<>(); + succeededJobUsers.put("alice1", 100d); + succeededJobUsers.put("bob1", 97d); + alertData.put(SUCCEEDED_JOB_USERS_KEY, succeededJobUsers); + + + Map<String,Double> finishedJobUsers = new TreeMap<>(); + finishedJobUsers.put("alice2", 100d); + finishedJobUsers.put("bob2", 97d); + alertData.put(FINISHED_JOB_USERS_KEY, finishedJobUsers); + + alertData.put(ALERT_TITLE_KEY, "Daily Job Report"); + alertData.put(NUM_TOP_USERS_KEY, 2); + alertData.put(JOB_OVERTIME_LIMIT_KEY, 6); + long currentTimestamp = System.currentTimeMillis(); + alertData.put(REPORT_RANGE_KEY, String.format(" %s ~ %s %s", + DateTimeUtil.millisecondsToHumanDateWithSeconds(currentTimestamp - 6 * DateTimeUtil.ONEHOUR), + DateTimeUtil.millisecondsToHumanDateWithSeconds(currentTimestamp), DateTimeUtil.CURRENT_TIME_ZONE.getID())); + alertData.put("joblink", "http://localhost:9090";); + return alertData; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/f430f521/eagle-jpm/eagle-jpm-mr-history/src/test/resources/JobReportTemplate.vm ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-mr-history/src/test/resources/JobReportTemplate.vm b/eagle-jpm/eagle-jpm-mr-history/src/test/resources/JobReportTemplate.vm new file mode 100644 index 0000000..fef83da --- /dev/null +++ b/eagle-jpm/eagle-jpm-mr-history/src/test/resources/JobReportTemplate.vm @@ -0,0 +1,141 @@ +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one or more + ~ contributor license agreements. See the NOTICE file distributed with + ~ this work for additional information regarding copyright ownership. + ~ The ASF licenses this file to You under the Apache License, Version 2.0 + ~ (the "License"); you may not use this file except in compliance with + ~ the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd";> +<html xmlns="http://www.w3.org/1999/xhtml";> + #set ( $alert = $alertList[0] ) +<head> + <meta http-equiv="Content-Type" content="text/html; charset=utf-8"/> + <meta name="viewport" content="width=device-width"/> + <title>$alert["alertTitle"]</title> +</head> +<body> +<table class="body"> + <tr> + <td align="center" valign="top"> + <!-- Eagle Body --> + <table width="580"> + <tr> + <!-- Title --> + <td align="center"> + <h2>$alert["alertTitle"]</h2> + </td> + </tr> + + <!-- Basic Information --> + <tr> + <td style="padding: 20px 0 10px 0;"> + <p><b>Summery ($alert["reportRange"])</b></p> + </td> + </tr> + <tr> + <td> + <table class="tableBordered" width="580"> + <tr> + <th>Metrics</th> + <th>Number of Jobs</th> + <th>Ratio</th> + </tr> + #foreach($item in $alert["summaryInfo"]) + <tr> + <td>$item.status</td> + <td>$item.numOfJobs</td> + <td>$item.ratio</td> + </tr> + #end + </table> + </td> + </tr> + + <!-- Top Users for Failed Jobs --> + <tr> + <td style="padding: 20px 0 10px 0;"> + <p><b>Top $alert["numTopUsers"] Users (Order by Number of Failed Jobs)</b></p> + </td> + </tr> + <tr> + <td> + <table class="tableBordered" width="580"> + <tr> + <th>Name</th> + <th>Number of Failed Jobs</th> + </tr> + #foreach($userItem in $alert["failedJobUsers"].entrySet()) + <tr> + <td>$userItem.key</td> + <td>$userItem.value</td> + </tr> + #end + </table> + </td> + </tr> + <tr> + <td style="padding: 20px 0 10px 0;"> + <p>View more job information on <a href="$alert["eagleJobLink"]">Eagle</a></p> + </td> + </tr> + + <!-- Top Users for Failed Jobs --> + <tr> + <td style="padding: 20px 0 10px 0;"> + <p><b>Top $alert["numTopUsers"] Users (Order by Number of Succeeded Jobs Running over $alert["jobOvertimeLimit"] Hours)</b></p> + </td> + </tr> + <tr> + <td> + <table class="tableBordered" width="580"> + <tr> + <th>Name</th> + <th>Number of Jobs Runing over $alert["jobOvertimeLimit"] hrs</th> + </tr> + #foreach($userItem in $alert["succeededJobUsers"].entrySet()) + <tr> + <td>$userItem.key</td> + <td>$userItem.value</td> + </tr> + #end + </table> + </td> + </tr> + + <!-- Top Users for All Jobs --> + <tr> + <td style="padding: 20px 0 10px 0;"> + <p><b>Top $alert["numTopUsers"] Users (Order by Number of Finished Jobs)</b></p> + </td> + </tr> + <tr> + <td> + <table class="tableBordered" width="580"> + <tr> + <th>Name</th> + <th>Number of Finished Jobs</th> + </tr> + #foreach($userItem in $alert["finishedJobUsers"].entrySet()) + <tr> + <td>$userItem.key</td> + <td>$userItem.value</td> + </tr> + #end + </table> + </td> + </tr> + </table> + </td> + </tr> +</table> +</body> +</html> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/f430f521/eagle-jpm/eagle-jpm-mr-history/src/test/resources/application-test.conf ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-mr-history/src/test/resources/application-test.conf b/eagle-jpm/eagle-jpm-mr-history/src/test/resources/application-test.conf new file mode 100644 index 0000000..057e189 --- /dev/null +++ b/eagle-jpm/eagle-jpm-mr-history/src/test/resources/application-test.conf @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +service { + env = "testing" + host = "localhost" + port = 9090 + username = "admin" + password = "secret" + readTimeOutSeconds = 60 + context = "/rest" + timezone = "UTC" +} + + +# --------------------------------------------- +# Eagle Application Configuration +# --------------------------------------------- +application { + sink { + type = org.apache.eagle.app.sink.KafkaStreamSink + } + storm { + nimbusHost = "server.eagle.apache.org" + nimbusThriftPort = 6627 + } + mailService { + mailSmtpServer = "localhost", + mailSmtpPort = 5025, + mailSmtpAuth = "false" + //mailSmtpConn = "plaintext", + //mailSmtpUsername = "" + //mailSmtpPassword = "" + //mailSmtpDebug = false + } + dailyJobReport { + reportHourTime: 0 + reportPeriodInHour: 12 + numTopUsers : 10 + jobOvertimeLimitInHour: 6 + subject: "Job Report For 12 hours" + recipients: "nob...@abc.com" + template: "JobReportTemplate.vm" + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/f430f521/eagle-jpm/eagle-jpm-mr-history/src/test/resources/log4j.properties ---------------------------------------------------------------------- diff --git a/eagle-jpm/eagle-jpm-mr-history/src/test/resources/log4j.properties b/eagle-jpm/eagle-jpm-mr-history/src/test/resources/log4j.properties new file mode 100644 index 0000000..71a5dac --- /dev/null +++ b/eagle-jpm/eagle-jpm-mr-history/src/test/resources/log4j.properties @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +log4j.rootLogger=INFO, DRFA, stdout +eagle.log.dir=./logs +eagle.log.file=eagle.log + +# standard output +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %p [%t] %c{2}[%L]: %m%n + +# Daily Rolling File Appender +log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender +log4j.appender.DRFA.File=${eagle.log.dir}/${eagle.log.file} +log4j.appender.DRFA.DatePattern=.yyyy-MM-dd +# 30-day backup +#log4j.appender.DRFA.MaxBackupIndex=30 +log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout + +# Pattern format: Date LogLevel LoggerName LogMessage +log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p [%t] %c{2}[%L]: %m%n \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/f430f521/eagle-server-assembly/src/main/conf/eagle.conf ---------------------------------------------------------------------- diff --git a/eagle-server-assembly/src/main/conf/eagle.conf b/eagle-server-assembly/src/main/conf/eagle.conf index 40a8c1e..af7e14a 100644 --- a/eagle-server-assembly/src/main/conf/eagle.conf +++ b/eagle-server-assembly/src/main/conf/eagle.conf @@ -90,6 +90,24 @@ application { nimbusHost = "server.eagle.apache.org" nimbusThriftPort = 6627 } + mailService { + mailSmtpServer = "", + mailSmtpPort = 25, + mailSmtpAuth = "false" + //mailSmtpConn = "plaintext", + //mailSmtpUsername = "" + //mailSmtpPassword = "" + //mailSmtpDebug = false + } + dailyJobReport { + reportHourTime: 1 + reportPeriodInHour: 12 + numTopUsers : 10 + jobOvertimeLimitInHour: 6 + subject: "Job Report For 12 hours" + recipients: "nob...@abc.com" + template: "JobReportTemplate.vm" + } } # --------------------------------------------- @@ -116,13 +134,6 @@ coordinator { host = "localhost", port = 9090, context = "/rest" - mailSmtpServer = "", - mailSmtpPort = 25, - mailSmtpAuth = "false" - //mailSmtpConn = "plaintext", - //mailSmtpUsername = "" - //mailSmtpPassword = "" - //mailSmtpDebug = false } metadataDynamicCheck { initDelayMillis = 1000 http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/f430f521/eagle-server/src/main/java/org/apache/eagle/server/ServerApplication.java ---------------------------------------------------------------------- diff --git a/eagle-server/src/main/java/org/apache/eagle/server/ServerApplication.java b/eagle-server/src/main/java/org/apache/eagle/server/ServerApplication.java index 495ec95..1d78ed1 100644 --- a/eagle-server/src/main/java/org/apache/eagle/server/ServerApplication.java +++ b/eagle-server/src/main/java/org/apache/eagle/server/ServerApplication.java @@ -19,6 +19,7 @@ package org.apache.eagle.server; import com.google.inject.Inject; import com.hubspot.dropwizard.guice.GuiceBundle; import com.sun.jersey.api.core.PackagesResourceConfig; +import com.typesafe.config.Config; import io.dropwizard.Application; import io.dropwizard.assets.AssetsBundle; import io.dropwizard.lifecycle.Managed; @@ -30,6 +31,7 @@ import org.apache.eagle.alert.coordinator.CoordinatorListener; import org.apache.eagle.alert.resource.SimpleCORSFiler; import org.apache.eagle.app.service.ApplicationHealthCheckService; import org.apache.eagle.common.Version; +import org.apache.eagle.jpm.mr.history.MRHistoryJobDailyReporter; import org.apache.eagle.log.base.taggedlog.EntityJsonModule; import org.apache.eagle.log.base.taggedlog.TaggedLogAPIEntity; import org.apache.eagle.metadata.service.ApplicationStatusUpdateService; @@ -48,6 +50,10 @@ class ServerApplication extends Application<ServerConfig> { private ApplicationStatusUpdateService applicationStatusUpdateService; @Inject private ApplicationHealthCheckService applicationHealthCheckService; + @Inject + private MRHistoryJobDailyReporter mrHistoryJobDailyReporter; + @Inject + private Config config; @Override public void initialize(Bootstrap<ServerConfig> bootstrap) { @@ -109,5 +115,10 @@ class ServerApplication extends Application<ServerConfig> { applicationHealthCheckService.init(environment); Managed appHealthCheckTask = new ApplicationTask(applicationHealthCheckService); environment.lifecycle().manage(appHealthCheckTask); + + if (config.hasPath(MRHistoryJobDailyReporter.SERVICE_PATH)) { + Managed jobReportTask = new ApplicationTask(mrHistoryJobDailyReporter); + environment.lifecycle().manage(jobReportTask); + } } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/f430f521/eagle-server/src/main/resources/application.conf ---------------------------------------------------------------------- diff --git a/eagle-server/src/main/resources/application.conf b/eagle-server/src/main/resources/application.conf index 84b467c..d657f54 100644 --- a/eagle-server/src/main/resources/application.conf +++ b/eagle-server/src/main/resources/application.conf @@ -110,6 +110,24 @@ application { mail.smtp.template = "HealthCheckTemplate.vm" } } + mailService { + mailSmtpServer = "", + mailSmtpPort = 25, + mailSmtpAuth = "false" + //mailSmtpConn = "plaintext", + //mailSmtpUsername = "" + //mailSmtpPassword = "" + //mailSmtpDebug = false + } + dailyJobReport { + reportHourTime: 1 + reportPeriodInHour: 12 + numTopUsers : 10 + jobOvertimeLimitInHour: 6 + subject: "Job Report For 12 hours" + recipients: "nob...@abc.com" + template: "JobReportTemplate.vm" + } } # --------------------------------------------- @@ -135,14 +153,7 @@ coordinator { metadataService { host = "localhost", port = 9090, - context = "/rest", - mailSmtpServer = "", - mailSmtpPort = 25, - mailSmtpAuth = "false" - //mailSmtpConn = "plaintext", - //mailSmtpUsername = "" - //mailSmtpPassword = "" - //mailSmtpDebug = false + context = "/rest" } metadataDynamicCheck { initDelayMillis = 1000
