Repository: airavata Updated Branches: refs/heads/master 11c01c6af -> 4f95a7908
Fixed AIRAVATA-1666 Project: http://git-wip-us.apache.org/repos/asf/airavata/repo Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/4f95a790 Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/4f95a790 Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/4f95a790 Branch: refs/heads/master Commit: 4f95a790825fab6ebb6749fe0e731734c520f338 Parents: 11c01c6 Author: shamrath <[email protected]> Authored: Mon Apr 13 17:05:32 2015 -0400 Committer: shamrath <[email protected]> Committed: Mon Apr 13 17:05:32 2015 -0400 ---------------------------------------------------------------------- .../airavata/common/utils/ServerSettings.java | 29 ++------------ .../main/resources/airavata-server.properties | 9 +---- .../gfac/monitor/email/EmailBasedMonitor.java | 18 ++++++--- .../gfac/monitor/email/EmailMonitorFactory.java | 18 +++++---- .../monitor/email/parser/PBSEmailParser.java | 42 -------------------- .../monitor/email/parser/SLURMEmailParser.java | 25 +----------- .../gfac/ssh/provider/impl/SSHProvider.java | 3 +- 7 files changed, 31 insertions(+), 113 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/airavata/blob/4f95a790/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/ServerSettings.java ---------------------------------------------------------------------- diff --git a/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/ServerSettings.java b/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/ServerSettings.java index 09360d4..e37ea42 100644 --- a/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/ServerSettings.java +++ b/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/ServerSettings.java @@ -68,12 +68,7 @@ public class ServerSettings extends ApplicationSettings { private static final String WORKFLOW_PARSER = "workflow.parser"; // email based monitoring configurations - private static final String EMAIL_BASED_MONITOR_HOST = "email.based.monitor.host"; - private static final String EMAIL_BASED_MONITOR_ADDRESS = "email.based.monitor.address"; - private static final String EMAIL_BASED_MONITOR_PASSWORD = "email.based.monitor.password"; - private static final String EMAIL_BASED_MONITOR_FOLDER_NAME = "email.based.monitor.folder.name"; - private static final String EMAIL_BASED_MONITOR_STORE_PROTOCOL = "email.based.monitor.store.protocol"; - private static final String ENABLE_EMAIL_BASED_MONITORING = "enable.email.based.monitoring"; + private static final String EMAIL_BASED_MONITORING_PERIOD = "email.based.monitoring.period"; private static boolean stopAllThreads = false; @@ -222,28 +217,10 @@ public class ServerSettings extends ApplicationSettings { return getSetting(WORKFLOW_PARSER); } - public static String getEmailBasedMonitorHost() throws ApplicationSettingsException { - return getSetting(EMAIL_BASED_MONITOR_HOST); - } - - public static String getEmailBasedMonitorAddress() throws ApplicationSettingsException { - return getSetting(EMAIL_BASED_MONITOR_ADDRESS); - } - - public static String getEmailBasedMonitorPassword() throws ApplicationSettingsException { - return getSetting(EMAIL_BASED_MONITOR_PASSWORD); - } - - public static String getEmailBasedMonitorFolderName() throws ApplicationSettingsException { - return getSetting(EMAIL_BASED_MONITOR_FOLDER_NAME); - } + public static int getEmailMonitorPeriod() throws ApplicationSettingsException { + return Integer.valueOf(getSetting(EMAIL_BASED_MONITORING_PERIOD, "100000")); - public static String getEmailBasedMonitorStoreProtocol() throws ApplicationSettingsException { - return getSetting(EMAIL_BASED_MONITOR_STORE_PROTOCOL); } - public static boolean isEmailBasedNotificationEnable() { - return Boolean.valueOf(getSetting(ENABLE_EMAIL_BASED_MONITORING, "false")); - } } http://git-wip-us.apache.org/repos/asf/airavata/blob/4f95a790/modules/configuration/server/src/main/resources/airavata-server.properties ---------------------------------------------------------------------- diff --git a/modules/configuration/server/src/main/resources/airavata-server.properties b/modules/configuration/server/src/main/resources/airavata-server.properties index 36f562c..7f18e3d 100644 --- a/modules/configuration/server/src/main/resources/airavata-server.properties +++ b/modules/configuration/server/src/main/resources/airavata-server.properties @@ -195,13 +195,8 @@ host.scheduler=org.apache.airavata.gfac.core.scheduler.impl.SimpleHostScheduler #mechanisms and one would be able to start a monitor monitors=org.apache.airavata.gfac.monitor.impl.pull.qstat.QstatMonitor,org.apache.airavata.gfac.monitor.impl.LocalJobMonitor -#These properties will used to enable email base monitoring -enable.email.based.monitoring=false -email.based.monitor.host=imap.gmail.com [email protected] -email.based.monitor.password=changeme -email.based.monitor.folder.name=INBOX -email.based.monitor.store.protocol=imaps +#These property will used to query the email server periodically. value in milliseconds(ms). +email.based.monitoring.period=10000 ########################################################################### # AMQP Notification Configuration ########################################################################### http://git-wip-us.apache.org/repos/asf/airavata/blob/4f95a790/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/EmailBasedMonitor.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/EmailBasedMonitor.java b/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/EmailBasedMonitor.java index affe156..631e1a4 100644 --- a/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/EmailBasedMonitor.java +++ b/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/EmailBasedMonitor.java @@ -137,11 +137,12 @@ public class EmailBasedMonitor implements Runnable{ if (!store.isConnected()) { store.connect(); } - Thread.sleep(2000); + Thread.sleep(ServerSettings.getEmailMonitorPeriod()); emailFolder = store.getFolder(emailMonitorProperty.getFolderName()); emailFolder.open(Folder.READ_WRITE); Message[] searchMessages = emailFolder.search(new FlagTerm(new Flags(Flags.Flag.SEEN), false)); List<Message> processedMessages = new ArrayList<>(); + List<Message> unreadMessages = new ArrayList<>(); for (Message message : searchMessages) { try { JobStatusResult jobStatusResult = parse(message); @@ -152,7 +153,8 @@ public class EmailBasedMonitor implements Runnable{ } else { // we can get JobExecutionContext null in multiple Gfac instances environment, // where this job is not submitted by this Gfac instance hence we ignore this message. - log.info("JobExecutionContext is not found for job Id " + jobStatusResult.getJobId()); + unreadMessages.add(message); +// log.info("JobExecutionContext is not found for job Id " + jobStatusResult.getJobId()); } } catch (AiravataException e) { log.error("Error parsing email message =====================================>", e); @@ -164,9 +166,15 @@ public class EmailBasedMonitor implements Runnable{ } } if (!processedMessages.isEmpty()) { - Message[] prosMessages = new Message[processedMessages.size()]; - processedMessages.toArray(prosMessages); - emailFolder.setFlags(prosMessages, new Flags(Flags.Flag.SEEN), true); + Message[] seenMessages = new Message[processedMessages.size()]; + processedMessages.toArray(seenMessages); + emailFolder.setFlags(seenMessages, new Flags(Flags.Flag.SEEN), true); + + } + if (!unreadMessages.isEmpty()) { + Message[] unseenMessages = new Message[unreadMessages.size()]; + unreadMessages.toArray(unseenMessages); + emailFolder.setFlags(unseenMessages, new Flags(Flags.Flag.SEEN), false); } emailFolder.close(false); } http://git-wip-us.apache.org/repos/asf/airavata/blob/4f95a790/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/EmailMonitorFactory.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/EmailMonitorFactory.java b/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/EmailMonitorFactory.java index 2c24973..5807460 100644 --- a/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/EmailMonitorFactory.java +++ b/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/EmailMonitorFactory.java @@ -27,27 +27,31 @@ import java.util.Map; public class EmailMonitorFactory { - private Map<String, EmailBasedMonitor> emailMonitors = new HashMap<String, EmailBasedMonitor>(); + private static Map<String, EmailBasedMonitor> emailMonitors = new HashMap<String, EmailBasedMonitor>(); - public synchronized EmailBasedMonitor getEmailBasedMonitor(EmailMonitorProperty emailMonitorProp) { + public static EmailBasedMonitor getEmailBasedMonitor(EmailMonitorProperty emailMonitorProp) { String key = getKey(emailMonitorProp); EmailBasedMonitor monitor = emailMonitors.get(key); if (monitor == null) { - monitor = new EmailBasedMonitor(emailMonitorProp); - emailMonitors.put(key, monitor); - new Thread(monitor).start(); + synchronized (emailMonitors){ + if (monitor == null) { + monitor = new EmailBasedMonitor(emailMonitorProp); + emailMonitors.put(key, monitor); + new Thread(monitor).start(); + } + } } return monitor; } - public void stopAllMonitors() { + public static void stopAllMonitors() { for (EmailBasedMonitor emailBasedMonitor : emailMonitors.values()) { emailBasedMonitor.stopMonitoring(); } } - private String getKey(EmailMonitorProperty emailMonitorProp) { + private static String getKey(EmailMonitorProperty emailMonitorProp) { StringBuffer sb = new StringBuffer(emailMonitorProp.getHost().trim()); sb.append("_").append(emailMonitorProp.getStoreProtocol().name()); sb.append("_").append(emailMonitorProp.getEmailAddress().trim()); http://git-wip-us.apache.org/repos/asf/airavata/blob/4f95a790/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/PBSEmailParser.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/PBSEmailParser.java b/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/PBSEmailParser.java index b748704..6cb961a 100644 --- a/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/PBSEmailParser.java +++ b/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/PBSEmailParser.java @@ -94,46 +94,4 @@ public class PBSEmailParser implements EmailParser { return -1; } - - -/* ----------------------- - This is the message envelope - --------------------------- - FROM: [email protected] - TO: [email protected] - SUBJECT: PBS JOB 2556782.trestles-fe1.local - ---------------------------- - CONTENT-TYPE: TEXT/PLAIN - This is plain text - --------------------------- - PBS Job Id: 2556782.trestles-fe1.local - Job Name: A1182004055 - Exec host: trestles-1-12/0+trestles-1-12/1+trestles-1-12/2+trestles-1-12/3 - Begun execution - */ -/* - ----------------------- - This is the message envelope - --------------------------- - FROM: [email protected] - TO: [email protected] - SUBJECT: PBS JOB 2556782.trestles-fe1.local - ---------------------------- - CONTENT-TYPE: TEXT/PLAIN - This is plain text - --------------------------- - PBS Job Id: 2556782.trestles-fe1.local - Job Name: A1182004055 - Exec host: trestles-1-12/0+trestles-1-12/1+trestles-1-12/2+trestles-1-12/3 - Execution terminated - Exit_status=0 - resources_used.cput=00:14:31 - resources_used.mem=124712kb - resources_used.vmem=3504116kb - resources_used.walltime=00:04:10 - Error_Path: trestles-login2.sdsc.edu:/oasis/scratch/trestles/ogce/temp_project/gta-work-dirs/MonitorTest_9169517d-e2d9-4ff5-bed5-dee6eb3eebb2/Amber_Sander.stderr - Output_Path: trestles-login2.sdsc.edu:/oasis/scratch/trestles/ogce/temp_project/gta-work-dirs/MonitorTest_9169517d-e2d9-4ff5-bed5-dee6eb3eebb2/Amber_Sander.stdout - */ - - } http://git-wip-us.apache.org/repos/asf/airavata/blob/4f95a790/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/SLURMEmailParser.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/SLURMEmailParser.java b/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/SLURMEmailParser.java index 643265e..327caa6 100644 --- a/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/SLURMEmailParser.java +++ b/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/SLURMEmailParser.java @@ -52,7 +52,7 @@ public class SLURMEmailParser implements EmailParser { // TODO remove following test lines String jobId = matcher.group(JOBID); String status = matcher.group(STATUS); - log.info("SLURM " + status + " message received -> " + jobId); +// log.info("SLURM " + status + " message received -> " + jobId); return jobStatusResult; } else { log.error("No matched found for subject -> " + subject); @@ -75,27 +75,4 @@ public class SLURMEmailParser implements EmailParser { } } - - -/* ----------------------- - This is the message envelope - --------------------------- - FROM: [email protected] - TO: [email protected] - SUBJECT: SLURM Job_id=5055468 Name=A433255759 Began, Queued time 00:00:01 - ---------------------------- - CONTENT-TYPE: TEXT/PLAIN; charset=us-ascii - This is plain text - ---------------------------*/ - -/* ----------------------- - This is the message envelope - --------------------------- - FROM: [email protected] - TO: [email protected] - SUBJECT: SLURM Job_id=5055468 Name=A433255759 Ended, Run time 00:02:40 - ---------------------------- - CONTENT-TYPE: TEXT/PLAIN; charset=us-ascii - This is plain text - ---------------------------*/ } http://git-wip-us.apache.org/repos/asf/airavata/blob/4f95a790/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/provider/impl/SSHProvider.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/provider/impl/SSHProvider.java b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/provider/impl/SSHProvider.java index 4dff9e1..b2fbcd2 100644 --- a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/provider/impl/SSHProvider.java +++ b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/provider/impl/SSHProvider.java @@ -382,8 +382,7 @@ public class SSHProvider extends AbstractProvider { if (monitorMode != null && monitorMode == MonitorMode.JOB_EMAIL_NOTIFICATION_MONITOR) { EmailMonitorProperty emailMonitorProp = sshJobSubmission.getEmailMonitorProperty(); if (emailMonitorProp != null) { - EmailMonitorFactory emailMonitorFactory = new EmailMonitorFactory(); - EmailBasedMonitor emailBasedMonitor = emailMonitorFactory.getEmailBasedMonitor(emailMonitorProp); + EmailBasedMonitor emailBasedMonitor = EmailMonitorFactory.getEmailBasedMonitor(emailMonitorProp); emailBasedMonitor.addToJobMonitorMap(jobExecutionContext); return; }
