Repository: airavata Updated Branches: refs/heads/master c4c03f3fb -> eaf941ae4
Fixed yaml configuration parsing issues Project: http://git-wip-us.apache.org/repos/asf/airavata/repo Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/eaf941ae Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/eaf941ae Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/eaf941ae Branch: refs/heads/master Commit: eaf941ae4112da96a5a15ed2dfa076de4002ea2d Parents: c4c03f3 Author: Shameera Rathanyaka <[email protected]> Authored: Wed Jul 8 16:29:44 2015 -0400 Committer: Shameera Rathanyaka <[email protected]> Committed: Wed Jul 8 16:29:44 2015 -0400 ---------------------------------------------------------------------- .../server/src/main/resources/gfac-config.yaml | 106 ++++++++++--------- .../core/config/DataTransferTaskConfig.java | 17 +++ .../gfac/core/config/GFacYamlConfigruation.java | 35 ++++-- .../core/config/JobSubmitterTaskConfig.java | 16 +++ .../org/apache/airavata/gfac/impl/Factory.java | 35 ++++-- 5 files changed, 144 insertions(+), 65 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/airavata/blob/eaf941ae/modules/configuration/server/src/main/resources/gfac-config.yaml ---------------------------------------------------------------------- diff --git a/modules/configuration/server/src/main/resources/gfac-config.yaml b/modules/configuration/server/src/main/resources/gfac-config.yaml index 5530547..46ece9c 100644 --- a/modules/configuration/server/src/main/resources/gfac-config.yaml +++ b/modules/configuration/server/src/main/resources/gfac-config.yaml @@ -18,62 +18,72 @@ # ################################################################ -config: - jobSubmitters: - - submissionProtocol: SSH - taskClass: org.apache.airavata.task.adapters.SSHJobSubmissionTask - properties: - - password: pwd123 - - passPhrase: test - - privateKey: key - - publicKey: pubkey +jobSubmitters: + - submissionProtocol: SSH + taskClass: org.apache.airavata.gfac.impl.task.SSHJobSubmissionTask + properties: + - password: pwd123 + passPhrase: test + privateKey: key + publicKey: pubk + # - submissionProtocol: GSISSH + # taskClass: org.apache.airavata.task.adapters.GSISSHJobSubmissionTask - - submissionProtocol: GSISSH - taskClass: org.apache.airavata.task.adapters.GSISSHJobSubmissionTask + - submissionProtocol: LOCAL + taskClass: org.apache.airavata.gfac.impl.task.LocalJobSubmissionTask - commonTasks: - - type: SETUP - taskClass: org.apache.airavata.task.common.SetupTask +commonTasks: + - type: SETUP + taskClass: org.apache.airavata.task.common.SetupTask - - type: CLEANUP - taskClass: org.apache.airavata.task.common.CleanupTask + - type: CLEANUP + taskClass: org.apache.airavata.task.common.CleanupTask - fileTransferTasks: - - transferProtocol: SCP - taskClass: org.apache.airavata.task.adapters.SCPFileTransferTask - properties: - - password: pwd123 - - passPhrase: test - - privateKey: key - - publicKey: pubkey +fileTransferTasks: + - transferProtocol: SCP + taskClass: org.apache.airavata.gfac.impl.task.SCPInputDataStageTask + properties: + - password: pwd123 + passPhrase: test + privateKey: key + publicKey: pubkey - - transferProtocol: HTTP - taskClass: org.apache.airavata.task.adapters.HTTPFileTransferTask + #- transferProtocol: SFTP + # taskClass: org.apache.airavata.task.adapters.SFTPFileTransferTask - - transferProtocol: GRIDFTP - taskClass: org.apache.airavata.task.adapters.GRIDFTPFileTransferTask + #- transferProtocol: GRIDFTP + # taskClass: org.apache.airavata.task.adapters.GRIDFTPFileTransferTask - resources: - - jobManagerType: PBS - commandOutputParser: org.apache.airavata.commmand.PBSOutputParser - emailParser: org.apache.airavata.email.parser.PBSEmailParser - resourceEmailAddresses: - - [email protected] # gordon - - [email protected] # Bigred2 - - root <[email protected]> # Bigred2 - - root <[email protected]> # alamo + #- transferProtocol: LOCAL + # taskClass: org.apache.airavata.task.adapters.LocalFileTransferTask - - jobManagerType: SLURM - commandOutputParser: org.apache.airavata.commmand.SLURMOutputParser - emailParser: org.apache.airavata.email.parser.SLURMEmailParser - resourceEmailAddresses: - - SDSC Admin <[email protected]> # comet - - [email protected] # stampede +resources: + - jobManagerType: PBS + commandOutputParser: org.apache.airavata.gfac.impl.job.PBSOutputParser + emailParser: org.apache.airavata.gfac.monitor.email.parser.PBSEmailParser + resourceEmailAddresses: + - [email protected] # gordon + - [email protected] # Bigred2 + - root <[email protected]> # Bigred2 + - root <[email protected]> # alamo - - jobManagerType: UGE - commandOutputParser: org.apache.airavata.commmand.UGEOutputParser - emailParser: org.apache.airavata.email.parser.UGEEmailParser - resourceEmailAddresses: - - ls4.tacc.utexas.edu # contain Lonestar + - jobManagerType: SLURM + commandOutputParser: org.apache.airavata.gfac.impl.job.SlurmOutputParser + emailParser: org.apache.airavata.gfac.monitor.email.parser.SLURMEmailParser + resourceEmailAddresses: + - SDSC Admin <[email protected]> # comet + - [email protected] # stampede + + - jobManagerType: UGE + commandOutputParser: org.apache.airavata.gfac.impl.job.UGEOutputParser + emailParser: org.apache.airavata.gfac.monitor.email.parser.UGEEmailParser + resourceEmailAddresses: + - ls4.tacc.utexas.edu # contain Lonestar + + - jobManagerType: LSF + commandOutputParser: org.apache.airavata.gfac.impl.job.LSFOutputParser + emailParser: org.apache.airavata.gfac.monitor.email.parser.LSFEmailParser + resourceEmailAddresses: + - iu.xsede.edu # test resource mail address http://git-wip-us.apache.org/repos/asf/airavata/blob/eaf941ae/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/config/DataTransferTaskConfig.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/config/DataTransferTaskConfig.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/config/DataTransferTaskConfig.java index a4817fb..76728f0 100644 --- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/config/DataTransferTaskConfig.java +++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/config/DataTransferTaskConfig.java @@ -22,9 +22,14 @@ package org.apache.airavata.gfac.core.config; import org.apache.airavata.model.appcatalog.computeresource.DataMovementProtocol; +import java.util.HashMap; +import java.util.Map; + public class DataTransferTaskConfig { private DataMovementProtocol transferProtocol; private String taskClass; + private Map<String,String> properties = new HashMap<>(); + public DataMovementProtocol getTransferProtocol() { return transferProtocol; @@ -41,4 +46,16 @@ public class DataTransferTaskConfig { public void setTaskClass(String taskClass) { this.taskClass = taskClass; } + + public void addProperty(String key, String value) { + properties.put(key, value); + } + + public void addProperties(Map<String, String> propMap) { + propMap.forEach(properties::put); + } + + public Map<String,String> getProperties(){ + return properties; + } } http://git-wip-us.apache.org/repos/asf/airavata/blob/eaf941ae/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/config/GFacYamlConfigruation.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/config/GFacYamlConfigruation.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/config/GFacYamlConfigruation.java index 5dbad87..fa8ce32 100644 --- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/config/GFacYamlConfigruation.java +++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/config/GFacYamlConfigruation.java @@ -46,6 +46,7 @@ public class GFacYamlConfigruation { private static final String COMMAND_OUTPUT_PARSER = "commandOutputParser"; private static final String EMAIL_PARSER = "emailParser"; private static final String RESOURCE_EMAIL_ADDRESS = "resourceEmailAddress"; + private static final String PROPERTIES = "properties"; private List<JobSubmitterTaskConfig> jobSubmitters = new ArrayList<>(); private List<DataTransferTaskConfig> fileTransferTasks = new ArrayList<>(); @@ -71,26 +72,44 @@ public class GFacYamlConfigruation { if (load instanceof Map) { Map<String, Object> loadMap = (Map<String, Object>) load; String identifier; - List<Map<String, String>> jobSubYamls = (List<Map<String, String>>) loadMap.get(JOB_SUBMITTERS); + List<Map<String,Object >> jobSubYamls = (List<Map<String, Object>>) loadMap.get(JOB_SUBMITTERS); JobSubmitterTaskConfig jobSubmitterTaskConfig; if (jobSubYamls != null) { - for (Map<String, String> jobSub : jobSubYamls) { + for (Map<String, Object> jobSub : jobSubYamls) { jobSubmitterTaskConfig = new JobSubmitterTaskConfig(); - identifier = jobSub.get (SUBMISSIO_PROTOCOL); + identifier = ((String) jobSub.get(SUBMISSIO_PROTOCOL)); jobSubmitterTaskConfig.setSubmissionProtocol(JobSubmissionProtocol.valueOf(identifier)); - jobSubmitterTaskConfig.setTaskClass(jobSub.get(TASK_CLASS)); + jobSubmitterTaskConfig.setTaskClass(((String) jobSub.get(TASK_CLASS))); + Object propertiesObj = jobSub.get(PROPERTIES); + List propertiesList; + if (propertiesObj instanceof List) { + propertiesList = ((List) propertiesObj); + if (propertiesList.size() > 0) { + Map<String, String> props = (Map<String, String>) propertiesList.get(0); + jobSubmitterTaskConfig.addProperties(props); + } + } jobSubmitters.add(jobSubmitterTaskConfig); } } - List<Map<String, String>> fileTransYamls = (List<Map<String, String>>) loadMap.get(FILE_TRANSFER_TASKS); + List<Map<String, Object>> fileTransYamls = (List<Map<String, Object>>) loadMap.get(FILE_TRANSFER_TASKS); DataTransferTaskConfig dataTransferTaskConfig; if (fileTransYamls != null) { - for (Map<String, String> fileTransConfig : fileTransYamls) { + for (Map<String, Object> fileTransConfig : fileTransYamls) { dataTransferTaskConfig = new DataTransferTaskConfig(); - identifier = fileTransConfig.get (TRANSFER_PROTOCOL); + identifier = ((String) fileTransConfig.get(TRANSFER_PROTOCOL)); dataTransferTaskConfig.setTransferProtocol(DataMovementProtocol.valueOf(identifier)); - dataTransferTaskConfig.setTaskClass(fileTransConfig.get(TASK_CLASS)); + dataTransferTaskConfig.setTaskClass(((String) fileTransConfig.get(TASK_CLASS))); + Object propertiesObj = fileTransConfig.get(PROPERTIES); + List propertiesList; + if (propertiesObj instanceof List) { + propertiesList = (List) propertiesObj; + if (propertiesList.size() > 0) { + Map<String, String> props = (Map<String, String>) propertiesList.get(0); + dataTransferTaskConfig.addProperties(props); + } + } fileTransferTasks.add(dataTransferTaskConfig); } } http://git-wip-us.apache.org/repos/asf/airavata/blob/eaf941ae/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/config/JobSubmitterTaskConfig.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/config/JobSubmitterTaskConfig.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/config/JobSubmitterTaskConfig.java index daea282..6377f88 100644 --- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/config/JobSubmitterTaskConfig.java +++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/config/JobSubmitterTaskConfig.java @@ -22,9 +22,13 @@ package org.apache.airavata.gfac.core.config; import org.apache.airavata.model.appcatalog.computeresource.JobSubmissionProtocol; +import java.util.HashMap; +import java.util.Map; + public class JobSubmitterTaskConfig { private JobSubmissionProtocol submissionProtocol; private String taskClass; + private Map<String,String> properties = new HashMap<>(); public JobSubmissionProtocol getSubmissionProtocol() { return submissionProtocol; @@ -41,4 +45,16 @@ public class JobSubmitterTaskConfig { public void setTaskClass(String taskClass) { this.taskClass = taskClass; } + + public void addProperty(String key, String value) { + properties.put(key, value); + } + + public void addProperties(Map<String, String> propMap) { + propMap.forEach(properties::put); + } + + public Map<String,String> getProperties(){ + return properties; + } } http://git-wip-us.apache.org/repos/asf/airavata/blob/eaf941ae/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/Factory.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/Factory.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/Factory.java index 2b629bb..fbb8f2f 100644 --- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/Factory.java +++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/Factory.java @@ -40,6 +40,7 @@ import org.apache.airavata.gfac.core.monitor.JobMonitor; import org.apache.airavata.gfac.core.scheduler.HostScheduler; import org.apache.airavata.gfac.core.task.JobSubmissionTask; import org.apache.airavata.gfac.core.task.Task; +import org.apache.airavata.gfac.core.task.TaskException; import org.apache.airavata.gfac.impl.job.LSFJobConfiguration; import org.apache.airavata.gfac.impl.job.LSFOutputParser; import org.apache.airavata.gfac.impl.job.PBSJobConfiguration; @@ -67,6 +68,8 @@ import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.retry.ExponentialBackoffRetry; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; import java.util.Calendar; import java.util.Date; import java.util.HashMap; @@ -259,18 +262,32 @@ public abstract class Factory { private static void loadConfiguration() throws GFacException { GFacYamlConfigruation config = new GFacYamlConfigruation(); - for (JobSubmitterTaskConfig jobSubmitterTaskConfig : config.getJobSbumitters()) { - jobSubmissionTask.put(jobSubmitterTaskConfig.getSubmissionProtocol(), null); - } + try { + for (JobSubmitterTaskConfig jobSubmitterTaskConfig : config.getJobSbumitters()) { + String taskClass = jobSubmitterTaskConfig.getTaskClass(); + Class<?> aClass = Class.forName(taskClass); + Constructor<?> constructor = aClass.getConstructor(); + JobSubmissionTask task = (JobSubmissionTask) constructor.newInstance(); + task.init(jobSubmitterTaskConfig.getProperties()); + jobSubmissionTask.put(jobSubmitterTaskConfig.getSubmissionProtocol(), task); + } - for (DataTransferTaskConfig dataTransferTaskConfig : config.getFileTransferTasks()) { - dataMovementTask.put(dataTransferTaskConfig.getTransferProtocol(), null); - } + for (DataTransferTaskConfig dataTransferTaskConfig : config.getFileTransferTasks()) { + String taskClass = dataTransferTaskConfig.getTaskClass(); + Class<?> aClass = Class.forName(taskClass); + Constructor<?> constructor = aClass.getConstructor(); + Task task = (Task) constructor.newInstance(); + task.init(dataTransferTaskConfig.getProperties()); + dataMovementTask.put(dataTransferTaskConfig.getTransferProtocol(), task); + } - for (ResourceConfig resourceConfig : config.getResourceConfiguration()) { - resources.put(resourceConfig.getJobManagerType(), resourceConfig); + for (ResourceConfig resourceConfig : config.getResourceConfiguration()) { + resources.put(resourceConfig.getJobManagerType(), resourceConfig); + } + readConfig = true; + } catch (Exception e) { + throw new GFacException("Gfac config issue", e); } - readConfig = true; }
