http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java new file mode 100644 index 0000000..e436ae9 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java @@ -0,0 +1,495 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.server.services.security; + +import com.google.inject.Singleton; +import org.apache.commons.io.FileUtils; +import org.apache.slider.common.SliderKeys; +import org.apache.slider.core.conf.MapOperations; +import org.apache.slider.core.exceptions.SliderException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.nio.charset.Charset; +import java.text.MessageFormat; + +@Singleton +public class CertificateManager { + + private static final Logger LOG = + LoggerFactory.getLogger(CertificateManager.class); + + private static final String GEN_SRVR_KEY = "openssl genrsa -des3 " + + "-passout pass:{0} -out {1}" + File.separator + "{2} 4096 "; + private static final String GEN_SRVR_REQ = "openssl req -passin pass:{0} " + + "-new -key {1}" + File.separator + "{2} -out {1}" + File.separator + + "{5} -config {1}" + File.separator + "ca.config " + + "-subj {6} -batch"; + private static final String SIGN_SRVR_CRT = "openssl ca -create_serial " + + "-out {1}" + File.separator + "{3} -days 365 -keyfile {1}" + File.separator + + "{2} -key {0} -selfsign -extensions jdk7_ca -config {1}" + File.separator + + "ca.config -batch -infiles {1}" + File.separator + "{5}"; + private static final String EXPRT_KSTR = "openssl pkcs12 -export" + + " -in {2}" + File.separator + "{4} -inkey {2}" + File.separator + + "{3} -certfile {2}" + File.separator + "{4} -out {2}" + File.separator + + "{5} -password pass:{1} -passin pass:{0} \n"; + private static final String REVOKE_AGENT_CRT = "openssl ca " + + "-config {0}" + File.separator + "ca.config -keyfile {0}" + + File.separator + "{4} -revoke {0}" + File.separator + "{2} -batch " + + "-passin pass:{3} -cert {0}" + File.separator + "{5}"; + private static final String SIGN_AGENT_CRT = "openssl ca -config " + + "{0}" + File.separator + "ca.config -in {0}" + File.separator + + "{1} -out {0}" + File.separator + "{2} -batch -passin pass:{3} " + + "-keyfile {0}" + File.separator + "{4} -cert {0}" + File.separator + "{5}"; + private static final String GEN_AGENT_KEY="openssl req -new -newkey " + + "rsa:1024 -nodes -keyout {0}" + File.separator + + "{2}.key -subj {1} -out {0}" + File.separator + "{2}.csr " + + "-config {3}" + File.separator + "ca.config "; + private String passphrase; + private String applicationName; + + + public void initialize(MapOperations compOperations) throws SliderException { + String hostname = null; + try { + hostname = InetAddress.getLocalHost().getCanonicalHostName(); + } catch (UnknownHostException e) { + hostname = "localhost"; + } + this.initialize(compOperations, hostname, null, null); + } + + /** + * Verify that root certificate exists, generate it otherwise. + */ + public void initialize(MapOperations compOperations, + String hostname, String containerId, + String appName) throws SliderException { + SecurityUtils.initializeSecurityParameters(compOperations); + + LOG.info("Initialization of root certificate"); + boolean certExists = isCertExists(); + LOG.info("Certificate exists:" + certExists); + + this.applicationName = appName; + + if (!certExists) { + generateAMKeystore(hostname, containerId); + } + + } + + /** + * Checks root certificate state. + * @return "true" if certificate exists + */ + private boolean isCertExists() { + + String srvrKstrDir = SecurityUtils.getSecurityDir(); + String srvrCrtName = SliderKeys.CRT_FILE_NAME; + File certFile = new File(srvrKstrDir + File.separator + srvrCrtName); + LOG.debug("srvrKstrDir = " + srvrKstrDir); + LOG.debug("srvrCrtName = " + srvrCrtName); + LOG.debug("certFile = " + certFile.getAbsolutePath()); + + return certFile.exists(); + } + + public void setPassphrase(String passphrase) { + this.passphrase = passphrase; + } + + class StreamConsumer extends Thread + { + InputStream is; + boolean logOutput; + + StreamConsumer(InputStream is, boolean logOutput) + { + this.is = is; + this.logOutput = logOutput; + } + + StreamConsumer(InputStream is) + { + this(is, false); + } + + public void run() + { + try + { + InputStreamReader isr = new InputStreamReader(is, + Charset.forName("UTF8")); + BufferedReader br = new BufferedReader(isr); + String line; + while ( (line = br.readLine()) != null) + if (logOutput) { + LOG.info(line); + } + } catch (IOException e) + { + LOG.error("Error during processing of process stream", e); + } + } + } + + + /** + * Runs os command + * + * @return command execution exit code + */ + private int runCommand(String command) throws SliderException { + int exitCode = -1; + String line = null; + Process process = null; + BufferedReader br= null; + try { + process = Runtime.getRuntime().exec(command); + StreamConsumer outputConsumer = + new StreamConsumer(process.getInputStream(), true); + StreamConsumer errorConsumer = + new StreamConsumer(process.getErrorStream(), true); + + outputConsumer.start(); + errorConsumer.start(); + + try { + process.waitFor(); + SecurityUtils.logOpenSslExitCode(command, process.exitValue()); + exitCode = process.exitValue(); + if (exitCode != 0) { + throw new SliderException(exitCode, "Error running command %s", command); + } + } catch (InterruptedException e) { + e.printStackTrace(); + } + } catch (IOException e) { + e.printStackTrace(); + } finally { + if (br != null) { + try { + br.close(); + } catch (IOException ioe) { + ioe.printStackTrace(); + } + } + } + + return exitCode;//some exception occurred + + } + + public synchronized void generateContainerCertificate(String hostname, + String identifier) { + LOG.info("Generation of certificate for {}", hostname); + + String srvrKstrDir = SecurityUtils.getSecurityDir(); + Object[] scriptArgs = {srvrKstrDir, getSubjectDN(hostname, identifier, + this.applicationName), identifier, SecurityUtils.getSecurityDir()}; + + try { + String command = MessageFormat.format(GEN_AGENT_KEY, scriptArgs); + runCommand(command); + + signAgentCertificate(identifier); + + } catch (SliderException e) { + LOG.error("Error generating the agent certificate", e); + } + } + + public synchronized SecurityStore generateContainerKeystore(String hostname, + String requesterId, + String role, + String keystorePass) + throws SliderException { + LOG.info("Generation of container keystore for container {} on {}", + requesterId, hostname); + + generateContainerCertificate(hostname, requesterId); + + // come up with correct args to invoke keystore command + String srvrCrtPass = SecurityUtils.getKeystorePass(); + String srvrKstrDir = SecurityUtils.getSecurityDir(); + String containerCrtName = requesterId + ".crt"; + String containerKeyName = requesterId + ".key"; + String kstrName = getKeystoreFileName(requesterId, role); + + Object[] scriptArgs = {srvrCrtPass, keystorePass, srvrKstrDir, + containerKeyName, containerCrtName, kstrName}; + + String command = MessageFormat.format(EXPRT_KSTR, scriptArgs); + runCommand(command); + + return new SecurityStore(new File(srvrKstrDir, kstrName), + SecurityStore.StoreType.keystore); + } + + private static String getKeystoreFileName(String containerId, + String role) { + return String.format("keystore-%s-%s.p12", containerId, + role != null ? role : ""); + } + + private void generateAMKeystore(String hostname, String containerId) + throws SliderException { + LOG.info("Generation of server certificate"); + + String srvrKstrDir = SecurityUtils.getSecurityDir(); + String srvrCrtName = SliderKeys.CRT_FILE_NAME; + String srvrCsrName = SliderKeys.CSR_FILE_NAME; + String srvrKeyName = SliderKeys.KEY_FILE_NAME; + String kstrName = SliderKeys.KEYSTORE_FILE_NAME; + String srvrCrtPass = SecurityUtils.getKeystorePass(); + + Object[] scriptArgs = {srvrCrtPass, srvrKstrDir, srvrKeyName, + srvrCrtName, kstrName, srvrCsrName, getSubjectDN(hostname, containerId, + this.applicationName)}; + + String command = MessageFormat.format(GEN_SRVR_KEY, scriptArgs); + runCommand(command); + + command = MessageFormat.format(GEN_SRVR_REQ, scriptArgs); + runCommand(command); + + command = MessageFormat.format(SIGN_SRVR_CRT, scriptArgs); + runCommand(command); + + Object[] keystoreArgs = {srvrCrtPass, srvrCrtPass, srvrKstrDir, srvrKeyName, + srvrCrtName, kstrName, srvrCsrName}; + command = MessageFormat.format(EXPRT_KSTR, keystoreArgs); + runCommand(command); + } + + public SecurityStore generateContainerTruststore(String containerId, + String role, + String truststorePass) + throws SliderException { + + String srvrKstrDir = SecurityUtils.getSecurityDir(); + String srvrCrtName = SliderKeys.CRT_FILE_NAME; + String srvrCsrName = SliderKeys.CSR_FILE_NAME; + String srvrKeyName = SliderKeys.KEY_FILE_NAME; + String kstrName = getTruststoreFileName(role, containerId); + String srvrCrtPass = SecurityUtils.getKeystorePass(); + + Object[] scriptArgs = {srvrCrtPass, truststorePass, srvrKstrDir, srvrKeyName, + srvrCrtName, kstrName, srvrCsrName}; + + String command = MessageFormat.format(EXPRT_KSTR, scriptArgs); + runCommand(command); + + return new SecurityStore(new File(srvrKstrDir, kstrName), + SecurityStore.StoreType.truststore); + } + + private static String getTruststoreFileName(String role, String containerId) { + return String.format("truststore-%s-%s.p12", containerId, + role != null ? role : ""); + } + + /** + * Returns server certificate content + * @return string with server certificate content + */ + public String getServerCert() { + File certFile = getServerCertficateFilePath(); + String srvrCrtContent = null; + try { + srvrCrtContent = FileUtils.readFileToString(certFile); + } catch (IOException e) { + LOG.error(e.getMessage()); + } + return srvrCrtContent; + } + + public static File getServerCertficateFilePath() { + return new File(String.format("%s%s%s", + SecurityUtils.getSecurityDir(), + File.separator, + SliderKeys.CRT_FILE_NAME)); + } + + public static File getAgentCertficateFilePath(String containerId) { + return new File(String.format("%s%s%s.crt", + SecurityUtils.getSecurityDir(), + File.separator, + containerId)); + } + + public static File getContainerKeystoreFilePath(String containerId, + String role) { + return new File(SecurityUtils.getSecurityDir(), getKeystoreFileName( + containerId, + role + )); + } + + public static File getContainerTruststoreFilePath(String role, + String containerId) { + return new File(SecurityUtils.getSecurityDir(), + getTruststoreFileName(role, containerId)); + } + + public static File getAgentKeyFilePath(String containerId) { + return new File(String.format("%s%s%s.key", + SecurityUtils.getSecurityDir(), + File.separator, + containerId)); + } + + /** + * Signs agent certificate + * Adds agent certificate to server keystore + * @return string with agent signed certificate content + */ + public synchronized SignCertResponse signAgentCrt(String agentHostname, + String agentCrtReqContent, + String passphraseAgent) { + SignCertResponse response = new SignCertResponse(); + LOG.info("Signing of agent certificate"); + LOG.info("Verifying passphrase"); + + if (!this.passphrase.equals(passphraseAgent.trim())) { + LOG.warn("Incorrect passphrase from the agent"); + response.setResult(SignCertResponse.ERROR_STATUS); + response.setMessage("Incorrect passphrase from the agent"); + return response; + } + + String srvrKstrDir = SecurityUtils.getSecurityDir(); + String srvrCrtPass = SecurityUtils.getKeystorePass(); + String srvrCrtName = SliderKeys.CRT_FILE_NAME; + String srvrKeyName = SliderKeys.KEY_FILE_NAME; + String agentCrtReqName = agentHostname + ".csr"; + String agentCrtName = agentHostname + ".crt"; + + Object[] scriptArgs = {srvrKstrDir, agentCrtReqName, agentCrtName, + srvrCrtPass, srvrKeyName, srvrCrtName}; + + //Revoke previous agent certificate if exists + File agentCrtFile = new File(srvrKstrDir + File.separator + agentCrtName); + + String command = null; + if (agentCrtFile.exists()) { + LOG.info("Revoking of " + agentHostname + " certificate."); + command = MessageFormat.format(REVOKE_AGENT_CRT, scriptArgs); + try { + runCommand(command); + } catch (SliderException e) { + int commandExitCode = e.getExitCode(); + response.setResult(SignCertResponse.ERROR_STATUS); + response.setMessage( + SecurityUtils.getOpenSslCommandResult(command, commandExitCode)); + return response; + } + } + + File agentCrtReqFile = new File(srvrKstrDir + File.separator + + agentCrtReqName); + try { + FileUtils.writeStringToFile(agentCrtReqFile, agentCrtReqContent); + } catch (IOException e1) { + // TODO Auto-generated catch block + e1.printStackTrace(); + } + + command = MessageFormat.format(SIGN_AGENT_CRT, scriptArgs); + + LOG.debug(SecurityUtils.hideOpenSslPassword(command)); + try { + runCommand(command); + } catch (SliderException e) { + int commandExitCode = e.getExitCode(); + response.setResult(SignCertResponse.ERROR_STATUS); + response.setMessage( + SecurityUtils.getOpenSslCommandResult(command, commandExitCode)); + return response; + } + + String agentCrtContent = ""; + try { + agentCrtContent = FileUtils.readFileToString(agentCrtFile); + } catch (IOException e) { + e.printStackTrace(); + LOG.error("Error reading signed agent certificate"); + response.setResult(SignCertResponse.ERROR_STATUS); + response.setMessage("Error reading signed agent certificate"); + return response; + } + response.setResult(SignCertResponse.OK_STATUS); + response.setSignedCa(agentCrtContent); + //LOG.info(ShellCommandUtil.getOpenSslCommandResult(command, commandExitCode)); + return response; + } + + private String signAgentCertificate (String containerId) + throws SliderException { + String srvrKstrDir = SecurityUtils.getSecurityDir(); + String srvrCrtPass = SecurityUtils.getKeystorePass(); + String srvrCrtName = SliderKeys.CRT_FILE_NAME; + String srvrKeyName = SliderKeys.KEY_FILE_NAME; + String agentCrtReqName = containerId + ".csr"; + String agentCrtName = containerId + ".crt"; + + // server certificate must exist already + if (!(new File(srvrKstrDir, srvrCrtName).exists())) { + throw new SliderException("CA certificate not generated"); + } + + Object[] scriptArgs = {srvrKstrDir, agentCrtReqName, agentCrtName, + srvrCrtPass, srvrKeyName, srvrCrtName}; + + //Revoke previous agent certificate if exists + File agentCrtFile = new File(srvrKstrDir + File.separator + agentCrtName); + + String command; + if (agentCrtFile.exists()) { + LOG.info("Revoking of " + containerId + " certificate."); + command = MessageFormat.format(REVOKE_AGENT_CRT, scriptArgs); + runCommand(command); + } + + command = MessageFormat.format(SIGN_AGENT_CRT, scriptArgs); + + LOG.debug(SecurityUtils.hideOpenSslPassword(command)); + runCommand(command); + + return agentCrtName; + + } + + private String getSubjectDN(String hostname, String containerId, + String appName) { + return String.format("/CN=%s%s%s", + hostname, + containerId != null ? "/OU=" + containerId : "", + appName != null ? "/OU=" + appName : ""); + + + } +}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/KeystoreGenerator.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/KeystoreGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/KeystoreGenerator.java new file mode 100644 index 0000000..e2339d5 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/KeystoreGenerator.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.server.services.security; + +import org.apache.slider.common.SliderKeys; +import org.apache.slider.core.conf.AggregateConf; +import org.apache.slider.core.conf.MapOperations; +import org.apache.slider.core.exceptions.SliderException; + +import java.io.File; +import java.io.IOException; + +/** + * + */ +public class KeystoreGenerator extends AbstractSecurityStoreGenerator { + + + public KeystoreGenerator(CertificateManager certificateMgr) { + super(certificateMgr); + } + + @Override + public SecurityStore generate(String hostname, String containerId, + AggregateConf instanceDefinition, + MapOperations compOps, String role) + throws SliderException, IOException { + SecurityStore keystore = null; + String password = getStorePassword( + instanceDefinition.getAppConf().credentials, compOps, role); + if (password != null) { + keystore = + certificateMgr.generateContainerKeystore(hostname, containerId, role, + password); + } + return keystore; + } + + @Override + String getPassword(MapOperations compOps) { + return compOps.get( + compOps.get(SliderKeys.COMP_KEYSTORE_PASSWORD_PROPERTY_KEY)); + } + + @Override + String getAlias(MapOperations compOps) { + return compOps.getOption(SliderKeys.COMP_KEYSTORE_PASSWORD_ALIAS_KEY, + SliderKeys.COMP_KEYSTORE_PASSWORD_ALIAS_DEFAULT); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityStore.java new file mode 100644 index 0000000..fc54267 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityStore.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.server.services.security; + +import java.io.File; + +/** + * + */ +public class SecurityStore { + private File file; + + public enum StoreType {truststore, keystore} + + private StoreType type; + + public String getType() { + return type.name(); + } + + public File getFile() { + return file; + } + + public SecurityStore(File file, + StoreType type) { + + this.file = file; + this.type = type; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + SecurityStore that = (SecurityStore) o; + + if (file != null ? !file.equals(that.file) : that.file != null) + return false; + if (type != that.type) return false; + + return true; + } + + @Override + public int hashCode() { + int result = file != null ? file.hashCode() : 0; + result = 31 * result + (type != null ? type.hashCode() : 0); + return result; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityStoreGenerator.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityStoreGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityStoreGenerator.java new file mode 100644 index 0000000..a814988 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityStoreGenerator.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.server.services.security; + +import org.apache.slider.core.conf.AggregateConf; +import org.apache.slider.core.conf.MapOperations; +import org.apache.slider.core.exceptions.SliderException; + +import java.io.File; +import java.io.IOException; + +/** + * + */ +public interface SecurityStoreGenerator { + + SecurityStore generate(String hostname, + String containerId, + AggregateConf instanceDefinition, + MapOperations compOps, + String role) + throws SliderException, IOException; + + boolean isStoreRequested(MapOperations compOps); +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java new file mode 100644 index 0000000..e82ad84 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java @@ -0,0 +1,256 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.server.services.security; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang.RandomStringUtils; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RawLocalFileSystem; +import org.apache.hadoop.fs.permission.FsAction; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.slider.common.SliderKeys; +import org.apache.slider.common.SliderXmlConfKeys; +import org.apache.slider.core.conf.MapOperations; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +//import java.nio.file.Files; +//import java.nio.file.Path; +//import java.nio.file.Paths; +//import java.nio.file.attribute.PosixFilePermission; +//import java.nio.file.attribute.PosixFilePermissions; + + +/** + * + */ +public class SecurityUtils { + private static final Logger LOG = + LoggerFactory.getLogger(SecurityUtils.class); + + private static String CA_CONFIG_CONTENTS = "HOME = .\n" + + "RANDFILE = $ENV::HOME/.rnd\n\n" + + "[ ca ]\n" + + "default_ca = CA_CLIENT\n" + + "[ CA_CLIENT ]\n" + + "dir = ${SEC_DIR}/db\n" + + "certs = $dir/certs\n" + + "new_certs_dir = $dir/newcerts\n" + + "\n" + + "database = $dir/index.txt\n" + + "serial = $dir/serial\n" + + "default_days = 365 \n" + + "\n" + + "default_crl_days = 7 \n" + + "default_md = sha256 \n" + + "\n" + + "policy = policy_anything \n" + + "\n" + + "[ policy_anything ]\n" + + "countryName = optional\n" + + "stateOrProvinceName = optional\n" + + "localityName = optional\n" + + "organizationName = optional\n" + + "organizationalUnitName = optional\n" + + "commonName = optional\n" + + "emailAddress = optional\n" + + "\n" + + "[req]\n" + + "distinguished_name = req_distinguished_name\n" + + "\n" + + "[ req_distinguished_name ]\n" + + "\n" + + "[ jdk7_ca ]\n" + + "subjectKeyIdentifier = hash\n" + + "authorityKeyIdentifier = keyid:always,issuer:always\n" + + "basicConstraints = CA:true\n"; + + private static final String PASS_TOKEN = "pass:"; + private static String keystorePass; + private static String securityDir; + + public static void logOpenSslExitCode(String command, int exitCode) { + if (exitCode == 0) { + LOG.info(getOpenSslCommandResult(command, exitCode)); + } else { + LOG.warn(getOpenSslCommandResult(command, exitCode)); + } + + } + + public static String hideOpenSslPassword(String command){ + int start = command.indexOf(PASS_TOKEN); + while (start >= 0) { + start += PASS_TOKEN.length(); + CharSequence cs = command.subSequence(start, command.indexOf(" ", start)); + command = command.replace(cs, "****"); + start = command.indexOf(PASS_TOKEN, start + 1); + } + return command; + } + + public static String getOpenSslCommandResult(String command, int exitCode) { + return new StringBuilder().append("Command ") + .append(hideOpenSslPassword(command)) + .append(" was finished with exit code: ") + .append(exitCode).append(" - ") + .append(getOpenSslExitCodeDescription(exitCode)).toString(); + } + + private static String getOpenSslExitCodeDescription(int exitCode) { + switch (exitCode) { + case 0: { + return "the operation was completed successfully."; + } + case 1: { + return "an error occurred parsing the command options."; + } + case 2: { + return "one of the input files could not be read."; + } + case 3: { + return "an error occurred creating the PKCS#7 file or when reading the MIME message."; + } + case 4: { + return "an error occurred decrypting or verifying the message."; + } + case 5: { + return "the message was verified correctly but an error occurred writing out the signers certificates."; + } + default: + return "unsupported code"; + } + } + + public static void writeCaConfigFile(String path) throws IOException { + String contents = CA_CONFIG_CONTENTS.replace("${SEC_DIR}", path); + FileUtils.writeStringToFile(new File(path, "ca.config"), contents); + } + + public static String getKeystorePass() { + return keystorePass; + } + + public static String getSecurityDir() { + return securityDir; + } + + public static void initializeSecurityParameters(MapOperations configMap) { + initializeSecurityParameters(configMap, false); + } + + public static void initializeSecurityParameters(MapOperations configMap, + boolean persistPassword) { + String keyStoreLocation = configMap.getOption( + SliderXmlConfKeys.KEY_KEYSTORE_LOCATION, getDefaultKeystoreLocation()); + if (keyStoreLocation == null) { + LOG.error(SliderXmlConfKeys.KEY_KEYSTORE_LOCATION + + " is not specified. Unable to initialize security params."); + return; + } + File secDirFile = new File(keyStoreLocation).getParentFile(); + if (!secDirFile.exists()) { + // create entire required directory structure + File dbDir = new File(secDirFile, "db"); + File newCertsDir = new File(dbDir, "newcerts"); + newCertsDir.mkdirs(); + RawLocalFileSystem fileSystem = null; + try { + fileSystem = new RawLocalFileSystem(); + FsPermission permissions = new FsPermission(FsAction.ALL, FsAction.NONE, + FsAction.NONE); + fileSystem.setPermission(new Path(dbDir.getAbsolutePath()), + permissions); + fileSystem.setPermission(new Path(dbDir.getAbsolutePath()), permissions); + fileSystem.setPermission(new Path(newCertsDir.getAbsolutePath()), + permissions); + File indexFile = new File(dbDir, "index.txt"); + indexFile.createNewFile(); + SecurityUtils.writeCaConfigFile(secDirFile.getAbsolutePath().replace('\\', '/')); + + } catch (IOException e) { + LOG.error("Unable to create SSL configuration directories/files", e); + } finally { + if (fileSystem != null) { + try { + fileSystem.close(); + } catch (IOException e) { + LOG.warn("Unable to close fileSystem", e); + } + } + } + // need to create the password + } + keystorePass = getKeystorePassword(secDirFile, persistPassword); + securityDir = secDirFile.getAbsolutePath(); + } + + private static String getKeystorePassword(File secDirFile, + boolean persistPassword) { + File passFile = new File(secDirFile, SliderKeys.CRT_PASS_FILE_NAME); + String password = null; + if (!passFile.exists()) { + LOG.info("Generating keystore password"); + password = RandomStringUtils.randomAlphanumeric( + Integer.valueOf(SliderKeys.PASS_LEN)); + if (persistPassword) { + try { + FileUtils.writeStringToFile(passFile, password); + passFile.setWritable(true); + passFile.setReadable(true); + } catch (IOException e) { + e.printStackTrace(); + throw new RuntimeException( + "Error creating certificate password file"); + } + } + } else { + LOG.info("Reading password from existing file"); + try { + password = FileUtils.readFileToString(passFile); + password = password.replaceAll("\\p{Cntrl}", ""); + } catch (IOException e) { + e.printStackTrace(); + } + } + + return password; + } + + private static String getDefaultKeystoreLocation() { + File workDir = null; + try { + workDir = new File(FileUtils.getTempDirectory().getAbsolutePath() + + "/sec" + System.currentTimeMillis()); + if (!workDir.mkdirs()) { + throw new IOException("Unable to create temporary security directory"); + } + } catch (IOException e) { + LOG.warn("Unable to create security directory"); + return null; + } + + return new StringBuilder().append(workDir.getAbsolutePath()) + .append(File.separator) + .append(SliderKeys.SECURITY_DIR) + .append(File.separator) + .append(SliderKeys.KEYSTORE_FILE_NAME).toString(); + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SignCertResponse.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SignCertResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SignCertResponse.java new file mode 100644 index 0000000..8437d88 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SignCertResponse.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.services.security; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.bind.annotation.XmlType; + +/** + * + * Sign certificate response data model. + * + */ +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +@XmlType(name = "", propOrder = {}) +public class SignCertResponse { + + public static final String ERROR_STATUS = "ERROR"; + public static final String OK_STATUS = "OK"; + + @XmlElement + private String result; + @XmlElement + private String signedCa; + @XmlElement + private String message; + + public String getResult() { + return result; + } + public void setResult(String result) { + this.result = result; + } + public String getSignedCa() { + return signedCa; + } + public void setSignedCa(String signedCa) { + this.signedCa = signedCa; + } + + public String getMessage() { + return message; + } + public void setMessage(String message) { + this.message = message; + } +} + http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SignMessage.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SignMessage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SignMessage.java new file mode 100644 index 0000000..4bccb87 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SignMessage.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.services.security; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.bind.annotation.XmlType; + +/** + * + * Sign certificate request data model. + * + */ +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +@XmlType(name = "", propOrder = {}) +public class SignMessage { + + @XmlElement + private String csr; + @XmlElement + private String passphrase; + public String getCsr() { + return csr; + } + public void setCsr(String csr) { + this.csr = csr; + } + public String getPassphrase() { + return passphrase; + } + public void setPassphrase(String passphrase) { + this.passphrase = passphrase; + } +} + http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/StoresGenerator.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/StoresGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/StoresGenerator.java new file mode 100644 index 0000000..226250f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/StoresGenerator.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.server.services.security; + +import org.apache.slider.core.conf.AggregateConf; +import org.apache.slider.core.conf.MapOperations; +import org.apache.slider.core.exceptions.SliderException; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * + */ +public class StoresGenerator { + + static CertificateManager certMgr = new CertificateManager(); + private static SecurityStoreGenerator[] GENERATORS = { + new KeystoreGenerator(certMgr), new TruststoreGenerator(certMgr) + }; + + public static SecurityStore[] generateSecurityStores(String hostname, + String containerId, + String role, + AggregateConf instanceDefinition, + MapOperations compOps) + throws SliderException, IOException { + //discover which stores need generation based on the passwords configured + List<SecurityStore> files = new ArrayList<SecurityStore>(); + for (SecurityStoreGenerator generator : GENERATORS) { + if (generator.isStoreRequested(compOps)) { + SecurityStore store = generator.generate(hostname, + containerId, + instanceDefinition, + compOps, + role); + if (store != null) { + files.add(store); + } + } + } + + if (files.isEmpty()) { + throw new SliderException("Security stores were requested but none were " + + "generated. Check the AM logs and ensure " + + "passwords are configured for the components " + + "requiring the stores."); + } + return files.toArray(new SecurityStore[files.size()]); + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/TruststoreGenerator.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/TruststoreGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/TruststoreGenerator.java new file mode 100644 index 0000000..d16dcbd --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/TruststoreGenerator.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.server.services.security; + +import org.apache.slider.common.SliderKeys; +import org.apache.slider.core.conf.AggregateConf; +import org.apache.slider.core.conf.MapOperations; +import org.apache.slider.core.exceptions.SliderException; + +import java.io.IOException; + +/** + * + */ +public class TruststoreGenerator extends AbstractSecurityStoreGenerator { + + + public TruststoreGenerator(CertificateManager certificateMgr) { + super(certificateMgr); + } + + @Override + public SecurityStore generate(String hostname, String containerId, + AggregateConf instanceDefinition, + MapOperations compOps, String role) + throws SliderException, IOException { + SecurityStore truststore = null; + String password = getStorePassword( + instanceDefinition.getAppConf().credentials, compOps, role); + if (password != null) { + truststore = certificateMgr.generateContainerTruststore(containerId, + role, password); + } + return truststore; + } + + @Override + String getPassword(MapOperations compOps) { + return compOps.get( + compOps.get(SliderKeys.COMP_TRUSTSTORE_PASSWORD_PROPERTY_KEY)); + } + + @Override + String getAlias(MapOperations compOps) { + return compOps.getOption(SliderKeys.COMP_TRUSTSTORE_PASSWORD_ALIAS_KEY, + SliderKeys.COMP_TRUSTSTORE_PASSWORD_ALIAS_DEFAULT); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java new file mode 100644 index 0000000..1622309 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.services.utility; + +import org.apache.hadoop.registry.client.api.RegistryConstants; +import org.apache.hadoop.registry.client.api.RegistryOperations; +import org.apache.hadoop.registry.client.api.RegistryOperationsFactory; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.slider.common.tools.ConfigHelper; +import org.apache.slider.common.tools.SliderUtils; +import org.apache.slider.core.exceptions.BadCommandArgumentsException; +import org.apache.slider.core.exceptions.BadConfigException; +import org.apache.slider.core.zk.ZookeeperUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Base service for the standard slider client/server services + */ +public abstract class AbstractSliderLaunchedService extends + LaunchedWorkflowCompositeService { + private static final Logger log = + LoggerFactory.getLogger(AbstractSliderLaunchedService.class); + + protected AbstractSliderLaunchedService(String name) { + super(name); + // make sure all the yarn configs get loaded + YarnConfiguration conf = new YarnConfiguration(); + ConfigHelper.registerDeprecatedConfigItems(); + } + + /** + * look up the registry quorum from the config + * @return the quorum string + * @throws BadConfigException if it is not there or invalid + */ + public String lookupZKQuorum() throws BadConfigException { + + String registryQuorum = getConfig().get(RegistryConstants.KEY_REGISTRY_ZK_QUORUM); + + // though if neither is set: trouble + if (SliderUtils.isUnset(registryQuorum)) { + throw new BadConfigException( + "No Zookeeper quorum provided in the" + + " configuration property " + RegistryConstants.KEY_REGISTRY_ZK_QUORUM + ); + } + ZookeeperUtils.splitToHostsAndPortsStrictly(registryQuorum); + return registryQuorum; + } + + /** + * Create, adopt ,and start the YARN registration service + * @return the registry operations service, already deployed as a child + * of the AbstractSliderLaunchedService instance. + */ + public RegistryOperations startRegistryOperationsService() + throws BadConfigException { + + // push back the slider registry entry if needed + String quorum = lookupZKQuorum(); + RegistryOperations registryWriterService = + createRegistryOperationsInstance(); + deployChildService(registryWriterService); + return registryWriterService; + } + + /** + * Create the registry operations instance. This is to allow + * subclasses to instantiate a subclass service + * @return an instance to match to the lifecycle of this service + */ + protected RegistryOperations createRegistryOperationsInstance() { + return RegistryOperationsFactory.createInstance("YarnRegistry", getConfig()); + } + + /** + * Utility method to require an argument to be set (non null, non-empty) + * @param argname argument name + * @param value value + * @throws BadCommandArgumentsException if the condition is not met + */ + protected static void requireArgumentSet(String argname, String value) + throws BadCommandArgumentsException { + require(isSet(value), "Required argument %s missing", argname ); + } + + /** + * Require a condition to hold; throw {@link BadCommandArgumentsException} if not. + * The exception text is the formatted message. + * @param condition condition + * @param message string to format + * @param args list of arguments to format. + * @throws BadCommandArgumentsException + */ + protected static void require(boolean condition, String message, + Object... args) + throws BadCommandArgumentsException { + if (!condition) { + throw new BadCommandArgumentsException(message, args); + } + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/EndOfServiceWaiter.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/EndOfServiceWaiter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/EndOfServiceWaiter.java new file mode 100644 index 0000000..40ceab8 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/EndOfServiceWaiter.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.services.utility; + +import org.apache.hadoop.service.Service; +import org.apache.hadoop.service.ServiceStateChangeListener; + +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * Wait for a service to stop. + * + * WARNING: the notification may come in as soon as the service enters + * the stopped state: it may take some time for the actual stop operation + * to complete. + */ +public class EndOfServiceWaiter implements ServiceStateChangeListener { + + private final AtomicBoolean finished = new AtomicBoolean(false); + private final String name; + private Service service; + + /** + * Wait for a service; use the service name as this instance's name + * @param service service + */ + public EndOfServiceWaiter(Service service) { + this(service.getName(), service); + } + + + /** + * Wait for a service + * @param name name for messages + * @param service service + */ + public EndOfServiceWaiter(String name, Service service) { + this.name = name; + this.service = service; + service.registerServiceListener(this); + } + + public synchronized void waitForServiceToStop(long timeout) throws + InterruptedException, TimeoutException { + service.waitForServiceToStop(timeout); + if (!finished.get()) { + wait(timeout); + if (!finished.get()) { + throw new TimeoutException(name + + " did not finish after " + timeout + + " milliseconds"); + } + } + } + + /** + * Wait for service state change callbacks; notify self if the service has + * now stopped + * @param service service + */ + @Override + public synchronized void stateChanged(Service service) { + if (service.isInState(Service.STATE.STOPPED)) { + finished.set(true); + notify(); + } + } + + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/LaunchedWorkflowCompositeService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/LaunchedWorkflowCompositeService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/LaunchedWorkflowCompositeService.java new file mode 100644 index 0000000..bcd1969 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/LaunchedWorkflowCompositeService.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.services.utility; + +import com.google.common.base.Preconditions; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.service.Service; +import org.apache.slider.core.main.LauncherExitCodes; +import org.apache.slider.core.main.RunService; +import org.apache.slider.server.services.workflow.WorkflowCompositeService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This is a workflow compositoe service which can be launched from the CLI + * ... catches the arguments and implements a stub runService operation. + */ +public class LaunchedWorkflowCompositeService extends WorkflowCompositeService + implements RunService { + private static final Logger log = LoggerFactory.getLogger( + LaunchedWorkflowCompositeService.class); + private String[] argv; + + public LaunchedWorkflowCompositeService(String name) { + super(name); + } + + public LaunchedWorkflowCompositeService(String name, Service... children) { + super(name, children); + } + + /** + * Implementation of set-ness, groovy definition of true/false for a string + * @param s + * @return true iff the string is non-null and non-empty + */ + protected static boolean isUnset(String s) { + return StringUtils.isEmpty(s); + } + + protected static boolean isSet(String s) { + return StringUtils.isNotEmpty(s); + } + + protected String[] getArgv() { + return argv; + } + + /** + * Pre-init argument binding + * @param config the initial configuration build up by the + * service launcher. + * @param args argument list list of arguments passed to the command line + * after any launcher-specific commands have been stripped. + * @return the configuration + * @throws Exception + */ + @Override + public Configuration bindArgs(Configuration config, String... args) throws + Exception { + this.argv = args; + if (log.isDebugEnabled()) { + log.debug("Binding {} Arguments:", args.length); + + StringBuilder builder = new StringBuilder(); + for (String arg : args) { + builder.append('"').append(arg).append("\" "); + } + log.debug(builder.toString()); + } + return config; + } + + @Override + public int runService() throws Throwable { + return LauncherExitCodes.EXIT_SUCCESS; + } + + @Override + public synchronized void addService(Service service) { + Preconditions.checkArgument(service != null, "null service argument"); + super.addService(service); + } + + /** + * Run a child service -initing and starting it if this + * service has already passed those parts of its own lifecycle + * @param service the service to start + */ + protected boolean deployChildService(Service service) { + service.init(getConfig()); + addService(service); + if (isInState(STATE.STARTED)) { + service.start(); + return true; + } + return false; + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/PatternValidator.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/PatternValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/PatternValidator.java new file mode 100644 index 0000000..6ab9de6 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/PatternValidator.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.services.utility; + +import org.apache.slider.server.appmaster.web.rest.RestPaths; + +import java.util.regex.Pattern; + +/** + * Utility class to validate strings against a predefined pattern. + */ +public class PatternValidator { + + public static final String E_INVALID_NAME = + "Invalid name %s does not match the pattern pattern %s "; + private final Pattern valid; + private final String pattern; + + public PatternValidator(String pattern) { + this.pattern = pattern; + valid = Pattern.compile(pattern); + } + + /** + * Validate the name -restricting it to the set defined in + * {@link RestPaths#PUBLISHED_CONFIGURATION_REGEXP} + * @param name name to validate + * @throws IllegalArgumentException if not a valid name + */ + public void validate(String name) { + if (!matches(name)) { + throw new IllegalArgumentException( + String.format(E_INVALID_NAME, name, pattern)); + } + } + + /** + * Query to see if the pattern matches + * @param name name to validate + * @return true if the string matches the pattern + */ + public boolean matches(String name) { + return valid.matcher(name).matches(); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/WebAppService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/WebAppService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/WebAppService.java new file mode 100644 index 0000000..ebfcb99 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/WebAppService.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.services.utility; + +import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.yarn.webapp.WebApp; + +/** + * Contains a webapp reference and stops it in teardown if non-null + * <p> + * It does not start the application. + * Access to the field is not synchronized across threads; it is the + * responsibility of the caller. + */ +public class WebAppService<T extends WebApp> extends AbstractService { + + private volatile T webApp; + + public WebAppService(String name) { + super(name); + } + + public WebAppService(String name, T app) { + super(name); + webApp = app; + } + + public T getWebApp() { + return webApp; + } + + public void setWebApp(T webApp) { + this.webApp = webApp; + } + + + @Override + protected void serviceStart() throws Exception { + + } + + /** + * Stop operation stops the webapp; sets the reference to null + * @throws Exception + */ + @Override + protected void serviceStop() throws Exception { + if (webApp != null) { + webApp.stop(); + webApp = null; + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/ClosingService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/ClosingService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/ClosingService.java new file mode 100644 index 0000000..8b711aa --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/ClosingService.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.services.workflow; + +import org.apache.hadoop.service.AbstractService; + +import java.io.Closeable; +import java.io.IOException; + +/** + * Service that closes the closeable supplied during shutdown, if not null. + * + * As the Service interface itself extends Closeable, this service + * can be used to shut down other services if desired. + */ +public class ClosingService<C extends Closeable> extends AbstractService { + + private C closeable; + + public ClosingService(String name) { + super(name); + } + + /** + * Construct an instance of the service + * @param name service name + * @param closeable closeable to close (may be null) + */ + public ClosingService(String name, + C closeable) { + super(name); + this.closeable = closeable; + } + + /** + * Construct an instance of the service, using the default name + * @param closeable closeable to close (may be null) + */ + public ClosingService(C closeable) { + this("ClosingService", closeable); + } + + + /** + * Get the closeable + * @return the closeable + */ + public synchronized C getCloseable() { + return closeable; + } + + /** + * Set or update the closeable. + * @param closeable + */ + public synchronized void setCloseable(C closeable) { + this.closeable = closeable; + } + + /** + * Stop routine will close the closeable -if not null - and set the + * reference to null afterwards + * This operation does raise any exception on the close, though it does + * record it + */ + @Override + protected void serviceStop() { + C target = getCloseable(); + if (target != null) { + try { + target.close(); + } catch (IOException ioe) { + noteFailure(ioe); + } + setCloseable(null); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/ForkedProcessService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/ForkedProcessService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/ForkedProcessService.java new file mode 100644 index 0000000..352be49 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/ForkedProcessService.java @@ -0,0 +1,301 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.services.workflow; + +import org.apache.hadoop.service.ServiceStateException; +import org.apache.slider.core.main.ServiceLaunchException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Service wrapper for an external program that is launched and can/will terminate. + * This service is notified when the subprocess terminates, and stops itself + * and converts a non-zero exit code into a failure exception. + * + * <p> + * Key Features: + * <ol> + * <li>The property {@link #executionTimeout} can be set to set a limit + * on the duration of a process</li> + * <li>Output is streamed to the output logger provided</li>. + * <li>The most recent lines of output are saved to a linked list</li>. + * <li>A synchronous callback, {@link LongLivedProcessLifecycleEvent}, is raised on the start + * and finish of a process.</li> + * </ol> + * + * Usage: + * <p></p> + * The service can be built in the constructor, {@link #ForkedProcessService(String, Map, List)}, + * or have its simple constructor used to instantiate the service, then the + * {@link #build(Map, List)} command used to define the environment variables + * and list of commands to execute. One of these two options MUST be exercised + * before calling the services's {@link #start()} method. + * <p></p> + * The forked process is executed in the service's {@link #serviceStart()} method; + * if still running when the service is stopped, {@link #serviceStop()} will + * attempt to stop it. + * <p></p> + * + * The service delegates process execution to {@link LongLivedProcess}, + * receiving callbacks via the {@link LongLivedProcessLifecycleEvent}. + * When the service receives a callback notifying that the process has completed, + * it calls its {@link #stop()} method. If the error code was non-zero, + * the service is logged as having failed. + */ +public class ForkedProcessService + extends WorkflowExecutorService<ExecutorService> + implements LongLivedProcessLifecycleEvent, Runnable { + + /** + * Log for the forked master process + */ + private static final Logger LOG = + LoggerFactory.getLogger(ForkedProcessService.class); + + private final AtomicBoolean processTerminated = new AtomicBoolean(false); + private boolean processStarted = false; + private LongLivedProcess process; + private int executionTimeout = -1; + private int timeoutCode = 1; + /** + log to log to; defaults to this service log + */ + private Logger processLog = LOG; + + /** + * Exit code set when the spawned process exits + */ + private AtomicInteger exitCode = new AtomicInteger(0); + + /** + * Create an instance of the service + * @param name a name + */ + public ForkedProcessService(String name) { + super(name); + } + + /** + * Create an instance of the service, set up the process + * @param name a name + * @param commandList list of commands is inserted on the front + * @param env environment variables above those generated by + * @throws IOException IO problems + */ + public ForkedProcessService(String name, + Map<String, String> env, + List<String> commandList) throws IOException { + super(name); + build(env, commandList); + } + + @Override //AbstractService + protected void serviceStart() throws Exception { + if (process == null) { + throw new ServiceStateException("Process not yet configured"); + } + //now spawn the process -expect updates via callbacks + process.start(); + } + + @Override //AbstractService + protected void serviceStop() throws Exception { + completed(); + stopForkedProcess(); + } + + private void stopForkedProcess() { + if (process != null) { + process.stop(); + } + } + + /** + * Set the process log. This may be null for "do not log" + * @param processLog process log + */ + public void setProcessLog(Logger processLog) { + this.processLog = processLog; + process.setProcessLog(processLog); + } + + /** + * Set the timeout by which time a process must have finished -or -1 for forever + * @param timeout timeout in milliseconds + */ + public void setTimeout(int timeout, int code) { + this.executionTimeout = timeout; + this.timeoutCode = code; + } + + /** + * Build the process to execute when the service is started + * @param commandList list of commands is inserted on the front + * @param env environment variables above those generated by + * @throws IOException IO problems + */ + public void build(Map<String, String> env, + List<String> commandList) + throws IOException { + assert process == null; + + process = new LongLivedProcess(getName(), processLog, commandList); + process.setLifecycleCallback(this); + //set the env variable mapping + process.putEnvMap(env); + } + + @Override // notification from executed process + public synchronized void onProcessStarted(LongLivedProcess process) { + LOG.debug("Process has started"); + processStarted = true; + if (executionTimeout > 0) { + setExecutor(ServiceThreadFactory.singleThreadExecutor(getName(), true)); + execute(this); + } + } + + @Override // notification from executed process + public void onProcessExited(LongLivedProcess process, + int uncorrected, + int code) { + try { + synchronized (this) { + completed(); + //note whether or not the service had already stopped + LOG.debug("Process has exited with exit code {}", code); + if (code != 0) { + reportFailure(code, getName() + " failed with code " + code); + } + } + } finally { + stop(); + } + } + + private void reportFailure(int code, String text) { + //error + ServiceLaunchException execEx = new ServiceLaunchException(code, text); + LOG.debug("Noting failure", execEx); + noteFailure(execEx); + } + + /** + * handle timeout response by escalating it to a failure + */ + @Override + public void run() { + try { + synchronized (processTerminated) { + if (!processTerminated.get()) { + processTerminated.wait(executionTimeout); + } + } + + } catch (InterruptedException e) { + //assume signalled; exit + } + //check the status; if the marker isn't true, bail + if (!processTerminated.getAndSet(true)) { + LOG.info("process timeout: reporting error code {}", timeoutCode); + + //timeout + if (isInState(STATE.STARTED)) { + //trigger a failure + stopForkedProcess(); + } + reportFailure(timeoutCode, getName() + ": timeout after " + executionTimeout + + " millis: exit code =" + timeoutCode); + } + } + + /** + * Note the process as having completed. + * The process marked as terminated + * -and anything synchronized on <code>processTerminated</code> + * is notified + */ + protected void completed() { + processTerminated.set(true); + synchronized (processTerminated) { + processTerminated.notify(); + } + } + + public boolean isProcessTerminated() { + return processTerminated.get(); + } + + public synchronized boolean isProcessStarted() { + return processStarted; + } + + /** + * Is a process running: between started and terminated + * @return true if the process is up. + */ + public synchronized boolean isProcessRunning() { + return processStarted && !isProcessTerminated(); + } + + + public Integer getExitCode() { + return process.getExitCode(); + } + + public int getExitCodeSignCorrected() { + Integer exitCode = process.getExitCodeSignCorrected(); + if (exitCode == null) return -1; + return exitCode; + } + + /** + * Get the recent output from the process, or [] if not defined + * @return a possibly empty list + */ + public List<String> getRecentOutput() { + return process != null + ? process.getRecentOutput() + : new LinkedList<String>(); + } + + /** + * Get the recent output from the process, or [] if not defined + * + * @param finalOutput flag to indicate "wait for the final output of the process" + * @param duration the duration, in ms, + * to wait for recent output to become non-empty + * @return a possibly empty list + */ + public List<String> getRecentOutput(boolean finalOutput, int duration) { + if (process == null) { + return new LinkedList<>(); + } + return process.getRecentOutput(finalOutput, duration); + } + +} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
