http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java new file mode 100644 index 0000000..1bdfb9c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.core.registry; + +import com.google.common.base.Preconditions; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.slider.client.SliderYarnClientImpl; +import org.apache.slider.api.types.SliderInstanceDescription; +import org.apache.slider.common.tools.CoreFileSystem; +import org.apache.slider.common.tools.SliderUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Client code for interacting with a list of service instances. + * The initial logic just enumerates service instances in the YARN RM + */ +public class YarnAppListClient { + + private final SliderYarnClientImpl yarnClient; + private final String username; + private final Configuration conf; + private static final Logger log = + LoggerFactory.getLogger(YarnAppListClient.class); + + public YarnAppListClient(SliderYarnClientImpl yarnClient, + String username, + Configuration conf) { + + Preconditions.checkArgument(yarnClient != null, + "yarn client is null: is app inited?"); + Preconditions.checkArgument(username != null, + "username is null"); + Preconditions.checkArgument(conf != null, + "conf parameter is null"); + this.yarnClient = yarnClient; + this.username = username; + this.conf = conf; + } + + /** + * find all live instances of a specific app -if there is more than one + * in the cluster, this returns them all. State should be running or earlier + * in the lifecycle + * @param appname application name + * @return the list of all matching application instances + */ + public List<ApplicationReport> findAllLiveInstances(String appname) + throws YarnException, IOException { + return yarnClient.findAllLiveInstances(username, appname); + } + + + /** + * Find an instance of a application belong to the current user + * @param appname application name + * @return the app report or null if none is found + * @throws YarnException YARN issues + * @throws IOException IO problems + */ + public ApplicationReport findInstance(String appname) throws + YarnException, + IOException { + List<ApplicationReport> instances = listInstances(null); + return yarnClient.findClusterInInstanceList(instances, appname); + } + + /** + * List instances belonging to the specific user + * @return a possibly empty list of AMs + */ + public List<ApplicationReport> listInstances() + throws YarnException, IOException { + return listInstances(null); + } + + /** + * List instances belonging to a specific user + * @return a possibly empty list of AMs + * @param user user if not the default. null means default, "" means all users, + * otherwise it is the name of a user + */ + public List<ApplicationReport> listInstances(String user) + throws YarnException, IOException { + String listUser = user == null ? username : user; + return yarnClient.listDeployedInstances(listUser); + } + + /** + * Enumerate slider instances for the current user, and the + * most recent app report, where available. + * @param listOnlyInState boolean to indicate that the instances should + * only include those in a YARN state + * <code> minAppState <= currentState <= maxAppState </code> + * + * @param minAppState minimum application state to include in enumeration. + * @param maxAppState maximum application state to include + * @return a map of application instance name to description + * @throws IOException Any IO problem + * @throws YarnException YARN problems + */ + public Map<String, SliderInstanceDescription> enumSliderInstances( + boolean listOnlyInState, + YarnApplicationState minAppState, + YarnApplicationState maxAppState) + throws IOException, YarnException { + + CoreFileSystem sliderFileSystem = new CoreFileSystem(conf); + Preconditions.checkArgument(!listOnlyInState || minAppState != null, + "null minAppState when listOnlyInState set"); + Preconditions.checkArgument(!listOnlyInState || maxAppState != null, + "null maxAppState when listOnlyInState set"); + if (!listOnlyInState) { + // if there's not filtering, ask for the entire range of states + minAppState = YarnApplicationState.NEW; + maxAppState = YarnApplicationState.KILLED; + } + // get the complete list of persistent instances + Map<String, Path> persistentInstances = + sliderFileSystem.listPersistentInstances(); + Map<String, SliderInstanceDescription> descriptions = + new HashMap<String, SliderInstanceDescription>(persistentInstances.size()); + + if (persistentInstances.isEmpty()) { + // an empty listing is a success if no cluster was named + log.debug("No application instances found"); + return descriptions; + } + + // enum those the RM knows about + List<ApplicationReport> rmInstances = listInstances(); + SliderUtils.sortApplicationsByMostRecent(rmInstances); + Map<String, ApplicationReport> reportMap = + SliderUtils.buildApplicationReportMap(rmInstances, minAppState, + maxAppState); + log.debug("Persisted {} deployed {} filtered[{}-{}] & de-duped to {}", + persistentInstances.size(), + rmInstances.size(), + minAppState, maxAppState, + reportMap.size()); + + // at this point there is a list of all persistent instances, and + // a (possibly filtered) list of application reports + + for (Map.Entry<String, Path> entry : persistentInstances.entrySet()) { + // loop through the persistent values + String name = entry.getKey(); + + // look up any report from the (possibly filtered) report set + ApplicationReport report = reportMap.get(name); + if (!listOnlyInState || report != null) { + // if the enum wants to filter in state, only add it if there is + // a report in that range. Otherwise: include all values + SliderInstanceDescription sid = new SliderInstanceDescription( + name, entry.getValue(), report); + descriptions.put(name, sid); + } + } + + return descriptions; + + } + +}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java new file mode 100644 index 0000000..ddab606 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.core.registry.docstore; + +public enum ConfigFormat { + + JSON("json"), + PROPERTIES("properties"), + XML("xml"), + HADOOP_XML("hadoop-xml"), + ENV("env"), + TEMPLATE("template"), + YAML("yaml"), + ; + ConfigFormat(String suffix) { + this.suffix = suffix; + } + + private final String suffix; + + public String getSuffix() { + return suffix; + } + + + @Override + public String toString() { + return suffix; + } + + /** + * Get a matching format or null + * @param type + * @return the format + */ + public static ConfigFormat resolve(String type) { + for (ConfigFormat format: values()) { + if (format.getSuffix().equals(type)) { + return format; + } + } + return null; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigUtils.java new file mode 100644 index 0000000..2e1615b --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigUtils.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.core.registry.docstore; + +import org.apache.hadoop.fs.Path; +import org.apache.slider.common.tools.SliderFileSystem; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class ConfigUtils { + public static final String TEMPLATE_FILE = "template.file"; + + public static String replaceProps(Map<String, String> config, String content) { + Map<String, String> tokens = new HashMap<>(); + for (Entry<String, String> entry : config.entrySet()) { + tokens.put("${" + entry.getKey() + "}", entry.getValue()); + tokens.put("{{" + entry.getKey() + "}}", entry.getValue()); + } + String value = content; + for (Map.Entry<String,String> token : tokens.entrySet()) { + value = value.replaceAll(Pattern.quote(token.getKey()), + Matcher.quoteReplacement(token.getValue())); + } + return value; + } + + public static Map<String, String> replacePropsInConfig( + Map<String, String> config, Map<String, String> env) { + Map<String, String> tokens = new HashMap<>(); + for (Entry<String, String> entry : env.entrySet()) { + tokens.put("${" + entry.getKey() + "}", entry.getValue()); + } + Map<String, String> newConfig = new HashMap<>(); + for (Entry<String, String> entry : config.entrySet()) { + String value = entry.getValue(); + for (Map.Entry<String,String> token : tokens.entrySet()) { + value = value.replaceAll(Pattern.quote(token.getKey()), + Matcher.quoteReplacement(token.getValue())); + } + newConfig.put(entry.getKey(), entry.getValue()); + } + return newConfig; + } + + public static void prepConfigForTemplateOutputter(ConfigFormat configFormat, + Map<String, String> config, SliderFileSystem fileSystem, + String clusterName, String fileName) throws IOException { + if (!configFormat.equals(ConfigFormat.TEMPLATE)) { + return; + } + Path templateFile = null; + if (config.containsKey(TEMPLATE_FILE)) { + templateFile = fileSystem.buildResourcePath(config.get(TEMPLATE_FILE)); + if (!fileSystem.isFile(templateFile)) { + templateFile = fileSystem.buildResourcePath(clusterName, + config.get(TEMPLATE_FILE)); + } + if (!fileSystem.isFile(templateFile)) { + throw new IOException("config specified template file " + config + .get(TEMPLATE_FILE) + " but " + templateFile + " doesn't exist"); + } + } + if (templateFile == null && fileName != null) { + templateFile = fileSystem.buildResourcePath(fileName); + if (!fileSystem.isFile(templateFile)) { + templateFile = fileSystem.buildResourcePath(clusterName, + fileName); + } + } + if (fileSystem.isFile(templateFile)) { + config.put("content", fileSystem.cat(templateFile)); + } else { + config.put("content", ""); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigurationResolver.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigurationResolver.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigurationResolver.java new file mode 100644 index 0000000..88bac77 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigurationResolver.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.core.registry.docstore; + +public class ConfigurationResolver { + + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java new file mode 100644 index 0000000..4bcf6c1 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.core.registry.docstore; + +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +/** + * JSON-serializable description of a published key-val configuration. + * + * The values themselves are not serialized in the external view; they have to be served up by the far end + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +public class ExportEntry { + + /** + * The value of the export + */ + private String value; + /** + * The container id of the container that is responsible for the export + */ + private String containerId; + /** + * Tag associated with the container - its usually an identifier different than container id + * that allows a soft serial id to all containers of a component - e.g. 1, 2, 3, ... + */ + private String tag; + /** + * An export can be at the level of a component or an application + */ + private String level; + /** + * The time when the export was updated + */ + private String updatedTime; + /** + * The time when the export expires + */ + private String validUntil; + + public ExportEntry() { + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + public String getContainerId() { + return containerId; + } + + public void setContainerId(String containerId) { + this.containerId = containerId; + } + + public String getTag() { + return tag; + } + + public void setTag(String tag) { + this.tag = tag; + } + + public String getLevel() { + return level; + } + + public void setLevel(String level) { + this.level = level; + } + public String getUpdatedTime() { + return updatedTime; + } + + public void setUpdatedTime(String updatedTime) { + this.updatedTime = updatedTime; + } + + public String getValidUntil() { + return validUntil; + } + + public void setValidUntil(String validUntil) { + this.validUntil = validUntil; + } + + @Override + public String toString() { + return new StringBuilder("ExportEntry{"). + append("value='").append(value).append("',"). + append("containerId='").append(containerId).append("',"). + append("tag='").append(tag).append("',"). + append("level='").append(level).append("'"). + append("updatedTime='").append(updatedTime).append("'"). + append("validUntil='").append(validUntil).append("'"). + append(" }").toString(); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigSet.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigSet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigSet.java new file mode 100644 index 0000000..edc129e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigSet.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.core.registry.docstore; + +import org.apache.slider.server.appmaster.web.rest.RestPaths; +import org.apache.slider.server.services.utility.PatternValidator; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; + +/** + * Represents a set of configurations for an application, component, etc. + * Json serialisable; accessors are synchronized + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +public class PublishedConfigSet { + + private static final PatternValidator validator = new PatternValidator( + RestPaths.PUBLISHED_CONFIGURATION_REGEXP); + + public Map<String, PublishedConfiguration> configurations = + new HashMap<>(); + + public PublishedConfigSet() { + } + + /** + * Put a name -it will be converted to lower case before insertion. + * Any existing entry will be overwritten (that includes an entry + * with a different case in the original name) + * @param name name of entry + * @param conf configuration + * @throws IllegalArgumentException if not a valid name + */ + public void put(String name, PublishedConfiguration conf) { + String name1 = name.toLowerCase(Locale.ENGLISH); + validateName(name1); + configurations.put(name1, conf); + } + + /** + * Validate the name -restricting it to the set defined in + * {@link RestPaths#PUBLISHED_CONFIGURATION_REGEXP} + * @param name name to validate + * @throws IllegalArgumentException if not a valid name + */ + public static void validateName(String name) { + validator.validate(name); + + } + + public PublishedConfiguration get(String name) { + return configurations.get(name); + } + + public boolean contains(String name) { + return configurations.containsKey(name); + } + + public int size() { + return configurations.size(); + } + + public Set<String> keys() { + TreeSet<String> keys = new TreeSet<>(); + keys.addAll(configurations.keySet()); + return keys; + } + + public PublishedConfigSet shallowCopy() { + PublishedConfigSet that = new PublishedConfigSet(); + for (Map.Entry<String, PublishedConfiguration> entry : + configurations.entrySet()) { + that.put(entry.getKey(), entry.getValue().shallowCopy()); + } + return that; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java new file mode 100644 index 0000000..50b522f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java @@ -0,0 +1,196 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.core.registry.docstore; + +import org.apache.hadoop.conf.Configuration; +import org.apache.slider.common.tools.ConfigHelper; +import org.apache.slider.core.exceptions.BadConfigException; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.ObjectMapper; +import org.codehaus.jackson.map.SerializationConfig; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +import java.io.IOException; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +/** + * JSON-serializable description of a published key-val configuration. + * + * The values themselves are not serialized in the external view; they have + * to be served up by the far end + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +public class PublishedConfiguration { + + public String description; + public long updated; + + public String updatedTime; + + public Map<String, String> entries = new HashMap<>(); + + public PublishedConfiguration() { + } + + /** + * build an empty published configuration + * @param description configuration description + */ + public PublishedConfiguration(String description) { + this.description = description; + } + + /** + * Build a configuration from the entries + * @param description configuration description + * @param entries entries to put + */ + public PublishedConfiguration(String description, + Iterable<Map.Entry<String, String>> entries) { + this.description = description; + putValues(entries); + } + + /** + * Build a published configuration, using the keys from keysource, + * but resolving the values from the value source, via Configuration.get() + * @param description configuration description + * @param keysource source of keys + * @param valuesource source of values + */ + public PublishedConfiguration(String description, + Iterable<Map.Entry<String, String>> keysource, + Configuration valuesource) { + this.description = description; + putValues(ConfigHelper.resolveConfiguration(keysource, valuesource)); + } + + + /** + * Is the configuration empty. This means either that it has not + * been given any values, or it is stripped down copy set down over the + * wire. + * @return true if it is empty + */ + public boolean isEmpty() { + return entries.isEmpty(); + } + + + public void setUpdated(long updated) { + this.updated = updated; + this.updatedTime = new Date(updated).toString(); + } + + public long getUpdated() { + return updated; + } + + /** + * Set the values from an iterable (this includes a Hadoop Configuration + * and Java properties object). + * Any existing value set is discarded + * @param entries entries to put + */ + public void putValues(Iterable<Map.Entry<String, String>> entries) { + this.entries = new HashMap<String, String>(); + for (Map.Entry<String, String> entry : entries) { + this.entries.put(entry.getKey(), entry.getValue()); + } + + } + + /** + * Convert to Hadoop XML + * @return the configuration as a Hadoop Configuratin + */ + public Configuration asConfiguration() { + Configuration conf = new Configuration(false); + try { + ConfigHelper.addConfigMap(conf, entries, ""); + } catch (BadConfigException e) { + // triggered on a null value; switch to a runtime (and discard the stack) + throw new RuntimeException(e.toString()); + } + return conf; + } + + public String asConfigurationXML() throws IOException { + return ConfigHelper.toXml(asConfiguration()); + } + + /** + * Convert values to properties + * @return a property file + */ + public Properties asProperties() { + Properties props = new Properties(); + props.putAll(entries); + return props; + } + + /** + * Return the values as json string + * @return the JSON representation + * @throws IOException marshalling failure + */ + public String asJson() throws IOException { + ObjectMapper mapper = new ObjectMapper(); + mapper.configure(SerializationConfig.Feature.INDENT_OUTPUT, true); + String json = mapper.writeValueAsString(entries); + return json; + } + + + /** + * This makes a copy without the nested content -so is suitable + * for returning as part of the list of a parent's values + * @return the copy + */ + public PublishedConfiguration shallowCopy() { + PublishedConfiguration that = new PublishedConfiguration(); + that.description = this.description; + that.updated = this.updated; + that.updatedTime = this.updatedTime; + return that; + } + + @Override + public String toString() { + final StringBuilder sb = + new StringBuilder("PublishedConfiguration{"); + sb.append("description='").append(description).append('\''); + sb.append(" entries = ").append(entries.size()); + sb.append('}'); + return sb.toString(); + } + + /** + * Create an outputter for a given format + * @param format format to use + * @return an instance of output + */ + public PublishedConfigurationOutputter createOutputter(ConfigFormat format) { + return PublishedConfigurationOutputter.createOutputter(format, this); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java new file mode 100644 index 0000000..9bdcfcb --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.core.registry.docstore; + +import com.google.common.base.Charsets; +import com.google.common.base.Preconditions; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.slider.common.tools.ConfigHelper; +import org.yaml.snakeyaml.DumperOptions; +import org.yaml.snakeyaml.DumperOptions.FlowStyle; +import org.yaml.snakeyaml.Yaml; + +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.io.StringWriter; +import java.util.Properties; + +/** + * Output a published configuration + */ +public abstract class PublishedConfigurationOutputter { + + protected final PublishedConfiguration owner; + + protected PublishedConfigurationOutputter(PublishedConfiguration owner) { + this.owner = owner; + } + + /** + * Save the config to a destination file, in the format of this outputter + * @param dest destination file + * @throws IOException + */ +/* JDK7 + public void save(File dest) throws IOException { + try(FileOutputStream out = new FileOutputStream(dest)) { + save(out); + out.close(); + } + } +*/ + public void save(File dest) throws IOException { + FileUtils.writeStringToFile(dest, asString(), Charsets.UTF_8); + } + + /** + * Save the content. The default saves the asString() value + * to the output stream + * @param out output stream + * @throws IOException + */ + public void save(OutputStream out) throws IOException { + IOUtils.write(asString(), out, Charsets.UTF_8); + } + /** + * Convert to a string + * @return the string form + * @throws IOException + */ + public abstract String asString() throws IOException; + + /** + * Create an outputter for the chosen format + * @param format format enumeration + * @param owner owning config + * @return the outputter + */ + + public static PublishedConfigurationOutputter createOutputter(ConfigFormat format, + PublishedConfiguration owner) { + Preconditions.checkNotNull(owner); + switch (format) { + case XML: + case HADOOP_XML: + return new XmlOutputter(owner); + case PROPERTIES: + return new PropertiesOutputter(owner); + case JSON: + return new JsonOutputter(owner); + case ENV: + return new EnvOutputter(owner); + case TEMPLATE: + return new TemplateOutputter(owner); + case YAML: + return new YamlOutputter(owner); + default: + throw new RuntimeException("Unsupported format :" + format); + } + } + + public static class XmlOutputter extends PublishedConfigurationOutputter { + + + private final Configuration configuration; + + public XmlOutputter(PublishedConfiguration owner) { + super(owner); + configuration = owner.asConfiguration(); + } + + @Override + public void save(OutputStream out) throws IOException { + configuration.writeXml(out); + } + + @Override + public String asString() throws IOException { + return ConfigHelper.toXml(configuration); + } + + public Configuration getConfiguration() { + return configuration; + } + } + + public static class PropertiesOutputter extends PublishedConfigurationOutputter { + + private final Properties properties; + + public PropertiesOutputter(PublishedConfiguration owner) { + super(owner); + properties = owner.asProperties(); + } + + @Override + public void save(OutputStream out) throws IOException { + properties.store(out, ""); + } + + + public String asString() throws IOException { + StringWriter sw = new StringWriter(); + properties.store(sw, ""); + return sw.toString(); + } + } + + + public static class JsonOutputter extends PublishedConfigurationOutputter { + + public JsonOutputter(PublishedConfiguration owner) { + super(owner); + } + + @Override + public String asString() throws IOException { + return owner.asJson(); + } + } + + + public static class EnvOutputter extends PublishedConfigurationOutputter { + + public EnvOutputter(PublishedConfiguration owner) { + super(owner); + } + + @Override + public String asString() throws IOException { + if (!owner.entries.containsKey("content")) { + throw new IOException("Configuration has no content field and cannot " + + "be retrieved as type 'env'"); + } + String content = owner.entries.get("content"); + return ConfigUtils.replaceProps(owner.entries, content); + } + } + + public static class TemplateOutputter extends EnvOutputter { + public TemplateOutputter(PublishedConfiguration owner) { + super(owner); + } + } + + public static class YamlOutputter extends PublishedConfigurationOutputter { + + private final Yaml yaml; + + public YamlOutputter(PublishedConfiguration owner) { + super(owner); + DumperOptions options = new DumperOptions(); + options.setDefaultFlowStyle(FlowStyle.BLOCK); + yaml = new Yaml(options); + } + + public String asString() throws IOException { + return yaml.dump(owner.entries); + } + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExports.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExports.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExports.java new file mode 100644 index 0000000..0759b4e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExports.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.core.registry.docstore; + +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.ObjectMapper; +import org.codehaus.jackson.map.SerializationConfig; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +import java.io.IOException; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * JSON-serializable description of a published key-val configuration. + * + * The values themselves are not serialized in the external view; they have to be served up by the far end + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +public class PublishedExports { + + public String description; + public long updated; + public String updatedTime; + public Map<String, List<ExportEntry>> entries = new HashMap<>(); + + public PublishedExports() { + } + + /** + * build an empty published configuration + * + * @param description configuration description + */ + public PublishedExports(String description) { + this.description = description; + } + + /** + * Build a configuration from the entries + * + * @param description configuration description + * @param entries entries to put + */ + public PublishedExports(String description, + Iterable<Map.Entry<String, List<ExportEntry>>> entries) { + this.description = description; + putValues(entries); + } + + /** + * Is the configuration empty. This means either that it has not been given any values, + * or it is stripped down copy + * set down over the wire. + * + * @return true if it is empty + */ + public boolean isEmpty() { + return entries.isEmpty(); + } + + public long getUpdated() { + return updated; + } + + public void setUpdated(long updated) { + this.updated = updated; + this.updatedTime = new Date(updated).toString(); + } + + /** + * Set the values from an iterable (this includes a Hadoop Configuration and Java properties + * object). Any existing value set is discarded + * + * @param entries entries to put + */ + public void putValues(Iterable<Map.Entry<String, List<ExportEntry>>> entries) { + this.entries = new HashMap<String, List<ExportEntry>>(); + for (Map.Entry<String, List<ExportEntry>> entry : entries) { + this.entries.put(entry.getKey(), entry.getValue()); + } + } + + /** + * Return the values as json string + * + * @return the JSON form + * + * @throws IOException mapping problems + */ + public String asJson() throws IOException { + ObjectMapper mapper = new ObjectMapper(); + mapper.configure(SerializationConfig.Feature.INDENT_OUTPUT, true); + String json = mapper.writeValueAsString(entries); + return json; + } + + /** + * This makes a copy without the nested content -so is suitable for returning as part of the list of a parent's + * values + * + * @return the copy + */ + public PublishedExports shallowCopy() { + PublishedExports that = new PublishedExports(); + that.description = this.description; + that.updated = this.updated; + that.updatedTime = this.updatedTime; + return that; + } + + @Override + public String toString() { + final StringBuilder sb = + new StringBuilder("PublishedConfiguration{"); + sb.append("description='").append(description).append('\''); + sb.append(" entries = ").append(entries.size()); + sb.append('}'); + return sb.toString(); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsOutputter.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsOutputter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsOutputter.java new file mode 100644 index 0000000..67cb094 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsOutputter.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.core.registry.docstore; + +import com.google.common.base.Charsets; +import com.google.common.base.Preconditions; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; + +/** Output a published configuration */ +public abstract class PublishedExportsOutputter { + + protected final PublishedExports exports; + + protected PublishedExportsOutputter(PublishedExports exports) { + this.exports = exports; + } + + /** + * Create an outputter for the chosen format + * + * @param format format enumeration + * @param exports owning config + * @return the outputter + */ + + public static PublishedExportsOutputter createOutputter(ConfigFormat format, + PublishedExports exports) { + Preconditions.checkNotNull(exports); + switch (format) { + case JSON: + return new JsonOutputter(exports); + default: + throw new RuntimeException("Unsupported format :" + format); + } + } + + public void save(File dest) throws IOException { + FileOutputStream out = null; + try { + out = new FileOutputStream(dest); + save(out); + out.close(); + } finally { + org.apache.hadoop.io.IOUtils.closeStream(out); + } + } + + /** + * Save the content. The default saves the asString() value to the output stream + * + * @param out output stream + * @throws IOException + */ + public void save(OutputStream out) throws IOException { + IOUtils.write(asString(), out, Charsets.UTF_8); + } + + /** + * Convert to a string + * + * @return the string form + * @throws IOException + */ + public abstract String asString() throws IOException; + + public static class JsonOutputter extends PublishedExportsOutputter { + + public JsonOutputter(PublishedExports exports) { + super(exports); + } + + @Override + public void save(File dest) throws IOException { + FileUtils.writeStringToFile(dest, asString(), Charsets.UTF_8); + } + + @Override + public String asString() throws IOException { + return exports.asJson(); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsSet.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsSet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsSet.java new file mode 100644 index 0000000..339d3d6 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsSet.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.core.registry.docstore; + +import org.apache.slider.server.appmaster.web.rest.RestPaths; +import org.apache.slider.server.services.utility.PatternValidator; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; + +/** + * Represents a set of configurations for an application, component, etc. + * Json serialisable; accessors are synchronized + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +public class PublishedExportsSet { + + private static final PatternValidator validator = new PatternValidator( + RestPaths.PUBLISHED_CONFIGURATION_REGEXP); + + public Map<String, PublishedExports> exports = new HashMap<>(); + + public PublishedExportsSet() { + } + + /** + * Put a name -it will be converted to lower case before insertion. + * Any existing entry will be overwritten (that includes an entry + * with a different case in the original name) + * @param name name of entry + * @param export published export + * @throws IllegalArgumentException if not a valid name + */ + public void put(String name, PublishedExports export) { + String name1 = name.toLowerCase(Locale.ENGLISH); + validateName(name1); + exports.put(name1, export); + } + + /** + * Validate the name -restricting it to the set defined in + * {@link RestPaths#PUBLISHED_CONFIGURATION_REGEXP} + * @param name name to validate + * @throws IllegalArgumentException if not a valid name + */ + public static void validateName(String name) { + validator.validate(name); + + } + + public PublishedExports get(String name) { + return exports.get(name); + } + + public boolean contains(String name) { + return exports.containsKey(name); + } + + public int size() { + return exports.size(); + } + + public Set<String> keys() { + TreeSet<String> keys = new TreeSet<>(); + keys.addAll(exports.keySet()); + return keys; + } + + public PublishedExportsSet shallowCopy() { + PublishedExportsSet that = new PublishedExportsSet(); + for (Map.Entry<String, PublishedExports> entry : exports.entrySet()) { + that.put(entry.getKey(), entry.getValue().shallowCopy()); + } + return that; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/UriMap.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/UriMap.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/UriMap.java new file mode 100644 index 0000000..120966f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/UriMap.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.core.registry.docstore; + +import org.codehaus.jackson.annotate.JsonIgnore; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +import java.util.HashMap; +import java.util.Map; + +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +public class UriMap { + + public Map<String, String> uris = new HashMap<>(); + + @JsonIgnore + public void put(String key, String value) { + uris.put(key, value); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java new file mode 100644 index 0000000..13ad5c5 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.core.registry.info; + +/** + * These are constants unique to the Slider AM + */ +public class CustomRegistryConstants { + + public static final String MANAGEMENT_REST_API = + "classpath:org.apache.slider.management"; + + public static final String REGISTRY_REST_API = + "classpath:org.apache.slider.registry"; + + public static final String PUBLISHER_REST_API = + "classpath:org.apache.slider.publisher"; + + public static final String PUBLISHER_CONFIGURATIONS_API = + "classpath:org.apache.slider.publisher.configurations"; + + public static final String PUBLISHER_EXPORTS_API = + "classpath:org.apache.slider.publisher.exports"; + + public static final String PUBLISHER_DOCUMENTS_API = + "classpath:org.apache.slider.publisher.documents"; + + public static final String AGENT_SECURE_REST_API = + "classpath:org.apache.slider.agents.secure"; + + public static final String AGENT_ONEWAY_REST_API = + "classpath:org.apache.slider.agents.oneway"; + + public static final String AM_IPC_PROTOCOL = + "classpath:org.apache.slider.appmaster.ipc"; + + public static final String AM_REST_BASE = + "classpath:org.apache.slider.client.rest"; + + public static final String WEB_UI = "http://";; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/retrieve/AMWebClient.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/retrieve/AMWebClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/retrieve/AMWebClient.java new file mode 100644 index 0000000..40fa217 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/retrieve/AMWebClient.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.core.registry.retrieve; + +import com.sun.jersey.api.client.Client; +import com.sun.jersey.api.client.GenericType; +import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.api.client.config.ClientConfig; +import com.sun.jersey.api.client.config.DefaultClientConfig; +import com.sun.jersey.api.json.JSONConfiguration; +import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory; +import com.sun.jersey.client.urlconnection.URLConnectionClientHandler; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.ssl.SSLFactory; +import org.apache.slider.client.rest.BaseRestClient; +import org.apache.slider.core.restclient.HttpVerb; +import org.apache.slider.core.restclient.UgiJerseyBinding; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.SSLSocketFactory; +import javax.ws.rs.core.HttpHeaders; +import javax.ws.rs.core.MediaType; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URI; +import java.net.URL; + +/** + * Class to retrieve artifacts from the AM's web site. This sets up + * the redirection and security logic properly + */ +public class AMWebClient { + + + private final BaseRestClient restClient; + private static final Logger + log = LoggerFactory.getLogger(AMWebClient.class); + + + public AMWebClient(Configuration conf) { + UgiJerseyBinding binding = new UgiJerseyBinding(conf); + + restClient = new BaseRestClient(binding.createJerseyClient()); + + } + + + private static URLConnectionClientHandler getUrlConnectionClientHandler() { + return new URLConnectionClientHandler(new HttpURLConnectionFactory() { + @Override + public HttpURLConnection getHttpURLConnection(URL url) + throws IOException { + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + if (connection.getResponseCode() == HttpURLConnection.HTTP_MOVED_TEMP) { + // is a redirect - are we changing schemes? + String redirectLocation = connection.getHeaderField(HttpHeaders.LOCATION); + String originalScheme = url.getProtocol(); + String redirectScheme = URI.create(redirectLocation).getScheme(); + if (!originalScheme.equals(redirectScheme)) { + // need to fake it out by doing redirect ourselves + log.info("Protocol change during redirect. Redirecting {} to URL {}", + url, redirectLocation); + URL redirectURL = new URL(redirectLocation); + connection = (HttpURLConnection) redirectURL.openConnection(); + } + } + if (connection instanceof HttpsURLConnection) { + log.debug("Attempting to configure HTTPS connection using client " + + "configuration"); + final SSLFactory factory; + final SSLSocketFactory sf; + final HostnameVerifier hv; + + try { + HttpsURLConnection c = (HttpsURLConnection) connection; + factory = new SSLFactory(SSLFactory.Mode.CLIENT, new Configuration()); + factory.init(); + sf = factory.createSSLSocketFactory(); + hv = factory.getHostnameVerifier(); + c.setSSLSocketFactory(sf); + c.setHostnameVerifier(hv); + } catch (Exception e) { + log.info("Unable to configure HTTPS connection from " + + "configuration. Using JDK properties."); + } + + } + return connection; + } + }); + } + + public WebResource resource(String url) { + return restClient.resource(url); + } + + public BaseRestClient getRestClient() { + return restClient; + } + + /** + * Execute the operation. Failures are raised as IOException subclasses + * @param method method to execute + * @param resource resource to work against + * @param c class to build + * @param <T> type expected + * @return an instance of the type T + * @throws IOException on any failure + */ + public <T> T exec(HttpVerb method, WebResource resource, Class<T> c) throws IOException { + return restClient.exec(method, resource, c); + } + + /** + * Execute the operation. Failures are raised as IOException subclasses + * @param method method to execute + * @param resource resource to work against + * @param t type to work with + * @param <T> type expected + * @return an instance of the type T + * @throws IOException on any failure + */ + public <T> T exec(HttpVerb method, WebResource resource, GenericType<T> t) + throws IOException { + return restClient.exec(method, resource, t); + } + + /** + * Execute the GET operation. Failures are raised as IOException subclasses + * @param resource resource to work against + * @param c class to build + * @param <T> type expected + * @return an instance of the type T + * @throws IOException on any failure + */ + public <T> T get(WebResource resource, Class<T> c) throws IOException { + return restClient.get(resource, c); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java new file mode 100644 index 0000000..b0eddb8 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.core.registry.retrieve; + +import com.beust.jcommander.Strings; +import com.sun.jersey.api.client.UniformInterfaceException; +import com.sun.jersey.api.client.WebResource; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.registry.client.exceptions.RegistryIOException; +import org.apache.hadoop.registry.client.types.ServiceRecord; +import static org.apache.slider.client.ClientRegistryBinder.*; +import org.apache.slider.common.tools.SliderUtils; +import org.apache.slider.core.exceptions.ExceptionConverter; +import org.apache.slider.core.registry.docstore.PublishedConfigSet; +import org.apache.slider.core.registry.docstore.PublishedConfiguration; +import org.apache.slider.core.registry.docstore.PublishedExports; +import org.apache.slider.core.registry.docstore.PublishedExportsSet; +import static org.apache.slider.core.registry.info.CustomRegistryConstants.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.FileNotFoundException; +import java.io.IOException; + +/** + * Registry retriever. + * This hides the HTTP operations that take place to + * get the actual content + */ +public class RegistryRetriever extends AMWebClient { + private static final Logger log = LoggerFactory.getLogger(RegistryRetriever.class); + + private final String externalConfigurationURL; + private final String internalConfigurationURL; + private final String externalExportsURL; + private final String internalExportsURL; + + /** + * Retrieve from a service by locating the + * exported {@link CustomRegistryConstants.PUBLISHER_CONFIGURATIONS_API} + * and working off it. + * + * @param conf configuration to work from + * @param record service record + * @throws RegistryIOException the address type of the endpoint does + * not match that expected (i.e. not a list of URLs), missing endpoint... + */ + public RegistryRetriever(Configuration conf, ServiceRecord record) throws RegistryIOException { + super(conf); + externalConfigurationURL = lookupRestAPI(record, + PUBLISHER_CONFIGURATIONS_API, true); + internalConfigurationURL = lookupRestAPI(record, + PUBLISHER_CONFIGURATIONS_API, false); + externalExportsURL = lookupRestAPI(record, + PUBLISHER_EXPORTS_API, true); + internalExportsURL = lookupRestAPI(record, + PUBLISHER_EXPORTS_API, false); + } + + /** + * Does a bonded registry retriever have a configuration? + * @param external flag to indicate that it is the external entries to fetch + * @return true if there is a URL to the configurations defined + */ + public boolean hasConfigurations(boolean external) { + return !Strings.isStringEmpty( + external ? externalConfigurationURL : internalConfigurationURL); + } + + /** + * Get the configurations of the registry + * @param external flag to indicate that it is the external entries to fetch + * @return the configuration sets + */ + public PublishedConfigSet getConfigurations(boolean external) throws + FileNotFoundException, IOException { + + String confURL = getConfigurationURL(external); + WebResource webResource = resource(confURL); + return get(webResource, PublishedConfigSet.class); + } + + protected String getConfigurationURL(boolean external) throws FileNotFoundException { + String confURL = external ? externalConfigurationURL: internalConfigurationURL; + if (Strings.isStringEmpty(confURL)) { + throw new FileNotFoundException("No configuration URL"); + } + return confURL; + } + + protected String getExportURL(boolean external) throws FileNotFoundException { + String confURL = external ? externalExportsURL: internalExportsURL; + if (Strings.isStringEmpty(confURL)) { + throw new FileNotFoundException("No configuration URL"); + } + return confURL; + } + + /** + * Get the configurations of the registry + * @param external flag to indicate that it is the external entries to fetch + * @return the configuration sets + */ + public PublishedExportsSet getExports(boolean external) throws + FileNotFoundException, IOException { + + String exportsUrl = getExportURL(external); + WebResource webResource = resource(exportsUrl); + return get(webResource, PublishedExportsSet.class); + } + + + /** + * Get a complete configuration, with all values + * @param configSet config set to ask for + * @param name name of the configuration + * @param external flag to indicate that it is an external configuration + * @return the retrieved config + * @throws IOException IO problems + */ + public PublishedConfiguration retrieveConfiguration(PublishedConfigSet configSet, + String name, + boolean external) throws IOException { + String confURL = getConfigurationURL(external); + if (!configSet.contains(name)) { + throw new FileNotFoundException("Unknown configuration " + name); + } + confURL = SliderUtils.appendToURL(confURL, name); + WebResource webResource = resource(confURL); + return get(webResource, PublishedConfiguration.class); + } + + /** + * Get a complete export, with all values + * @param exportSet + * @param name name of the configuration + * @param external flag to indicate that it is an external configuration + * @return the retrieved config + * @throws IOException IO problems + */ + public PublishedExports retrieveExports(PublishedExportsSet exportSet, + String name, + boolean external) throws IOException { + if (!exportSet.contains(name)) { + throw new FileNotFoundException("Unknown export " + name); + } + String exportsURL = getExportURL(external); + exportsURL = SliderUtils.appendToURL(exportsURL, name); + return get(resource(exportsURL), PublishedExports.class); + } + + @Override + public String toString() { + final StringBuilder sb = + new StringBuilder("RegistryRetriever{"); + sb.append("externalConfigurationURL='") + .append(externalConfigurationURL) + .append('\''); + sb.append(", internalConfigurationURL='") + .append(internalConfigurationURL) + .append('\''); + sb.append(", externalExportsURL='").append(externalExportsURL).append('\''); + sb.append(", internalExportsURL='").append(internalExportsURL).append('\''); + sb.append('}'); + return sb.toString(); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/HttpOperationResponse.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/HttpOperationResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/HttpOperationResponse.java new file mode 100644 index 0000000..0266223 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/HttpOperationResponse.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.core.restclient; + +import java.util.List; +import java.util.Map; + +/** + * A response for use as a return value from operations + */ +public class HttpOperationResponse { + + public int responseCode; + public long lastModified; + public String contentType; + public byte[] data; + public Map<String, List<String>> headers; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/HttpVerb.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/HttpVerb.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/HttpVerb.java new file mode 100644 index 0000000..c040345 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/HttpVerb.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.core.restclient; + +/** + * Http verbs with details on what they support in terms of submit and + * response bodies. + * <p> + * Those verbs which do support bodies in the response MAY NOT return it; + * if the response code is 204 then the answer is "no body", but the operation + * is considered a success. + */ +public enum HttpVerb { + GET("GET", false, true), + POST("POST", true, true), + PUT("PUT", true, true), + DELETE("DELETE", false, true), + HEAD("HEAD", false, false); + + private final String verb; + private final boolean hasUploadBody; + private final boolean hasResponseBody; + + HttpVerb(String verb, boolean hasUploadBody, boolean hasResponseBody) { + this.verb = verb; + this.hasUploadBody = hasUploadBody; + this.hasResponseBody = hasResponseBody; + } + + public String getVerb() { + return verb; + } + + public boolean hasUploadBody() { + return hasUploadBody; + } + + public boolean hasResponseBody() { + return hasResponseBody; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/SliderURLConnectionFactory.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/SliderURLConnectionFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/SliderURLConnectionFactory.java new file mode 100644 index 0000000..e453f52 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/SliderURLConnectionFactory.java @@ -0,0 +1,176 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.core.restclient; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.web.KerberosUgiAuthenticator; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.client.AuthenticatedURL; +import org.apache.hadoop.security.authentication.client.AuthenticationException; +import org.apache.hadoop.security.authentication.client.ConnectionConfigurator; +import org.apache.hadoop.security.ssl.SSLFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.SSLSocketFactory; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.net.URLConnection; +import java.security.GeneralSecurityException; + +/** + * Factory for URL connections; used behind the scenes in the Jersey integration. + * <p> + * Derived from the WebHDFS implementation. + */ +public class SliderURLConnectionFactory { + private static final Logger log = + LoggerFactory.getLogger(SliderURLConnectionFactory.class); + + /** + * Timeout for socket connects and reads + */ + public final static int DEFAULT_SOCKET_TIMEOUT = 60 * 1000; // 1 minute + private final ConnectionConfigurator connConfigurator; + + private static final ConnectionConfigurator DEFAULT_CONFIGURATOR = new BasicConfigurator(); + + /** + * Construct a new URLConnectionFactory based on the configuration. It will + * try to load SSL certificates when it is specified. + */ + public static SliderURLConnectionFactory newInstance(Configuration conf) { + ConnectionConfigurator conn; + try { + conn = newSslConnConfigurator(DEFAULT_SOCKET_TIMEOUT, conf); + } catch (Exception e) { + log.debug("Cannot load customized SSL configuration.", e); + conn = DEFAULT_CONFIGURATOR; + } + return new SliderURLConnectionFactory(conn); + } + + private SliderURLConnectionFactory(ConnectionConfigurator connConfigurator) { + this.connConfigurator = connConfigurator; + } + + /** + * Create a new ConnectionConfigurator for SSL connections + */ + private static ConnectionConfigurator newSslConnConfigurator(final int timeout, + Configuration conf) throws IOException, GeneralSecurityException { + final SSLFactory factory; + final SSLSocketFactory sf; + final HostnameVerifier hv; + + factory = new SSLFactory(SSLFactory.Mode.CLIENT, conf); + factory.init(); + sf = factory.createSSLSocketFactory(); + hv = factory.getHostnameVerifier(); + + return new ConnectionConfigurator() { + @Override + public HttpURLConnection configure(HttpURLConnection conn) + throws IOException { + if (conn instanceof HttpsURLConnection) { + HttpsURLConnection c = (HttpsURLConnection) conn; + c.setSSLSocketFactory(sf); + c.setHostnameVerifier(hv); + } + SliderURLConnectionFactory.setupConnection(conn, timeout); + return conn; + } + }; + } + + /** + * Opens a url with read and connect timeouts + * + * @param url + * to open + * @return URLConnection + * @throws IOException + */ + public URLConnection openConnection(URL url) throws IOException { + try { + return openConnection(url, false); + } catch (AuthenticationException e) { + // Unreachable + return null; + } + } + + /** + * Opens a url with read and connect timeouts + * + * @param url + * URL to open + * @param isSpnego + * whether the url should be authenticated via SPNEGO + * @return URLConnection + * @throws IOException + * @throws AuthenticationException + */ + public URLConnection openConnection(URL url, boolean isSpnego) + throws IOException, AuthenticationException { + if (isSpnego) { + log.debug("open AuthenticatedURL connection {}", url); + UserGroupInformation.getCurrentUser().checkTGTAndReloginFromKeytab(); + final AuthenticatedURL.Token authToken = new AuthenticatedURL.Token(); + return new AuthenticatedURL(new KerberosUgiAuthenticator(), + connConfigurator).openConnection(url, authToken); + } else { + log.debug("open URL connection {}", url); + URLConnection connection = url.openConnection(); + if (connection instanceof HttpURLConnection) { + connConfigurator.configure((HttpURLConnection) connection); + } + return connection; + } + } + + /** + * Sets connection parameters on the given URLConnection + * + * @param connection + * URLConnection to set + * @param socketTimeout + * the connection and read timeout of the connection. + */ + private static void setupConnection(URLConnection connection, int socketTimeout) { + connection.setConnectTimeout(socketTimeout); + connection.setReadTimeout(socketTimeout); + connection.setUseCaches(false); + if (connection instanceof HttpURLConnection) { + ((HttpURLConnection) connection).setInstanceFollowRedirects(true); + } + } + + private static class BasicConfigurator implements ConnectionConfigurator { + @Override + public HttpURLConnection configure(HttpURLConnection conn) + throws IOException { + SliderURLConnectionFactory.setupConnection(conn, DEFAULT_SOCKET_TIMEOUT); + return conn; + } + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
