Author: stevel
Date: Mon May 19 13:32:10 2014
New Revision: 1595897
URL: http://svn.apache.org/r1595897
Log:
SLIDER-52 PublishedConfiguration XML conf values are not resolved until
client-side
Added:
incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigurationResolver.java
(with props)
Modified:
incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java
incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy
Modified:
incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
URL:
http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/client/SliderClient.java?rev=1595897&r1=1595896&r2=1595897&view=diff
==============================================================================
---
incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
(original)
+++
incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
Mon May 19 13:32:10 2014
@@ -1978,7 +1978,7 @@ public class SliderClient extends Abstra
}
} catch (FileNotFoundException e) {
log.info("{}", e.toString());
- log.debug("{}", e);
+ log.debug("{}",e, e);
return EXIT_NOT_FOUND;
}
return EXIT_SUCCESS;
Modified:
incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
URL:
http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java?rev=1595897&r1=1595896&r2=1595897&view=diff
==============================================================================
---
incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
(original)
+++
incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
Mon May 19 13:32:10 2014
@@ -18,6 +18,7 @@
package org.apache.slider.common.tools;
+import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -509,4 +510,25 @@ public class ConfigHelper {
Map<String, String> map = new HashMap<String, String>();
return SliderUtils.mergeEntries(map, conf);
}
+
+ /**
+ * This goes through the keyset of one configuration and retrieves each value
+ * from a value source -a different or the same configuration. This triggers
+ * the property resolution process of the value, resolving any variables
against
+ * in-config or inherited configurations
+ * @param keysource source of keys
+ * @param valuesource the source of values
+ * @return a new configuration where <code>foreach key in keysource,
get(key)==valuesource.get(key)</code>
+ */
+ public static Configuration resolveConfiguration(Iterable<Map.Entry<String,
String>> keysource,
+ Configuration valuesource) {
+ Configuration result = new Configuration(false);
+ for (Map.Entry<String, String> entry : keysource) {
+ String key = entry.getKey();
+ String value = valuesource.get(key);
+ Preconditions.checkState(value!=null, "no reference for \"%s\" in
values", key);
+ result.set(key, value);
+ }
+ return result;
+ }
}
Added:
incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigurationResolver.java
URL:
http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigurationResolver.java?rev=1595897&view=auto
==============================================================================
---
incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigurationResolver.java
(added)
+++
incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigurationResolver.java
Mon May 19 13:32:10 2014
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.docstore;
+
+public class ConfigurationResolver {
+
+
+}
Propchange:
incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigurationResolver.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified:
incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java
URL:
http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java?rev=1595897&r1=1595896&r2=1595897&view=diff
==============================================================================
---
incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java
(original)
+++
incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java
Mon May 19 13:32:10 2014
@@ -51,15 +51,39 @@ public class PublishedConfiguration {
public PublishedConfiguration() {
}
+ /**
+ * build an empty published configuration
+ * @param description configuration description
+ */
public PublishedConfiguration(String description) {
this.description = description;
}
+
+ /**
+ * Build a configuration from the entries
+ * @param description configuration description
+ * @param entries entries to put
+ */
public PublishedConfiguration(String description,
Iterable<Map.Entry<String, String>> entries) {
this.description = description;
putValues(entries);
}
+ /**
+ * Build a published configuration, using the keys from keysource,
+ * but resolving the values from the value source, via Configuration.get()
+ * @param description configuration description
+ * @param keysource source of keys
+ * @param valuesource source of values
+ */
+ public PublishedConfiguration(String description,
+ Iterable<Map.Entry<String, String>> keysource,
+ Configuration valuesource) {
+ this.description = description;
+ putValues(ConfigHelper.resolveConfiguration(keysource, valuesource));
+ }
+
/**
* Is the configuration empty. This means either that it has not
@@ -158,7 +182,12 @@ public class PublishedConfiguration {
sb.append('}');
return sb.toString();
}
-
+
+ /**
+ * Create an outputter for a given format
+ * @param format format to use
+ * @return an instance of output
+ */
public PublishedConfigurationOutputter createOutputter(ConfigFormat format) {
return PublishedConfigurationOutputter.createOutputter(format, this);
}
Modified:
incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
URL:
http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java?rev=1595897&r1=1595896&r2=1595897&view=diff
==============================================================================
---
incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
(original)
+++
incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
Mon May 19 13:32:10 2014
@@ -23,6 +23,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.ipc.ProtocolSignature;
import org.apache.hadoop.security.Credentials;
@@ -714,27 +715,31 @@ public class SliderAppMaster extends Abs
// now publish site.xml files
+ YarnConfiguration defaultYarnConfig = new YarnConfiguration();
appState.getPublishedConfigurations().put(
PublishedArtifacts.COMPLETE_CONFIG,
new PublishedConfiguration(
- "Complete site settings",
- new YarnConfiguration()));
+ "Complete slider application settings",
+ getConfig(), getConfig()));
appState.getPublishedConfigurations().put(
PublishedArtifacts.YARN_SITE_CONFIG,
new PublishedConfiguration(
"YARN site settings",
- ConfigHelper.loadFromResource("yarn-site.xml")));
+ ConfigHelper.loadFromResource("yarn-site.xml"),
+ defaultYarnConfig));
appState.getPublishedConfigurations().put(
PublishedArtifacts.CORE_SITE_CONFIG,
new PublishedConfiguration(
"Core site settings",
- ConfigHelper.loadFromResource("core-site.xml")));
+ ConfigHelper.loadFromResource("core-site.xml"),
+ defaultYarnConfig));
appState.getPublishedConfigurations().put(
PublishedArtifacts.HDFS_SITE_CONFIG,
new PublishedConfiguration(
"HDFS site settings",
- ConfigHelper.loadFromResource("hdfs-site.xml")));
+ ConfigHelper.loadFromResource("hdfs-site.xml"),
+ new HdfsConfiguration(true)));
ServiceInstanceData instanceData = new ServiceInstanceData();
Modified:
incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy
URL:
http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy?rev=1595897&r1=1595896&r2=1595897&view=diff
==============================================================================
---
incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy
(original)
+++
incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy
Mon May 19 13:32:10 2014
@@ -29,8 +29,6 @@ import org.apache.slider.client.SliderCl
import org.apache.slider.common.SliderExitCodes
import org.apache.slider.common.SliderKeys
import org.apache.slider.common.params.ActionRegistryArgs
-import org.apache.slider.core.exceptions.SliderException
-import org.apache.slider.core.exceptions.UnknownApplicationInstanceException
import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.core.persist.JsonSerDeser
import org.apache.slider.core.registry.docstore.PublishedConfigSet
@@ -39,7 +37,6 @@ import org.apache.slider.core.registry.i
import org.apache.slider.core.registry.info.ServiceInstanceData
import org.apache.slider.core.registry.retrieve.RegistryRetriever
import org.apache.slider.server.appmaster.PublishedArtifacts
-import org.apache.slider.server.appmaster.web.rest.RestPaths
import org.apache.slider.server.services.curator.CuratorServiceInstance
import org.apache.slider.server.services.registry.SliderRegistryService
import org.junit.Test
@@ -212,6 +209,7 @@ class TestStandaloneRegistryAM extends A
assert externalConfSet[ARTIFACT_NAME]
+ describe "verify SLIDER-52 processing"
def yarnSite = retriever.retrieveConfiguration(
externalConfSet,
ARTIFACT_NAME,
@@ -223,9 +221,11 @@ class TestStandaloneRegistryAM extends A
assert rmHostnameViaClientSideXML == rmHostnameFromDownloadedProperties
def rmAddrViaClientSideXML = siteXML.get(YarnConfiguration.RM_ADDRESS)
- /* TODO SLIDER-52 PublishedConfiguration XML conf values are not resolved
until client-side
- assert rmAddrViaClientSideXML == rmAddrFromDownloadedProperties
- */
+ //TODO SLIDER-52 PublishedConfiguration XML conf values are not resolved
until client-side
+
+ log.info("RM from downloaded props = $rmAddrFromDownloadedProperties")
+ assert rmAddrViaClientSideXML == rmAddrFromDownloadedProperties
+
describe "fetch missing artifact"
try {
retriever.retrieveConfiguration(externalConfSet, "no-such-artifact",
true)
