Merge branch 'develop' into feature/SLIDER-38_berlin-buzz_cleanup
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/6adb3410 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/6adb3410 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/6adb3410 Branch: refs/heads/develop Commit: 6adb341087896e065c4300e5474f53ec1fcad99c Parents: 5281b04 3a5cf1a Author: Steve Loughran <ste...@hortonworks.com> Authored: Thu May 22 15:41:39 2014 +0100 Committer: Steve Loughran <ste...@hortonworks.com> Committed: Thu May 22 15:41:39 2014 +0100 ---------------------------------------------------------------------- app-packages/accumulo-v1_5/metainfo.xml | 10 +- app-packages/hbase-v0_96/README.txt | 4 +- app-packages/hbase-v0_96/appConfig.json | 1 + app-packages/hbase-v0_96/metainfo.xml | 14 +- app-packages/hbase-v0_96/metrics.json | 110 +++++ .../org/apache/slider/client/SliderClient.java | 2 +- .../apache/slider/common/tools/SliderUtils.java | 21 + .../slider/core/build/InstanceBuilder.java | 4 +- .../conf/AbstractInputPropertiesValidator.java | 4 +- .../core/registry/zk/BlockingZKWatcher.java | 63 --- .../slider/core/registry/zk/ZKCallback.java | 31 -- .../slider/core/registry/zk/ZKIntegration.java | 280 ------------ .../slider/core/registry/zk/ZKPathBuilder.java | 82 ---- .../slider/core/registry/zk/ZookeeperUtils.java | 134 ------ .../slider/core/zk/BlockingZKWatcher.java | 63 +++ .../slider/core/zk/MiniZooKeeperCluster.java | 423 +++++++++++++++++++ .../org/apache/slider/core/zk/ZKCallback.java | 31 ++ .../apache/slider/core/zk/ZKIntegration.java | 280 ++++++++++++ .../apache/slider/core/zk/ZKPathBuilder.java | 82 ++++ .../apache/slider/core/zk/ZookeeperUtils.java | 134 ++++++ .../providers/agent/AgentClientProvider.java | 2 +- .../providers/agent/AgentProviderService.java | 4 +- .../utility/AbstractSliderLaunchedService.java | 2 +- .../common/tools/GroovyZKIntegration.groovy | 2 +- .../common/tools/TestZKIntegration.groovy | 3 +- .../apache/slider/test/MicroZKCluster.groovy | 17 +- .../slider/test/YarnMiniClusterTestBase.groovy | 6 +- .../test/YarnZKMiniClusterTestBase.groovy | 4 +- .../slider/common/tools/TestSliderUtils.java | 18 +- .../accumulo/AccumuloProviderService.java | 3 +- .../live/TestAccCorrectInstanceName.groovy | 2 +- .../accumulo/live/TestAccFreezeThaw.groovy | 2 +- .../accumulo/live/TestAccLiveHDFSArchive.groovy | 2 +- .../live/TestAccLiveLocalArchive.groovy | 2 +- .../accumulo/live/TestAccM1T1GC1Mon1.groovy | 2 +- .../accumulo/live/TestAccM2T2GC1Mon1.groovy | 2 +- .../accumulo/live/TestAccumuloAMWebApp.groovy | 2 +- .../providers/hbase/HBaseClientProvider.java | 2 +- .../minicluster/live/TestHBaseMaster.groovy | 2 +- .../live/TestHBaseMasterWithBadHeap.groovy | 3 - .../live/TestLiveRegionServiceOnHDFS.groovy | 2 +- .../slider_specs/application_definition.md | 12 +- .../application_instance_configuration.md | 92 ++-- .../slider_specs/creating_app_definitions.md | 89 +--- src/site/markdown/slider_specs/index.md | 14 +- .../slider_specs/resource_specification.md | 46 +- 46 files changed, 1328 insertions(+), 782 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6adb3410/slider-core/src/main/java/org/apache/slider/core/zk/ZookeeperUtils.java ---------------------------------------------------------------------- diff --cc slider-core/src/main/java/org/apache/slider/core/zk/ZookeeperUtils.java index 0000000,90029ca..ea56bc2 mode 000000,100644..100644 --- a/slider-core/src/main/java/org/apache/slider/core/zk/ZookeeperUtils.java +++ b/slider-core/src/main/java/org/apache/slider/core/zk/ZookeeperUtils.java @@@ -1,0 -1,136 +1,134 @@@ + /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + package org.apache.slider.core.zk; + + import com.google.common.net.HostAndPort; + import org.apache.hadoop.util.StringUtils; + import org.apache.slider.common.tools.SliderUtils; + import org.apache.slider.core.exceptions.BadConfigException; + + import java.util.ArrayList; + import java.util.List; + + public class ZookeeperUtils { + + public static String buildConnectionString(String zkHosts, int port) { + String zkPort = Integer.toString(port); + //parse the hosts + String[] hostlist = zkHosts.split(",", 0); - String quorum = SliderUtils.join(hostlist, ":" + zkPort + ","); - //this quorum has a trailing comma - quorum = quorum.substring(0, quorum.length() - 1); ++ String quorum = SliderUtils.join(hostlist, ":" + zkPort + ",", false); + return quorum; + } + + /** + * Take a quorum list and split it to (trimmed) pairs + * @param hostPortQuorumList list of form h1:port, h2:port2,... + * @return a possibly empty list of values between commas. They may not be + * valid hostname:port pairs + */ + public static List<String> splitToPairs(String hostPortQuorumList) { + // split an address hot + String[] strings = StringUtils.getStrings(hostPortQuorumList); + List<String> tuples = new ArrayList<>(strings.length); + for (String s : strings) { + tuples.add(s.trim()); + } + return tuples; + } + + /** + * Split a quorum list into a list of hostnames and ports + * @param hostPortQuorumList split to a list of hosts and ports + * @return a list of values + */ + public static List<HostAndPort> splitToHostsAndPorts(String hostPortQuorumList) { + // split an address hot + String[] strings = StringUtils.getStrings(hostPortQuorumList); + List<HostAndPort> list = new ArrayList<>(strings.length); + for (String s : strings) { + list.add(HostAndPort.fromString(s.trim())); + } + return list; + } + + /** + * Build up to a hosts only list + * @param hostAndPorts + * @return a list of the hosts only + */ + public static String buildHostsOnlyList(List<HostAndPort> hostAndPorts) { + StringBuilder sb = new StringBuilder(); + for (HostAndPort hostAndPort : hostAndPorts) { + sb.append(hostAndPort.getHostText()).append(","); + } + if (sb.length() > 0) { + sb.delete(sb.length() - 1, sb.length()); + } + return sb.toString(); + } + + public static String buildQuorumEntry(HostAndPort hostAndPort, + int defaultPort) { + String s = hostAndPort.toString(); + if (hostAndPort.hasPort()) { + return s; + } else { + return s + ":" + defaultPort; + } + } + + /** + * Build a quorum list, injecting a ":defaultPort" ref if needed on + * any entry without one + * @param hostAndPorts + * @param defaultPort + * @return + */ + public static String buildQuorum(List<HostAndPort> hostAndPorts, int defaultPort) { + List<String> entries = new ArrayList<>(hostAndPorts.size()); + for (HostAndPort hostAndPort : hostAndPorts) { + entries.add(buildQuorumEntry(hostAndPort, defaultPort)); + } + return SliderUtils.join(entries, ",", false); + } + + public static String convertToHostsOnlyList(String quorum) throws + BadConfigException { + List<HostAndPort> hostAndPorts = splitToHostsAndPortsStrictly(quorum); + return ZookeeperUtils.buildHostsOnlyList(hostAndPorts); + } + + public static List<HostAndPort> splitToHostsAndPortsStrictly(String quorum) throws + BadConfigException { + List<HostAndPort> hostAndPorts = + ZookeeperUtils.splitToHostsAndPorts(quorum); + if (hostAndPorts.isEmpty()) { + throw new BadConfigException("empty zookeeper quorum"); + } + return hostAndPorts; + } + + public static int getFirstPort(String quorum, int defVal) throws + BadConfigException { + List<HostAndPort> hostAndPorts = splitToHostsAndPortsStrictly(quorum); + int port = hostAndPorts.get(0).getPortOrDefault(defVal); + return port; + + } + } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6adb3410/slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy ---------------------------------------------------------------------- diff --cc slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy index 09a5a8f,2228580..691cd2f --- a/slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy @@@ -21,11 -21,10 +21,11 @@@ package org.apache.slider.tes import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.io.IOUtils import org.apache.hadoop.yarn.conf.YarnConfiguration import org.apache.slider.common.SliderXmlConfKeys - import org.apache.slider.core.registry.zk.BlockingZKWatcher - import org.apache.slider.core.registry.zk.ZKIntegration + import org.apache.slider.core.zk.BlockingZKWatcher + import org.apache.slider.core.zk.ZKIntegration import java.util.concurrent.atomic.AtomicBoolean
