[
https://issues.apache.org/jira/browse/ACCUMULO-4173?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15218415#comment-15218415
]
ASF GitHub Bot commented on ACCUMULO-4173:
------------------------------------------
Github user dlmarion commented on a diff in the pull request:
https://github.com/apache/accumulo/pull/83#discussion_r57930648
--- Diff:
server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java
---
@@ -0,0 +1,314 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.master.balancer;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.regex.Pattern;
+
+import org.apache.accumulo.core.client.admin.TableOperations;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.impl.KeyExtent;
+import org.apache.accumulo.core.master.thrift.TabletServerStatus;
+import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.master.state.TabletMigration;
+import org.apache.thrift.TException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This balancer will create pools of tablet servers by grouping tablet
servers that match a regex into the same pool and calling the balancer set on
the table
+ * to balance within the set of matching tablet servers. <br>
+ * Regex properties for this balancer are specified as:<br>
+ *
<b>table.custom.balancer.host.regex.<tablename>=<regex></b><br>
+ * Periodically (default 5m) this balancer will check to see if a tablet
server is hosting tablets that it should not be according to the regex
configuration.
+ * If this occurs then the offending tablets will be reassigned. This
would cover the case where the configuration is changed and the master is
restarted while
+ * the tablet servers are up. To change the out of bounds check time
period, set the following property:<br>
+ * <b>table.custom.balancer.host.regex.oob.period=5m</b><br>
+ * Periodically (default 5m) this balancer will regroup the set of current
tablet servers into pools based on regexes applied to the tserver host names.
This
+ * would cover the case of tservers dying or coming online. To change the
host pool check time period, set the following property: <br>
+ * <b>table.custom.balancer.host.regex.pool.check=5m</b><br>
+ * Regex matching can be based on either the host name (default) or host
ip address. To set this balancer to match the regular expressions to the tablet
server
+ * IP address, then set the following property:<br>
+ * <b>table.custom.balancer.host.regex.is.ip=true</b>
+ *
+ */
+public class HostRegexTableLoadBalancer extends TableLoadBalancer {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(HostRegexTableLoadBalancer.class);
+ public static final String HOST_BALANCER_PREFIX =
Property.TABLE_ARBITRARY_PROP_PREFIX.getKey() + "balancer.host.regex.";
+ public static final String HOST_BALANCER_OOB_CHECK =
Property.TABLE_ARBITRARY_PROP_PREFIX.getKey() +
"balancer.host.regex.oob.period";
+ private static final String HOST_BALANCER_OOB_DEFAULT = "5m";
+ public static final String HOST_BALANCER_POOL_RECHECK_KEY =
Property.TABLE_ARBITRARY_PROP_PREFIX.getKey() +
"balancer.host.regex.pool.check";
+ private static final String HOST_BALANCER_POOL_RECHECK_DEFAULT = "5m";
+ public static final String HOST_BALANCER_REGEX_USING_IPS =
Property.TABLE_ARBITRARY_PROP_PREFIX.getKey() + "balancer.host.regex.is.ip";
+ protected static final String DEFAULT_POOL = "HostTableLoadBalancer.ALL";
+
+ protected long oobCheckMillis =
AccumuloConfiguration.getTimeInMillis(HOST_BALANCER_OOB_DEFAULT);
+ protected long poolRecheckMillis =
AccumuloConfiguration.getTimeInMillis(HOST_BALANCER_POOL_RECHECK_DEFAULT);
+
+ private Map<String,String> tableIdToTableName = null;
+ private Map<String,Pattern> poolNameToRegexPattern = null;
+ private long lastOOBCheck = System.currentTimeMillis();
+ private long lastPoolRecheck = 0;
+ private boolean isIpBasedRegex = false;
+ private Map<String,SortedMap<TServerInstance,TabletServerStatus>> pools
= new HashMap<String,SortedMap<TServerInstance,TabletServerStatus>>();
+
+ /**
+ * Group the set of current tservers by pool name. Tservers that don't
match a regex are put into a default ppol.
+ *
+ * @param current
+ * map of current tservers
+ * @return current servers grouped by pool name, if not a match it is
put into a default pool.
+ */
+ protected synchronized
Map<String,SortedMap<TServerInstance,TabletServerStatus>>
splitCurrentByRegex(SortedMap<TServerInstance,TabletServerStatus> current) {
+ if ((System.currentTimeMillis() - lastPoolRecheck) >
poolRecheckMillis) {
+ Map<String,SortedMap<TServerInstance,TabletServerStatus>> newPools =
new HashMap<String,SortedMap<TServerInstance,TabletServerStatus>>();
+ for (Entry<TServerInstance,TabletServerStatus> e :
current.entrySet()) {
+ String tableName = getPoolNameForHost(e.getKey().host());
+ if (!newPools.containsKey(tableName)) {
+ newPools.put(tableName, new
TreeMap<TServerInstance,TabletServerStatus>());
+ }
+ newPools.get(tableName).put(e.getKey(), e.getValue());
+ }
+ // Ensure that no host is in more than one pool
+ // TODO: I'm not sure that I need to check for disjoint as the call
to getPoolNameForHost checks for more than one match
+ boolean error = false;
+ for (SortedMap<TServerInstance,TabletServerStatus> s1 :
newPools.values()) {
--- End diff --
I have no issue removing this check. However, getPoolNameForHost only
returns one pool name; either the name of the pool that matches one regex, or
the DEFAULT. I believe to support this example, getPoolNameForHost would have
to change to return all matching pool names, and return DEFAULT when it matches
none.
> Balance table within a set of hosts
> -----------------------------------
>
> Key: ACCUMULO-4173
> URL: https://issues.apache.org/jira/browse/ACCUMULO-4173
> Project: Accumulo
> Issue Type: Bug
> Components: master
> Reporter: Dave Marion
> Assignee: Dave Marion
> Labels: balancer
> Fix For: 1.8.0
>
> Attachments: ACCUMULO-4173-1.patch
>
>
> Create a table balancer that will provide a set of hosts for the table tablet
> balancer to use.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
