Merge branch '1.8'
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/8c3c0783 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/8c3c0783 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/8c3c0783 Branch: refs/heads/master Commit: 8c3c0783a1273b8237a4c358d8f0d39067f360d4 Parents: dddbba9 a205559 Author: Christopher Tubbs <ctubb...@apache.org> Authored: Thu Jul 27 20:27:10 2017 -0400 Committer: Christopher Tubbs <ctubb...@apache.org> Committed: Thu Jul 27 20:27:10 2017 -0400 ---------------------------------------------------------------------- .../org/apache/accumulo/core/client/impl/ThriftScanner.java | 1 - .../java/org/apache/accumulo/core/client/mock/MockTable.java | 2 +- .../accumulo/core/client/summary/SummarizerConfiguration.java | 2 +- .../accumulo/core/iterators/system/LocalityGroupIterator.java | 4 ++-- .../org/apache/accumulo/core/summary/SummaryCollection.java | 1 - .../accumulo/core/iterators/system/ColumnFilterTest.java | 6 +++--- .../apache/accumulo/core/summary/SummaryCollectionTest.java | 4 ++-- .../server/master/balancer/HostRegexTableLoadBalancer.java | 2 +- .../org/apache/accumulo/server/security/UserImpersonation.java | 1 - .../master/balancer/BaseHostRegexTableLoadBalancerTest.java | 4 ++-- .../src/main/java/org/apache/accumulo/master/Master.java | 2 +- .../apache/accumulo/tserver/TabletServerResourceManager.java | 2 +- .../apache/accumulo/tserver/LargestFirstMemoryManagerTest.java | 2 +- .../org/apache/accumulo/test/BalanceWithOfflineTableIT.java | 1 - .../apache/accumulo/test/functional/FunctionalTestUtils.java | 2 +- 15 files changed, 16 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/8c3c0783/core/src/main/java/org/apache/accumulo/core/client/impl/ThriftScanner.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/8c3c0783/core/src/main/java/org/apache/accumulo/core/client/mock/MockTable.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/8c3c0783/core/src/main/java/org/apache/accumulo/core/client/summary/SummarizerConfiguration.java ---------------------------------------------------------------------- diff --cc core/src/main/java/org/apache/accumulo/core/client/summary/SummarizerConfiguration.java index ec98695,0000000..45fb7b1 mode 100644,000000..100644 --- a/core/src/main/java/org/apache/accumulo/core/client/summary/SummarizerConfiguration.java +++ b/core/src/main/java/org/apache/accumulo/core/client/summary/SummarizerConfiguration.java @@@ -1,285 -1,0 +1,285 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.core.client.summary; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.accumulo.core.summary.SummarizerConfigurationUtil; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import com.google.common.hash.Hasher; +import com.google.common.hash.Hashing; + +/** + * This class encapsulates the configuration needed to instantiate a {@link Summarizer}. It also provides methods and documentation for setting the table + * properties that configure a Summarizer. + * + * @since 2.0.0 + */ +public class SummarizerConfiguration { + + private final String className; + private final Map<String,String> options; + private int hashCode = 0; + private final String configId; + + private SummarizerConfiguration(String className, String configId, Map<String,String> options) { + this.className = className; + this.options = ImmutableMap.copyOf(options); + + if (configId == null) { + ArrayList<String> keys = new ArrayList<>(this.options.keySet()); + Collections.sort(keys); + Hasher hasher = Hashing.murmur3_32().newHasher(); + hasher.putString(className); + for (String key : keys) { + hasher.putString(key); + hasher.putString(options.get(key)); + } + + this.configId = hasher.hash().toString(); + } else { + this.configId = configId; + } + } + + /** + * @return the name of a class that implements @link {@link Summarizer}. + */ + public String getClassName() { + return className; + } + + /** + * @return custom options for a {link @Summarizer} + */ + public Map<String,String> getOptions() { + return options; + } + + /** + * The propertyId is used to when creating table properties for a summarizer. Its not used for equality or hashCode for this class. + */ + public String getPropertyId() { + return configId; + } + + @Override + public String toString() { + return className + " " + configId + " " + options; + } + + /** + * Compares the classname and options to determine equality. + */ + @Override + public boolean equals(Object o) { + if (o instanceof SummarizerConfiguration) { + SummarizerConfiguration osc = (SummarizerConfiguration) o; + return className.equals(osc.className) && options.equals(osc.options); + } + + return false; + } + + /** + * Hashes the classname and options to create a hashcode. + */ + @Override + public int hashCode() { + if (hashCode == 0) { + hashCode = 31 * options.hashCode() + className.hashCode(); + } + return hashCode; + } + + /** + * Converts this configuration to Accumulo per table properties. The returned map has the following key values. The {@code <configId>} below is from + * {@link #getPropertyId()}. The {@code <optionKey>} and {@code <optionValue>} below are derived from the key values of {@link #getOptions()}. + * + * <pre> + * {@code + * table.summarizer.<configId>=<classname> + * table.summarizer.<configId>.opt.<optionKey1>=<optionValue1> + * table.summarizer.<configId>.opt.<optionKey2>=<optionValue2> + * . + * . + * . + * table.summarizer.<configId>.opt.<optionKeyN>=<optionValueN> + * } + * </pre> + */ + public Map<String,String> toTableProperties() { + return SummarizerConfigurationUtil.toTablePropertiesMap(Collections.singletonList(this)); + } + + /** + * Encodes each configuration in the same way as {@link #toTableProperties()}. + * + * @throws IllegalArgumentException + * when there are duplicate values for {@link #getPropertyId()} + */ + public static Map<String,String> toTableProperties(SummarizerConfiguration... configurations) { + return SummarizerConfigurationUtil.toTablePropertiesMap(Arrays.asList(configurations)); + } + + /** + * Encodes each configuration in the same way as {@link #toTableProperties()}. + * + * @throws IllegalArgumentException + * when there are duplicate values for {@link #getPropertyId()} + */ + public static Map<String,String> toTableProperties(Collection<SummarizerConfiguration> configurations) { - return SummarizerConfigurationUtil.toTablePropertiesMap(new ArrayList<SummarizerConfiguration>(configurations)); ++ return SummarizerConfigurationUtil.toTablePropertiesMap(new ArrayList<>(configurations)); + } + + /** + * Decodes table properties with the prefix {@code table.summarizer} into {@link SummarizerConfiguration} objects. Table properties with prefixes other than + * {@code table.summarizer} are ignored. + */ + public static Collection<SummarizerConfiguration> fromTableProperties(Map<String,String> props) { + return fromTableProperties(props.entrySet()); + } + + /** + * @see #fromTableProperties(Map) + */ + public static Collection<SummarizerConfiguration> fromTableProperties(Iterable<Entry<String,String>> props) { + return SummarizerConfigurationUtil.getSummarizerConfigs(props); + } + + public static class Builder { + private String className; + private ImmutableMap.Builder<String,String> imBuilder; + private String configId = null; + + private Builder(String className) { + this.className = className; + this.imBuilder = ImmutableMap.builder(); + } + + /** + * Sets the id used when generating table properties. Setting this is optional. If not set, an id is generated using hashing that will likely be unique. + * + * @param propId + * This id is used when converting a {@link SummarizerConfiguration} to table properties. Since tables can have multiple summarizers, make sure its + * unique. + * + * @see SummarizerConfiguration#toTableProperties() + */ + public Builder setPropertyId(String propId) { + Preconditions.checkArgument(propId.matches("\\w+"), "Config Id %s is not alphanum", propId); + this.configId = propId; + return this; + } + + /** + * Adds an option that Summarizers can use when constructing Collectors and Combiners. + * + * @return this + * + * @see SummarizerConfiguration#getOptions() + */ + public Builder addOption(String key, String value) { + Preconditions.checkArgument(key.matches("\\w+"), "Option Id %s is not alphanum", key); + imBuilder.put(key, value); + return this; + } + + /** + * Adds an option that Summarizers can use when constructing Collectors and Combiners. + * + * @return this + * + * @see SummarizerConfiguration#getOptions() + */ + public Builder addOption(String key, long value) { + return addOption(key, Long.toString(value)); + } + + /** + * Convenience method for adding multiple options. The following + * + * <pre> + * {@code builder.addOptions("opt1","val1","opt2","val2","opt3","val3")} + * </pre> + * + * <p> + * is equivalent to + * + * <pre> + * {@code + * builder.addOption("opt1","val1"); + * builder.addOption("opt2","val2"); + * builder.addOption("opt3","val3"); + * } + * </pre> + * + * @param keyValuePairs + * This array must have an even and positive number of elements. + * @return this + * @see SummarizerConfiguration#getOptions() + */ + public Builder addOptions(String... keyValuePairs) { + Preconditions.checkArgument(keyValuePairs.length % 2 == 0 && keyValuePairs.length > 0, "Require an even, positive number of arguments, got %s", + keyValuePairs.length); + for (int i = 0; i < keyValuePairs.length; i += 2) { + addOption(keyValuePairs[i], keyValuePairs[i + 1]); + } + return this; + } + + /** + * @param options + * Each entry in the map is passed to {@link #addOption(String, String)} + * @return this + * + * @see SummarizerConfiguration#getOptions() + */ + public Builder addOptions(Map<String,String> options) { + options.entrySet().forEach(e -> addOption(e.getKey(), e.getValue())); + return this; + } + + public SummarizerConfiguration build() { + return new SummarizerConfiguration(className, configId, imBuilder.build()); + } + } + + /** + * Call this method to initiate a chain of fluent method calls to a create an immutable {@link SummarizerConfiguration} + * + * @param className + * The fully qualified name of a class that implements {@link Summarizer}. + */ + public static Builder builder(String className) { + return new Builder(className); + } + + /** + * @see #builder(String) + */ + public static Builder builder(Class<? extends Summarizer> clazz) { + return new Builder(clazz.getName()); + } +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/8c3c0783/core/src/main/java/org/apache/accumulo/core/summary/SummaryCollection.java ---------------------------------------------------------------------- diff --cc core/src/main/java/org/apache/accumulo/core/summary/SummaryCollection.java index cc688c9,0000000..cf216ed mode 100644,000000..100644 --- a/core/src/main/java/org/apache/accumulo/core/summary/SummaryCollection.java +++ b/core/src/main/java/org/apache/accumulo/core/summary/SummaryCollection.java @@@ -1,188 -1,0 +1,187 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.accumulo.core.summary; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.accumulo.core.client.summary.Summarizer; +import org.apache.accumulo.core.client.summary.SummarizerConfiguration; +import org.apache.accumulo.core.client.summary.Summary; +import org.apache.accumulo.core.data.thrift.TSummaries; +import org.apache.accumulo.core.data.thrift.TSummarizerConfiguration; +import org.apache.accumulo.core.data.thrift.TSummary; + +import com.google.common.base.Preconditions; + +/** + * This class facilitates merging, storing, and serializing (to/from thrift) intermediate summary information. + */ +public class SummaryCollection { + + private static class MergedSummary { + Map<String,Long> summary; + long filesContaining; + long filesExceedingBoundry; + long filesLarge; + + public MergedSummary(FileSummary entry) { + this.summary = entry.summary; + this.filesContaining = 1; + this.filesExceedingBoundry = entry.exceededBoundry ? 1 : 0; + this.filesLarge = entry.exceededMaxSize ? 1 : 0; + } + + public MergedSummary(TSummary tSummary) { + this.summary = new HashMap<>(tSummary.getSummary()); + this.filesContaining = tSummary.getFilesContaining(); + this.filesExceedingBoundry = tSummary.getFilesExceeding(); + this.filesLarge = tSummary.getFilesLarge(); + } + + public void merge(MergedSummary other, SummarizerConfiguration config, SummarizerFactory factory) { + + if (summary == null && other.summary != null) { + summary = new HashMap<>(other.summary); + } else if (summary != null && other.summary != null) { + Summarizer summarizer = factory.getSummarizer(config); + summarizer.combiner(config).merge(summary, other.summary); + } + + filesContaining += other.filesContaining; + filesExceedingBoundry += other.filesExceedingBoundry; + filesLarge += other.filesLarge; + } + + public TSummary toThrift(SummarizerConfiguration key) { + TSummarizerConfiguration tsumConf = SummarizerConfigurationUtil.toThrift(key); + return new TSummary(summary, tsumConf, filesContaining, filesExceedingBoundry, filesLarge); + } + + } + + private Map<SummarizerConfiguration,MergedSummary> mergedSummaries; + private long totalFiles; + private long deletedFiles; + + public SummaryCollection() { + mergedSummaries = new HashMap<>(); + totalFiles = 0; + } + + public SummaryCollection(TSummaries tsums) { + mergedSummaries = new HashMap<>(); + for (TSummary tSummary : tsums.getSummaries()) { + SummarizerConfiguration sconf = SummarizerConfigurationUtil.fromThrift(tSummary.getConfig()); + mergedSummaries.put(sconf, new MergedSummary(tSummary)); + } + + totalFiles = tsums.getTotalFiles(); + deletedFiles = tsums.getDeletedFiles(); + } + + SummaryCollection(Collection<FileSummary> initialEntries) { + this(initialEntries, false); + } + + SummaryCollection(Collection<FileSummary> initialEntries, boolean deleted) { + if (deleted) { + Preconditions.checkArgument(initialEntries.size() == 0); + } + mergedSummaries = new HashMap<>(); + for (FileSummary entry : initialEntries) { + mergedSummaries.put(entry.conf, new MergedSummary(entry)); + } + totalFiles = 1; + this.deletedFiles = deleted ? 1 : 0; + } + + static class FileSummary { + + private SummarizerConfiguration conf; + private Map<String,Long> summary; + private boolean exceededBoundry; + private boolean exceededMaxSize; + + FileSummary(SummarizerConfiguration conf, Map<String,Long> summary, boolean exceededBoundry) { + this.conf = conf; + this.summary = summary; + this.exceededBoundry = exceededBoundry; + this.exceededMaxSize = false; + } + + FileSummary(SummarizerConfiguration conf) { + this.conf = conf; + this.summary = new HashMap<>(); - ; + this.exceededBoundry = false; + this.exceededMaxSize = true; + } + } + + public void merge(SummaryCollection other, SummarizerFactory factory) { + for (Entry<SummarizerConfiguration,MergedSummary> entry : other.mergedSummaries.entrySet()) { + MergedSummary ms = mergedSummaries.get(entry.getKey()); + if (ms == null) { + mergedSummaries.put(entry.getKey(), entry.getValue()); + } else { + ms.merge(entry.getValue(), entry.getKey(), factory); + } + } + + this.totalFiles += other.totalFiles; + this.deletedFiles += other.deletedFiles; + } + + public static SummaryCollection merge(SummaryCollection sc1, SummaryCollection sc2, SummarizerFactory factory) { + SummaryCollection ret = new SummaryCollection(); + ret.merge(sc1, factory); + ret.merge(sc2, factory); + return ret; + } + + public List<Summary> getSummaries() { + ArrayList<Summary> ret = new ArrayList<>(mergedSummaries.size()); + + for (Entry<SummarizerConfiguration,MergedSummary> entry : mergedSummaries.entrySet()) { + SummarizerConfiguration config = entry.getKey(); + MergedSummary ms = entry.getValue(); + + ret.add(new Summary(ms.summary, config, totalFiles, (totalFiles - deletedFiles) - ms.filesContaining, ms.filesExceedingBoundry, ms.filesLarge, + deletedFiles)); + } + + return ret; + } + + public long getTotalFiles() { + return totalFiles; + } + + public TSummaries toThrift() { + List<TSummary> summaries = new ArrayList<>(mergedSummaries.size()); + for (Entry<SummarizerConfiguration,MergedSummary> entry : mergedSummaries.entrySet()) { + summaries.add(entry.getValue().toThrift(entry.getKey())); + } + + return new TSummaries(true, -1l, totalFiles, deletedFiles, summaries); + } +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/8c3c0783/core/src/test/java/org/apache/accumulo/core/iterators/system/ColumnFilterTest.java ---------------------------------------------------------------------- diff --cc core/src/test/java/org/apache/accumulo/core/iterators/system/ColumnFilterTest.java index ab9c975,4b03826..dd776af --- a/core/src/test/java/org/apache/accumulo/core/iterators/system/ColumnFilterTest.java +++ b/core/src/test/java/org/apache/accumulo/core/iterators/system/ColumnFilterTest.java @@@ -46,9 -47,9 +46,9 @@@ public class ColumnFilterTest extends T } public void test1() { - TreeMap<Key,Value> data = new TreeMap<Key,Value>(); + TreeMap<Key,Value> data = new TreeMap<>(); - data.put(newKey("r1", "cf1", "cq1"), new Value()); - data.put(newKey("r1", "cf2", "cq1"), new Value()); + data.put(newKey("r1", "cf1", "cq1"), new Value("")); + data.put(newKey("r1", "cf2", "cq1"), new Value("")); HashSet<Column> columns = new HashSet<>(); columns.add(newColumn("cf1")); @@@ -61,10 -62,10 +61,10 @@@ public void test2() throws Exception { - TreeMap<Key,Value> data = new TreeMap<Key,Value>(); + TreeMap<Key,Value> data = new TreeMap<>(); - data.put(newKey("r1", "cf1", "cq1"), new Value()); - data.put(newKey("r1", "cf2", "cq1"), new Value()); - data.put(newKey("r1", "cf2", "cq2"), new Value()); + data.put(newKey("r1", "cf1", "cq1"), new Value("")); + data.put(newKey("r1", "cf2", "cq1"), new Value("")); + data.put(newKey("r1", "cf2", "cq2"), new Value("")); HashSet<Column> columns = new HashSet<>(); @@@ -85,10 -86,10 +85,10 @@@ public void test3() throws Exception { - TreeMap<Key,Value> data = new TreeMap<Key,Value>(); + TreeMap<Key,Value> data = new TreeMap<>(); - data.put(newKey("r1", "cf1", "cq1"), new Value()); - data.put(newKey("r1", "cf2", "cq1"), new Value()); - data.put(newKey("r1", "cf2", "cq2"), new Value()); + data.put(newKey("r1", "cf1", "cq1"), new Value("")); + data.put(newKey("r1", "cf2", "cq1"), new Value("")); + data.put(newKey("r1", "cf2", "cq2"), new Value("")); HashSet<Column> columns = new HashSet<>(); http://git-wip-us.apache.org/repos/asf/accumulo/blob/8c3c0783/core/src/test/java/org/apache/accumulo/core/summary/SummaryCollectionTest.java ---------------------------------------------------------------------- diff --cc core/src/test/java/org/apache/accumulo/core/summary/SummaryCollectionTest.java index 95702d3,0000000..ae1dab5 mode 100644,000000..100644 --- a/core/src/test/java/org/apache/accumulo/core/summary/SummaryCollectionTest.java +++ b/core/src/test/java/org/apache/accumulo/core/summary/SummaryCollectionTest.java @@@ -1,72 -1,0 +1,72 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.accumulo.core.summary; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; + +import org.apache.accumulo.core.client.summary.SummarizerConfiguration; +import org.apache.accumulo.core.client.summary.Summary; +import org.apache.accumulo.core.client.summary.Summary.FileStatistics; +import org.apache.accumulo.core.client.summary.summarizers.FamilySummarizer; +import org.apache.accumulo.core.summary.SummaryCollection.FileSummary; +import org.junit.Assert; +import org.junit.Test; + +public class SummaryCollectionTest { + @Test + public void testDeleted() { + SummarizerConfiguration conf = SummarizerConfiguration.builder(FamilySummarizer.class).build(); + - HashMap<String,Long> stats = new HashMap<String,Long>(); ++ HashMap<String,Long> stats = new HashMap<>(); + stats.put("c:foo", 9L); + FileSummary fs1 = new FileSummary(conf, stats, false); + SummaryCollection sc1 = new SummaryCollection(Collections.singleton(fs1)); + - stats = new HashMap<String,Long>(); ++ stats = new HashMap<>(); + stats.put("c:foo", 5L); + stats.put("c:bar", 3L); + FileSummary fs2 = new FileSummary(conf, stats, true); + SummaryCollection sc2 = new SummaryCollection(Collections.singleton(fs2)); + + SummaryCollection sc3 = new SummaryCollection(Collections.emptyList()); + + SummaryCollection sc4 = new SummaryCollection(Collections.emptyList(), true); + + SummarizerFactory factory = new SummarizerFactory(); + SummaryCollection mergeSc = new SummaryCollection(); + for (SummaryCollection sc : Arrays.asList(sc1, sc2, sc3, sc4, sc4)) { + mergeSc.merge(sc, factory); + } + + for (SummaryCollection sc : Arrays.asList(mergeSc, new SummaryCollection(mergeSc.toThrift()))) { + List<Summary> summaries = sc.getSummaries(); + Assert.assertEquals(1, summaries.size()); + Summary summary = summaries.get(0); + FileStatistics filestats = summary.getFileStatistics(); + Assert.assertEquals(5, filestats.getTotal()); + Assert.assertEquals(1, filestats.getExtra()); + Assert.assertEquals(0, filestats.getLarge()); + Assert.assertEquals(1, filestats.getMissing()); + Assert.assertEquals(2, filestats.getDeleted()); + Assert.assertEquals(4, filestats.getInaccurate()); + } + } +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/8c3c0783/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/8c3c0783/server/base/src/test/java/org/apache/accumulo/server/master/balancer/BaseHostRegexTableLoadBalancerTest.java ---------------------------------------------------------------------- diff --cc server/base/src/test/java/org/apache/accumulo/server/master/balancer/BaseHostRegexTableLoadBalancerTest.java index 4606c04,44cf76b..e6791f4 --- a/server/base/src/test/java/org/apache/accumulo/server/master/balancer/BaseHostRegexTableLoadBalancerTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/master/balancer/BaseHostRegexTableLoadBalancerTest.java @@@ -338,10 -340,10 +338,10 @@@ public abstract class BaseHostRegexTabl TreeMap<TServerInstance,TabletServerStatus> current = new TreeMap<>(); for (int i = 1; i <= numTservers; i++) { TabletServerStatus status = new TabletServerStatus(); - Map<String,TableInfo> tableMap = new HashMap<String,TableInfo>(); + Map<String,TableInfo> tableMap = new HashMap<>(); - tableMap.put(FOO.getId(), new TableInfo()); - tableMap.put(BAR.getId(), new TableInfo()); - tableMap.put(BAZ.getId(), new TableInfo()); + tableMap.put(FOO.getId().canonicalID(), new TableInfo()); + tableMap.put(BAR.getId().canonicalID(), new TableInfo()); + tableMap.put(BAZ.getId().canonicalID(), new TableInfo()); status.setTableMap(tableMap); current.put(new TServerInstance(base + i + ":9997", 1), status); } http://git-wip-us.apache.org/repos/asf/accumulo/blob/8c3c0783/server/master/src/main/java/org/apache/accumulo/master/Master.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/8c3c0783/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java ---------------------------------------------------------------------- diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java index 7397c06,d8ae94c..cc498fb --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java @@@ -146,16 -139,8 +146,16 @@@ public class TabletServerResourceManage return createEs(max, name, new LinkedBlockingQueue<Runnable>()); } + private ExecutorService createIdlingEs(Property max, String name, long timeout, TimeUnit timeUnit) { - LinkedBlockingQueue<Runnable> queue = new LinkedBlockingQueue<Runnable>(); ++ LinkedBlockingQueue<Runnable> queue = new LinkedBlockingQueue<>(); + int maxThreads = conf.getSystemConfiguration().getCount(max); + ThreadPoolExecutor tp = new ThreadPoolExecutor(maxThreads, maxThreads, timeout, timeUnit, queue, new NamingThreadFactory(name)); + tp.allowCoreThreadTimeOut(true); + return addEs(max, name, tp); + } + private ExecutorService createEs(Property max, String name, BlockingQueue<Runnable> queue) { - int maxThreads = conf.getConfiguration().getCount(max); + int maxThreads = conf.getSystemConfiguration().getCount(max); ThreadPoolExecutor tp = new ThreadPoolExecutor(maxThreads, maxThreads, 0L, TimeUnit.MILLISECONDS, queue, new NamingThreadFactory(name)); return addEs(max, name, tp); } http://git-wip-us.apache.org/repos/asf/accumulo/blob/8c3c0783/server/tserver/src/test/java/org/apache/accumulo/tserver/LargestFirstMemoryManagerTest.java ---------------------------------------------------------------------- diff --cc server/tserver/src/test/java/org/apache/accumulo/tserver/LargestFirstMemoryManagerTest.java index 27045c3,82ec8ec..42b70a5 --- a/server/tserver/src/test/java/org/apache/accumulo/tserver/LargestFirstMemoryManagerTest.java +++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/LargestFirstMemoryManagerTest.java @@@ -172,7 -176,12 +172,7 @@@ public class LargestFirstMemoryManagerT @Test public void testDeletedTable() throws Exception { final String deletedTableId = "1"; - Function<Table.ID,Boolean> existenceCheck = tableId -> !deletedTableId.equals(tableId); - Function<String,Boolean> existenceCheck = new Function<String,Boolean>() { - @Override - public Boolean apply(String tableId) { - return !deletedTableId.equals(tableId); - } - }; ++ Function<Table.ID,Boolean> existenceCheck = tableId -> !deletedTableId.contentEquals(tableId.canonicalID()); LargestFirstMemoryManagerWithExistenceCheck mgr = new LargestFirstMemoryManagerWithExistenceCheck(existenceCheck); ServerConfiguration config = new ServerConfiguration() { ServerConfigurationFactory delegate = new ServerConfigurationFactory(inst); http://git-wip-us.apache.org/repos/asf/accumulo/blob/8c3c0783/test/src/main/java/org/apache/accumulo/test/functional/FunctionalTestUtils.java ----------------------------------------------------------------------
