[2/3] accumulo git commit: Merge branch '1.7' into 1.8
Merge branch '1.7' into 1.8 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/5da9a3af Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/5da9a3af Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/5da9a3af Branch: refs/heads/master Commit: 5da9a3afaa324b4e2cfbcfbf6132854c586ef1bd Parents: 591ba9f f2b9e24 Author: Mike MillerAuthored: Wed Aug 2 12:15:59 2017 -0400 Committer: Mike Miller Committed: Wed Aug 2 12:15:59 2017 -0400 -- .../org/apache/accumulo/core/util/Version.java | 73 .../apache/accumulo/core/util/TestVersion.java | 64 - .../org/apache/accumulo/server/Accumulo.java| 4 +- 3 files changed, 1 insertion(+), 140 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/5da9a3af/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java -- diff --cc server/base/src/main/java/org/apache/accumulo/server/Accumulo.java index 43b9209,e4c944f..362cd0e --- a/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java +++ b/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java @@@ -36,7 -34,7 +36,6 @@@ import org.apache.accumulo.core.client. import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.util.AddressUtil; - import org.apache.accumulo.core.util.Version; -import org.apache.accumulo.core.util.UtilWaitThread; import org.apache.accumulo.core.volume.Volume; import org.apache.accumulo.core.zookeeper.ZooUtil; import org.apache.accumulo.fate.ReadOnlyStore;
[2/3] accumulo git commit: Merge branch '1.7' into 1.8
Merge branch '1.7' into 1.8 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/cdafd020 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/cdafd020 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/cdafd020 Branch: refs/heads/master Commit: cdafd0203469aa51ba120b1ac0ed34cc579eaa18 Parents: fab5589 b7d9be0 Author: Mike MillerAuthored: Mon Jul 10 14:28:42 2017 -0400 Committer: Mike Miller Committed: Mon Jul 10 14:28:42 2017 -0400 -- .../accumulo/test/AssignmentThreadsIT.java | 20 1 file changed, 12 insertions(+), 8 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/cdafd020/test/src/main/java/org/apache/accumulo/test/AssignmentThreadsIT.java -- diff --cc test/src/main/java/org/apache/accumulo/test/AssignmentThreadsIT.java index bbc1d47,000..fafe349 mode 100644,00..100644 --- a/test/src/main/java/org/apache/accumulo/test/AssignmentThreadsIT.java +++ b/test/src/main/java/org/apache/accumulo/test/AssignmentThreadsIT.java @@@ -1,99 -1,0 +1,103 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.test; + +import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly; +import static org.junit.Assert.assertTrue; + +import java.util.Random; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.concurrent.TimeUnit; + +import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl; +import org.apache.accumulo.test.categories.MiniClusterOnlyTests; +import org.apache.accumulo.test.categories.PerformanceTests; +import org.apache.accumulo.test.functional.ConfigurableMacBase; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.Text; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +// ACCUMULO-1177 +@Category({MiniClusterOnlyTests.class, PerformanceTests.class}) +public class AssignmentThreadsIT extends ConfigurableMacBase { + + @Override + public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) { +cfg.setNumTservers(1); +cfg.setProperty(Property.TSERV_ASSIGNMENT_MAXCONCURRENT, "1"); + } + + // [0-9a-f] + private final static byte[] HEXCHARS = {0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66}; + private final static Random random = new Random(); + + public static byte[] randomHex(int n) { +byte[] binary = new byte[n]; +byte[] hex = new byte[n * 2]; +random.nextBytes(binary); +int count = 0; +for (byte x : binary) { + hex[count++] = HEXCHARS[(x >> 4) & 0xf]; + hex[count++] = HEXCHARS[x & 0xf]; +} +return hex; + } + + @Test(timeout = 5 * 60 * 1000) + public void testConcurrentAssignmentPerformance() throws Exception { +// make a table with a lot of splits +String tableName = getUniqueNames(1)[0]; +Connector c = getConnector(); +log.info("Creating table"); +c.tableOperations().create(tableName); +SortedSet splits = new TreeSet<>(); +for (int i = 0; i < 1000; i++) { + splits.add(new Text(randomHex(8))); +} +log.info("Adding splits"); +c.tableOperations().addSplits(tableName, splits); +log.info("Taking table offline"); +c.tableOperations().offline(tableName, true); ++ +// time how long it takes to load +log.info("Bringing the table online"); - long now = System.currentTimeMillis(); ++long now = System.nanoTime(); +c.tableOperations().online(tableName, true); - long diff = System.currentTimeMillis() - now; - log.info("Loaded " + splits.size() + " tablets in " + diff + " ms"); ++long diff = System.nanoTime() - now; ++ ++log.info("Loaded " + splits.size() + " tablets
[2/3] accumulo git commit: Merge branch '1.7' into 1.8
Merge branch '1.7' into 1.8 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/d38685c7 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/d38685c7 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/d38685c7 Branch: refs/heads/master Commit: d38685c70dea7378e6386f5d98903e9d382eb21e Parents: 15055b4 722d5eb Author: Mike MillerAuthored: Thu Jun 22 11:55:23 2017 -0400 Committer: Mike Miller Committed: Thu Jun 22 11:55:23 2017 -0400 -- .../accumulo/server/master/balancer/ChaoticLoadBalancer.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) --
[2/3] accumulo git commit: Merge branch '1.7' into 1.8
Merge branch '1.7' into 1.8 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/0dee0d85 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/0dee0d85 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/0dee0d85 Branch: refs/heads/master Commit: 0dee0d854e8f382b1837dcd77c7fabc6c91db0d5 Parents: e7e5b6b ede9dae Author: Dave MarionAuthored: Mon Jun 19 15:14:38 2017 -0400 Committer: Dave Marion Committed: Mon Jun 19 15:14:38 2017 -0400 -- .../server/master/balancer/HostRegexTableLoadBalancer.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/0dee0d85/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java --
[2/3] accumulo git commit: Merge branch '1.7' into 1.8
Merge branch '1.7' into 1.8 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/02f24e06 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/02f24e06 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/02f24e06 Branch: refs/heads/master Commit: 02f24e066a875ec41a390eab452b136e2ac24b71 Parents: bca75d3 1061fb5 Author: Mike MillerAuthored: Tue May 2 15:40:42 2017 -0400 Committer: Mike Miller Committed: Tue May 2 15:40:42 2017 -0400 -- test/src/main/java/org/apache/accumulo/test/ShellServerIT.java | 4 1 file changed, 4 insertions(+) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/02f24e06/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java -- diff --cc test/src/main/java/org/apache/accumulo/test/ShellServerIT.java index b23a014,000..e3d823c mode 100644,00..100644 --- a/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java +++ b/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java @@@ -1,1890 -1,0 +1,1894 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.test; + +import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.PrintWriter; +import java.lang.reflect.Constructor; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Random; +import java.util.concurrent.TimeUnit; + +import org.apache.accumulo.core.Constants; +import org.apache.accumulo.core.client.ClientConfiguration; +import org.apache.accumulo.core.client.ClientConfiguration.ClientProperty; +import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.client.IteratorSetting; +import org.apache.accumulo.core.client.Scanner; +import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.client.admin.TableOperations; +import org.apache.accumulo.core.client.impl.Namespaces; +import org.apache.accumulo.core.client.sample.RowSampler; +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; +import org.apache.accumulo.core.client.security.tokens.KerberosToken; +import org.apache.accumulo.core.client.security.tokens.PasswordToken; +import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.file.FileOperations; +import org.apache.accumulo.core.file.FileSKVWriter; +import org.apache.accumulo.core.metadata.MetadataTable; +import org.apache.accumulo.core.security.Authorizations; +import org.apache.accumulo.core.util.format.Formatter; +import org.apache.accumulo.core.util.format.FormatterConfig; +import org.apache.accumulo.harness.MiniClusterConfigurationCallback; +import org.apache.accumulo.harness.SharedMiniClusterBase; +import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl; +import org.apache.accumulo.shell.Shell; +import org.apache.accumulo.test.categories.MiniClusterOnlyTests; +import org.apache.accumulo.test.categories.SunnyDayTests; +import org.apache.accumulo.test.functional.SlowIterator; +import org.apache.accumulo.tracer.TraceServer; +import org.apache.commons.configuration.ConfigurationException; +import org.apache.commons.io.FileUtils; +import
[2/3] accumulo git commit: Merge branch '1.7' into 1.8
Merge branch '1.7' into 1.8 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/ba3dea16 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/ba3dea16 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/ba3dea16 Branch: refs/heads/master Commit: ba3dea165b42c3150eed60489ee8a8b98c8b47be Parents: 8d6b729 8c0f03a Author: Keith TurnerAuthored: Fri Apr 7 19:03:49 2017 -0400 Committer: Keith Turner Committed: Fri Apr 7 19:03:49 2017 -0400 -- .../core/client/impl/TableOperationsImpl.java | 39 +++- 1 file changed, 21 insertions(+), 18 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/ba3dea16/core/src/main/java/org/apache/accumulo/core/client/impl/TableOperationsImpl.java --
[2/3] accumulo git commit: Merge branch '1.7' into 1.8
Merge branch '1.7' into 1.8 Conflicts: core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/7ca6332d Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/7ca6332d Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/7ca6332d Branch: refs/heads/master Commit: 7ca6332df2f0d26d15d03ed4a74893bc0593ffbc Parents: 4a87fc9 de80cf5 Author: Keith TurnerAuthored: Tue Mar 7 15:48:46 2017 -0500 Committer: Keith Turner Committed: Tue Mar 7 15:48:46 2017 -0500 -- .../accumulo/core/file/rfile/PrintInfo.java | 7 +- .../apache/accumulo/core/file/rfile/RFile.java | 18 -- .../accumulo/core/util/LocalityGroupUtil.java | 35 ++ .../core/file/rfile/RFileMetricsTest.java | 68 ++-- 4 files changed, 116 insertions(+), 12 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/7ca6332d/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java -- diff --cc core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java index cf0d046,366e4a8..7d2a054 --- a/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java +++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java @@@ -28,11 -27,10 +27,12 @@@ import org.apache.accumulo.core.data.By import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.file.FileSKVIterator; import org.apache.accumulo.core.file.blockfile.impl.CachableBlockFile; import org.apache.accumulo.core.file.rfile.RFile.Reader; + import org.apache.accumulo.core.util.LocalityGroupUtil; import org.apache.accumulo.start.spi.KeywordExecutable; +import org.apache.commons.math3.stat.descriptive.SummaryStatistics; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@@ -161,47 -116,24 +161,46 @@@ public class PrintInfo implements Keywo Map localityGroupCF = null; - if (opts.histogram || opts.dump || opts.vis || opts.hash) { + if (opts.histogram || opts.dump || opts.vis || opts.hash || opts.keyStats) { localityGroupCF = iter.getLocalityGroupCF(); +FileSKVIterator dataIter; +if (opts.useSample) { + dataIter = iter.getSample(); + + if (dataIter == null) { +System.out.println("ERROR : This rfile has no sample data"); +return; + } +} else { + dataIter = iter; +} + +if (opts.keyStats) { + FileSKVIterator indexIter = iter.getIndex(); + while (indexIter.hasTop()) { +indexKeyStats.add(indexIter.getTopKey()); +indexIter.next(); + } +} + - for (Entry cf : localityGroupCF.entrySet()) { - - dataIter.seek(new Range((Key) null, (Key) null), cf.getValue(), true); + for (String lgName : localityGroupCF.keySet()) { - LocalityGroupUtil.seek(iter, new Range(), lgName, localityGroupCF); - while (iter.hasTop()) { -Key key = iter.getTopKey(); -Value value = iter.getTopValue(); -if (opts.dump) ++ LocalityGroupUtil.seek(dataIter, new Range(), lgName, localityGroupCF); + while (dataIter.hasTop()) { +Key key = dataIter.getTopKey(); +Value value = dataIter.getTopValue(); +if (opts.dump) { System.out.println(key + " -> " + value); + if (System.out.checkError()) +return; +} if (opts.histogram) { - long size = key.getSize() + value.getSize(); - int bucket = (int) Math.log10(size); - countBuckets[bucket]++; - sizeBuckets[bucket] += size; - totalSize += size; + kvHistogram.add(key.getSize() + value.getSize()); } -iter.next(); +if (opts.keyStats) { + dataKeyStats.add(key); +} +dataIter.next(); } } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/7ca6332d/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java -- diff --cc core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java index b11cf1a,bab2266..26343ba ---
[2/3] accumulo git commit: Merge branch '1.7' into 1.8
Merge branch '1.7' into 1.8 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/04847d97 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/04847d97 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/04847d97 Branch: refs/heads/master Commit: 04847d97d2e9e1fc6de30cccd13a705fc2294950 Parents: a4ac4b9 15ae69c Author: Keith TurnerAuthored: Mon Feb 6 14:38:24 2017 -0500 Committer: Keith Turner Committed: Mon Feb 6 14:38:24 2017 -0500 -- .../core/client/impl/TableOperationsImpl.java | 11 + .../accumulo/core/client/impl/Tables.java | 45 ++ .../test/functional/TableChangeStateIT.java | 439 +++ 3 files changed, 495 insertions(+) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/04847d97/core/src/main/java/org/apache/accumulo/core/client/impl/TableOperationsImpl.java -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/04847d97/core/src/main/java/org/apache/accumulo/core/client/impl/Tables.java -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/04847d97/test/src/main/java/org/apache/accumulo/test/functional/TableChangeStateIT.java -- diff --cc test/src/main/java/org/apache/accumulo/test/functional/TableChangeStateIT.java index 000,000..7ab9f3f new file mode 100644 --- /dev/null +++ b/test/src/main/java/org/apache/accumulo/test/functional/TableChangeStateIT.java @@@ -1,0 -1,0 +1,439 @@@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++package org.apache.accumulo.test.functional; ++ ++import org.apache.accumulo.core.Constants; ++import org.apache.accumulo.core.client.AccumuloException; ++import org.apache.accumulo.core.client.AccumuloSecurityException; ++import org.apache.accumulo.core.client.BatchWriter; ++import org.apache.accumulo.core.client.BatchWriterConfig; ++import org.apache.accumulo.core.client.Connector; ++import org.apache.accumulo.core.client.Instance; ++import org.apache.accumulo.core.client.IteratorSetting; ++import org.apache.accumulo.core.client.Scanner; ++import org.apache.accumulo.core.client.TableExistsException; ++import org.apache.accumulo.core.client.TableNotFoundException; ++import org.apache.accumulo.core.client.impl.Tables; ++import org.apache.accumulo.core.conf.Property; ++import org.apache.accumulo.core.data.Key; ++import org.apache.accumulo.core.data.Mutation; ++import org.apache.accumulo.core.data.Value; ++import org.apache.accumulo.core.master.state.tables.TableState; ++import org.apache.accumulo.core.security.Authorizations; ++import org.apache.accumulo.core.zookeeper.ZooUtil; ++import org.apache.accumulo.fate.AdminUtil; ++import org.apache.accumulo.fate.ZooStore; ++import org.apache.accumulo.fate.zookeeper.IZooReaderWriter; ++import org.apache.accumulo.harness.AccumuloClusterHarness; ++import org.apache.accumulo.server.zookeeper.ZooReaderWriterFactory; ++import org.apache.hadoop.io.Text; ++import org.apache.zookeeper.KeeperException; ++import org.junit.Before; ++import org.junit.Test; ++import org.slf4j.Logger; ++import org.slf4j.LoggerFactory; ++ ++import java.util.ArrayList; ++import java.util.List; ++import java.util.Map; ++import java.util.concurrent.Callable; ++import java.util.concurrent.ExecutorService; ++import java.util.concurrent.Executors; ++import java.util.concurrent.Future; ++import java.util.concurrent.TimeUnit; ++ ++import static java.nio.charset.StandardCharsets.UTF_8; ++import static org.junit.Assert.assertEquals; ++import static org.junit.Assert.assertTrue; ++ ++/** ++ * ACCUMULO-4574. Test to verify that changing table state to online / offline {@link org.apache.accumulo.core.client.admin.TableOperations#online(String)} when ++ * the table is already in that state returns without blocking. ++ */ ++public class TableChangeStateIT extends AccumuloClusterHarness {
[2/3] accumulo git commit: Merge branch '1.7' into 1.8
Merge branch '1.7' into 1.8 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/a4ac4b9d Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/a4ac4b9d Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/a4ac4b9d Branch: refs/heads/master Commit: a4ac4b9d415b8a7bb6dc36788c3a4d461d15b935 Parents: 09fcd14 33712bb Author: Michael WallAuthored: Thu Feb 2 10:39:52 2017 -0500 Committer: Michael Wall Committed: Thu Feb 2 10:39:52 2017 -0500 -- --
[2/3] accumulo git commit: Merge branch '1.7' into 1.8
Merge branch '1.7' into 1.8 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/09fcd142 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/09fcd142 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/09fcd142 Branch: refs/heads/master Commit: 09fcd142ee4279a4c726d40f16b82eb8dae89d18 Parents: 1f31ca6 5f664cd Author: Keith TurnerAuthored: Mon Jan 30 17:03:24 2017 -0500 Committer: Keith Turner Committed: Mon Jan 30 17:03:24 2017 -0500 -- fate/src/main/java/org/apache/accumulo/fate/AdminUtil.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) --
[2/3] accumulo git commit: Merge branch '1.7' into 1.8
Merge branch '1.7' into 1.8 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/bf5b6e0f Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/bf5b6e0f Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/bf5b6e0f Branch: refs/heads/master Commit: bf5b6e0fae4974bb77b6409c21c5770d815b991f Parents: 8668c7f 7b9a11a Author: Keith TurnerAuthored: Thu Jan 26 19:24:14 2017 -0500 Committer: Keith Turner Committed: Thu Jan 26 19:24:14 2017 -0500 -- .../accumulo/master/FateServiceHandler.java | 15 +- .../master/tableOps/CancelCompactions.java | 19 ++- .../master/tableOps/ChangeTableState.java | 20 ++- .../accumulo/master/tableOps/CompactRange.java | 30 ++-- .../master/tableOps/CompactionDriver.java | 14 +- .../accumulo/master/tableOps/DeleteTable.java | 29 ++-- .../accumulo/master/tableOps/RenameTable.java | 19 ++- .../accumulo/master/tableOps/TableRangeOp.java | 20 ++- .../master/tableOps/TableRangeOpWait.java | 9 +- .../apache/accumulo/master/tableOps/Utils.java | 9 +- .../functional/ConcurrentDeleteTableIT.java | 167 ++- 11 files changed, 262 insertions(+), 89 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/bf5b6e0f/server/master/src/main/java/org/apache/accumulo/master/FateServiceHandler.java -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/bf5b6e0f/server/master/src/main/java/org/apache/accumulo/master/tableOps/CompactRange.java -- diff --cc server/master/src/main/java/org/apache/accumulo/master/tableOps/CompactRange.java index d7d5b14,e641479..f3d53a7 --- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/CompactRange.java +++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/CompactRange.java @@@ -24,11 -24,10 +24,10 @@@ import java.util.List import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.client.admin.CompactionStrategyConfig; +import org.apache.accumulo.core.client.impl.AcceptableThriftTableOperationException; import org.apache.accumulo.core.client.impl.CompactionStrategyConfigUtil; - import org.apache.accumulo.core.client.impl.Tables; import org.apache.accumulo.core.client.impl.thrift.TableOperation; import org.apache.accumulo.core.client.impl.thrift.TableOperationExceptionType; -import org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException; import org.apache.accumulo.fate.Repo; import org.apache.accumulo.fate.zookeeper.IZooReaderWriter; import org.apache.accumulo.fate.zookeeper.IZooReaderWriter.Mutator; @@@ -51,9 -51,14 +51,14 @@@ public class CompactRange extends Maste private byte[] endRow; private byte[] config; - public CompactRange(String tableId, byte[] startRow, byte[] endRow, List iterators, CompactionStrategyConfig compactionStrategy) - throws AcceptableThriftTableOperationException { + private String getNamespaceId(Master env) throws Exception { + return Utils.getNamespaceId(env.getInstance(), tableId, TableOperation.COMPACT, this.namespaceId); + } + + public CompactRange(String namespaceId, String tableId, byte[] startRow, byte[] endRow, List iterators, - CompactionStrategyConfig compactionStrategy) throws ThriftTableOperationException { ++ CompactionStrategyConfig compactionStrategy) throws AcceptableThriftTableOperationException { + requireNonNull(namespaceId, "Invalid argument: null namespaceId"); requireNonNull(tableId, "Invalid argument: null tableId"); requireNonNull(iterators, "Invalid argument: null iterator list"); requireNonNull(compactionStrategy, "Invalid argument: null compactionStrategy"); @@@ -122,9 -127,9 +127,9 @@@ } }); - return new CompactionDriver(Long.parseLong(new String(cid, UTF_8).split(",")[0]), tableId, startRow, endRow); + return new CompactionDriver(Long.parseLong(new String(cid, UTF_8).split(",")[0]), getNamespaceId(env), tableId, startRow, endRow); } catch (NoNodeException nne) { - throw new ThriftTableOperationException(tableId, null, TableOperation.COMPACT, TableOperationExceptionType.NOTFOUND, null); + throw new AcceptableThriftTableOperationException(tableId, null, TableOperation.COMPACT, TableOperationExceptionType.NOTFOUND, null); } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/bf5b6e0f/server/master/src/main/java/org/apache/accumulo/master/tableOps/CompactionDriver.java --
[2/3] accumulo git commit: Merge branch '1.7' into 1.8
Merge branch '1.7' into 1.8 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/8668c7f3 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/8668c7f3 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/8668c7f3 Branch: refs/heads/master Commit: 8668c7f381832128d6bf76e8b6097778a42b71c4 Parents: b903766 ff134d2 Author: Dave MarionAuthored: Wed Jan 25 16:11:20 2017 -0500 Committer: Dave Marion Committed: Wed Jan 25 16:11:20 2017 -0500 -- .../balancer/HostRegexTableLoadBalancer.java| 86 ++-- .../BaseHostRegexTableLoadBalancerTest.java | 14 +++- ...gexTableLoadBalancerReconfigurationTest.java | 5 +- .../HostRegexTableLoadBalancerTest.java | 7 +- 4 files changed, 60 insertions(+), 52 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/8668c7f3/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/8668c7f3/server/base/src/test/java/org/apache/accumulo/server/master/balancer/BaseHostRegexTableLoadBalancerTest.java --
[2/3] accumulo git commit: Merge branch '1.7' into 1.8
Merge branch '1.7' into 1.8 Conflicts: server/master/src/main/java/org/apache/accumulo/master/tableOps/DeleteTable.java Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/b903766e Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/b903766e Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/b903766e Branch: refs/heads/master Commit: b903766ec14b36ac4185ab658d2155970cc7ffb4 Parents: 95be9f3 df400c5 Author: Keith TurnerAuthored: Wed Jan 25 12:42:41 2017 -0500 Committer: Keith Turner Committed: Wed Jan 25 12:42:41 2017 -0500 -- .../org/apache/accumulo/fate/AdminUtil.java | 104 +++-- .../accumulo/cluster/AccumuloCluster.java | 6 + .../standalone/StandaloneAccumuloCluster.java | 12 ++ .../impl/MiniAccumuloClusterImpl.java | 8 + .../accumulo/master/FateServiceHandler.java | 2 +- .../accumulo/master/tableOps/DeleteTable.java | 49 +++ .../apache/accumulo/master/tableOps/Utils.java | 13 ++ .../test/functional/BackupMasterIT.java | 7 +- .../functional/ConcurrentDeleteTableIT.java | 147 +++ 9 files changed, 307 insertions(+), 41 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/b903766e/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java -- diff --cc minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java index a3180a7,79ad527..829b321 --- a/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java +++ b/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java @@@ -110,12 -111,12 +111,13 @@@ import org.slf4j.LoggerFactory import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; import com.google.common.base.Predicate; + import com.google.common.collect.Iterables; import com.google.common.collect.Maps; +import com.google.common.util.concurrent.Uninterruptibles; /** - * A utility class that will create Zookeeper and Accumulo processes that write all of their data to a single local directory. This class makes it easy to test - * code against a real Accumulo instance. Its much more accurate for testing than {@link org.apache.accumulo.core.client.mock.MockAccumulo}, but much slower. + * This class provides the backing implementation for {@link MiniAccumuloCluster}, and may contain features for internal testing which have not yet been + * promoted to the public API. It's best to use {@link MiniAccumuloCluster} whenever possible. Use of this class risks API breakage between versions. * * @since 1.6.0 */ http://git-wip-us.apache.org/repos/asf/accumulo/blob/b903766e/server/master/src/main/java/org/apache/accumulo/master/FateServiceHandler.java -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/b903766e/server/master/src/main/java/org/apache/accumulo/master/tableOps/Utils.java -- diff --cc server/master/src/main/java/org/apache/accumulo/master/tableOps/Utils.java index 2baf7ac,9b921e2..aa02559 --- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/Utils.java +++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/Utils.java @@@ -117,6 -116,19 +117,19 @@@ public class Utils return 100; } + public static String getNamespaceId(Instance instance, String tableId, TableOperation op) throws Exception { + try { + return Tables.getNamespaceId(instance, tableId); + } catch (RuntimeException e) { + // see if this was caused because the table does not exists + IZooReaderWriter zk = ZooReaderWriter.getInstance(); + if (!zk.exists(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + tableId)) -throw new ThriftTableOperationException(tableId, "", op, TableOperationExceptionType.NOTFOUND, "Table does not exist"); ++throw new AcceptableThriftTableOperationException(tableId, "", op, TableOperationExceptionType.NOTFOUND, "Table does not exist"); + else + throw e; + } + } + public static long reserveHdfsDirectory(String directory, long tid) throws KeeperException, InterruptedException { Instance instance = HdfsZooInstance.getInstance(); http://git-wip-us.apache.org/repos/asf/accumulo/blob/b903766e/test/src/main/java/org/apache/accumulo/test/functional/BackupMasterIT.java -- diff --cc
[2/3] accumulo git commit: Merge branch '1.7' into 1.8
Merge branch '1.7' into 1.8 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/cd43958f Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/cd43958f Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/cd43958f Branch: refs/heads/master Commit: cd43958fe43ef82d9d584fbed3af8d0bd4f29795 Parents: 225 a361f63 Author: Mike MillerAuthored: Tue Nov 8 14:44:21 2016 -0500 Committer: Mike Miller Committed: Tue Nov 8 14:44:21 2016 -0500 -- .../main/java/org/apache/accumulo/server/init/Initialize.java | 3 ++- shell/src/main/java/org/apache/accumulo/shell/Shell.java | 3 +-- .../main/java/org/apache/accumulo/shell/ShellOptionsJC.java | 7 +++ 3 files changed, 10 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd43958f/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd43958f/shell/src/main/java/org/apache/accumulo/shell/Shell.java -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd43958f/shell/src/main/java/org/apache/accumulo/shell/ShellOptionsJC.java --
[2/3] accumulo git commit: Merge branch '1.7' into 1.8
Merge branch '1.7' into 1.8 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/2255 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/2255 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/2255 Branch: refs/heads/master Commit: 2255848271bb05368f8f97f69c8f486d98fd Parents: 09ee7b0 32fd100 Author: Mike MillerAuthored: Mon Nov 7 12:15:38 2016 -0500 Committer: Mike Miller Committed: Mon Nov 7 12:15:38 2016 -0500 -- .../main/java/org/apache/accumulo/shell/ShellOptionsJC.java | 9 ++--- 1 file changed, 6 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/2255/shell/src/main/java/org/apache/accumulo/shell/ShellOptionsJC.java --
[2/3] accumulo git commit: Merge branch '1.7' into 1.8
Merge branch '1.7' into 1.8 Conflicts: server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/1bf8a636 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/1bf8a636 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/1bf8a636 Branch: refs/heads/1.8 Commit: 1bf8a6362c685f316d04facb26faa29939cad9ef Parents: 8c85c32 48733ce Author: Sean BusbeyAuthored: Thu Sep 15 08:41:13 2016 -0700 Committer: Sean Busbey Committed: Thu Sep 15 08:41:13 2016 -0700 -- .../org/apache/accumulo/core/conf/Property.java | 7 .../accumulo/core/conf/SiteConfiguration.java | 24 - .../accumulo/server/client/HdfsZooInstance.java | 36 +--- 3 files changed, 53 insertions(+), 14 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/1bf8a636/core/src/main/java/org/apache/accumulo/core/conf/Property.java -- diff --cc core/src/main/java/org/apache/accumulo/core/conf/Property.java index cdaebef,96adaeb..34ef953 --- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java +++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java @@@ -822,9 -800,16 +822,16 @@@ public enum Property return validTableProperties.contains(key) || key.startsWith(Property.TABLE_CONSTRAINT_PREFIX.getKey()) || key.startsWith(Property.TABLE_ITERATOR_PREFIX.getKey()) || key.startsWith(Property.TABLE_LOCALITY_GROUP_PREFIX.getKey()) || key.startsWith(Property.TABLE_COMPACTION_STRATEGY_PREFIX.getKey()) || key.startsWith(Property.TABLE_REPLICATION_TARGET.getKey()) -|| key.startsWith(Property.TABLE_ARBITRARY_PROP_PREFIX.getKey()); +|| key.startsWith(Property.TABLE_ARBITRARY_PROP_PREFIX.getKey()) || key.startsWith(TABLE_SAMPLER_OPTS.getKey()); } + /** +* Properties we check the value of within the TabletServer request handling or maintenance processing loops. +*/ + public static final EnumSet HOT_PATH_PROPERTIES = EnumSet.of(Property.TSERV_CLIENT_TIMEOUT, Property.TSERV_TOTAL_MUTATION_QUEUE_MAX, + Property.TSERV_ARCHIVE_WALOGS, Property.GC_TRASH_IGNORE, Property.TSERV_MAJC_DELAY, Property.TABLE_MINC_LOGS_MAX, Property.TSERV_MAJC_MAXCONCURRENT, + Property.REPLICATION_WORKER_THREADS, Property.TABLE_DURABILITY, Property.INSTANCE_ZK_TIMEOUT, Property.TABLE_CLASSPATH); + private static final EnumSet fixedProperties = EnumSet.of(Property.TSERV_CLIENTPORT, Property.TSERV_NATIVEMAP_ENABLED, Property.TSERV_SCAN_MAX_OPENFILES, Property.MASTER_CLIENTPORT, Property.GC_PORT); http://git-wip-us.apache.org/repos/asf/accumulo/blob/1bf8a636/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java -- diff --cc server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java index e4e73d2,2dacf61..49bf1f4 --- a/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java +++ b/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java @@@ -23,7 -23,7 +23,8 @@@ import java.nio.ByteBuffer import java.util.Collections; import java.util.List; import java.util.UUID; +import java.util.concurrent.TimeUnit; + import java.util.concurrent.atomic.AtomicReference; import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.AccumuloException; @@@ -76,9 -76,9 +76,9 @@@ public class HdfsZooInstance implement return cachedHdfsZooInstance; } - private static ZooCache zooCache; + private final ZooCache zooCache; private static String instanceId = null; - private static final Logger log = Logger.getLogger(HdfsZooInstance.class); + private static final Logger log = LoggerFactory.getLogger(HdfsZooInstance.class); @Override public String getRootTabletLocation() {
[2/3] accumulo git commit: Merge branch '1.7' into 1.8
Merge branch '1.7' into 1.8 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/c5d9d493 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/c5d9d493 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/c5d9d493 Branch: refs/heads/master Commit: c5d9d493ecdd94bdb00dc038b1cda0f2fb1e0f2b Parents: ea07523 0978c7e Author: Dave MarionAuthored: Mon May 23 14:17:29 2016 -0400 Committer: Dave Marion Committed: Mon May 23 14:17:29 2016 -0400 -- .../providers/ReadOnlyHdfsFileProviderTest.java | 248 --- 1 file changed, 248 deletions(-) --