accumulo git commit: ACCUMULO-4399 WIP, print makefile actions instead of running.
Repository: accumulo Updated Branches: refs/heads/ACCUMULO-4399 [created] 700c4680b ACCUMULO-4399 WIP, print makefile actions instead of running. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/700c4680 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/700c4680 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/700c4680 Branch: refs/heads/ACCUMULO-4399 Commit: 700c4680b41ddc8506369da9618b8dc81c206aa7 Parents: 71fde9e Author: Sean Busbey Authored: Wed Aug 10 16:25:19 2016 -0500 Committer: Sean Busbey Committed: Wed Aug 10 16:25:19 2016 -0500 -- server/native/pom.xml | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/700c4680/server/native/pom.xml -- diff --git a/server/native/pom.xml b/server/native/pom.xml index 8a1ff4c..7a0ed1c 100644 --- a/server/native/pom.xml +++ b/server/native/pom.xml @@ -121,6 +121,7 @@ make ${project.build.directory}/${project.artifactId}-${project.version}/${project.artifactId}-${project.version} +--just-print test CXX=${CXX} OUTPUT_DIR=${project.build.directory}
[1/3] accumulo git commit: ACCUMULO-4422 Initialize logging as first step in server initialization.
Repository: accumulo Updated Branches: refs/heads/1.6 82f173496 -> 679a1c3eb refs/heads/1.7 ffd50280b -> 371db68cf ACCUMULO-4422 Initialize logging as first step in server initialization. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/679a1c3e Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/679a1c3e Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/679a1c3e Branch: refs/heads/1.6 Commit: 679a1c3eb9013d8bb0640a617392f921f096b158 Parents: 82f1734 Author: Sean Busbey Authored: Wed Aug 24 10:04:01 2016 -0500 Committer: Sean Busbey Committed: Wed Aug 24 10:04:01 2016 -0500 -- .../main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java | 2 +- .../master/src/main/java/org/apache/accumulo/master/Master.java | 4 ++-- .../src/main/java/org/apache/accumulo/monitor/Monitor.java | 4 ++-- .../src/main/java/org/apache/accumulo/tracer/TraceServer.java| 4 ++-- .../src/main/java/org/apache/accumulo/tserver/TabletServer.java | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/679a1c3e/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java -- diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java index 6d50e94..0191f38 100644 --- a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java +++ b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java @@ -136,9 +136,9 @@ public class SimpleGarbageCollector implements Iface { private Instance instance; public static void main(String[] args) throws UnknownHostException, IOException { -SecurityUtil.serverLogin(ServerConfiguration.getSiteConfiguration()); final String app = "gc"; Accumulo.setupLogging(app); +SecurityUtil.serverLogin(ServerConfiguration.getSiteConfiguration()); Instance instance = HdfsZooInstance.getInstance(); log.info("Version " + Constants.VERSION); log.info("Instance " + instance.getInstanceID()); http://git-wip-us.apache.org/repos/asf/accumulo/blob/679a1c3e/server/master/src/main/java/org/apache/accumulo/master/Master.java -- diff --git a/server/master/src/main/java/org/apache/accumulo/master/Master.java b/server/master/src/main/java/org/apache/accumulo/master/Master.java index cc4a02f..14bdd6b 100644 --- a/server/master/src/main/java/org/apache/accumulo/master/Master.java +++ b/server/master/src/main/java/org/apache/accumulo/master/Master.java @@ -1156,13 +1156,13 @@ public class Master implements LiveTServerSet.Listener, TableObserver, CurrentSt public static void main(String[] args) throws Exception { try { + final String app = "master"; + Accumulo.setupLogging(app); SecurityUtil.serverLogin(ServerConfiguration.getSiteConfiguration()); ServerOpts opts = new ServerOpts(); - final String app = "master"; opts.parseArgs(app, args); String hostname = opts.getAddress(); - Accumulo.setupLogging(app); Instance instance = HdfsZooInstance.getInstance(); ServerConfiguration conf = new ServerConfiguration(instance); VolumeManager fs = VolumeManagerImpl.get(); http://git-wip-us.apache.org/repos/asf/accumulo/blob/679a1c3e/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java -- diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java index 4e162c3..e5ba9e4 100644 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java @@ -422,14 +422,14 @@ public class Monitor { } public static void main(String[] args) throws Exception { +final String app = "monitor"; +Accumulo.setupLogging(app); SecurityUtil.serverLogin(ServerConfiguration.getSiteConfiguration()); ServerOpts opts = new ServerOpts(); -final String app = "monitor"; opts.parseArgs(app, args); String hostname = opts.getAddress(); -Accumulo.setupLogging(app); VolumeManager fs = VolumeManagerImpl.get(); instance = HdfsZooInstance.getInstance(); config = new ServerConfiguration(instance); http://git-wip-us.apache.org/repos/asf/accumulo/blob/679a1c3e/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
[3/3] accumulo git commit: Merge branch '1.6' into 1.7
Merge branch '1.6' into 1.7 Conflicts: server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java server/master/src/main/java/org/apache/accumulo/master/Master.java server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/371db68c Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/371db68c Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/371db68c Branch: refs/heads/1.7 Commit: 371db68cfc4a82921a0b47c886be3e88c4058cdf Parents: ffd5028 679a1c3 Author: Sean Busbey Authored: Wed Aug 24 10:22:25 2016 -0500 Committer: Sean Busbey Committed: Wed Aug 24 10:22:25 2016 -0500 -- .../main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java | 2 +- .../master/src/main/java/org/apache/accumulo/master/Master.java | 4 ++-- .../src/main/java/org/apache/accumulo/monitor/Monitor.java | 4 ++-- .../src/main/java/org/apache/accumulo/tracer/TraceServer.java| 4 ++-- .../src/main/java/org/apache/accumulo/tserver/TabletServer.java | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/371db68c/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java -- diff --cc server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java index da25d55,0191f38..98acf9a --- a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java +++ b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java @@@ -139,14 -133,16 +139,14 @@@ public class SimpleGarbageCollector ext private GCStatus status = new GCStatus(new GcCycleStats(), new GcCycleStats(), new GcCycleStats(), new GcCycleStats()); - private Instance instance; - public static void main(String[] args) throws UnknownHostException, IOException { - SecurityUtil.serverLogin(SiteConfiguration.getInstance()); final String app = "gc"; Accumulo.setupLogging(app); -SecurityUtil.serverLogin(ServerConfiguration.getSiteConfiguration()); ++SecurityUtil.serverLogin(SiteConfiguration.getInstance()); Instance instance = HdfsZooInstance.getInstance(); +ServerConfigurationFactory conf = new ServerConfigurationFactory(instance); log.info("Version " + Constants.VERSION); log.info("Instance " + instance.getInstanceID()); -ServerConfiguration conf = new ServerConfiguration(instance); final VolumeManager fs = VolumeManagerImpl.get(); Accumulo.init(fs, conf, app); Opts opts = new Opts(); http://git-wip-us.apache.org/repos/asf/accumulo/blob/371db68c/server/master/src/main/java/org/apache/accumulo/master/Master.java -- diff --cc server/master/src/main/java/org/apache/accumulo/master/Master.java index 5a2a346,14bdd6b..dc26a11 --- a/server/master/src/main/java/org/apache/accumulo/master/Master.java +++ b/server/master/src/main/java/org/apache/accumulo/master/Master.java @@@ -1345,14 -1156,15 +1345,14 @@@ public class Master extends AccumuloSer public static void main(String[] args) throws Exception { try { + final String app = "master"; + Accumulo.setupLogging(app); - SecurityUtil.serverLogin(ServerConfiguration.getSiteConfiguration()); + SecurityUtil.serverLogin(SiteConfiguration.getInstance()); ServerOpts opts = new ServerOpts(); - final String app = "master"; opts.parseArgs(app, args); String hostname = opts.getAddress(); - Accumulo.setupLogging(app); - Instance instance = HdfsZooInstance.getInstance(); - ServerConfiguration conf = new ServerConfiguration(instance); + ServerConfigurationFactory conf = new ServerConfigurationFactory(HdfsZooInstance.getInstance()); VolumeManager fs = VolumeManagerImpl.get(); Accumulo.init(fs, conf, app); Master master = new Master(conf, fs, hostname); http://git-wip-us.apache.org/repos/asf/accumulo/blob/371db68c/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java -- diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java index c3dd773,e5ba9e4..58cf921 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java @@@ -417,18 -422,17 +417,18 @@@ publi
[2/3] accumulo git commit: ACCUMULO-4422 Initialize logging as first step in server initialization.
ACCUMULO-4422 Initialize logging as first step in server initialization. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/679a1c3e Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/679a1c3e Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/679a1c3e Branch: refs/heads/1.7 Commit: 679a1c3eb9013d8bb0640a617392f921f096b158 Parents: 82f1734 Author: Sean Busbey Authored: Wed Aug 24 10:04:01 2016 -0500 Committer: Sean Busbey Committed: Wed Aug 24 10:04:01 2016 -0500 -- .../main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java | 2 +- .../master/src/main/java/org/apache/accumulo/master/Master.java | 4 ++-- .../src/main/java/org/apache/accumulo/monitor/Monitor.java | 4 ++-- .../src/main/java/org/apache/accumulo/tracer/TraceServer.java| 4 ++-- .../src/main/java/org/apache/accumulo/tserver/TabletServer.java | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/679a1c3e/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java -- diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java index 6d50e94..0191f38 100644 --- a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java +++ b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java @@ -136,9 +136,9 @@ public class SimpleGarbageCollector implements Iface { private Instance instance; public static void main(String[] args) throws UnknownHostException, IOException { -SecurityUtil.serverLogin(ServerConfiguration.getSiteConfiguration()); final String app = "gc"; Accumulo.setupLogging(app); +SecurityUtil.serverLogin(ServerConfiguration.getSiteConfiguration()); Instance instance = HdfsZooInstance.getInstance(); log.info("Version " + Constants.VERSION); log.info("Instance " + instance.getInstanceID()); http://git-wip-us.apache.org/repos/asf/accumulo/blob/679a1c3e/server/master/src/main/java/org/apache/accumulo/master/Master.java -- diff --git a/server/master/src/main/java/org/apache/accumulo/master/Master.java b/server/master/src/main/java/org/apache/accumulo/master/Master.java index cc4a02f..14bdd6b 100644 --- a/server/master/src/main/java/org/apache/accumulo/master/Master.java +++ b/server/master/src/main/java/org/apache/accumulo/master/Master.java @@ -1156,13 +1156,13 @@ public class Master implements LiveTServerSet.Listener, TableObserver, CurrentSt public static void main(String[] args) throws Exception { try { + final String app = "master"; + Accumulo.setupLogging(app); SecurityUtil.serverLogin(ServerConfiguration.getSiteConfiguration()); ServerOpts opts = new ServerOpts(); - final String app = "master"; opts.parseArgs(app, args); String hostname = opts.getAddress(); - Accumulo.setupLogging(app); Instance instance = HdfsZooInstance.getInstance(); ServerConfiguration conf = new ServerConfiguration(instance); VolumeManager fs = VolumeManagerImpl.get(); http://git-wip-us.apache.org/repos/asf/accumulo/blob/679a1c3e/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java -- diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java index 4e162c3..e5ba9e4 100644 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java @@ -422,14 +422,14 @@ public class Monitor { } public static void main(String[] args) throws Exception { +final String app = "monitor"; +Accumulo.setupLogging(app); SecurityUtil.serverLogin(ServerConfiguration.getSiteConfiguration()); ServerOpts opts = new ServerOpts(); -final String app = "monitor"; opts.parseArgs(app, args); String hostname = opts.getAddress(); -Accumulo.setupLogging(app); VolumeManager fs = VolumeManagerImpl.get(); instance = HdfsZooInstance.getInstance(); config = new ServerConfiguration(instance); http://git-wip-us.apache.org/repos/asf/accumulo/blob/679a1c3e/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java -- diff --git a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java b/server/
[1/2] accumulo git commit: ACCUMULO-4421 Check if the Trace User is expected to use Kerberos before attempting to login to Kerberos as the trace user.
Repository: accumulo Updated Branches: refs/heads/1.7 40d5a722b -> 2be85ade3 ACCUMULO-4421 Check if the Trace User is expected to use Kerberos before attempting to login to Kerberos as the trace user. Signed-off-by: Josh Elser Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/d66a8d08 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/d66a8d08 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/d66a8d08 Branch: refs/heads/1.7 Commit: d66a8d08627e98e9bbdd2bd0b1ab4f4658a84d9f Parents: 40d5a72 Author: Sean Busbey Authored: Thu Aug 25 14:47:38 2016 -0500 Committer: Sean Busbey Committed: Fri Aug 26 19:08:30 2016 -0500 -- .../org/apache/accumulo/tracer/TraceServer.java | 61 +--- 1 file changed, 41 insertions(+), 20 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/d66a8d08/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java -- diff --git a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java index 4b07dcc..2a06dc3 100644 --- a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java +++ b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java @@ -36,6 +36,7 @@ import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.client.MutationsRejectedException; import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Properties; +import org.apache.accumulo.core.client.security.tokens.KerberosToken; import org.apache.accumulo.core.client.security.tokens.PasswordToken; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.Property; @@ -306,30 +307,50 @@ public class TraceServer implements Watcher { } private static void loginTracer(AccumuloConfiguration acuConf) { -Map loginMap = acuConf.getAllPropertiesWithPrefix(Property.TRACE_TOKEN_PROPERTY_PREFIX); -String keyTab = loginMap.get(Property.TRACE_TOKEN_PROPERTY_PREFIX.getKey() + "keytab"); -if (keyTab == null || keyTab.length() == 0) { - keyTab = acuConf.getPath(Property.GENERAL_KERBEROS_KEYTAB); -} -if (keyTab == null || keyTab.length() == 0) - return; +try { + Class traceTokenType = AccumuloVFSClassLoader.getClassLoader().loadClass(acuConf.get(Property.TRACE_TOKEN_TYPE)) + .asSubclass(AuthenticationToken.class); + + if (!(KerberosToken.class.isAssignableFrom(traceTokenType))) { +// We're not using Kerberos to talk to Accumulo, but we might still need it for talking to HDFS/ZK for +// instance information. +log.info("Handling login under the assumption that Accumulo users are not using Kerberos."); +SecurityUtil.serverLogin(acuConf); + } else { +// We're using Kerberos to talk to Accumulo, so check for trace user specific auth details. +// We presume this same user will have the needed access for the service to interact with HDFS/ZK for +// instance information. +log.info("Handling login under the assumption that Accumulo users are using Kerberos."); +Map loginMap = acuConf.getAllPropertiesWithPrefix(Property.TRACE_TOKEN_PROPERTY_PREFIX); +String keyTab = loginMap.get(Property.TRACE_TOKEN_PROPERTY_PREFIX.getKey() + "keytab"); +if (keyTab == null || keyTab.length() == 0) { + keyTab = acuConf.getPath(Property.GENERAL_KERBEROS_KEYTAB); +} +if (keyTab == null || keyTab.length() == 0) + return; -String principalConfig = acuConf.get(Property.TRACE_USER); -if (principalConfig == null || principalConfig.length() == 0) - return; +String principalConfig = acuConf.get(Property.TRACE_USER); +if (principalConfig == null || principalConfig.length() == 0) + return; -log.info("Attempting to login as {} with {}", principalConfig, keyTab); -if (SecurityUtil.login(principalConfig, keyTab)) { - try { -// This spawns a thread to periodically renew the logged in (trace) user -UserGroupInformation.getLoginUser(); -return; - } catch (IOException io) { -log.error("Error starting up renewal thread. This shouldn't be happening.", io); +log.info("Attempting to login as {} with {}", principalConfig, keyTab); +if (SecurityUtil.login(principalConfig, keyTab)) { + try { +// This spawns a thread to periodically renew the logge
[2/2] accumulo git commit: ACCUMULO-4421 Ensure that TraceServer launches the Kerberos ticket renewal thread
ACCUMULO-4421 Ensure that TraceServer launches the Kerberos ticket renewal thread The previous method that TraceServer was using on SecurityUtil only performed the login and was relying on incorrect functionality in Hadoop's UGI to launch a renewal thread. This logic is incorrect. Refactored SecurityUtil a little to prevent other callers from making the same mistake in the future. Signed-off-by: Sean Busbey Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/2be85ade Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/2be85ade Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/2be85ade Branch: refs/heads/1.7 Commit: 2be85ade34c413cc32db838e6125e582b829ef03 Parents: d66a8d0 Author: Josh Elser Authored: Fri Aug 26 17:33:42 2016 -0400 Committer: Sean Busbey Committed: Fri Aug 26 19:08:56 2016 -0500 -- .../accumulo/server/security/SecurityUtil.java | 30 ++-- .../org/apache/accumulo/tracer/TraceServer.java | 13 + 2 files changed, 23 insertions(+), 20 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/2be85ade/server/base/src/main/java/org/apache/accumulo/server/security/SecurityUtil.java -- diff --git a/server/base/src/main/java/org/apache/accumulo/server/security/SecurityUtil.java b/server/base/src/main/java/org/apache/accumulo/server/security/SecurityUtil.java index 73f671b..38afa31 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/security/SecurityUtil.java +++ b/server/base/src/main/java/org/apache/accumulo/server/security/SecurityUtil.java @@ -40,17 +40,31 @@ public class SecurityUtil { * {@link #login(String, String)} */ public static void serverLogin(AccumuloConfiguration acuConf) { -String keyTab = acuConf.getPath(Property.GENERAL_KERBEROS_KEYTAB); +serverLogin(acuConf, acuConf.getPath(Property.GENERAL_KERBEROS_KEYTAB), acuConf.get(Property.GENERAL_KERBEROS_PRINCIPAL)); + } + + /** + * Performs a Kerberos login using the given Kerberos principal and keytab if they are non-null and positive length Strings. This method automaticallys spawns + * a thread to renew the given ticket upon successful login using {@link Property#GENERAL_KERBEROS_RENEWAL_PERIOD} as the renewal period. This method does + * nothing if either {@code keyTab} or {@code principal} are null or of zero length. + * + * @param acuConf + * The Accumulo configuration + * @param keyTab + * The path to the Kerberos keytab file + * @param principal + * The Kerberos principal + */ + public static void serverLogin(AccumuloConfiguration acuConf, String keyTab, String principal) { if (keyTab == null || keyTab.length() == 0) return; -usingKerberos = true; - -String principalConfig = acuConf.get(Property.GENERAL_KERBEROS_PRINCIPAL); -if (principalConfig == null || principalConfig.length() == 0) +if (principal == null || principal.length() == 0) return; -if (login(principalConfig, keyTab)) { +usingKerberos = true; + +if (login(principal, keyTab)) { try { startTicketRenewalThread(UserGroupInformation.getCurrentUser(), acuConf.getTimeInMillis(Property.GENERAL_KERBEROS_RENEWAL_PERIOD)); return; @@ -59,7 +73,7 @@ public class SecurityUtil { } } -throw new RuntimeException("Failed to perform Kerberos login for " + principalConfig + " using " + keyTab); +throw new RuntimeException("Failed to perform Kerberos login for " + principal + " using " + keyTab); } /** @@ -70,7 +84,7 @@ public class SecurityUtil { * replaced by the systems host name. * @return true if login succeeded, otherwise false */ - public static boolean login(String principalConfig, String keyTabPath) { + static boolean login(String principalConfig, String keyTabPath) { try { String principalName = getServerPrincipal(principalConfig); if (keyTabPath != null && principalName != null && keyTabPath.length() != 0 && principalName.length() != 0) { http://git-wip-us.apache.org/repos/asf/accumulo/blob/2be85ade/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java -- diff --git a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java index 2a06dc3..10601ba 100644 --- a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java +++ b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java @@ -61,7
[3/3] accumulo git commit: Merge branch '1.7' into 1.8
Merge branch '1.7' into 1.8 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/e68b8dbc Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/e68b8dbc Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/e68b8dbc Branch: refs/heads/1.8 Commit: e68b8dbc61a204e73452f254a5f130ac813c9b78 Parents: c2900a3 2be85ad Author: Sean Busbey Authored: Fri Aug 26 19:23:13 2016 -0500 Committer: Sean Busbey Committed: Fri Aug 26 19:23:13 2016 -0500 -- .../accumulo/server/security/SecurityUtil.java | 30 --- .../org/apache/accumulo/tracer/TraceServer.java | 52 2 files changed, 53 insertions(+), 29 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/e68b8dbc/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java --
[1/3] accumulo git commit: ACCUMULO-4421 Check if the Trace User is expected to use Kerberos before attempting to login to Kerberos as the trace user.
Repository: accumulo Updated Branches: refs/heads/1.8 c2900a380 -> e68b8dbc6 ACCUMULO-4421 Check if the Trace User is expected to use Kerberos before attempting to login to Kerberos as the trace user. Signed-off-by: Josh Elser Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/d66a8d08 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/d66a8d08 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/d66a8d08 Branch: refs/heads/1.8 Commit: d66a8d08627e98e9bbdd2bd0b1ab4f4658a84d9f Parents: 40d5a72 Author: Sean Busbey Authored: Thu Aug 25 14:47:38 2016 -0500 Committer: Sean Busbey Committed: Fri Aug 26 19:08:30 2016 -0500 -- .../org/apache/accumulo/tracer/TraceServer.java | 61 +--- 1 file changed, 41 insertions(+), 20 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/d66a8d08/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java -- diff --git a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java index 4b07dcc..2a06dc3 100644 --- a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java +++ b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java @@ -36,6 +36,7 @@ import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.client.MutationsRejectedException; import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Properties; +import org.apache.accumulo.core.client.security.tokens.KerberosToken; import org.apache.accumulo.core.client.security.tokens.PasswordToken; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.Property; @@ -306,30 +307,50 @@ public class TraceServer implements Watcher { } private static void loginTracer(AccumuloConfiguration acuConf) { -Map loginMap = acuConf.getAllPropertiesWithPrefix(Property.TRACE_TOKEN_PROPERTY_PREFIX); -String keyTab = loginMap.get(Property.TRACE_TOKEN_PROPERTY_PREFIX.getKey() + "keytab"); -if (keyTab == null || keyTab.length() == 0) { - keyTab = acuConf.getPath(Property.GENERAL_KERBEROS_KEYTAB); -} -if (keyTab == null || keyTab.length() == 0) - return; +try { + Class traceTokenType = AccumuloVFSClassLoader.getClassLoader().loadClass(acuConf.get(Property.TRACE_TOKEN_TYPE)) + .asSubclass(AuthenticationToken.class); + + if (!(KerberosToken.class.isAssignableFrom(traceTokenType))) { +// We're not using Kerberos to talk to Accumulo, but we might still need it for talking to HDFS/ZK for +// instance information. +log.info("Handling login under the assumption that Accumulo users are not using Kerberos."); +SecurityUtil.serverLogin(acuConf); + } else { +// We're using Kerberos to talk to Accumulo, so check for trace user specific auth details. +// We presume this same user will have the needed access for the service to interact with HDFS/ZK for +// instance information. +log.info("Handling login under the assumption that Accumulo users are using Kerberos."); +Map loginMap = acuConf.getAllPropertiesWithPrefix(Property.TRACE_TOKEN_PROPERTY_PREFIX); +String keyTab = loginMap.get(Property.TRACE_TOKEN_PROPERTY_PREFIX.getKey() + "keytab"); +if (keyTab == null || keyTab.length() == 0) { + keyTab = acuConf.getPath(Property.GENERAL_KERBEROS_KEYTAB); +} +if (keyTab == null || keyTab.length() == 0) + return; -String principalConfig = acuConf.get(Property.TRACE_USER); -if (principalConfig == null || principalConfig.length() == 0) - return; +String principalConfig = acuConf.get(Property.TRACE_USER); +if (principalConfig == null || principalConfig.length() == 0) + return; -log.info("Attempting to login as {} with {}", principalConfig, keyTab); -if (SecurityUtil.login(principalConfig, keyTab)) { - try { -// This spawns a thread to periodically renew the logged in (trace) user -UserGroupInformation.getLoginUser(); -return; - } catch (IOException io) { -log.error("Error starting up renewal thread. This shouldn't be happening.", io); +log.info("Attempting to login as {} with {}", principalConfig, keyTab); +if (SecurityUtil.login(principalConfig, keyTab)) { + try { +// This spawns a thread to periodically renew the logge
[2/3] accumulo git commit: ACCUMULO-4421 Ensure that TraceServer launches the Kerberos ticket renewal thread
ACCUMULO-4421 Ensure that TraceServer launches the Kerberos ticket renewal thread The previous method that TraceServer was using on SecurityUtil only performed the login and was relying on incorrect functionality in Hadoop's UGI to launch a renewal thread. This logic is incorrect. Refactored SecurityUtil a little to prevent other callers from making the same mistake in the future. Signed-off-by: Sean Busbey Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/2be85ade Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/2be85ade Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/2be85ade Branch: refs/heads/1.8 Commit: 2be85ade34c413cc32db838e6125e582b829ef03 Parents: d66a8d0 Author: Josh Elser Authored: Fri Aug 26 17:33:42 2016 -0400 Committer: Sean Busbey Committed: Fri Aug 26 19:08:56 2016 -0500 -- .../accumulo/server/security/SecurityUtil.java | 30 ++-- .../org/apache/accumulo/tracer/TraceServer.java | 13 + 2 files changed, 23 insertions(+), 20 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/2be85ade/server/base/src/main/java/org/apache/accumulo/server/security/SecurityUtil.java -- diff --git a/server/base/src/main/java/org/apache/accumulo/server/security/SecurityUtil.java b/server/base/src/main/java/org/apache/accumulo/server/security/SecurityUtil.java index 73f671b..38afa31 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/security/SecurityUtil.java +++ b/server/base/src/main/java/org/apache/accumulo/server/security/SecurityUtil.java @@ -40,17 +40,31 @@ public class SecurityUtil { * {@link #login(String, String)} */ public static void serverLogin(AccumuloConfiguration acuConf) { -String keyTab = acuConf.getPath(Property.GENERAL_KERBEROS_KEYTAB); +serverLogin(acuConf, acuConf.getPath(Property.GENERAL_KERBEROS_KEYTAB), acuConf.get(Property.GENERAL_KERBEROS_PRINCIPAL)); + } + + /** + * Performs a Kerberos login using the given Kerberos principal and keytab if they are non-null and positive length Strings. This method automaticallys spawns + * a thread to renew the given ticket upon successful login using {@link Property#GENERAL_KERBEROS_RENEWAL_PERIOD} as the renewal period. This method does + * nothing if either {@code keyTab} or {@code principal} are null or of zero length. + * + * @param acuConf + * The Accumulo configuration + * @param keyTab + * The path to the Kerberos keytab file + * @param principal + * The Kerberos principal + */ + public static void serverLogin(AccumuloConfiguration acuConf, String keyTab, String principal) { if (keyTab == null || keyTab.length() == 0) return; -usingKerberos = true; - -String principalConfig = acuConf.get(Property.GENERAL_KERBEROS_PRINCIPAL); -if (principalConfig == null || principalConfig.length() == 0) +if (principal == null || principal.length() == 0) return; -if (login(principalConfig, keyTab)) { +usingKerberos = true; + +if (login(principal, keyTab)) { try { startTicketRenewalThread(UserGroupInformation.getCurrentUser(), acuConf.getTimeInMillis(Property.GENERAL_KERBEROS_RENEWAL_PERIOD)); return; @@ -59,7 +73,7 @@ public class SecurityUtil { } } -throw new RuntimeException("Failed to perform Kerberos login for " + principalConfig + " using " + keyTab); +throw new RuntimeException("Failed to perform Kerberos login for " + principal + " using " + keyTab); } /** @@ -70,7 +84,7 @@ public class SecurityUtil { * replaced by the systems host name. * @return true if login succeeded, otherwise false */ - public static boolean login(String principalConfig, String keyTabPath) { + static boolean login(String principalConfig, String keyTabPath) { try { String principalName = getServerPrincipal(principalConfig); if (keyTabPath != null && principalName != null && keyTabPath.length() != 0 && principalName.length() != 0) { http://git-wip-us.apache.org/repos/asf/accumulo/blob/2be85ade/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java -- diff --git a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java index 2a06dc3..10601ba 100644 --- a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java +++ b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java @@ -61,7
[1/4] accumulo git commit: ACCUMULO-4421 Check if the Trace User is expected to use Kerberos before attempting to login to Kerberos as the trace user.
Repository: accumulo Updated Branches: refs/heads/master 97ff25db2 -> 59c6a8e19 ACCUMULO-4421 Check if the Trace User is expected to use Kerberos before attempting to login to Kerberos as the trace user. Signed-off-by: Josh Elser Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/d66a8d08 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/d66a8d08 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/d66a8d08 Branch: refs/heads/master Commit: d66a8d08627e98e9bbdd2bd0b1ab4f4658a84d9f Parents: 40d5a72 Author: Sean Busbey Authored: Thu Aug 25 14:47:38 2016 -0500 Committer: Sean Busbey Committed: Fri Aug 26 19:08:30 2016 -0500 -- .../org/apache/accumulo/tracer/TraceServer.java | 61 +--- 1 file changed, 41 insertions(+), 20 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/d66a8d08/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java -- diff --git a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java index 4b07dcc..2a06dc3 100644 --- a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java +++ b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java @@ -36,6 +36,7 @@ import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.client.MutationsRejectedException; import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Properties; +import org.apache.accumulo.core.client.security.tokens.KerberosToken; import org.apache.accumulo.core.client.security.tokens.PasswordToken; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.Property; @@ -306,30 +307,50 @@ public class TraceServer implements Watcher { } private static void loginTracer(AccumuloConfiguration acuConf) { -Map loginMap = acuConf.getAllPropertiesWithPrefix(Property.TRACE_TOKEN_PROPERTY_PREFIX); -String keyTab = loginMap.get(Property.TRACE_TOKEN_PROPERTY_PREFIX.getKey() + "keytab"); -if (keyTab == null || keyTab.length() == 0) { - keyTab = acuConf.getPath(Property.GENERAL_KERBEROS_KEYTAB); -} -if (keyTab == null || keyTab.length() == 0) - return; +try { + Class traceTokenType = AccumuloVFSClassLoader.getClassLoader().loadClass(acuConf.get(Property.TRACE_TOKEN_TYPE)) + .asSubclass(AuthenticationToken.class); + + if (!(KerberosToken.class.isAssignableFrom(traceTokenType))) { +// We're not using Kerberos to talk to Accumulo, but we might still need it for talking to HDFS/ZK for +// instance information. +log.info("Handling login under the assumption that Accumulo users are not using Kerberos."); +SecurityUtil.serverLogin(acuConf); + } else { +// We're using Kerberos to talk to Accumulo, so check for trace user specific auth details. +// We presume this same user will have the needed access for the service to interact with HDFS/ZK for +// instance information. +log.info("Handling login under the assumption that Accumulo users are using Kerberos."); +Map loginMap = acuConf.getAllPropertiesWithPrefix(Property.TRACE_TOKEN_PROPERTY_PREFIX); +String keyTab = loginMap.get(Property.TRACE_TOKEN_PROPERTY_PREFIX.getKey() + "keytab"); +if (keyTab == null || keyTab.length() == 0) { + keyTab = acuConf.getPath(Property.GENERAL_KERBEROS_KEYTAB); +} +if (keyTab == null || keyTab.length() == 0) + return; -String principalConfig = acuConf.get(Property.TRACE_USER); -if (principalConfig == null || principalConfig.length() == 0) - return; +String principalConfig = acuConf.get(Property.TRACE_USER); +if (principalConfig == null || principalConfig.length() == 0) + return; -log.info("Attempting to login as {} with {}", principalConfig, keyTab); -if (SecurityUtil.login(principalConfig, keyTab)) { - try { -// This spawns a thread to periodically renew the logged in (trace) user -UserGroupInformation.getLoginUser(); -return; - } catch (IOException io) { -log.error("Error starting up renewal thread. This shouldn't be happening.", io); +log.info("Attempting to login as {} with {}", principalConfig, keyTab); +if (SecurityUtil.login(principalConfig, keyTab)) { + try { +// This spawns a thread to periodically renew the logge
[2/4] accumulo git commit: ACCUMULO-4421 Ensure that TraceServer launches the Kerberos ticket renewal thread
ACCUMULO-4421 Ensure that TraceServer launches the Kerberos ticket renewal thread The previous method that TraceServer was using on SecurityUtil only performed the login and was relying on incorrect functionality in Hadoop's UGI to launch a renewal thread. This logic is incorrect. Refactored SecurityUtil a little to prevent other callers from making the same mistake in the future. Signed-off-by: Sean Busbey Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/2be85ade Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/2be85ade Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/2be85ade Branch: refs/heads/master Commit: 2be85ade34c413cc32db838e6125e582b829ef03 Parents: d66a8d0 Author: Josh Elser Authored: Fri Aug 26 17:33:42 2016 -0400 Committer: Sean Busbey Committed: Fri Aug 26 19:08:56 2016 -0500 -- .../accumulo/server/security/SecurityUtil.java | 30 ++-- .../org/apache/accumulo/tracer/TraceServer.java | 13 + 2 files changed, 23 insertions(+), 20 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/2be85ade/server/base/src/main/java/org/apache/accumulo/server/security/SecurityUtil.java -- diff --git a/server/base/src/main/java/org/apache/accumulo/server/security/SecurityUtil.java b/server/base/src/main/java/org/apache/accumulo/server/security/SecurityUtil.java index 73f671b..38afa31 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/security/SecurityUtil.java +++ b/server/base/src/main/java/org/apache/accumulo/server/security/SecurityUtil.java @@ -40,17 +40,31 @@ public class SecurityUtil { * {@link #login(String, String)} */ public static void serverLogin(AccumuloConfiguration acuConf) { -String keyTab = acuConf.getPath(Property.GENERAL_KERBEROS_KEYTAB); +serverLogin(acuConf, acuConf.getPath(Property.GENERAL_KERBEROS_KEYTAB), acuConf.get(Property.GENERAL_KERBEROS_PRINCIPAL)); + } + + /** + * Performs a Kerberos login using the given Kerberos principal and keytab if they are non-null and positive length Strings. This method automaticallys spawns + * a thread to renew the given ticket upon successful login using {@link Property#GENERAL_KERBEROS_RENEWAL_PERIOD} as the renewal period. This method does + * nothing if either {@code keyTab} or {@code principal} are null or of zero length. + * + * @param acuConf + * The Accumulo configuration + * @param keyTab + * The path to the Kerberos keytab file + * @param principal + * The Kerberos principal + */ + public static void serverLogin(AccumuloConfiguration acuConf, String keyTab, String principal) { if (keyTab == null || keyTab.length() == 0) return; -usingKerberos = true; - -String principalConfig = acuConf.get(Property.GENERAL_KERBEROS_PRINCIPAL); -if (principalConfig == null || principalConfig.length() == 0) +if (principal == null || principal.length() == 0) return; -if (login(principalConfig, keyTab)) { +usingKerberos = true; + +if (login(principal, keyTab)) { try { startTicketRenewalThread(UserGroupInformation.getCurrentUser(), acuConf.getTimeInMillis(Property.GENERAL_KERBEROS_RENEWAL_PERIOD)); return; @@ -59,7 +73,7 @@ public class SecurityUtil { } } -throw new RuntimeException("Failed to perform Kerberos login for " + principalConfig + " using " + keyTab); +throw new RuntimeException("Failed to perform Kerberos login for " + principal + " using " + keyTab); } /** @@ -70,7 +84,7 @@ public class SecurityUtil { * replaced by the systems host name. * @return true if login succeeded, otherwise false */ - public static boolean login(String principalConfig, String keyTabPath) { + static boolean login(String principalConfig, String keyTabPath) { try { String principalName = getServerPrincipal(principalConfig); if (keyTabPath != null && principalName != null && keyTabPath.length() != 0 && principalName.length() != 0) { http://git-wip-us.apache.org/repos/asf/accumulo/blob/2be85ade/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java -- diff --git a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java index 2a06dc3..10601ba 100644 --- a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java +++ b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java @@ -61,7
[3/4] accumulo git commit: Merge branch '1.7' into 1.8
Merge branch '1.7' into 1.8 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/e68b8dbc Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/e68b8dbc Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/e68b8dbc Branch: refs/heads/master Commit: e68b8dbc61a204e73452f254a5f130ac813c9b78 Parents: c2900a3 2be85ad Author: Sean Busbey Authored: Fri Aug 26 19:23:13 2016 -0500 Committer: Sean Busbey Committed: Fri Aug 26 19:23:13 2016 -0500 -- .../accumulo/server/security/SecurityUtil.java | 30 --- .../org/apache/accumulo/tracer/TraceServer.java | 52 2 files changed, 53 insertions(+), 29 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/e68b8dbc/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java --
[4/4] accumulo git commit: Merge branch '1.8'
Merge branch '1.8' Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/59c6a8e1 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/59c6a8e1 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/59c6a8e1 Branch: refs/heads/master Commit: 59c6a8e1f11cd3fc2db1416f8eb82c7de480 Parents: 97ff25d e68b8db Author: Sean Busbey Authored: Fri Aug 26 21:46:04 2016 -0500 Committer: Sean Busbey Committed: Fri Aug 26 21:46:04 2016 -0500 -- .../accumulo/server/security/SecurityUtil.java | 30 --- .../org/apache/accumulo/tracer/TraceServer.java | 52 2 files changed, 53 insertions(+), 29 deletions(-) --
accumulo git commit: ACCUMULO-4458 reduce contention on config retrievals.
Repository: accumulo Updated Branches: refs/heads/1.7 4136bfafc -> 48733ce8c ACCUMULO-4458 reduce contention on config retrievals. We retrieve configuration values in a few hot code paths, so contention on getting the site configuration and values that are only read from the filesystem once causes a perf impact. We try to reduce that here in a few ways: * Since SiteConfiguration is a singleton, we only retrieve it once when HdfsZooInstance is made instead of on each lookup. * Since HdfsZooInstance is a singleton, we create it once on initialization and avoid synchronizing on retrieval * Since the Hadoop Configuration object uses a single lock for all gets against the shared instance, we copy out the values it reads when the SiteConfiguration singleton is created. This has the side-effect of causing our for-test-only set method to only work on values not set in the Configuration (essentially we treat all configs as final). * Additionally, for those configurations that are read within any of the processing loops of the TabletServer, we have the SiteConfiguration singleton cache the fact that they weren't in the Hadoop Configuration object so that we will default to the parent AccumuloConfiguration without contending for the Hadoop Configuration object lock. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/48733ce8 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/48733ce8 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/48733ce8 Branch: refs/heads/1.7 Commit: 48733ce8cb9e89f1243e5e46d792010900086c7b Parents: 4136bfa Author: Sean Busbey Authored: Tue Sep 6 11:52:37 2016 -0500 Committer: Sean Busbey Committed: Wed Sep 14 16:35:31 2016 -0700 -- .../org/apache/accumulo/core/conf/Property.java | 7 .../accumulo/core/conf/SiteConfiguration.java | 24 - .../accumulo/server/client/HdfsZooInstance.java | 36 +--- 3 files changed, 53 insertions(+), 14 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/48733ce8/core/src/main/java/org/apache/accumulo/core/conf/Property.java -- diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java index 9d145e1..96adaeb 100644 --- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java +++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java @@ -803,6 +803,13 @@ public enum Property { || key.startsWith(Property.TABLE_ARBITRARY_PROP_PREFIX.getKey()); } + /** + * Properties we check the value of within the TabletServer request handling or maintenance processing loops. + */ + public static final EnumSet HOT_PATH_PROPERTIES = EnumSet.of(Property.TSERV_CLIENT_TIMEOUT, Property.TSERV_TOTAL_MUTATION_QUEUE_MAX, + Property.TSERV_ARCHIVE_WALOGS, Property.GC_TRASH_IGNORE, Property.TSERV_MAJC_DELAY, Property.TABLE_MINC_LOGS_MAX, Property.TSERV_MAJC_MAXCONCURRENT, + Property.REPLICATION_WORKER_THREADS, Property.TABLE_DURABILITY, Property.INSTANCE_ZK_TIMEOUT, Property.TABLE_CLASSPATH); + private static final EnumSet fixedProperties = EnumSet.of(Property.TSERV_CLIENTPORT, Property.TSERV_NATIVEMAP_ENABLED, Property.TSERV_SCAN_MAX_OPENFILES, Property.MASTER_CLIENTPORT, Property.GC_PORT); http://git-wip-us.apache.org/repos/asf/accumulo/blob/48733ce8/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java -- diff --git a/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java b/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java index b2f5a18..07aaf9f 100644 --- a/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java +++ b/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java @@ -17,6 +17,8 @@ package org.apache.accumulo.core.conf; import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; @@ -45,12 +47,31 @@ public class SiteConfiguration extends AccumuloConfiguration { private static SiteConfiguration instance = null; private static Configuration xmlConfig; + private final Map staticConfigs; /** * Not for consumers. Call {@link SiteConfiguration#getInstance(AccumuloConfiguration)} instead */ SiteConfiguration(AccumuloConfiguration parent) { this.parent = parent; +/* + * Make a read-only copy of static configs so we can avoid lock contention on the Hadoop Configuration object + */ +final Configuration conf = getXmlConfig(); +Map temp = new HashMap<>((int) (Math.
[1/3] accumulo git commit: ACCUMULO-4458 reduce contention on config retrievals.
Repository: accumulo Updated Branches: refs/heads/1.8 8c85c32af -> 923c87e2f ACCUMULO-4458 reduce contention on config retrievals. We retrieve configuration values in a few hot code paths, so contention on getting the site configuration and values that are only read from the filesystem once causes a perf impact. We try to reduce that here in a few ways: * Since SiteConfiguration is a singleton, we only retrieve it once when HdfsZooInstance is made instead of on each lookup. * Since HdfsZooInstance is a singleton, we create it once on initialization and avoid synchronizing on retrieval * Since the Hadoop Configuration object uses a single lock for all gets against the shared instance, we copy out the values it reads when the SiteConfiguration singleton is created. This has the side-effect of causing our for-test-only set method to only work on values not set in the Configuration (essentially we treat all configs as final). * Additionally, for those configurations that are read within any of the processing loops of the TabletServer, we have the SiteConfiguration singleton cache the fact that they weren't in the Hadoop Configuration object so that we will default to the parent AccumuloConfiguration without contending for the Hadoop Configuration object lock. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/48733ce8 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/48733ce8 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/48733ce8 Branch: refs/heads/1.8 Commit: 48733ce8cb9e89f1243e5e46d792010900086c7b Parents: 4136bfa Author: Sean Busbey Authored: Tue Sep 6 11:52:37 2016 -0500 Committer: Sean Busbey Committed: Wed Sep 14 16:35:31 2016 -0700 -- .../org/apache/accumulo/core/conf/Property.java | 7 .../accumulo/core/conf/SiteConfiguration.java | 24 - .../accumulo/server/client/HdfsZooInstance.java | 36 +--- 3 files changed, 53 insertions(+), 14 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/48733ce8/core/src/main/java/org/apache/accumulo/core/conf/Property.java -- diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java index 9d145e1..96adaeb 100644 --- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java +++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java @@ -803,6 +803,13 @@ public enum Property { || key.startsWith(Property.TABLE_ARBITRARY_PROP_PREFIX.getKey()); } + /** + * Properties we check the value of within the TabletServer request handling or maintenance processing loops. + */ + public static final EnumSet HOT_PATH_PROPERTIES = EnumSet.of(Property.TSERV_CLIENT_TIMEOUT, Property.TSERV_TOTAL_MUTATION_QUEUE_MAX, + Property.TSERV_ARCHIVE_WALOGS, Property.GC_TRASH_IGNORE, Property.TSERV_MAJC_DELAY, Property.TABLE_MINC_LOGS_MAX, Property.TSERV_MAJC_MAXCONCURRENT, + Property.REPLICATION_WORKER_THREADS, Property.TABLE_DURABILITY, Property.INSTANCE_ZK_TIMEOUT, Property.TABLE_CLASSPATH); + private static final EnumSet fixedProperties = EnumSet.of(Property.TSERV_CLIENTPORT, Property.TSERV_NATIVEMAP_ENABLED, Property.TSERV_SCAN_MAX_OPENFILES, Property.MASTER_CLIENTPORT, Property.GC_PORT); http://git-wip-us.apache.org/repos/asf/accumulo/blob/48733ce8/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java -- diff --git a/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java b/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java index b2f5a18..07aaf9f 100644 --- a/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java +++ b/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java @@ -17,6 +17,8 @@ package org.apache.accumulo.core.conf; import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; @@ -45,12 +47,31 @@ public class SiteConfiguration extends AccumuloConfiguration { private static SiteConfiguration instance = null; private static Configuration xmlConfig; + private final Map staticConfigs; /** * Not for consumers. Call {@link SiteConfiguration#getInstance(AccumuloConfiguration)} instead */ SiteConfiguration(AccumuloConfiguration parent) { this.parent = parent; +/* + * Make a read-only copy of static configs so we can avoid lock contention on the Hadoop Configuration object + */ +final Configuration conf = getXmlConfig(); +Map temp = new HashMap<>((int) (Math.
[3/3] accumulo git commit: ACCUMULO-4458 addendum for 1.8
ACCUMULO-4458 addendum for 1.8 - Adds an additional property to HOT_PATH_PROPERTIES based on use in 1.8 tablet server. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/923c87e2 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/923c87e2 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/923c87e2 Branch: refs/heads/1.8 Commit: 923c87e2fbdfd49409e8099293cc87371da8f6fd Parents: 1bf8a63 Author: Sean Busbey Authored: Thu Sep 15 08:47:24 2016 -0700 Committer: Sean Busbey Committed: Thu Sep 15 08:47:24 2016 -0700 -- core/src/main/java/org/apache/accumulo/core/conf/Property.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/923c87e2/core/src/main/java/org/apache/accumulo/core/conf/Property.java -- diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java index 34ef953..ede1c6f 100644 --- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java +++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java @@ -830,7 +830,8 @@ public enum Property { */ public static final EnumSet HOT_PATH_PROPERTIES = EnumSet.of(Property.TSERV_CLIENT_TIMEOUT, Property.TSERV_TOTAL_MUTATION_QUEUE_MAX, Property.TSERV_ARCHIVE_WALOGS, Property.GC_TRASH_IGNORE, Property.TSERV_MAJC_DELAY, Property.TABLE_MINC_LOGS_MAX, Property.TSERV_MAJC_MAXCONCURRENT, - Property.REPLICATION_WORKER_THREADS, Property.TABLE_DURABILITY, Property.INSTANCE_ZK_TIMEOUT, Property.TABLE_CLASSPATH); + Property.REPLICATION_WORKER_THREADS, Property.TABLE_DURABILITY, Property.INSTANCE_ZK_TIMEOUT, Property.TABLE_CLASSPATH, + Property.MASTER_METADATA_SUSPENDABLE); private static final EnumSet fixedProperties = EnumSet.of(Property.TSERV_CLIENTPORT, Property.TSERV_NATIVEMAP_ENABLED, Property.TSERV_SCAN_MAX_OPENFILES, Property.MASTER_CLIENTPORT, Property.GC_PORT);
[2/3] accumulo git commit: Merge branch '1.7' into 1.8
Merge branch '1.7' into 1.8 Conflicts: server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/1bf8a636 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/1bf8a636 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/1bf8a636 Branch: refs/heads/1.8 Commit: 1bf8a6362c685f316d04facb26faa29939cad9ef Parents: 8c85c32 48733ce Author: Sean Busbey Authored: Thu Sep 15 08:41:13 2016 -0700 Committer: Sean Busbey Committed: Thu Sep 15 08:41:13 2016 -0700 -- .../org/apache/accumulo/core/conf/Property.java | 7 .../accumulo/core/conf/SiteConfiguration.java | 24 - .../accumulo/server/client/HdfsZooInstance.java | 36 +--- 3 files changed, 53 insertions(+), 14 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/1bf8a636/core/src/main/java/org/apache/accumulo/core/conf/Property.java -- diff --cc core/src/main/java/org/apache/accumulo/core/conf/Property.java index cdaebef,96adaeb..34ef953 --- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java +++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java @@@ -822,9 -800,16 +822,16 @@@ public enum Property return validTableProperties.contains(key) || key.startsWith(Property.TABLE_CONSTRAINT_PREFIX.getKey()) || key.startsWith(Property.TABLE_ITERATOR_PREFIX.getKey()) || key.startsWith(Property.TABLE_LOCALITY_GROUP_PREFIX.getKey()) || key.startsWith(Property.TABLE_COMPACTION_STRATEGY_PREFIX.getKey()) || key.startsWith(Property.TABLE_REPLICATION_TARGET.getKey()) -|| key.startsWith(Property.TABLE_ARBITRARY_PROP_PREFIX.getKey()); +|| key.startsWith(Property.TABLE_ARBITRARY_PROP_PREFIX.getKey()) || key.startsWith(TABLE_SAMPLER_OPTS.getKey()); } + /** +* Properties we check the value of within the TabletServer request handling or maintenance processing loops. +*/ + public static final EnumSet HOT_PATH_PROPERTIES = EnumSet.of(Property.TSERV_CLIENT_TIMEOUT, Property.TSERV_TOTAL_MUTATION_QUEUE_MAX, + Property.TSERV_ARCHIVE_WALOGS, Property.GC_TRASH_IGNORE, Property.TSERV_MAJC_DELAY, Property.TABLE_MINC_LOGS_MAX, Property.TSERV_MAJC_MAXCONCURRENT, + Property.REPLICATION_WORKER_THREADS, Property.TABLE_DURABILITY, Property.INSTANCE_ZK_TIMEOUT, Property.TABLE_CLASSPATH); + private static final EnumSet fixedProperties = EnumSet.of(Property.TSERV_CLIENTPORT, Property.TSERV_NATIVEMAP_ENABLED, Property.TSERV_SCAN_MAX_OPENFILES, Property.MASTER_CLIENTPORT, Property.GC_PORT); http://git-wip-us.apache.org/repos/asf/accumulo/blob/1bf8a636/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java -- diff --cc server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java index e4e73d2,2dacf61..49bf1f4 --- a/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java +++ b/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java @@@ -23,7 -23,7 +23,8 @@@ import java.nio.ByteBuffer import java.util.Collections; import java.util.List; import java.util.UUID; +import java.util.concurrent.TimeUnit; + import java.util.concurrent.atomic.AtomicReference; import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.AccumuloException; @@@ -76,9 -76,9 +76,9 @@@ public class HdfsZooInstance implement return cachedHdfsZooInstance; } - private static ZooCache zooCache; + private final ZooCache zooCache; private static String instanceId = null; - private static final Logger log = Logger.getLogger(HdfsZooInstance.class); + private static final Logger log = LoggerFactory.getLogger(HdfsZooInstance.class); @Override public String getRootTabletLocation() {
[1/8] accumulo git commit: ACCUMULO-4467 Ensure commons-math is on the classpath for RandomWalk MR jobs.
Repository: accumulo Updated Branches: refs/heads/1.7 dc53d51dc -> 985651b48 refs/heads/1.8 080a3ba5b -> 6a677adaf refs/heads/master b919a5cfe -> 0bf8f8e1c ACCUMULO-4467 Ensure commons-math is on the classpath for RandomWalk MR jobs. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/985651b4 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/985651b4 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/985651b4 Branch: refs/heads/1.7 Commit: 985651b48451025db8a1bd00e79730f9d10ded6f Parents: dc53d51 Author: Sean Busbey Authored: Wed Sep 21 14:44:56 2016 -0500 Committer: Sean Busbey Committed: Thu Sep 22 12:58:26 2016 -0500 -- test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/985651b4/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java -- diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java b/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java index 6df5aed..f8cb0b9 100644 --- a/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java +++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java @@ -91,7 +91,7 @@ public abstract class Node { } File libdir = new File(acuHome + "/lib"); -for (String jar : "accumulo-core accumulo-server-base accumulo-fate accumulo-trace libthrift htrace-core".split(" ")) { +for (String jar : "accumulo-core accumulo-server-base accumulo-fate accumulo-trace commons-math libthrift htrace-core".split(" ")) { retval += String.format(",%s/%s.jar", libdir.getAbsolutePath(), jar); }
[4/8] accumulo git commit: Merge branch '1.7' into 1.8
Merge branch '1.7' into 1.8 -sours merge, since 1.7 and 1.8 have different classpath needs for ACCUMULO-4467 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/7ab49808 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/7ab49808 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/7ab49808 Branch: refs/heads/1.8 Commit: 7ab498087b5cf6ed032e8d9d72f31830198ee37a Parents: 080a3ba 985651b Author: Sean Busbey Authored: Thu Sep 22 12:59:14 2016 -0500 Committer: Sean Busbey Committed: Thu Sep 22 12:59:20 2016 -0500 -- --
[5/8] accumulo git commit: Merge branch '1.7' into 1.8
Merge branch '1.7' into 1.8 -sours merge, since 1.7 and 1.8 have different classpath needs for ACCUMULO-4467 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/7ab49808 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/7ab49808 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/7ab49808 Branch: refs/heads/master Commit: 7ab498087b5cf6ed032e8d9d72f31830198ee37a Parents: 080a3ba 985651b Author: Sean Busbey Authored: Thu Sep 22 12:59:14 2016 -0500 Committer: Sean Busbey Committed: Thu Sep 22 12:59:20 2016 -0500 -- --
[8/8] accumulo git commit: Merge branch '1.8'
Merge branch '1.8' Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/0bf8f8e1 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/0bf8f8e1 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/0bf8f8e1 Branch: refs/heads/master Commit: 0bf8f8e1c42e1cc0f1487ad96398999c8796a4a3 Parents: b919a5c 6a677ad Author: Sean Busbey Authored: Thu Sep 22 13:02:30 2016 -0500 Committer: Sean Busbey Committed: Thu Sep 22 13:02:30 2016 -0500 -- test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) --
[2/8] accumulo git commit: ACCUMULO-4467 Ensure commons-math is on the classpath for RandomWalk MR jobs.
ACCUMULO-4467 Ensure commons-math is on the classpath for RandomWalk MR jobs. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/985651b4 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/985651b4 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/985651b4 Branch: refs/heads/1.8 Commit: 985651b48451025db8a1bd00e79730f9d10ded6f Parents: dc53d51 Author: Sean Busbey Authored: Wed Sep 21 14:44:56 2016 -0500 Committer: Sean Busbey Committed: Thu Sep 22 12:58:26 2016 -0500 -- test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/985651b4/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java -- diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java b/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java index 6df5aed..f8cb0b9 100644 --- a/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java +++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java @@ -91,7 +91,7 @@ public abstract class Node { } File libdir = new File(acuHome + "/lib"); -for (String jar : "accumulo-core accumulo-server-base accumulo-fate accumulo-trace libthrift htrace-core".split(" ")) { +for (String jar : "accumulo-core accumulo-server-base accumulo-fate accumulo-trace commons-math libthrift htrace-core".split(" ")) { retval += String.format(",%s/%s.jar", libdir.getAbsolutePath(), jar); }
[7/8] accumulo git commit: ACCUMULO-4467 Ensure commons-math3 is on the classpath for RandomWalk MR jobs.
ACCUMULO-4467 Ensure commons-math3 is on the classpath for RandomWalk MR jobs. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/6a677ada Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/6a677ada Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/6a677ada Branch: refs/heads/1.8 Commit: 6a677adafe0fe03a6a331d281c1a27b4854f94e7 Parents: 7ab4980 Author: Sean Busbey Authored: Thu Sep 22 13:01:52 2016 -0500 Committer: Sean Busbey Committed: Thu Sep 22 13:02:01 2016 -0500 -- test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/6a677ada/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java -- diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java b/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java index 6df5aed..282ceeb 100644 --- a/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java +++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java @@ -91,7 +91,7 @@ public abstract class Node { } File libdir = new File(acuHome + "/lib"); -for (String jar : "accumulo-core accumulo-server-base accumulo-fate accumulo-trace libthrift htrace-core".split(" ")) { +for (String jar : "accumulo-core accumulo-server-base accumulo-fate accumulo-trace commons-math3 libthrift htrace-core".split(" ")) { retval += String.format(",%s/%s.jar", libdir.getAbsolutePath(), jar); }
[3/8] accumulo git commit: ACCUMULO-4467 Ensure commons-math is on the classpath for RandomWalk MR jobs.
ACCUMULO-4467 Ensure commons-math is on the classpath for RandomWalk MR jobs. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/985651b4 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/985651b4 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/985651b4 Branch: refs/heads/master Commit: 985651b48451025db8a1bd00e79730f9d10ded6f Parents: dc53d51 Author: Sean Busbey Authored: Wed Sep 21 14:44:56 2016 -0500 Committer: Sean Busbey Committed: Thu Sep 22 12:58:26 2016 -0500 -- test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/985651b4/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java -- diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java b/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java index 6df5aed..f8cb0b9 100644 --- a/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java +++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java @@ -91,7 +91,7 @@ public abstract class Node { } File libdir = new File(acuHome + "/lib"); -for (String jar : "accumulo-core accumulo-server-base accumulo-fate accumulo-trace libthrift htrace-core".split(" ")) { +for (String jar : "accumulo-core accumulo-server-base accumulo-fate accumulo-trace commons-math libthrift htrace-core".split(" ")) { retval += String.format(",%s/%s.jar", libdir.getAbsolutePath(), jar); }
[6/8] accumulo git commit: ACCUMULO-4467 Ensure commons-math3 is on the classpath for RandomWalk MR jobs.
ACCUMULO-4467 Ensure commons-math3 is on the classpath for RandomWalk MR jobs. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/6a677ada Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/6a677ada Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/6a677ada Branch: refs/heads/master Commit: 6a677adafe0fe03a6a331d281c1a27b4854f94e7 Parents: 7ab4980 Author: Sean Busbey Authored: Thu Sep 22 13:01:52 2016 -0500 Committer: Sean Busbey Committed: Thu Sep 22 13:02:01 2016 -0500 -- test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/6a677ada/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java -- diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java b/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java index 6df5aed..282ceeb 100644 --- a/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java +++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java @@ -91,7 +91,7 @@ public abstract class Node { } File libdir = new File(acuHome + "/lib"); -for (String jar : "accumulo-core accumulo-server-base accumulo-fate accumulo-trace libthrift htrace-core".split(" ")) { +for (String jar : "accumulo-core accumulo-server-base accumulo-fate accumulo-trace commons-math3 libthrift htrace-core".split(" ")) { retval += String.format(",%s/%s.jar", libdir.getAbsolutePath(), jar); }
[6/6] accumulo git commit: Merge branch '1.8'
Merge branch '1.8' Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/6823140e Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/6823140e Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/6823140e Branch: refs/heads/master Commit: 6823140eb5f3fded282eb02daf8d3f6f9d80c3b4 Parents: ababa81 38606a8 Author: Sean Busbey Authored: Mon Sep 26 11:05:38 2016 -0500 Committer: Sean Busbey Committed: Mon Sep 26 11:05:38 2016 -0500 -- docs/src/main/asciidoc/chapters/replication.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/6823140e/docs/src/main/asciidoc/chapters/replication.txt --
[5/6] accumulo git commit: Merge branch '1.7' into 1.8
Merge branch '1.7' into 1.8 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/38606a86 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/38606a86 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/38606a86 Branch: refs/heads/1.8 Commit: 38606a86e66830c0f5e8083311c062af4a5a42e4 Parents: c1dda77 e76f3a2 Author: Sean Busbey Authored: Mon Sep 26 10:55:12 2016 -0500 Committer: Sean Busbey Committed: Mon Sep 26 10:55:12 2016 -0500 -- docs/src/main/asciidoc/chapters/replication.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) --
[2/6] accumulo git commit: ACCUMULO-4471 Typo in user manual replication instructions
ACCUMULO-4471 Typo in user manual replication instructions Signed-off-by: Sean Busbey Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/e76f3a23 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/e76f3a23 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/e76f3a23 Branch: refs/heads/1.8 Commit: e76f3a236a8c91553974a8f1981d838de2933ba2 Parents: 064ae0a Author: Dima Spivak Authored: Fri Sep 23 20:29:15 2016 -0700 Committer: Sean Busbey Committed: Mon Sep 26 10:48:34 2016 -0500 -- docs/src/main/asciidoc/chapters/replication.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/e76f3a23/docs/src/main/asciidoc/chapters/replication.txt -- diff --git a/docs/src/main/asciidoc/chapters/replication.txt b/docs/src/main/asciidoc/chapters/replication.txt index ad48cd2..82fcf44 100644 --- a/docs/src/main/asciidoc/chapters/replication.txt +++ b/docs/src/main/asciidoc/chapters/replication.txt @@ -121,10 +121,10 @@ cluster, this is a table ID. In this example, we want to enable replication on +my_table+ and configure our peer +accumulo_peer+ as a target, sending the data to the table with an ID of +2+ in +accumulo_peer+. -\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim} + root@accumulo_primary> config -t my_table -s table.replication=true -root@accumulo_primary> config -t my_table -s table.replication.target.acccumulo_peer=2 -\end{verbatim}\endgroup +root@accumulo_primary> config -t my_table -s table.replication.target.accumulo_peer=2 + To replicate a single table on the primary to multiple peers, the second command in the above shell snippet can be issued, for each peer and remote identifier pair.
[1/6] accumulo git commit: ACCUMULO-4471 Typo in user manual replication instructions
Repository: accumulo Updated Branches: refs/heads/1.7 064ae0aa7 -> e76f3a236 refs/heads/1.8 c1dda7786 -> 38606a86e refs/heads/master ababa81c3 -> 6823140eb ACCUMULO-4471 Typo in user manual replication instructions Signed-off-by: Sean Busbey Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/e76f3a23 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/e76f3a23 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/e76f3a23 Branch: refs/heads/1.7 Commit: e76f3a236a8c91553974a8f1981d838de2933ba2 Parents: 064ae0a Author: Dima Spivak Authored: Fri Sep 23 20:29:15 2016 -0700 Committer: Sean Busbey Committed: Mon Sep 26 10:48:34 2016 -0500 -- docs/src/main/asciidoc/chapters/replication.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/e76f3a23/docs/src/main/asciidoc/chapters/replication.txt -- diff --git a/docs/src/main/asciidoc/chapters/replication.txt b/docs/src/main/asciidoc/chapters/replication.txt index ad48cd2..82fcf44 100644 --- a/docs/src/main/asciidoc/chapters/replication.txt +++ b/docs/src/main/asciidoc/chapters/replication.txt @@ -121,10 +121,10 @@ cluster, this is a table ID. In this example, we want to enable replication on +my_table+ and configure our peer +accumulo_peer+ as a target, sending the data to the table with an ID of +2+ in +accumulo_peer+. -\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim} + root@accumulo_primary> config -t my_table -s table.replication=true -root@accumulo_primary> config -t my_table -s table.replication.target.acccumulo_peer=2 -\end{verbatim}\endgroup +root@accumulo_primary> config -t my_table -s table.replication.target.accumulo_peer=2 + To replicate a single table on the primary to multiple peers, the second command in the above shell snippet can be issued, for each peer and remote identifier pair.
[3/6] accumulo git commit: ACCUMULO-4471 Typo in user manual replication instructions
ACCUMULO-4471 Typo in user manual replication instructions Signed-off-by: Sean Busbey Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/e76f3a23 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/e76f3a23 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/e76f3a23 Branch: refs/heads/master Commit: e76f3a236a8c91553974a8f1981d838de2933ba2 Parents: 064ae0a Author: Dima Spivak Authored: Fri Sep 23 20:29:15 2016 -0700 Committer: Sean Busbey Committed: Mon Sep 26 10:48:34 2016 -0500 -- docs/src/main/asciidoc/chapters/replication.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/e76f3a23/docs/src/main/asciidoc/chapters/replication.txt -- diff --git a/docs/src/main/asciidoc/chapters/replication.txt b/docs/src/main/asciidoc/chapters/replication.txt index ad48cd2..82fcf44 100644 --- a/docs/src/main/asciidoc/chapters/replication.txt +++ b/docs/src/main/asciidoc/chapters/replication.txt @@ -121,10 +121,10 @@ cluster, this is a table ID. In this example, we want to enable replication on +my_table+ and configure our peer +accumulo_peer+ as a target, sending the data to the table with an ID of +2+ in +accumulo_peer+. -\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim} + root@accumulo_primary> config -t my_table -s table.replication=true -root@accumulo_primary> config -t my_table -s table.replication.target.acccumulo_peer=2 -\end{verbatim}\endgroup +root@accumulo_primary> config -t my_table -s table.replication.target.accumulo_peer=2 + To replicate a single table on the primary to multiple peers, the second command in the above shell snippet can be issued, for each peer and remote identifier pair.
[4/6] accumulo git commit: Merge branch '1.7' into 1.8
Merge branch '1.7' into 1.8 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/38606a86 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/38606a86 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/38606a86 Branch: refs/heads/master Commit: 38606a86e66830c0f5e8083311c062af4a5a42e4 Parents: c1dda77 e76f3a2 Author: Sean Busbey Authored: Mon Sep 26 10:55:12 2016 -0500 Committer: Sean Busbey Committed: Mon Sep 26 10:55:12 2016 -0500 -- docs/src/main/asciidoc/chapters/replication.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) --
[3/6] accumulo git commit: ACCUMULO-4488 update kerberos section in the user manual to fill some gaps.
ACCUMULO-4488 update kerberos section in the user manual to fill some gaps. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/a17390db Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/a17390db Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/a17390db Branch: refs/heads/master Commit: a17390db11c69dea0c7f1fbf75c9b28f8e1d561c Parents: 52f53f5 Author: Sean Busbey Authored: Thu Oct 6 11:43:50 2016 -0500 Committer: Sean Busbey Committed: Fri Oct 7 00:22:11 2016 -0500 -- docs/src/main/asciidoc/chapters/kerberos.txt | 104 +- 1 file changed, 101 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/a17390db/docs/src/main/asciidoc/chapters/kerberos.txt -- diff --git a/docs/src/main/asciidoc/chapters/kerberos.txt b/docs/src/main/asciidoc/chapters/kerberos.txt index a165221..bfb7cb7 100644 --- a/docs/src/main/asciidoc/chapters/kerberos.txt +++ b/docs/src/main/asciidoc/chapters/kerberos.txt @@ -102,6 +102,10 @@ numerous guidelines already exist on the subject of configuring Hadoop and ZooKe use with Kerberos and won't be covered here. It is assumed that you have functional Hadoop and ZooKeeper already installed. +Note that on an existing cluster the server side changes will require a full cluster shutdown and restart. You should +wait to restart the TraceServers until after you've completed the rest of the cluster set up and provisioned +a trace user with appropriate permissions. + Servers The first step is to obtain a Kerberos identity for the Accumulo server processes. @@ -190,6 +194,10 @@ creating a SystemToken from the provided keytab and principal, we can still use keytab/principal to serialize traces. Like non-Kerberized instances, the table must be created and permissions granted to the trace.user. The same +_HOST+ replacement is performed on this value, substituted the FQDN for +_HOST+. +|trace.token.property.keytab || +You can optionally specify the path to a keytab file for the principal given in the +trace.user+ property. If you don't +set this path, it will default to the value given in +general.kerberos.principal+. + |general.delegation.token.lifetime |7d | The length of time that the server-side secret used to create delegation tokens is valid. After a server-side secret expires, a delegation token created with that secret is no longer valid. @@ -233,7 +241,7 @@ The KDC is still the authoritative entity for user management. The previously me are provided as they simplify management of users within Accumulo, especially with respect to granting Authorizations and Permissions to new users. -= Accumulo Initialization += Administrative User Out of the box (without Kerberos enabled), Accumulo has a single user with administrative permissions "root". This users is used to "bootstrap" other users, creating less-privileged users for applications using @@ -246,6 +254,24 @@ enabled, Accumulo will prompt for the name of a user to grant the same permissio user would normally have. The name of the Accumulo user to grant administrative permissions to can also be given by the `-u` or `--user` options. +If you are enabling Kerberos on an existing cluster, you will need to reinitialize the security system in +order to replace the existing "root" user with one that can be used with Kerberos. These steps should be +completed after you have done the previously described configuration changes and will require access to +a complete +accumulo-site.xml+, including the instance secret. Note that this process will delete all +existing users in the system; you will need to reassign user permissions based on Kerberos principals. + +1. Ensure Accumulo is not running. +2. Given the path to a +accumulo-site.xml+ with the instance secret, run the security reset tool. If you are +prompted for a password you can just hit return, since it won't be used. + +$ ACCUMULO_CONF_DIR=/path/to/server/conf/ accumulo init --reset-security +Running against secured HDFS +Principal (user) to grant administrative privileges to : acculumo_ad...@example.com +Enter initial password for accumulo_ad...@example.com (this may not be applicable for your security setup): +Confirm initial password for accumulo_ad...@example.com: + +3. Start the Accumulo cluster + = Verifying secure access To verify that servers have correctly started with Kerberos enabled, ensure that the processes @@ -369,18 +395,55 @@ Valid starting Expires Service principal = Configuration The second thing clients need to do is to set up their client config
[1/6] accumulo git commit: ACCUMULO-4488 update kerberos section in the user manual to fill some gaps.
Repository: accumulo Updated Branches: refs/heads/1.7 52f53f56e -> a17390db1 refs/heads/1.8 f7989c678 -> 91269bf31 refs/heads/master 1c218de97 -> 9dddeed88 ACCUMULO-4488 update kerberos section in the user manual to fill some gaps. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/a17390db Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/a17390db Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/a17390db Branch: refs/heads/1.7 Commit: a17390db11c69dea0c7f1fbf75c9b28f8e1d561c Parents: 52f53f5 Author: Sean Busbey Authored: Thu Oct 6 11:43:50 2016 -0500 Committer: Sean Busbey Committed: Fri Oct 7 00:22:11 2016 -0500 -- docs/src/main/asciidoc/chapters/kerberos.txt | 104 +- 1 file changed, 101 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/a17390db/docs/src/main/asciidoc/chapters/kerberos.txt -- diff --git a/docs/src/main/asciidoc/chapters/kerberos.txt b/docs/src/main/asciidoc/chapters/kerberos.txt index a165221..bfb7cb7 100644 --- a/docs/src/main/asciidoc/chapters/kerberos.txt +++ b/docs/src/main/asciidoc/chapters/kerberos.txt @@ -102,6 +102,10 @@ numerous guidelines already exist on the subject of configuring Hadoop and ZooKe use with Kerberos and won't be covered here. It is assumed that you have functional Hadoop and ZooKeeper already installed. +Note that on an existing cluster the server side changes will require a full cluster shutdown and restart. You should +wait to restart the TraceServers until after you've completed the rest of the cluster set up and provisioned +a trace user with appropriate permissions. + Servers The first step is to obtain a Kerberos identity for the Accumulo server processes. @@ -190,6 +194,10 @@ creating a SystemToken from the provided keytab and principal, we can still use keytab/principal to serialize traces. Like non-Kerberized instances, the table must be created and permissions granted to the trace.user. The same +_HOST+ replacement is performed on this value, substituted the FQDN for +_HOST+. +|trace.token.property.keytab || +You can optionally specify the path to a keytab file for the principal given in the +trace.user+ property. If you don't +set this path, it will default to the value given in +general.kerberos.principal+. + |general.delegation.token.lifetime |7d | The length of time that the server-side secret used to create delegation tokens is valid. After a server-side secret expires, a delegation token created with that secret is no longer valid. @@ -233,7 +241,7 @@ The KDC is still the authoritative entity for user management. The previously me are provided as they simplify management of users within Accumulo, especially with respect to granting Authorizations and Permissions to new users. -= Accumulo Initialization += Administrative User Out of the box (without Kerberos enabled), Accumulo has a single user with administrative permissions "root". This users is used to "bootstrap" other users, creating less-privileged users for applications using @@ -246,6 +254,24 @@ enabled, Accumulo will prompt for the name of a user to grant the same permissio user would normally have. The name of the Accumulo user to grant administrative permissions to can also be given by the `-u` or `--user` options. +If you are enabling Kerberos on an existing cluster, you will need to reinitialize the security system in +order to replace the existing "root" user with one that can be used with Kerberos. These steps should be +completed after you have done the previously described configuration changes and will require access to +a complete +accumulo-site.xml+, including the instance secret. Note that this process will delete all +existing users in the system; you will need to reassign user permissions based on Kerberos principals. + +1. Ensure Accumulo is not running. +2. Given the path to a +accumulo-site.xml+ with the instance secret, run the security reset tool. If you are +prompted for a password you can just hit return, since it won't be used. + +$ ACCUMULO_CONF_DIR=/path/to/server/conf/ accumulo init --reset-security +Running against secured HDFS +Principal (user) to grant administrative privileges to : acculumo_ad...@example.com +Enter initial password for accumulo_ad...@example.com (this may not be applicable for your security setup): +Confirm initial password for accumulo_ad...@example.com: + +3. Start the Accumulo cluster + = Verifying secure access To verify that servers have correctly started with Kerberos enabled, ensure that the processes @@ -369,18 +
[4/6] accumulo git commit: Merge branch '1.7' into 1.8
Merge branch '1.7' into 1.8 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/91269bf3 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/91269bf3 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/91269bf3 Branch: refs/heads/master Commit: 91269bf318586d31756fc3aecc1d42a20e602f76 Parents: f7989c6 a17390d Author: Sean Busbey Authored: Fri Oct 7 00:26:10 2016 -0500 Committer: Sean Busbey Committed: Fri Oct 7 00:26:10 2016 -0500 -- docs/src/main/asciidoc/chapters/kerberos.txt | 104 +- 1 file changed, 101 insertions(+), 3 deletions(-) --
[5/6] accumulo git commit: Merge branch '1.7' into 1.8
Merge branch '1.7' into 1.8 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/91269bf3 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/91269bf3 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/91269bf3 Branch: refs/heads/1.8 Commit: 91269bf318586d31756fc3aecc1d42a20e602f76 Parents: f7989c6 a17390d Author: Sean Busbey Authored: Fri Oct 7 00:26:10 2016 -0500 Committer: Sean Busbey Committed: Fri Oct 7 00:26:10 2016 -0500 -- docs/src/main/asciidoc/chapters/kerberos.txt | 104 +- 1 file changed, 101 insertions(+), 3 deletions(-) --
[6/6] accumulo git commit: Merge branch '1.8'
Merge branch '1.8' Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9dddeed8 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9dddeed8 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9dddeed8 Branch: refs/heads/master Commit: 9dddeed885d8ae4c6ba34eca561dcdb747131d68 Parents: 1c218de 91269bf Author: Sean Busbey Authored: Fri Oct 7 00:28:21 2016 -0500 Committer: Sean Busbey Committed: Fri Oct 7 00:28:21 2016 -0500 -- docs/src/main/asciidoc/chapters/kerberos.txt | 104 +- 1 file changed, 101 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/9dddeed8/docs/src/main/asciidoc/chapters/kerberos.txt -- diff --cc docs/src/main/asciidoc/chapters/kerberos.txt index 7817c8d,bfb7cb7..48553b7 --- a/docs/src/main/asciidoc/chapters/kerberos.txt +++ b/docs/src/main/asciidoc/chapters/kerberos.txt @@@ -558,5 -621,40 +621,40 @@@ Caused by: org.apache.thrift.transport. on, is extremely important when negotiating an SASL connection. This problem commonly arises when the Accumulo servers are not configured to listen on the address denoted by their FQDN. -The values in the Accumulo "hosts" files (In +$ACCUMULO_CONF_DIR+: +masters+, +monitors+, +slaves+, +tracers+, +The values in the Accumulo "hosts" files (In +$ACCUMULO_CONF_DIR+: +masters+, +monitors+, +tservers+, +tracers+, and +gc+) should match the instance componentof the Kerberos server principal (e.g. +host+ in +accumulo/h...@example.com+). + + *Q*: After configuring my system for Kerberos, server processes come up normally and I can interact with the system. However, + when I attempt to use the "Recent Traces" page on the Monitor UI I get a stacktrace similar to: + + + java.lang.AssertionError: AuthenticationToken should not be null +at org.apache.accumulo.monitor.servlets.trace.Basic.getScanner(Basic.java:139) + at org.apache.accumulo.monitor.servlets.trace.Summary.pageBody(Summary.java:164) + at org.apache.accumulo.monitor.servlets.BasicServlet.doGet(BasicServlet.java:63) +at javax.servlet.http.HttpServlet.service(HttpServlet.java:687) +at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) + at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:738) + at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:551) + at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143) +at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:568) + at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:221) + at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:) + at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:478) + at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:183) + at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1045) + at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) + at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97) + at org.eclipse.jetty.server.Server.handle(Server.java:462) + at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:279) +at org.eclipse.jetty.server.HttpConne
[2/6] accumulo git commit: ACCUMULO-4488 update kerberos section in the user manual to fill some gaps.
ACCUMULO-4488 update kerberos section in the user manual to fill some gaps. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/a17390db Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/a17390db Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/a17390db Branch: refs/heads/1.8 Commit: a17390db11c69dea0c7f1fbf75c9b28f8e1d561c Parents: 52f53f5 Author: Sean Busbey Authored: Thu Oct 6 11:43:50 2016 -0500 Committer: Sean Busbey Committed: Fri Oct 7 00:22:11 2016 -0500 -- docs/src/main/asciidoc/chapters/kerberos.txt | 104 +- 1 file changed, 101 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/a17390db/docs/src/main/asciidoc/chapters/kerberos.txt -- diff --git a/docs/src/main/asciidoc/chapters/kerberos.txt b/docs/src/main/asciidoc/chapters/kerberos.txt index a165221..bfb7cb7 100644 --- a/docs/src/main/asciidoc/chapters/kerberos.txt +++ b/docs/src/main/asciidoc/chapters/kerberos.txt @@ -102,6 +102,10 @@ numerous guidelines already exist on the subject of configuring Hadoop and ZooKe use with Kerberos and won't be covered here. It is assumed that you have functional Hadoop and ZooKeeper already installed. +Note that on an existing cluster the server side changes will require a full cluster shutdown and restart. You should +wait to restart the TraceServers until after you've completed the rest of the cluster set up and provisioned +a trace user with appropriate permissions. + Servers The first step is to obtain a Kerberos identity for the Accumulo server processes. @@ -190,6 +194,10 @@ creating a SystemToken from the provided keytab and principal, we can still use keytab/principal to serialize traces. Like non-Kerberized instances, the table must be created and permissions granted to the trace.user. The same +_HOST+ replacement is performed on this value, substituted the FQDN for +_HOST+. +|trace.token.property.keytab || +You can optionally specify the path to a keytab file for the principal given in the +trace.user+ property. If you don't +set this path, it will default to the value given in +general.kerberos.principal+. + |general.delegation.token.lifetime |7d | The length of time that the server-side secret used to create delegation tokens is valid. After a server-side secret expires, a delegation token created with that secret is no longer valid. @@ -233,7 +241,7 @@ The KDC is still the authoritative entity for user management. The previously me are provided as they simplify management of users within Accumulo, especially with respect to granting Authorizations and Permissions to new users. -= Accumulo Initialization += Administrative User Out of the box (without Kerberos enabled), Accumulo has a single user with administrative permissions "root". This users is used to "bootstrap" other users, creating less-privileged users for applications using @@ -246,6 +254,24 @@ enabled, Accumulo will prompt for the name of a user to grant the same permissio user would normally have. The name of the Accumulo user to grant administrative permissions to can also be given by the `-u` or `--user` options. +If you are enabling Kerberos on an existing cluster, you will need to reinitialize the security system in +order to replace the existing "root" user with one that can be used with Kerberos. These steps should be +completed after you have done the previously described configuration changes and will require access to +a complete +accumulo-site.xml+, including the instance secret. Note that this process will delete all +existing users in the system; you will need to reassign user permissions based on Kerberos principals. + +1. Ensure Accumulo is not running. +2. Given the path to a +accumulo-site.xml+ with the instance secret, run the security reset tool. If you are +prompted for a password you can just hit return, since it won't be used. + +$ ACCUMULO_CONF_DIR=/path/to/server/conf/ accumulo init --reset-security +Running against secured HDFS +Principal (user) to grant administrative privileges to : acculumo_ad...@example.com +Enter initial password for accumulo_ad...@example.com (this may not be applicable for your security setup): +Confirm initial password for accumulo_ad...@example.com: + +3. Start the Accumulo cluster + = Verifying secure access To verify that servers have correctly started with Kerberos enabled, ensure that the processes @@ -369,18 +395,55 @@ Valid starting Expires Service principal = Configuration The second thing clients need to do is to set up their client config
[2/2] accumulo git commit: Jekyll build from gh-pages:af8b0e5
Jekyll build from gh-pages:af8b0e5 update 1.7 and 1.8 user manuals with recent changes Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/fc21741f Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/fc21741f Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/fc21741f Branch: refs/heads/asf-site Commit: fc21741fa75d4d2fe2b56b01ad58d6ce1e4bd4aa Parents: a26a875 Author: Sean Busbey Authored: Fri Oct 7 01:18:09 2016 -0500 Committer: Sean Busbey Committed: Fri Oct 7 01:42:53 2016 -0500 -- 1.7/accumulo_user_manual.html | 229 +++-- 1.8/accumulo_user_manual.html | 137 +- feed.xml | 4 +- 3 files changed, 329 insertions(+), 41 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/fc21741f/1.7/accumulo_user_manual.html -- diff --git a/1.7/accumulo_user_manual.html b/1.7/accumulo_user_manual.html index 9f34918..a4f4213 100644 --- a/1.7/accumulo_user_manual.html +++ b/1.7/accumulo_user_manual.html @@ -6,7 +6,7 @@ -Apache Accumulo User Manual Version 1.7 +Apache Accumulo® User Manual Version 1.7 https://fonts.googleapis.com/css?family=Open+Sans:300,300italic,400,400italic,600,600italic%7CNoto+Serif:400,400italic,700,700italic%7CDroid+Sans+Mono:400";>
[1/2] accumulo git commit: update 1.7 and 1.8 user manuals with recent changes
Repository: accumulo Updated Branches: refs/heads/asf-site a26a875ef -> fc21741fa refs/heads/gh-pages d1a74a8de -> af8b0e58a update 1.7 and 1.8 user manuals with recent changes Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/af8b0e58 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/af8b0e58 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/af8b0e58 Branch: refs/heads/gh-pages Commit: af8b0e58a9ef146c5d522b5ebf09d60a627de507 Parents: d1a74a8 Author: Sean Busbey Authored: Fri Oct 7 01:18:09 2016 -0500 Committer: Sean Busbey Committed: Fri Oct 7 01:42:36 2016 -0500 -- 1.7/accumulo_user_manual.html | 229 +++-- 1.8/accumulo_user_manual.html | 137 +- 2 files changed, 327 insertions(+), 39 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/af8b0e58/1.7/accumulo_user_manual.html -- diff --git a/1.7/accumulo_user_manual.html b/1.7/accumulo_user_manual.html index 9f34918..a4f4213 100644 --- a/1.7/accumulo_user_manual.html +++ b/1.7/accumulo_user_manual.html @@ -6,7 +6,7 @@ -Apache Accumulo User Manual Version 1.7 +Apache Accumulo® User Manual Version 1.7 https://fonts.googleapis.com/css?family=Open+Sans:300,300italic,400,400italic,600,600italic%7CNoto+Serif:400,400italic,700,700italic%7CDroid+Sans+Mono:400";>
[3/6] accumulo git commit: ACCUMULO-4489 Monitor login for access to trace table should fall back to general kerberos keytab when trace keytab is not set.
ACCUMULO-4489 Monitor login for access to trace table should fall back to general kerberos keytab when trace keytab is not set. Signed-off-by: Josh Elser Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/0d41ec6e Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/0d41ec6e Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/0d41ec6e Branch: refs/heads/master Commit: 0d41ec6e64f8c661b476ba74e1670aabd21d85b2 Parents: 9e236f2 Author: Sean Busbey Authored: Thu Oct 6 09:21:59 2016 -0500 Committer: Sean Busbey Committed: Mon Oct 17 14:41:32 2016 -0700 -- .../java/org/apache/accumulo/monitor/servlets/trace/Basic.java| 3 +++ 1 file changed, 3 insertions(+) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/0d41ec6e/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java -- diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java index 2143766..1839abe 100644 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java @@ -83,6 +83,9 @@ abstract class Basic extends BasicServlet { Map loginMap = conf.getAllPropertiesWithPrefix(Property.TRACE_TOKEN_PROPERTY_PREFIX); // May be null String keytab = loginMap.get(Property.TRACE_TOKEN_PROPERTY_PREFIX.getKey() + "keytab"); +if (keytab == null || keytab.length() == 0) { + keytab = conf.getPath(Property.GENERAL_KERBEROS_KEYTAB); +} if (saslEnabled && null != keytab) { principal = SecurityUtil.getServerPrincipal(conf.get(Property.TRACE_USER));
[5/6] accumulo git commit: Merge branch '1.7' into 1.8
Merge branch '1.7' into 1.8 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/c7431092 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/c7431092 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/c7431092 Branch: refs/heads/1.8 Commit: c7431092e132e070c27e464b473100552255ba45 Parents: 9c11f72 0d41ec6 Author: Sean Busbey Authored: Mon Oct 17 14:41:50 2016 -0700 Committer: Sean Busbey Committed: Mon Oct 17 14:41:50 2016 -0700 -- .../java/org/apache/accumulo/monitor/servlets/trace/Basic.java| 3 +++ 1 file changed, 3 insertions(+) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/c7431092/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java --
[1/6] accumulo git commit: ACCUMULO-4489 Monitor login for access to trace table should fall back to general kerberos keytab when trace keytab is not set.
Repository: accumulo Updated Branches: refs/heads/1.7 9e236f289 -> 0d41ec6e6 refs/heads/1.8 9c11f72e3 -> c7431092e refs/heads/master 3d11cab03 -> c4749ac35 ACCUMULO-4489 Monitor login for access to trace table should fall back to general kerberos keytab when trace keytab is not set. Signed-off-by: Josh Elser Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/0d41ec6e Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/0d41ec6e Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/0d41ec6e Branch: refs/heads/1.7 Commit: 0d41ec6e64f8c661b476ba74e1670aabd21d85b2 Parents: 9e236f2 Author: Sean Busbey Authored: Thu Oct 6 09:21:59 2016 -0500 Committer: Sean Busbey Committed: Mon Oct 17 14:41:32 2016 -0700 -- .../java/org/apache/accumulo/monitor/servlets/trace/Basic.java| 3 +++ 1 file changed, 3 insertions(+) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/0d41ec6e/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java -- diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java index 2143766..1839abe 100644 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java @@ -83,6 +83,9 @@ abstract class Basic extends BasicServlet { Map loginMap = conf.getAllPropertiesWithPrefix(Property.TRACE_TOKEN_PROPERTY_PREFIX); // May be null String keytab = loginMap.get(Property.TRACE_TOKEN_PROPERTY_PREFIX.getKey() + "keytab"); +if (keytab == null || keytab.length() == 0) { + keytab = conf.getPath(Property.GENERAL_KERBEROS_KEYTAB); +} if (saslEnabled && null != keytab) { principal = SecurityUtil.getServerPrincipal(conf.get(Property.TRACE_USER));
[4/6] accumulo git commit: Merge branch '1.7' into 1.8
Merge branch '1.7' into 1.8 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/c7431092 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/c7431092 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/c7431092 Branch: refs/heads/master Commit: c7431092e132e070c27e464b473100552255ba45 Parents: 9c11f72 0d41ec6 Author: Sean Busbey Authored: Mon Oct 17 14:41:50 2016 -0700 Committer: Sean Busbey Committed: Mon Oct 17 14:41:50 2016 -0700 -- .../java/org/apache/accumulo/monitor/servlets/trace/Basic.java| 3 +++ 1 file changed, 3 insertions(+) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/c7431092/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java --
[6/6] accumulo git commit: Merge branch '1.8'
Merge branch '1.8' Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/c4749ac3 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/c4749ac3 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/c4749ac3 Branch: refs/heads/master Commit: c4749ac35840813b11d3c386136b9ee257f19bd1 Parents: 3d11cab c743109 Author: Sean Busbey Authored: Mon Oct 17 14:42:22 2016 -0700 Committer: Sean Busbey Committed: Mon Oct 17 14:42:22 2016 -0700 -- .../java/org/apache/accumulo/monitor/servlets/trace/Basic.java| 3 +++ 1 file changed, 3 insertions(+) --
[2/6] accumulo git commit: ACCUMULO-4489 Monitor login for access to trace table should fall back to general kerberos keytab when trace keytab is not set.
ACCUMULO-4489 Monitor login for access to trace table should fall back to general kerberos keytab when trace keytab is not set. Signed-off-by: Josh Elser Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/0d41ec6e Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/0d41ec6e Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/0d41ec6e Branch: refs/heads/1.8 Commit: 0d41ec6e64f8c661b476ba74e1670aabd21d85b2 Parents: 9e236f2 Author: Sean Busbey Authored: Thu Oct 6 09:21:59 2016 -0500 Committer: Sean Busbey Committed: Mon Oct 17 14:41:32 2016 -0700 -- .../java/org/apache/accumulo/monitor/servlets/trace/Basic.java| 3 +++ 1 file changed, 3 insertions(+) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/0d41ec6e/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java -- diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java index 2143766..1839abe 100644 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java @@ -83,6 +83,9 @@ abstract class Basic extends BasicServlet { Map loginMap = conf.getAllPropertiesWithPrefix(Property.TRACE_TOKEN_PROPERTY_PREFIX); // May be null String keytab = loginMap.get(Property.TRACE_TOKEN_PROPERTY_PREFIX.getKey() + "keytab"); +if (keytab == null || keytab.length() == 0) { + keytab = conf.getPath(Property.GENERAL_KERBEROS_KEYTAB); +} if (saslEnabled && null != keytab) { principal = SecurityUtil.getServerPrincipal(conf.get(Property.TRACE_USER));
[3/6] accumulo git commit: ACCUMULO-4533 TraceServer shouldn't abort given problems with trace table checks.
ACCUMULO-4533 TraceServer shouldn't abort given problems with trace table checks. Signed-off-by: Mike Drob Signed-off-by: Josh Elser Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/47b57f73 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/47b57f73 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/47b57f73 Branch: refs/heads/master Commit: 47b57f7300ae5d1df9536cf07d99c83bfb8e2af6 Parents: 3c45441 Author: Sean Busbey Authored: Wed Dec 7 10:45:51 2016 -0600 Committer: Sean Busbey Committed: Fri Dec 9 15:21:54 2016 -0600 -- .../org/apache/accumulo/tracer/TraceServer.java | 50 ++-- 1 file changed, 35 insertions(+), 15 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/47b57f73/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java -- diff --git a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java index dbb593d..67bd9d5 100644 --- a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java +++ b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java @@ -28,12 +28,16 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import org.apache.accumulo.core.Constants; +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.BatchWriter; import org.apache.accumulo.core.client.BatchWriterConfig; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.Instance; import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.client.MutationsRejectedException; +import org.apache.accumulo.core.client.TableExistsException; +import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Properties; import org.apache.accumulo.core.client.security.tokens.KerberosToken; @@ -182,6 +186,35 @@ public class TraceServer implements Watcher { log.info("Instance " + serverConfiguration.getInstance().getInstanceID()); AccumuloConfiguration conf = serverConfiguration.getConfiguration(); table = conf.get(Property.TRACE_TABLE); +connector = ensureTraceTableExists(conf); + +int port = conf.getPort(Property.TRACE_PORT); +final ServerSocket sock = ServerSocketChannel.open().socket(); +sock.setReuseAddress(true); +sock.bind(new InetSocketAddress(hostname, port)); +final TServerTransport transport = new TServerSocket(sock); +TThreadPoolServer.Args options = new TThreadPoolServer.Args(transport); +options.processor(new Processor(new Receiver())); +server = new TThreadPoolServer(options); +registerInZooKeeper(sock.getInetAddress().getHostAddress() + ":" + sock.getLocalPort(), conf.get(Property.TRACE_ZK_PATH)); +writer = new AtomicReference<>(this.connector.createBatchWriter(table, new BatchWriterConfig().setMaxLatency(BATCH_WRITER_MAX_LATENCY, TimeUnit.SECONDS))); + } + + /** + * Exceptions thrown out of here should be things that cause service failure (e.g. misconfigurations that aren't likely to change on retry). + * + * @return a working Connection that can be reused + * @throws ClassNotFoundException + * if TRACE_TOKEN_TYPE is set to a class that we can't load. + * @throws InstantiationException + * if we fail to create an instance of TRACE_TOKEN_TYPE. + * @throws IllegalAccessException + * if the class pointed to by TRACE_TOKEN_TYPE is private. + * @throws AccumuloSecurityException + * if the trace user has the wrong permissions + */ + private Connector ensureTraceTableExists(final AccumuloConfiguration conf) throws AccumuloSecurityException, ClassNotFoundException, InstantiationException, + IllegalAccessException { Connector connector = null; while (true) { try { @@ -221,25 +254,12 @@ public class TraceServer implements Watcher { } connector.tableOperations().setProperty(table, Property.TABLE_FORMATTER_CLASS.getKey(), TraceFormatter.class.getName()); break; - } catch (RuntimeException ex) { + } catch (AccumuloException | TableExistsException | TableNotFoundException | IOException | RuntimeException ex) { log.info("Waiting to checking/create the trace table.", ex); UtilWaitThread.sleep(1000); } } -this.connector = connecto
[5/6] accumulo git commit: Merge branch '1.7' into 1.8
Merge branch '1.7' into 1.8 Conflicts: server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/037c1384 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/037c1384 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/037c1384 Branch: refs/heads/1.8 Commit: 037c1384ac861be95d8c9d1c5d1acbf61e40e2fe Parents: 7c7bbab 47b57f7 Author: Sean Busbey Authored: Fri Dec 9 15:33:37 2016 -0600 Committer: Sean Busbey Committed: Fri Dec 9 15:33:37 2016 -0600 -- .../org/apache/accumulo/tracer/TraceServer.java | 76 1 file changed, 48 insertions(+), 28 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/037c1384/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java -- diff --cc server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java index 2b6bcbf,67bd9d5..7c0d9b2 --- a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java +++ b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java @@@ -183,7 -185,36 +187,49 @@@ public class TraceServer implements Wat log.info("Version " + Constants.VERSION); log.info("Instance " + serverConfiguration.getInstance().getInstanceID()); AccumuloConfiguration conf = serverConfiguration.getConfiguration(); -table = conf.get(Property.TRACE_TABLE); +tableName = conf.get(Property.TRACE_TABLE); + connector = ensureTraceTableExists(conf); + -int port = conf.getPort(Property.TRACE_PORT); -final ServerSocket sock = ServerSocketChannel.open().socket(); -sock.setReuseAddress(true); -sock.bind(new InetSocketAddress(hostname, port)); ++int ports[] = conf.getPort(Property.TRACE_PORT); ++ServerSocket sock = null; ++for (int port : ports) { ++ ServerSocket s = ServerSocketChannel.open().socket(); ++ s.setReuseAddress(true); ++ try { ++s.bind(new InetSocketAddress(hostname, port)); ++sock = s; ++break; ++ } catch (Exception e) { ++log.warn("Unable to start trace server on port {}", port); ++ } ++} ++if (null == sock) { ++ throw new RuntimeException("Unable to start trace server on configured ports: " + Arrays.toString(ports)); ++} + final TServerTransport transport = new TServerSocket(sock); + TThreadPoolServer.Args options = new TThreadPoolServer.Args(transport); + options.processor(new Processor(new Receiver())); + server = new TThreadPoolServer(options); + registerInZooKeeper(sock.getInetAddress().getHostAddress() + ":" + sock.getLocalPort(), conf.get(Property.TRACE_ZK_PATH)); -writer = new AtomicReference<>(this.connector.createBatchWriter(table, new BatchWriterConfig().setMaxLatency(BATCH_WRITER_MAX_LATENCY, TimeUnit.SECONDS))); ++writer = new AtomicReference<>(this.connector.createBatchWriter(tableName, ++new BatchWriterConfig().setMaxLatency(BATCH_WRITER_MAX_LATENCY, TimeUnit.SECONDS))); + } + + /** +* Exceptions thrown out of here should be things that cause service failure (e.g. misconfigurations that aren't likely to change on retry). +* +* @return a working Connection that can be reused +* @throws ClassNotFoundException +* if TRACE_TOKEN_TYPE is set to a class that we can't load. +* @throws InstantiationException +* if we fail to create an instance of TRACE_TOKEN_TYPE. +* @throws IllegalAccessException +* if the class pointed to by TRACE_TOKEN_TYPE is private. +* @throws AccumuloSecurityException +* if the trace user has the wrong permissions +*/ + private Connector ensureTraceTableExists(final AccumuloConfiguration conf) throws AccumuloSecurityException, ClassNotFoundException, InstantiationException, + IllegalAccessException { Connector connector = null; while (true) { try { @@@ -215,46 -246,20 +261,20 @@@ } connector = serverConfiguration.getInstance().getConnector(principal, at); -if (!connector.tableOperations().exists(table)) { - connector.tableOperations().create(table); +if (!connector.tableOperations().exists(tableName)) { + connector.tableOperations().create(tableName); IteratorSetting setting = new IteratorSetting(10, "ageoff", AgeOffFilter.class.getName()); AgeOffFilter.setTTL(setting, 7 * 24 * 60 * 60 * 1000l); - connector.tableOperations().attachIterator(table, setting); + connector.tableOperations().attachItera
[1/6] accumulo git commit: ACCUMULO-4533 TraceServer shouldn't abort given problems with trace table checks.
Repository: accumulo Updated Branches: refs/heads/1.7 3c4544144 -> 47b57f730 refs/heads/1.8 7c7bbabb8 -> 037c1384a refs/heads/master 8e0f19a1c -> 3dc9b373d ACCUMULO-4533 TraceServer shouldn't abort given problems with trace table checks. Signed-off-by: Mike Drob Signed-off-by: Josh Elser Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/47b57f73 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/47b57f73 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/47b57f73 Branch: refs/heads/1.7 Commit: 47b57f7300ae5d1df9536cf07d99c83bfb8e2af6 Parents: 3c45441 Author: Sean Busbey Authored: Wed Dec 7 10:45:51 2016 -0600 Committer: Sean Busbey Committed: Fri Dec 9 15:21:54 2016 -0600 -- .../org/apache/accumulo/tracer/TraceServer.java | 50 ++-- 1 file changed, 35 insertions(+), 15 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/47b57f73/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java -- diff --git a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java index dbb593d..67bd9d5 100644 --- a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java +++ b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java @@ -28,12 +28,16 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import org.apache.accumulo.core.Constants; +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.BatchWriter; import org.apache.accumulo.core.client.BatchWriterConfig; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.Instance; import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.client.MutationsRejectedException; +import org.apache.accumulo.core.client.TableExistsException; +import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Properties; import org.apache.accumulo.core.client.security.tokens.KerberosToken; @@ -182,6 +186,35 @@ public class TraceServer implements Watcher { log.info("Instance " + serverConfiguration.getInstance().getInstanceID()); AccumuloConfiguration conf = serverConfiguration.getConfiguration(); table = conf.get(Property.TRACE_TABLE); +connector = ensureTraceTableExists(conf); + +int port = conf.getPort(Property.TRACE_PORT); +final ServerSocket sock = ServerSocketChannel.open().socket(); +sock.setReuseAddress(true); +sock.bind(new InetSocketAddress(hostname, port)); +final TServerTransport transport = new TServerSocket(sock); +TThreadPoolServer.Args options = new TThreadPoolServer.Args(transport); +options.processor(new Processor(new Receiver())); +server = new TThreadPoolServer(options); +registerInZooKeeper(sock.getInetAddress().getHostAddress() + ":" + sock.getLocalPort(), conf.get(Property.TRACE_ZK_PATH)); +writer = new AtomicReference<>(this.connector.createBatchWriter(table, new BatchWriterConfig().setMaxLatency(BATCH_WRITER_MAX_LATENCY, TimeUnit.SECONDS))); + } + + /** + * Exceptions thrown out of here should be things that cause service failure (e.g. misconfigurations that aren't likely to change on retry). + * + * @return a working Connection that can be reused + * @throws ClassNotFoundException + * if TRACE_TOKEN_TYPE is set to a class that we can't load. + * @throws InstantiationException + * if we fail to create an instance of TRACE_TOKEN_TYPE. + * @throws IllegalAccessException + * if the class pointed to by TRACE_TOKEN_TYPE is private. + * @throws AccumuloSecurityException + * if the trace user has the wrong permissions + */ + private Connector ensureTraceTableExists(final AccumuloConfiguration conf) throws AccumuloSecurityException, ClassNotFoundException, InstantiationException, + IllegalAccessException { Connector connector = null; while (true) { try { @@ -221,25 +254,12 @@ public class TraceServer implements Watcher { } connector.tableOperations().setProperty(table, Property.TABLE_FORMATTER_CLASS.getKey(), TraceFormatter.class.getName()); break; - } catch (RuntimeException ex) { + } catch (AccumuloException | TableExistsException | TableNotFoundException | IOException | RuntimeException e
[6/6] accumulo git commit: Merge branch '1.8'
Merge branch '1.8' Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/3dc9b373 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/3dc9b373 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/3dc9b373 Branch: refs/heads/master Commit: 3dc9b373d0fb9a78efefefc640e0e395e42ce1c7 Parents: 8e0f19a 037c138 Author: Sean Busbey Authored: Fri Dec 9 15:39:14 2016 -0600 Committer: Sean Busbey Committed: Fri Dec 9 15:39:14 2016 -0600 -- .../org/apache/accumulo/tracer/TraceServer.java | 76 1 file changed, 48 insertions(+), 28 deletions(-) --
[4/6] accumulo git commit: Merge branch '1.7' into 1.8
Merge branch '1.7' into 1.8 Conflicts: server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/037c1384 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/037c1384 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/037c1384 Branch: refs/heads/master Commit: 037c1384ac861be95d8c9d1c5d1acbf61e40e2fe Parents: 7c7bbab 47b57f7 Author: Sean Busbey Authored: Fri Dec 9 15:33:37 2016 -0600 Committer: Sean Busbey Committed: Fri Dec 9 15:33:37 2016 -0600 -- .../org/apache/accumulo/tracer/TraceServer.java | 76 1 file changed, 48 insertions(+), 28 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/037c1384/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java -- diff --cc server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java index 2b6bcbf,67bd9d5..7c0d9b2 --- a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java +++ b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java @@@ -183,7 -185,36 +187,49 @@@ public class TraceServer implements Wat log.info("Version " + Constants.VERSION); log.info("Instance " + serverConfiguration.getInstance().getInstanceID()); AccumuloConfiguration conf = serverConfiguration.getConfiguration(); -table = conf.get(Property.TRACE_TABLE); +tableName = conf.get(Property.TRACE_TABLE); + connector = ensureTraceTableExists(conf); + -int port = conf.getPort(Property.TRACE_PORT); -final ServerSocket sock = ServerSocketChannel.open().socket(); -sock.setReuseAddress(true); -sock.bind(new InetSocketAddress(hostname, port)); ++int ports[] = conf.getPort(Property.TRACE_PORT); ++ServerSocket sock = null; ++for (int port : ports) { ++ ServerSocket s = ServerSocketChannel.open().socket(); ++ s.setReuseAddress(true); ++ try { ++s.bind(new InetSocketAddress(hostname, port)); ++sock = s; ++break; ++ } catch (Exception e) { ++log.warn("Unable to start trace server on port {}", port); ++ } ++} ++if (null == sock) { ++ throw new RuntimeException("Unable to start trace server on configured ports: " + Arrays.toString(ports)); ++} + final TServerTransport transport = new TServerSocket(sock); + TThreadPoolServer.Args options = new TThreadPoolServer.Args(transport); + options.processor(new Processor(new Receiver())); + server = new TThreadPoolServer(options); + registerInZooKeeper(sock.getInetAddress().getHostAddress() + ":" + sock.getLocalPort(), conf.get(Property.TRACE_ZK_PATH)); -writer = new AtomicReference<>(this.connector.createBatchWriter(table, new BatchWriterConfig().setMaxLatency(BATCH_WRITER_MAX_LATENCY, TimeUnit.SECONDS))); ++writer = new AtomicReference<>(this.connector.createBatchWriter(tableName, ++new BatchWriterConfig().setMaxLatency(BATCH_WRITER_MAX_LATENCY, TimeUnit.SECONDS))); + } + + /** +* Exceptions thrown out of here should be things that cause service failure (e.g. misconfigurations that aren't likely to change on retry). +* +* @return a working Connection that can be reused +* @throws ClassNotFoundException +* if TRACE_TOKEN_TYPE is set to a class that we can't load. +* @throws InstantiationException +* if we fail to create an instance of TRACE_TOKEN_TYPE. +* @throws IllegalAccessException +* if the class pointed to by TRACE_TOKEN_TYPE is private. +* @throws AccumuloSecurityException +* if the trace user has the wrong permissions +*/ + private Connector ensureTraceTableExists(final AccumuloConfiguration conf) throws AccumuloSecurityException, ClassNotFoundException, InstantiationException, + IllegalAccessException { Connector connector = null; while (true) { try { @@@ -215,46 -246,20 +261,20 @@@ } connector = serverConfiguration.getInstance().getConnector(principal, at); -if (!connector.tableOperations().exists(table)) { - connector.tableOperations().create(table); +if (!connector.tableOperations().exists(tableName)) { + connector.tableOperations().create(tableName); IteratorSetting setting = new IteratorSetting(10, "ageoff", AgeOffFilter.class.getName()); AgeOffFilter.setTTL(setting, 7 * 24 * 60 * 60 * 1000l); - connector.tableOperations().attachIterator(table, setting); + connector.tableOperations().attachItera
[2/6] accumulo git commit: ACCUMULO-4533 TraceServer shouldn't abort given problems with trace table checks.
ACCUMULO-4533 TraceServer shouldn't abort given problems with trace table checks. Signed-off-by: Mike Drob Signed-off-by: Josh Elser Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/47b57f73 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/47b57f73 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/47b57f73 Branch: refs/heads/1.8 Commit: 47b57f7300ae5d1df9536cf07d99c83bfb8e2af6 Parents: 3c45441 Author: Sean Busbey Authored: Wed Dec 7 10:45:51 2016 -0600 Committer: Sean Busbey Committed: Fri Dec 9 15:21:54 2016 -0600 -- .../org/apache/accumulo/tracer/TraceServer.java | 50 ++-- 1 file changed, 35 insertions(+), 15 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/47b57f73/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java -- diff --git a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java index dbb593d..67bd9d5 100644 --- a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java +++ b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java @@ -28,12 +28,16 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import org.apache.accumulo.core.Constants; +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.BatchWriter; import org.apache.accumulo.core.client.BatchWriterConfig; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.Instance; import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.client.MutationsRejectedException; +import org.apache.accumulo.core.client.TableExistsException; +import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Properties; import org.apache.accumulo.core.client.security.tokens.KerberosToken; @@ -182,6 +186,35 @@ public class TraceServer implements Watcher { log.info("Instance " + serverConfiguration.getInstance().getInstanceID()); AccumuloConfiguration conf = serverConfiguration.getConfiguration(); table = conf.get(Property.TRACE_TABLE); +connector = ensureTraceTableExists(conf); + +int port = conf.getPort(Property.TRACE_PORT); +final ServerSocket sock = ServerSocketChannel.open().socket(); +sock.setReuseAddress(true); +sock.bind(new InetSocketAddress(hostname, port)); +final TServerTransport transport = new TServerSocket(sock); +TThreadPoolServer.Args options = new TThreadPoolServer.Args(transport); +options.processor(new Processor(new Receiver())); +server = new TThreadPoolServer(options); +registerInZooKeeper(sock.getInetAddress().getHostAddress() + ":" + sock.getLocalPort(), conf.get(Property.TRACE_ZK_PATH)); +writer = new AtomicReference<>(this.connector.createBatchWriter(table, new BatchWriterConfig().setMaxLatency(BATCH_WRITER_MAX_LATENCY, TimeUnit.SECONDS))); + } + + /** + * Exceptions thrown out of here should be things that cause service failure (e.g. misconfigurations that aren't likely to change on retry). + * + * @return a working Connection that can be reused + * @throws ClassNotFoundException + * if TRACE_TOKEN_TYPE is set to a class that we can't load. + * @throws InstantiationException + * if we fail to create an instance of TRACE_TOKEN_TYPE. + * @throws IllegalAccessException + * if the class pointed to by TRACE_TOKEN_TYPE is private. + * @throws AccumuloSecurityException + * if the trace user has the wrong permissions + */ + private Connector ensureTraceTableExists(final AccumuloConfiguration conf) throws AccumuloSecurityException, ClassNotFoundException, InstantiationException, + IllegalAccessException { Connector connector = null; while (true) { try { @@ -221,25 +254,12 @@ public class TraceServer implements Watcher { } connector.tableOperations().setProperty(table, Property.TABLE_FORMATTER_CLASS.getKey(), TraceFormatter.class.getName()); break; - } catch (RuntimeException ex) { + } catch (AccumuloException | TableExistsException | TableNotFoundException | IOException | RuntimeException ex) { log.info("Waiting to checking/create the trace table.", ex); UtilWaitThread.sleep(1000); } } -this.connector = connector; -
svn commit: r1558294 - /accumulo/site/trunk/content/people.mdtext
Author: busbey Date: Wed Jan 15 05:10:16 2014 New Revision: 1558294 URL: http://svn.apache.org/r1558294 Log: Moved Sean Busbey from contributors to committers Modified: accumulo/site/trunk/content/people.mdtext Modified: accumulo/site/trunk/content/people.mdtext URL: http://svn.apache.org/viewvc/accumulo/site/trunk/content/people.mdtext?rev=1558294&r1=1558293&r2=1558294&view=diff == --- accumulo/site/trunk/content/people.mdtext (original) +++ accumulo/site/trunk/content/people.mdtext Wed Jan 15 05:10:16 2014 @@ -65,6 +65,7 @@ PMC and Committers elserjJosh Elserhttp://hortonworks.com/>HortonworksET (http://www.timeanddate.com/library/abbreviations/timezones/na/est.html>-5 / http://www.timeanddate.com/library/abbreviations/timezones/na/edt.html>-4) kturnerKeith Turnerhttp://www.ptech-llc.com/>Peterson TechnologiesET (http://www.timeanddate.com/library/abbreviations/timezones/na/est.html>-5 / http://www.timeanddate.com/library/abbreviations/timezones/na/edt.html>-4) mdrobMike Drobhttp://www.cloudera.com";>ClouderaET (http://www.timeanddate.com/library/abbreviations/timezones/na/est.html>-5 / http://www.timeanddate.com/library/abbreviations/timezones/na/edt.html>-4) + busbeySean Busbeyhttp://www.cloudera.com";>ClouderaCT (http://www.timeanddate.com/library/abbreviations/timezones/na/cst.html>-6 / http://www.timeanddate.com/library/abbreviations/timezones/na/cdt.html>-5) ujustgotbilldWilliam Slacumhttp://www.koverse.com";>KoverseET (http://www.timeanddate.com/library/abbreviations/timezones/na/est.html>-5 / http://www.timeanddate.com/library/abbreviations/timezones/na/edt.html>-4) @@ -113,8 +114,7 @@ Contributors Ryan Fishelhttp://www.cloudera.com";>Cloudera Ryan Leary Sapah Shah - Scott Kuehn - Sean Busbeyhttp://www.cloudera.com";>ClouderaCT (http://www.timeanddate.com/library/abbreviations/timezones/na/cst.html>-6 / http://www.timeanddate.com/library/abbreviations/timezones/na/cdt.html>-5) + Scott Kuehn Sean Hickey stevelSteve Loughranhttp://hortonworks.com/>Hortonworks(http://www.timeanddate.com/library/abbreviations/timezones/eu/gmt.html>0 / http://www.timeanddate.com/library/abbreviations/timezones/eu/bst.html>+1) supunSupun Kamburugamuva
svn commit: r894217 - in /websites/production/accumulo/content: ./ 1.4/apidocs/ 1.5/apidocs/
Author: busbey Date: Wed Jan 15 05:13:46 2014 New Revision: 894217 Log: Moved Sean Busbey from contributors to committers Added: websites/production/accumulo/content/ - copied from r894216, websites/staging/accumulo/trunk/content/ websites/production/accumulo/content/1.4/apidocs/ - copied from r894216, websites/production/accumulo/content/1.4/apidocs/ websites/production/accumulo/content/1.5/apidocs/ - copied from r894216, websites/production/accumulo/content/1.5/apidocs/
svn commit: r1558295 - /accumulo/site/trunk/content/governance/releasing.mdtext
Author: busbey Date: Wed Jan 15 05:27:48 2014 New Revision: 1558295 URL: http://svn.apache.org/r1558295 Log: ACCUMULO-1797 Updates release guide to include collecting information about testing. clarifies some test details. Modified: accumulo/site/trunk/content/governance/releasing.mdtext Modified: accumulo/site/trunk/content/governance/releasing.mdtext URL: http://svn.apache.org/viewvc/accumulo/site/trunk/content/governance/releasing.mdtext?rev=1558295&r1=1558294&r2=1558295&view=diff == --- accumulo/site/trunk/content/governance/releasing.mdtext (original) +++ accumulo/site/trunk/content/governance/releasing.mdtext Wed Jan 15 05:27:48 2014 @@ -47,12 +47,30 @@ Note: There will be times where API comp ## Testing +Testing for an Accumulo release includes a few steps that a developer may take without a Hadoop cluster and several that require a working cluster. + +### Stand alone +The following steps can be taken without having an underlying cluster. They SHOULD be handled with each Hadoop profile available for a given release version. To activate an alternative profile specify e.g. "-Dhadoop.profile=2" for the Hadoop 2 profile on the Maven commandline. Some older versions of Accumulo referred to Hadoop profiles diferently; see the README that came with said versions for details on building against different Hadoop versions. + 1. All JUnit tests must pass. This should be a requirement of any patch so it should never be an issue of the codebase. - - All functional tests must complete successfully. +- Use "mvn package" to run against the default profile of a particular release +- Use "mvn -Dhadoop.profile=2 package" to test against the Hadoop 2 profile on e.g. 1.4 or 1.5 +- Use "mvn -Dhadoop.profile=1 package" to test against the Hadoop 1 profile on e.g. 1.6 or later - Analyze output of static analysis tools like Findbugs and PMD. - - Two 24-hour periods of the randomwalk LongClean test with and without agitation need to be run successfully on a cluster. - - Two 24-hour periods of continuous ingest with and without agitation need to be validated successfully on a cluster. - - Two 72-hour periods of continuous ingest with and without agitation on a cluster. No validation is necessary but the cluster should be checked to ensure it is still functional. + - For versions 1.6 and later, all functional tests must pass via the Maven failsafe plugin. +- Use "mvn verify" to run against the default profile of a particular release +- Use "mvn -Dhadoop.profile=1 verify" to run the functional tests against the Hadoop 1 profile + +### Cluster based +The following tests require a Hadoop cluster running a minimum of HDFS, MapReduce, and ZooKeeper. The cluster MAY have any number of worker nodes; it can even be a single node in psuedo-distributed mode. A cluster with multiple tablet servers SHOULD be used so that more of the code base will be exercised. For the purposes of release testing, you should note the number of nodes and versions used. See the Releasing section for more details. + + 1. For versions prior to 1.6, all functional tests must complete successfully. +- See $ACCUMULO_HOME/test/system/auto/README for details on running the functional tests. + - Two 24-hour periods of the randomwalk LongClean test with and without agitation need to be run successfully. +- See $ACCUMULO_HOME/test/system/randomwalk/README for details on running the LongClean module. + - Two 24-hour periods of continuous ingest with and without agitation need to be validated successfully. +- See $ACCUMULO_HOME/test/system/continuous/README for details on running and verifying the continous ingest test. + - Two 72-hour periods of continuous ingest with and without agitation. No validation is necessary but the cluster should be checked to ensure it is still functional. ## Releasing @@ -61,5 +79,6 @@ Note: There will be times where API comp - Be fully built, including a tar.gz of the entire project as well as the documentation. - PGP Signatures of the tarball must be signed to a separate file and made available in the public_html folder of the user creating the release on people.apache.org, along with the tarball and MD5 and SHA512 checksums. - A vote must be made on dev@accumulo. Lazy consensus is not sufficient for a release; at least 3 +1 votes from PMC members are required. All checksums and signatures need to be verified before any voter can +1 it. Voting shall last 72 hours. +- Voters SHOULD include with their vote details on the tests from the testing section they have successfully run. If given, said details for each test MUST include: the number of worker nodes in the cluster, the operating system and version, the Hadoop version, and the Zookeeper version. All such gather
svn commit: r894220 - in /websites/production/accumulo/content: ./ 1.4/apidocs/ 1.5/apidocs/
Author: busbey Date: Wed Jan 15 05:31:44 2014 New Revision: 894220 Log: Publishing svnmucc operation to accumulo site by busbey Added: websites/production/accumulo/content/ - copied from r894219, websites/staging/accumulo/trunk/content/ websites/production/accumulo/content/1.4/apidocs/ - copied from r894219, websites/production/accumulo/content/1.4/apidocs/ websites/production/accumulo/content/1.5/apidocs/ - copied from r894219, websites/production/accumulo/content/1.5/apidocs/
[6/6] git commit: Merge branch '1.6.0-SNAPSHOT'
Merge branch '1.6.0-SNAPSHOT' Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/66cccbe4 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/66cccbe4 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/66cccbe4 Branch: refs/heads/master Commit: 66cccbe48febd25b94cf0e9d6aa73be030139e9f Parents: 7ecb50a 3931cce Author: Sean Busbey Authored: Tue Jan 21 12:17:15 2014 -0600 Committer: Sean Busbey Committed: Tue Jan 21 12:17:15 2014 -0600 -- .../apache/accumulo/test/continuous/ContinuousVerify.java | 10 +- 1 file changed, 5 insertions(+), 5 deletions(-) --
[2/6] git commit: ACCUMULO-2230 Make sure offline scan uses the cloned table.
ACCUMULO-2230 Make sure offline scan uses the cloned table. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/a4384884 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/a4384884 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/a4384884 Branch: refs/heads/1.6.0-SNAPSHOT Commit: a43848849e78aa9bc6edb5d6d87e2a285d8fccf2 Parents: 78c5f46 Author: Sean Busbey Authored: Mon Jan 20 23:30:01 2014 -0600 Committer: Sean Busbey Committed: Tue Jan 21 11:52:14 2014 -0600 -- .../apache/accumulo/test/continuous/ContinuousVerify.java | 10 +- 1 file changed, 5 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/a4384884/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java -- diff --git a/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java b/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java index 1ace774..08d8ff1 100644 --- a/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java +++ b/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java @@ -183,6 +183,9 @@ public class ContinuousVerify extends Configured implements Tool { Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis()); job.setJarByClass(this.getClass()); +job.setInputFormatClass(AccumuloInputFormat.class); +opts.setAccumuloConfigs(job); + String clone = opts.getTableName(); Connector conn = null; if (opts.scanOffline) { @@ -191,13 +194,10 @@ public class ContinuousVerify extends Configured implements Tool { conn = opts.getConnector(); conn.tableOperations().clone(opts.getTableName(), clone, true, new HashMap(), new HashSet()); conn.tableOperations().offline(clone); + AccumuloInputFormat.setInputTableName(job, clone); + AccumuloInputFormat.setOfflineTableScan(job, true); } -job.setInputFormatClass(AccumuloInputFormat.class); - -opts.setAccumuloConfigs(job); -AccumuloInputFormat.setOfflineTableScan(job, opts.scanOffline); - // set up ranges try { Set ranges = opts.getConnector().tableOperations().splitRangeByTablets(opts.getTableName(), new Range(), opts.maxMaps);
[4/6] git commit: Merge branch '1.5.1-SNAPSHOT' into 1.6.0-SNAPSHOT
Merge branch '1.5.1-SNAPSHOT' into 1.6.0-SNAPSHOT Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/3931cceb Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/3931cceb Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/3931cceb Branch: refs/heads/master Commit: 3931cceb1731c835d16674a5c8fa72374bf16dcd Parents: 8cfdc1f a438488 Author: Sean Busbey Authored: Tue Jan 21 12:16:15 2014 -0600 Committer: Sean Busbey Committed: Tue Jan 21 12:16:15 2014 -0600 -- .../apache/accumulo/test/continuous/ContinuousVerify.java | 10 +- 1 file changed, 5 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/3931cceb/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java --
[5/6] git commit: Merge branch '1.5.1-SNAPSHOT' into 1.6.0-SNAPSHOT
Merge branch '1.5.1-SNAPSHOT' into 1.6.0-SNAPSHOT Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/3931cceb Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/3931cceb Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/3931cceb Branch: refs/heads/1.6.0-SNAPSHOT Commit: 3931cceb1731c835d16674a5c8fa72374bf16dcd Parents: 8cfdc1f a438488 Author: Sean Busbey Authored: Tue Jan 21 12:16:15 2014 -0600 Committer: Sean Busbey Committed: Tue Jan 21 12:16:15 2014 -0600 -- .../apache/accumulo/test/continuous/ContinuousVerify.java | 10 +- 1 file changed, 5 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/3931cceb/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java --
[3/6] git commit: ACCUMULO-2230 Make sure offline scan uses the cloned table.
ACCUMULO-2230 Make sure offline scan uses the cloned table. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/a4384884 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/a4384884 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/a4384884 Branch: refs/heads/master Commit: a43848849e78aa9bc6edb5d6d87e2a285d8fccf2 Parents: 78c5f46 Author: Sean Busbey Authored: Mon Jan 20 23:30:01 2014 -0600 Committer: Sean Busbey Committed: Tue Jan 21 11:52:14 2014 -0600 -- .../apache/accumulo/test/continuous/ContinuousVerify.java | 10 +- 1 file changed, 5 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/a4384884/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java -- diff --git a/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java b/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java index 1ace774..08d8ff1 100644 --- a/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java +++ b/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java @@ -183,6 +183,9 @@ public class ContinuousVerify extends Configured implements Tool { Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis()); job.setJarByClass(this.getClass()); +job.setInputFormatClass(AccumuloInputFormat.class); +opts.setAccumuloConfigs(job); + String clone = opts.getTableName(); Connector conn = null; if (opts.scanOffline) { @@ -191,13 +194,10 @@ public class ContinuousVerify extends Configured implements Tool { conn = opts.getConnector(); conn.tableOperations().clone(opts.getTableName(), clone, true, new HashMap(), new HashSet()); conn.tableOperations().offline(clone); + AccumuloInputFormat.setInputTableName(job, clone); + AccumuloInputFormat.setOfflineTableScan(job, true); } -job.setInputFormatClass(AccumuloInputFormat.class); - -opts.setAccumuloConfigs(job); -AccumuloInputFormat.setOfflineTableScan(job, opts.scanOffline); - // set up ranges try { Set ranges = opts.getConnector().tableOperations().splitRangeByTablets(opts.getTableName(), new Range(), opts.maxMaps);
[1/6] git commit: ACCUMULO-2230 Make sure offline scan uses the cloned table.
Updated Branches: refs/heads/1.5.1-SNAPSHOT 78c5f460b -> a43848849 refs/heads/1.6.0-SNAPSHOT 8cfdc1ffc -> 3931cceb1 refs/heads/master 7ecb50a97 -> 66cccbe48 ACCUMULO-2230 Make sure offline scan uses the cloned table. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/a4384884 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/a4384884 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/a4384884 Branch: refs/heads/1.5.1-SNAPSHOT Commit: a43848849e78aa9bc6edb5d6d87e2a285d8fccf2 Parents: 78c5f46 Author: Sean Busbey Authored: Mon Jan 20 23:30:01 2014 -0600 Committer: Sean Busbey Committed: Tue Jan 21 11:52:14 2014 -0600 -- .../apache/accumulo/test/continuous/ContinuousVerify.java | 10 +- 1 file changed, 5 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/a4384884/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java -- diff --git a/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java b/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java index 1ace774..08d8ff1 100644 --- a/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java +++ b/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java @@ -183,6 +183,9 @@ public class ContinuousVerify extends Configured implements Tool { Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis()); job.setJarByClass(this.getClass()); +job.setInputFormatClass(AccumuloInputFormat.class); +opts.setAccumuloConfigs(job); + String clone = opts.getTableName(); Connector conn = null; if (opts.scanOffline) { @@ -191,13 +194,10 @@ public class ContinuousVerify extends Configured implements Tool { conn = opts.getConnector(); conn.tableOperations().clone(opts.getTableName(), clone, true, new HashMap(), new HashSet()); conn.tableOperations().offline(clone); + AccumuloInputFormat.setInputTableName(job, clone); + AccumuloInputFormat.setOfflineTableScan(job, true); } -job.setInputFormatClass(AccumuloInputFormat.class); - -opts.setAccumuloConfigs(job); -AccumuloInputFormat.setOfflineTableScan(job, opts.scanOffline); - // set up ranges try { Set ranges = opts.getConnector().tableOperations().splitRangeByTablets(opts.getTableName(), new Range(), opts.maxMaps);
[03/23] git commit: ACCUMULO-2213 Make writer recovery in the Tracer more robust.
ACCUMULO-2213 Make writer recovery in the Tracer more robust. * Cleans up writer reseting in the TraceServer, avoids overly broad catching. * tones down log levels in TraceServer to WARN because trace information is transient and we retry everything. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/dfafd9c1 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/dfafd9c1 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/dfafd9c1 Branch: refs/heads/1.6.0-SNAPSHOT Commit: dfafd9c1104d8359ca1eabe345661c541766d57f Parents: 57f9b6c Author: Sean Busbey Authored: Tue Jan 21 14:05:56 2014 -0600 Committer: Sean Busbey Committed: Wed Jan 22 16:36:34 2014 -0600 -- .../accumulo/server/trace/TraceServer.java | 68 ++-- 1 file changed, 49 insertions(+), 19 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/dfafd9c1/src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java -- diff --git a/src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java b/src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java index 4d89e9c..0e2010a 100644 --- a/src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java +++ b/src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java @@ -19,6 +19,7 @@ package org.apache.accumulo.server.trace; import java.net.InetSocketAddress; import java.net.ServerSocket; import java.nio.channels.ServerSocketChannel; +import java.util.concurrent.atomic.AtomicReference; import java.util.TimerTask; import org.apache.accumulo.cloudtrace.instrument.Span; @@ -27,6 +28,7 @@ import org.apache.accumulo.cloudtrace.thrift.SpanReceiver; import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.BatchWriter; import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.client.MutationsRejectedException; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.Mutation; @@ -64,8 +66,8 @@ public class TraceServer implements Watcher { final private static Logger log = Logger.getLogger(TraceServer.class); final private AccumuloConfiguration conf; final private TServer server; - private BatchWriter writer = null; - private Connector connector; + final private AtomicReference writer; + final private Connector connector; final String table; private static void put(Mutation m, String cf, String cq, byte[] bytes, int len) { @@ -128,16 +130,27 @@ public class TraceServer implements Watcher { put(timeMutation, "id", idString, transport.get(), transport.len()); } try { -if (writer == null) - resetWriter(); -if (writer == null) +final BatchWriter writer = TraceServer.this.writer.get(); +/* Check for null, because we expect spans to come in much faster than flush calls. + In the case of failure, we'd rather avoid logging tons of NPEs. + */ +if (null == writer) { + log.warn("writer is not ready; discarding span."); return; +} writer.addMutation(spanMutation); writer.addMutation(indexMutation); if (timeMutation != null) writer.addMutation(timeMutation); - } catch (Exception ex) { -log.error("Unable to write mutation to table: " + spanMutation, ex); + } catch (MutationsRejectedException exception) { +log.warn("Unable to write mutation to table; discarding span. set log level to DEBUG for span information and stacktrace. cause: " + exception); +if (log.isDebugEnabled()) { + log.debug("discarded span due to rejection of mutation: " + spanMutation, exception); +} + /* XXX this could be e.g. an IllegalArgumentExceptoion if we're trying to write this mutation to a writer that has been closed since we retrieved it */ + } catch (RuntimeException exception) { +log.warn("Unable to write mutation to table; discarding span. set log level to DEBUG for stacktrace. cause: " + exception); +log.debug("unable to write mutation to table due to exception.", exception); } } @@ -147,6 +160,7 @@ public class TraceServer implements Watcher { Accumulo.init("tracer"); conf = ServerConfiguration.getSystemConfiguration(); table = conf.get(Property.TRACE_TABLE); +Connector connector = null; while (true) { try { connector = HdfsZooInstance.getInstance().getConnector(conf.get(Property.TRA
[07/23] git commit: ACCUMULO-2224 Make ZooSession more resiliant in the face of transient DNS issues.
ACCUMULO-2224 Make ZooSession more resiliant in the face of transient DNS issues. * retries if host is not found, up to 2xZK timeout (same as other IOExceptions), rather than bailing on any host name problem. * adds utility method for getting the max time the JVM will cache host failures * add test for said method Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/f42ead0d Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/f42ead0d Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/f42ead0d Branch: refs/heads/1.6.0-SNAPSHOT Commit: f42ead0d39e34578c6fe9636af4cfbd9d91e47a5 Parents: dfafd9c Author: Sean Busbey Authored: Mon Jan 20 14:26:20 2014 -0600 Committer: Sean Busbey Committed: Wed Jan 22 23:12:24 2014 -0600 -- .../apache/accumulo/core/util/AddressUtil.java | 39 +++ .../accumulo/core/zookeeper/ZooSession.java | 11 ++-- .../accumulo/core/util/AddressUtilTest.java | 69 +++- src/core/src/test/resources/log4j.properties| 23 +++ 4 files changed, 137 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/f42ead0d/src/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java -- diff --git a/src/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java b/src/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java index 0b82128..96c2e18 100644 --- a/src/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java +++ b/src/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java @@ -16,12 +16,20 @@ */ package org.apache.accumulo.core.util; +import java.net.InetAddress; // workaround to enable @see/@link hyperlink import java.net.InetSocketAddress; +import java.net.UnknownHostException; +import java.security.Security; import org.apache.hadoop.io.Text; import org.apache.thrift.transport.TSocket; +import org.apache.log4j.Logger; + public class AddressUtil { + + private static final Logger log = Logger.getLogger(AddressUtil.class); + static public InetSocketAddress parseAddress(String address, int defaultPort) throws NumberFormatException { final String[] parts = address.split(":", 2); if (parts.length == 2) { @@ -44,5 +52,36 @@ public class AddressUtil { static public String toString(InetSocketAddress addr) { return addr.getAddress().getHostAddress() + ":" + addr.getPort(); } + + /** + * Fetch the security value that determines how long DNS failures are cached. + * Looks up the security property 'networkaddress.cache.negative.ttl'. Should that fail returns + * the default value used in the Oracle JVM 1.4+, which is 10 seconds. + * + * @param originalException the host lookup that is the source of needing this lookup. maybe be null. + * @return positive integer number of seconds + * @see java.net.InetAddress + * @throws IllegalArgumentException if dns failures are cached forever + */ + static public int getAddressCacheNegativeTtl(UnknownHostException originalException) { +int negativeTtl = 10; +try { + negativeTtl = Integer.parseInt(Security.getProperty("networkaddress.cache.negative.ttl")); +} catch (NumberFormatException exception) { + log.warn("Failed to get JVM negative DNS respones cache TTL due to format problem (e.g. this JVM might not have the " + +"property). Falling back to default based on Oracle JVM 1.6 (10s)", exception); +} catch (SecurityException exception) { + log.warn("Failed to get JVM negative DNS response cache TTL due to security manager. Falling back to default based on Oracle JVM 1.6 (10s)", exception); +} +if (-1 == negativeTtl) { + log.error("JVM negative DNS repsonse cache TTL is set to 'forever' and host lookup failed. TTL can be changed with security property " + +"'networkaddress.cache.negative.ttl', see java.net.InetAddress.", originalException); + throw new IllegalArgumentException(originalException); +} else if (0 > negativeTtl) { + log.warn("JVM specified negative DNS response cache TTL was negative (and not 'forever'). Falling back to default based on Oracle JVM 1.6 (10s)"); + negativeTtl = 10; +} +return negativeTtl; + } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/f42ead0d/src/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooSession.java -- diff --git a/src/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooSession.java b/src/core/src/main/java/org/apache/
[15/23] Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c4cd3b1b/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java -- diff --cc server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java index 0777d03,000..116f134 mode 100644,00..100644 --- a/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java +++ b/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java @@@ -1,292 -1,0 +1,323 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.server.trace; + +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.nio.channels.ServerSocketChannel; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.TimeUnit; ++import java.util.concurrent.atomic.AtomicReference; + +import org.apache.accumulo.core.Constants; +import org.apache.accumulo.core.client.BatchWriter; +import org.apache.accumulo.core.client.BatchWriterConfig; +import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.client.Instance; +import org.apache.accumulo.core.client.IteratorSetting; +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Properties; +import org.apache.accumulo.core.client.security.tokens.PasswordToken; ++import org.apache.accumulo.core.client.MutationsRejectedException; +import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.data.Mutation; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.file.FileUtil; +import org.apache.accumulo.core.iterators.user.AgeOffFilter; +import org.apache.accumulo.core.security.SecurityUtil; +import org.apache.accumulo.core.trace.TraceFormatter; +import org.apache.accumulo.core.util.AddressUtil; +import org.apache.accumulo.core.util.CachedConfiguration; +import org.apache.accumulo.core.util.UtilWaitThread; +import org.apache.accumulo.core.zookeeper.ZooUtil; +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter; +import org.apache.accumulo.server.Accumulo; +import org.apache.accumulo.server.client.HdfsZooInstance; +import org.apache.accumulo.server.conf.ServerConfiguration; +import org.apache.accumulo.server.util.time.SimpleTimer; +import org.apache.accumulo.server.zookeeper.ZooReaderWriter; +import org.apache.accumulo.start.classloader.AccumuloClassLoader; +import org.apache.accumulo.trace.instrument.Span; +import org.apache.accumulo.trace.thrift.RemoteSpan; +import org.apache.accumulo.trace.thrift.SpanReceiver.Iface; +import org.apache.accumulo.trace.thrift.SpanReceiver.Processor; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.io.Text; +import org.apache.log4j.Logger; +import org.apache.thrift.TByteArrayOutputStream; +import org.apache.thrift.TException; +import org.apache.thrift.protocol.TCompactProtocol; +import org.apache.thrift.server.TServer; +import org.apache.thrift.server.TThreadPoolServer; +import org.apache.thrift.transport.TServerSocket; +import org.apache.thrift.transport.TServerTransport; +import org.apache.thrift.transport.TTransport; +import org.apache.thrift.transport.TTransportException; +import org.apache.zookeeper.WatchedEvent; +import org.apache.zookeeper.Watcher; +import org.apache.zookeeper.Watcher.Event.EventType; +import org.apache.zookeeper.Watcher.Event.KeeperState; + +public class TraceServer implements Watcher { + + final private static Logger log = Logger.getLogger(TraceServer.class); + final private ServerConfiguration serverConfiguration; + final private TServer server; - private BatchWriter writer = null; - private Connector connector; ++ final private AtomicReference writer; ++ final private Connector connector; + final String table; + + private static void put(Mutation m, String cf, String cq, byte[] bytes, int len) { +m.put(new Text(cf), new Text(cq), new Value(bytes, 0, len)); + } + + static class ByteArrayTransport extends TTransport { +
[23/23] git commit: Merge branch '1.6.0-SNAPSHOT'
Merge branch '1.6.0-SNAPSHOT' Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/353badb0 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/353badb0 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/353badb0 Branch: refs/heads/master Commit: 353badb056681d0ea014810af68128c849070743 Parents: bfe39f6 e7e5c00 Author: Sean Busbey Authored: Thu Jan 23 01:26:44 2014 -0600 Committer: Sean Busbey Committed: Thu Jan 23 01:26:44 2014 -0600 -- .../apache/accumulo/core/util/AddressUtil.java | 2 +- .../apache/accumulo/core/zookeeper/ZooUtil.java | 10 ++- .../apache/accumulo/fate/util/AddressUtil.java | 60 + .../accumulo/fate/zookeeper/ZooSession.java | 14 +-- .../accumulo/fate/util/AddressUtilTest.java | 95 .../org/apache/accumulo/server/Accumulo.java| 22 - .../accumulo/server/fs/VolumeManagerImpl.java | 3 + .../accumulo/server/util/TabletOperations.java | 10 ++- .../accumulo/master/tableOps/DeleteTable.java | 8 ++ .../org/apache/accumulo/tracer/TraceServer.java | 67 ++ .../org/apache/accumulo/tserver/Compactor.java | 2 + 11 files changed, 264 insertions(+), 29 deletions(-) --
[06/23] git commit: ACCUMULO-2224 Make ZooSession more resiliant in the face of transient DNS issues.
ACCUMULO-2224 Make ZooSession more resiliant in the face of transient DNS issues. * retries if host is not found, up to 2xZK timeout (same as other IOExceptions), rather than bailing on any host name problem. * adds utility method for getting the max time the JVM will cache host failures * add test for said method Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/f42ead0d Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/f42ead0d Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/f42ead0d Branch: refs/heads/1.5.1-SNAPSHOT Commit: f42ead0d39e34578c6fe9636af4cfbd9d91e47a5 Parents: dfafd9c Author: Sean Busbey Authored: Mon Jan 20 14:26:20 2014 -0600 Committer: Sean Busbey Committed: Wed Jan 22 23:12:24 2014 -0600 -- .../apache/accumulo/core/util/AddressUtil.java | 39 +++ .../accumulo/core/zookeeper/ZooSession.java | 11 ++-- .../accumulo/core/util/AddressUtilTest.java | 69 +++- src/core/src/test/resources/log4j.properties| 23 +++ 4 files changed, 137 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/f42ead0d/src/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java -- diff --git a/src/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java b/src/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java index 0b82128..96c2e18 100644 --- a/src/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java +++ b/src/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java @@ -16,12 +16,20 @@ */ package org.apache.accumulo.core.util; +import java.net.InetAddress; // workaround to enable @see/@link hyperlink import java.net.InetSocketAddress; +import java.net.UnknownHostException; +import java.security.Security; import org.apache.hadoop.io.Text; import org.apache.thrift.transport.TSocket; +import org.apache.log4j.Logger; + public class AddressUtil { + + private static final Logger log = Logger.getLogger(AddressUtil.class); + static public InetSocketAddress parseAddress(String address, int defaultPort) throws NumberFormatException { final String[] parts = address.split(":", 2); if (parts.length == 2) { @@ -44,5 +52,36 @@ public class AddressUtil { static public String toString(InetSocketAddress addr) { return addr.getAddress().getHostAddress() + ":" + addr.getPort(); } + + /** + * Fetch the security value that determines how long DNS failures are cached. + * Looks up the security property 'networkaddress.cache.negative.ttl'. Should that fail returns + * the default value used in the Oracle JVM 1.4+, which is 10 seconds. + * + * @param originalException the host lookup that is the source of needing this lookup. maybe be null. + * @return positive integer number of seconds + * @see java.net.InetAddress + * @throws IllegalArgumentException if dns failures are cached forever + */ + static public int getAddressCacheNegativeTtl(UnknownHostException originalException) { +int negativeTtl = 10; +try { + negativeTtl = Integer.parseInt(Security.getProperty("networkaddress.cache.negative.ttl")); +} catch (NumberFormatException exception) { + log.warn("Failed to get JVM negative DNS respones cache TTL due to format problem (e.g. this JVM might not have the " + +"property). Falling back to default based on Oracle JVM 1.6 (10s)", exception); +} catch (SecurityException exception) { + log.warn("Failed to get JVM negative DNS response cache TTL due to security manager. Falling back to default based on Oracle JVM 1.6 (10s)", exception); +} +if (-1 == negativeTtl) { + log.error("JVM negative DNS repsonse cache TTL is set to 'forever' and host lookup failed. TTL can be changed with security property " + +"'networkaddress.cache.negative.ttl', see java.net.InetAddress.", originalException); + throw new IllegalArgumentException(originalException); +} else if (0 > negativeTtl) { + log.warn("JVM specified negative DNS response cache TTL was negative (and not 'forever'). Falling back to default based on Oracle JVM 1.6 (10s)"); + negativeTtl = 10; +} +return negativeTtl; + } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/f42ead0d/src/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooSession.java -- diff --git a/src/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooSession.java b/src/core/src/main/java/org/apache/
[09/23] git commit: ACCUMULO-2225 handle IllegalArgumentExceptions from Hadoop on host lookup failure.
ACCUMULO-2225 handle IllegalArgumentExceptions from Hadoop on host lookup failure. Looks for cases where we treat IOExceptions out of Hadoop specially, then attempts to replicate for UnknownHostExceptions that have been wrapped in IllegalArgumentExceptions. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/f778dcf5 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/f778dcf5 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/f778dcf5 Branch: refs/heads/1.4.5-SNAPSHOT Commit: f778dcf50cd7163722f63471d1585333704e639c Parents: f42ead0 Author: Sean Busbey Authored: Tue Jan 21 08:26:16 2014 -0600 Committer: Sean Busbey Committed: Wed Jan 22 23:12:29 2014 -0600 -- .../accumulo/core/client/ZooKeeperInstance.java | 10 ++- .../org/apache/accumulo/server/Accumulo.java| 28 +--- .../server/master/tableOps/DeleteTable.java | 8 ++ .../accumulo/server/tabletserver/Compactor.java | 2 ++ .../randomwalk/security/SecurityHelper.java | 11 ++-- .../accumulo/server/util/TabletOperations.java | 10 ++- 6 files changed, 62 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/f778dcf5/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java -- diff --git a/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java b/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java index e02c197..05982e4 100644 --- a/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java +++ b/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java @@ -18,6 +18,7 @@ package org.apache.accumulo.core.client; import java.io.FileNotFoundException; import java.io.IOException; +import java.net.UnknownHostException; import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; @@ -276,7 +277,14 @@ public class ZooKeeperInstance implements Instance { return result; } } catch (IOException e) { - throw new RuntimeException("Accumulo not initialized, there is no instance id at " + instanceDirectory, e); + log.error("Problem reading instance id out of hdfs at " + instanceDirectory, e); + throw new RuntimeException("Can't tell if Accumulo is initialized; can't read instance id at " + instanceDirectory, e); +} catch (IllegalArgumentException exception) { + /* HDFS throws this when there's a UnknownHostException due to DNS troubles. */ + if (exception.getCause() instanceof UnknownHostException) { +log.error("Problem reading instance id out of hdfs at " + instanceDirectory, exception); + } + throw exception; } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/f778dcf5/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java -- diff --git a/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java b/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java index 253962b..184692c 100644 --- a/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java +++ b/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java @@ -30,6 +30,7 @@ import java.util.TreeMap; import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.file.FileUtil; import org.apache.accumulo.core.trace.DistributedTrace; +import org.apache.accumulo.core.util.AddressUtil; import org.apache.accumulo.core.util.CachedConfiguration; import org.apache.accumulo.core.util.UtilWaitThread; import org.apache.accumulo.core.util.Version; @@ -209,6 +210,7 @@ public class Accumulo { } log.info("Zookeeper connected and initialized, attemping to talk to HDFS"); long sleep = 1000; +int unknownHostTries = 3; while (true) { try { FileSystem fs = FileSystem.get(CachedConfiguration.getInstance()); @@ -216,10 +218,26 @@ public class Accumulo { break; log.warn("Waiting for the NameNode to leave safemode"); } catch (IOException ex) { -log.warn("Unable to connect to HDFS"); +log.warn("Unable to connect to HDFS", ex); + } catch (IllegalArgumentException exception) { +/* Unwrap the UnknownHostException so we can deal with it directly */ +if (exception.getCause() instanceof UnknownHostException) { + if (unknownHostTries > 0) { +log.warn("Unable to connect to HDFS, will retry. cause: " + exception.getCause()); +/* We need to make sure our sleep period
[13/23] Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c4cd3b1b/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java -- diff --cc server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java index 0777d03,000..116f134 mode 100644,00..100644 --- a/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java +++ b/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java @@@ -1,292 -1,0 +1,323 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.server.trace; + +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.nio.channels.ServerSocketChannel; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.TimeUnit; ++import java.util.concurrent.atomic.AtomicReference; + +import org.apache.accumulo.core.Constants; +import org.apache.accumulo.core.client.BatchWriter; +import org.apache.accumulo.core.client.BatchWriterConfig; +import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.client.Instance; +import org.apache.accumulo.core.client.IteratorSetting; +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Properties; +import org.apache.accumulo.core.client.security.tokens.PasswordToken; ++import org.apache.accumulo.core.client.MutationsRejectedException; +import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.data.Mutation; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.file.FileUtil; +import org.apache.accumulo.core.iterators.user.AgeOffFilter; +import org.apache.accumulo.core.security.SecurityUtil; +import org.apache.accumulo.core.trace.TraceFormatter; +import org.apache.accumulo.core.util.AddressUtil; +import org.apache.accumulo.core.util.CachedConfiguration; +import org.apache.accumulo.core.util.UtilWaitThread; +import org.apache.accumulo.core.zookeeper.ZooUtil; +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter; +import org.apache.accumulo.server.Accumulo; +import org.apache.accumulo.server.client.HdfsZooInstance; +import org.apache.accumulo.server.conf.ServerConfiguration; +import org.apache.accumulo.server.util.time.SimpleTimer; +import org.apache.accumulo.server.zookeeper.ZooReaderWriter; +import org.apache.accumulo.start.classloader.AccumuloClassLoader; +import org.apache.accumulo.trace.instrument.Span; +import org.apache.accumulo.trace.thrift.RemoteSpan; +import org.apache.accumulo.trace.thrift.SpanReceiver.Iface; +import org.apache.accumulo.trace.thrift.SpanReceiver.Processor; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.io.Text; +import org.apache.log4j.Logger; +import org.apache.thrift.TByteArrayOutputStream; +import org.apache.thrift.TException; +import org.apache.thrift.protocol.TCompactProtocol; +import org.apache.thrift.server.TServer; +import org.apache.thrift.server.TThreadPoolServer; +import org.apache.thrift.transport.TServerSocket; +import org.apache.thrift.transport.TServerTransport; +import org.apache.thrift.transport.TTransport; +import org.apache.thrift.transport.TTransportException; +import org.apache.zookeeper.WatchedEvent; +import org.apache.zookeeper.Watcher; +import org.apache.zookeeper.Watcher.Event.EventType; +import org.apache.zookeeper.Watcher.Event.KeeperState; + +public class TraceServer implements Watcher { + + final private static Logger log = Logger.getLogger(TraceServer.class); + final private ServerConfiguration serverConfiguration; + final private TServer server; - private BatchWriter writer = null; - private Connector connector; ++ final private AtomicReference writer; ++ final private Connector connector; + final String table; + + private static void put(Mutation m, String cf, String cq, byte[] bytes, int len) { +m.put(new Text(cf), new Text(cq), new Value(bytes, 0, len)); + } + + static class ByteArrayTransport extends TTransport { +
[19/23] Merge branch '1.5.1-SNAPSHOT' into 1.6.0-SNAPSHOT
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e7e5c009/server/tserver/src/main/java/org/apache/accumulo/tserver/Compactor.java -- diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/Compactor.java index 0843c13,000..df476ec mode 100644,00..100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/Compactor.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/Compactor.java @@@ -1,546 -1,0 +1,548 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.tserver; + +import java.io.IOException; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.accumulo.core.client.IteratorSetting; +import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.data.ByteSequence; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.KeyExtent; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.data.thrift.IterInfo; +import org.apache.accumulo.core.file.FileOperations; +import org.apache.accumulo.core.file.FileSKVIterator; +import org.apache.accumulo.core.file.FileSKVWriter; +import org.apache.accumulo.core.iterators.IteratorEnvironment; +import org.apache.accumulo.core.iterators.IteratorUtil; +import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; +import org.apache.accumulo.core.iterators.SortedKeyValueIterator; +import org.apache.accumulo.core.iterators.WrappingIterator; +import org.apache.accumulo.core.iterators.system.ColumnFamilySkippingIterator; +import org.apache.accumulo.core.iterators.system.DeletingIterator; +import org.apache.accumulo.core.iterators.system.MultiIterator; +import org.apache.accumulo.core.iterators.system.TimeSettingIterator; +import org.apache.accumulo.core.metadata.schema.DataFileValue; +import org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction; +import org.apache.accumulo.core.tabletserver.thrift.CompactionReason; +import org.apache.accumulo.core.tabletserver.thrift.CompactionType; +import org.apache.accumulo.core.util.LocalityGroupUtil; +import org.apache.accumulo.core.util.LocalityGroupUtil.LocalityGroupConfigurationError; +import org.apache.accumulo.server.fs.FileRef; +import org.apache.accumulo.server.fs.VolumeManager; +import org.apache.accumulo.server.problems.ProblemReport; +import org.apache.accumulo.server.problems.ProblemReportingIterator; +import org.apache.accumulo.server.problems.ProblemReports; +import org.apache.accumulo.server.problems.ProblemType; +import org.apache.accumulo.trace.instrument.Span; +import org.apache.accumulo.trace.instrument.Trace; +import org.apache.accumulo.tserver.Tablet.MinorCompactionReason; +import org.apache.accumulo.tserver.compaction.MajorCompactionReason; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.log4j.Logger; + +public class Compactor implements Callable { + + public static class CountingIterator extends WrappingIterator { + +private long count; +private ArrayList deepCopies; +private AtomicLong entriesRead; + +@Override +public CountingIterator deepCopy(IteratorEnvironment env) { + return new CountingIterator(this, env); +} + +private CountingIterator(CountingIterator other, IteratorEnvironment env) { + setSource(other.getSource().deepCopy(env)); + count = 0; + this.deepCopies = other.deepCopies; + this.entriesRead = other.entriesRead; + deepCopies.add(this); +} + +public CountingIterator(SortedKeyValueIterator source, AtomicLong entriesRead) { + deepCopies = new ArrayList(); + this.setSource(source); + count = 0; + this.entriesRead = entriesR
[08/23] git commit: ACCUMULO-2224 Make ZooSession more resiliant in the face of transient DNS issues.
ACCUMULO-2224 Make ZooSession more resiliant in the face of transient DNS issues. * retries if host is not found, up to 2xZK timeout (same as other IOExceptions), rather than bailing on any host name problem. * adds utility method for getting the max time the JVM will cache host failures * add test for said method Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/f42ead0d Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/f42ead0d Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/f42ead0d Branch: refs/heads/master Commit: f42ead0d39e34578c6fe9636af4cfbd9d91e47a5 Parents: dfafd9c Author: Sean Busbey Authored: Mon Jan 20 14:26:20 2014 -0600 Committer: Sean Busbey Committed: Wed Jan 22 23:12:24 2014 -0600 -- .../apache/accumulo/core/util/AddressUtil.java | 39 +++ .../accumulo/core/zookeeper/ZooSession.java | 11 ++-- .../accumulo/core/util/AddressUtilTest.java | 69 +++- src/core/src/test/resources/log4j.properties| 23 +++ 4 files changed, 137 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/f42ead0d/src/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java -- diff --git a/src/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java b/src/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java index 0b82128..96c2e18 100644 --- a/src/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java +++ b/src/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java @@ -16,12 +16,20 @@ */ package org.apache.accumulo.core.util; +import java.net.InetAddress; // workaround to enable @see/@link hyperlink import java.net.InetSocketAddress; +import java.net.UnknownHostException; +import java.security.Security; import org.apache.hadoop.io.Text; import org.apache.thrift.transport.TSocket; +import org.apache.log4j.Logger; + public class AddressUtil { + + private static final Logger log = Logger.getLogger(AddressUtil.class); + static public InetSocketAddress parseAddress(String address, int defaultPort) throws NumberFormatException { final String[] parts = address.split(":", 2); if (parts.length == 2) { @@ -44,5 +52,36 @@ public class AddressUtil { static public String toString(InetSocketAddress addr) { return addr.getAddress().getHostAddress() + ":" + addr.getPort(); } + + /** + * Fetch the security value that determines how long DNS failures are cached. + * Looks up the security property 'networkaddress.cache.negative.ttl'. Should that fail returns + * the default value used in the Oracle JVM 1.4+, which is 10 seconds. + * + * @param originalException the host lookup that is the source of needing this lookup. maybe be null. + * @return positive integer number of seconds + * @see java.net.InetAddress + * @throws IllegalArgumentException if dns failures are cached forever + */ + static public int getAddressCacheNegativeTtl(UnknownHostException originalException) { +int negativeTtl = 10; +try { + negativeTtl = Integer.parseInt(Security.getProperty("networkaddress.cache.negative.ttl")); +} catch (NumberFormatException exception) { + log.warn("Failed to get JVM negative DNS respones cache TTL due to format problem (e.g. this JVM might not have the " + +"property). Falling back to default based on Oracle JVM 1.6 (10s)", exception); +} catch (SecurityException exception) { + log.warn("Failed to get JVM negative DNS response cache TTL due to security manager. Falling back to default based on Oracle JVM 1.6 (10s)", exception); +} +if (-1 == negativeTtl) { + log.error("JVM negative DNS repsonse cache TTL is set to 'forever' and host lookup failed. TTL can be changed with security property " + +"'networkaddress.cache.negative.ttl', see java.net.InetAddress.", originalException); + throw new IllegalArgumentException(originalException); +} else if (0 > negativeTtl) { + log.warn("JVM specified negative DNS response cache TTL was negative (and not 'forever'). Falling back to default based on Oracle JVM 1.6 (10s)"); + negativeTtl = 10; +} +return negativeTtl; + } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/f42ead0d/src/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooSession.java -- diff --git a/src/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooSession.java b/src/core/src/main/java/org/apache/
[05/23] git commit: ACCUMULO-2224 Make ZooSession more resiliant in the face of transient DNS issues.
ACCUMULO-2224 Make ZooSession more resiliant in the face of transient DNS issues. * retries if host is not found, up to 2xZK timeout (same as other IOExceptions), rather than bailing on any host name problem. * adds utility method for getting the max time the JVM will cache host failures * add test for said method Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/f42ead0d Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/f42ead0d Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/f42ead0d Branch: refs/heads/1.4.5-SNAPSHOT Commit: f42ead0d39e34578c6fe9636af4cfbd9d91e47a5 Parents: dfafd9c Author: Sean Busbey Authored: Mon Jan 20 14:26:20 2014 -0600 Committer: Sean Busbey Committed: Wed Jan 22 23:12:24 2014 -0600 -- .../apache/accumulo/core/util/AddressUtil.java | 39 +++ .../accumulo/core/zookeeper/ZooSession.java | 11 ++-- .../accumulo/core/util/AddressUtilTest.java | 69 +++- src/core/src/test/resources/log4j.properties| 23 +++ 4 files changed, 137 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/f42ead0d/src/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java -- diff --git a/src/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java b/src/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java index 0b82128..96c2e18 100644 --- a/src/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java +++ b/src/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java @@ -16,12 +16,20 @@ */ package org.apache.accumulo.core.util; +import java.net.InetAddress; // workaround to enable @see/@link hyperlink import java.net.InetSocketAddress; +import java.net.UnknownHostException; +import java.security.Security; import org.apache.hadoop.io.Text; import org.apache.thrift.transport.TSocket; +import org.apache.log4j.Logger; + public class AddressUtil { + + private static final Logger log = Logger.getLogger(AddressUtil.class); + static public InetSocketAddress parseAddress(String address, int defaultPort) throws NumberFormatException { final String[] parts = address.split(":", 2); if (parts.length == 2) { @@ -44,5 +52,36 @@ public class AddressUtil { static public String toString(InetSocketAddress addr) { return addr.getAddress().getHostAddress() + ":" + addr.getPort(); } + + /** + * Fetch the security value that determines how long DNS failures are cached. + * Looks up the security property 'networkaddress.cache.negative.ttl'. Should that fail returns + * the default value used in the Oracle JVM 1.4+, which is 10 seconds. + * + * @param originalException the host lookup that is the source of needing this lookup. maybe be null. + * @return positive integer number of seconds + * @see java.net.InetAddress + * @throws IllegalArgumentException if dns failures are cached forever + */ + static public int getAddressCacheNegativeTtl(UnknownHostException originalException) { +int negativeTtl = 10; +try { + negativeTtl = Integer.parseInt(Security.getProperty("networkaddress.cache.negative.ttl")); +} catch (NumberFormatException exception) { + log.warn("Failed to get JVM negative DNS respones cache TTL due to format problem (e.g. this JVM might not have the " + +"property). Falling back to default based on Oracle JVM 1.6 (10s)", exception); +} catch (SecurityException exception) { + log.warn("Failed to get JVM negative DNS response cache TTL due to security manager. Falling back to default based on Oracle JVM 1.6 (10s)", exception); +} +if (-1 == negativeTtl) { + log.error("JVM negative DNS repsonse cache TTL is set to 'forever' and host lookup failed. TTL can be changed with security property " + +"'networkaddress.cache.negative.ttl', see java.net.InetAddress.", originalException); + throw new IllegalArgumentException(originalException); +} else if (0 > negativeTtl) { + log.warn("JVM specified negative DNS response cache TTL was negative (and not 'forever'). Falling back to default based on Oracle JVM 1.6 (10s)"); + negativeTtl = 10; +} +return negativeTtl; + } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/f42ead0d/src/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooSession.java -- diff --git a/src/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooSession.java b/src/core/src/main/java/org/apache/
[17/23] Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c4cd3b1b/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java -- diff --cc server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java index 0777d03,000..116f134 mode 100644,00..100644 --- a/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java +++ b/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java @@@ -1,292 -1,0 +1,323 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.server.trace; + +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.nio.channels.ServerSocketChannel; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.TimeUnit; ++import java.util.concurrent.atomic.AtomicReference; + +import org.apache.accumulo.core.Constants; +import org.apache.accumulo.core.client.BatchWriter; +import org.apache.accumulo.core.client.BatchWriterConfig; +import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.client.Instance; +import org.apache.accumulo.core.client.IteratorSetting; +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Properties; +import org.apache.accumulo.core.client.security.tokens.PasswordToken; ++import org.apache.accumulo.core.client.MutationsRejectedException; +import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.data.Mutation; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.file.FileUtil; +import org.apache.accumulo.core.iterators.user.AgeOffFilter; +import org.apache.accumulo.core.security.SecurityUtil; +import org.apache.accumulo.core.trace.TraceFormatter; +import org.apache.accumulo.core.util.AddressUtil; +import org.apache.accumulo.core.util.CachedConfiguration; +import org.apache.accumulo.core.util.UtilWaitThread; +import org.apache.accumulo.core.zookeeper.ZooUtil; +import org.apache.accumulo.fate.zookeeper.IZooReaderWriter; +import org.apache.accumulo.server.Accumulo; +import org.apache.accumulo.server.client.HdfsZooInstance; +import org.apache.accumulo.server.conf.ServerConfiguration; +import org.apache.accumulo.server.util.time.SimpleTimer; +import org.apache.accumulo.server.zookeeper.ZooReaderWriter; +import org.apache.accumulo.start.classloader.AccumuloClassLoader; +import org.apache.accumulo.trace.instrument.Span; +import org.apache.accumulo.trace.thrift.RemoteSpan; +import org.apache.accumulo.trace.thrift.SpanReceiver.Iface; +import org.apache.accumulo.trace.thrift.SpanReceiver.Processor; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.io.Text; +import org.apache.log4j.Logger; +import org.apache.thrift.TByteArrayOutputStream; +import org.apache.thrift.TException; +import org.apache.thrift.protocol.TCompactProtocol; +import org.apache.thrift.server.TServer; +import org.apache.thrift.server.TThreadPoolServer; +import org.apache.thrift.transport.TServerSocket; +import org.apache.thrift.transport.TServerTransport; +import org.apache.thrift.transport.TTransport; +import org.apache.thrift.transport.TTransportException; +import org.apache.zookeeper.WatchedEvent; +import org.apache.zookeeper.Watcher; +import org.apache.zookeeper.Watcher.Event.EventType; +import org.apache.zookeeper.Watcher.Event.KeeperState; + +public class TraceServer implements Watcher { + + final private static Logger log = Logger.getLogger(TraceServer.class); + final private ServerConfiguration serverConfiguration; + final private TServer server; - private BatchWriter writer = null; - private Connector connector; ++ final private AtomicReference writer; ++ final private Connector connector; + final String table; + + private static void put(Mutation m, String cf, String cq, byte[] bytes, int len) { +m.put(new Text(cf), new Text(cq), new Value(bytes, 0, len)); + } + + static class ByteArrayTransport extends TTransport { +
[02/23] git commit: ACCUMULO-2213 Make writer recovery in the Tracer more robust.
ACCUMULO-2213 Make writer recovery in the Tracer more robust. * Cleans up writer reseting in the TraceServer, avoids overly broad catching. * tones down log levels in TraceServer to WARN because trace information is transient and we retry everything. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/dfafd9c1 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/dfafd9c1 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/dfafd9c1 Branch: refs/heads/1.5.1-SNAPSHOT Commit: dfafd9c1104d8359ca1eabe345661c541766d57f Parents: 57f9b6c Author: Sean Busbey Authored: Tue Jan 21 14:05:56 2014 -0600 Committer: Sean Busbey Committed: Wed Jan 22 16:36:34 2014 -0600 -- .../accumulo/server/trace/TraceServer.java | 68 ++-- 1 file changed, 49 insertions(+), 19 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/dfafd9c1/src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java -- diff --git a/src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java b/src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java index 4d89e9c..0e2010a 100644 --- a/src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java +++ b/src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java @@ -19,6 +19,7 @@ package org.apache.accumulo.server.trace; import java.net.InetSocketAddress; import java.net.ServerSocket; import java.nio.channels.ServerSocketChannel; +import java.util.concurrent.atomic.AtomicReference; import java.util.TimerTask; import org.apache.accumulo.cloudtrace.instrument.Span; @@ -27,6 +28,7 @@ import org.apache.accumulo.cloudtrace.thrift.SpanReceiver; import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.BatchWriter; import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.client.MutationsRejectedException; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.Mutation; @@ -64,8 +66,8 @@ public class TraceServer implements Watcher { final private static Logger log = Logger.getLogger(TraceServer.class); final private AccumuloConfiguration conf; final private TServer server; - private BatchWriter writer = null; - private Connector connector; + final private AtomicReference writer; + final private Connector connector; final String table; private static void put(Mutation m, String cf, String cq, byte[] bytes, int len) { @@ -128,16 +130,27 @@ public class TraceServer implements Watcher { put(timeMutation, "id", idString, transport.get(), transport.len()); } try { -if (writer == null) - resetWriter(); -if (writer == null) +final BatchWriter writer = TraceServer.this.writer.get(); +/* Check for null, because we expect spans to come in much faster than flush calls. + In the case of failure, we'd rather avoid logging tons of NPEs. + */ +if (null == writer) { + log.warn("writer is not ready; discarding span."); return; +} writer.addMutation(spanMutation); writer.addMutation(indexMutation); if (timeMutation != null) writer.addMutation(timeMutation); - } catch (Exception ex) { -log.error("Unable to write mutation to table: " + spanMutation, ex); + } catch (MutationsRejectedException exception) { +log.warn("Unable to write mutation to table; discarding span. set log level to DEBUG for span information and stacktrace. cause: " + exception); +if (log.isDebugEnabled()) { + log.debug("discarded span due to rejection of mutation: " + spanMutation, exception); +} + /* XXX this could be e.g. an IllegalArgumentExceptoion if we're trying to write this mutation to a writer that has been closed since we retrieved it */ + } catch (RuntimeException exception) { +log.warn("Unable to write mutation to table; discarding span. set log level to DEBUG for stacktrace. cause: " + exception); +log.debug("unable to write mutation to table due to exception.", exception); } } @@ -147,6 +160,7 @@ public class TraceServer implements Watcher { Accumulo.init("tracer"); conf = ServerConfiguration.getSystemConfiguration(); table = conf.get(Property.TRACE_TABLE); +Connector connector = null; while (true) { try { connector = HdfsZooInstance.getInstance().getConnector(conf.get(Property.TRA
[12/23] git commit: ACCUMULO-2225 handle IllegalArgumentExceptions from Hadoop on host lookup failure.
ACCUMULO-2225 handle IllegalArgumentExceptions from Hadoop on host lookup failure. Looks for cases where we treat IOExceptions out of Hadoop specially, then attempts to replicate for UnknownHostExceptions that have been wrapped in IllegalArgumentExceptions. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/f778dcf5 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/f778dcf5 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/f778dcf5 Branch: refs/heads/master Commit: f778dcf50cd7163722f63471d1585333704e639c Parents: f42ead0 Author: Sean Busbey Authored: Tue Jan 21 08:26:16 2014 -0600 Committer: Sean Busbey Committed: Wed Jan 22 23:12:29 2014 -0600 -- .../accumulo/core/client/ZooKeeperInstance.java | 10 ++- .../org/apache/accumulo/server/Accumulo.java| 28 +--- .../server/master/tableOps/DeleteTable.java | 8 ++ .../accumulo/server/tabletserver/Compactor.java | 2 ++ .../randomwalk/security/SecurityHelper.java | 11 ++-- .../accumulo/server/util/TabletOperations.java | 10 ++- 6 files changed, 62 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/f778dcf5/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java -- diff --git a/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java b/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java index e02c197..05982e4 100644 --- a/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java +++ b/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java @@ -18,6 +18,7 @@ package org.apache.accumulo.core.client; import java.io.FileNotFoundException; import java.io.IOException; +import java.net.UnknownHostException; import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; @@ -276,7 +277,14 @@ public class ZooKeeperInstance implements Instance { return result; } } catch (IOException e) { - throw new RuntimeException("Accumulo not initialized, there is no instance id at " + instanceDirectory, e); + log.error("Problem reading instance id out of hdfs at " + instanceDirectory, e); + throw new RuntimeException("Can't tell if Accumulo is initialized; can't read instance id at " + instanceDirectory, e); +} catch (IllegalArgumentException exception) { + /* HDFS throws this when there's a UnknownHostException due to DNS troubles. */ + if (exception.getCause() instanceof UnknownHostException) { +log.error("Problem reading instance id out of hdfs at " + instanceDirectory, exception); + } + throw exception; } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/f778dcf5/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java -- diff --git a/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java b/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java index 253962b..184692c 100644 --- a/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java +++ b/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java @@ -30,6 +30,7 @@ import java.util.TreeMap; import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.file.FileUtil; import org.apache.accumulo.core.trace.DistributedTrace; +import org.apache.accumulo.core.util.AddressUtil; import org.apache.accumulo.core.util.CachedConfiguration; import org.apache.accumulo.core.util.UtilWaitThread; import org.apache.accumulo.core.util.Version; @@ -209,6 +210,7 @@ public class Accumulo { } log.info("Zookeeper connected and initialized, attemping to talk to HDFS"); long sleep = 1000; +int unknownHostTries = 3; while (true) { try { FileSystem fs = FileSystem.get(CachedConfiguration.getInstance()); @@ -216,10 +218,26 @@ public class Accumulo { break; log.warn("Waiting for the NameNode to leave safemode"); } catch (IOException ex) { -log.warn("Unable to connect to HDFS"); +log.warn("Unable to connect to HDFS", ex); + } catch (IllegalArgumentException exception) { +/* Unwrap the UnknownHostException so we can deal with it directly */ +if (exception.getCause() instanceof UnknownHostException) { + if (unknownHostTries > 0) { +log.warn("Unable to connect to HDFS, will retry. cause: " + exception.getCause()); +/* We need to make sure our sleep period
[10/23] git commit: ACCUMULO-2225 handle IllegalArgumentExceptions from Hadoop on host lookup failure.
ACCUMULO-2225 handle IllegalArgumentExceptions from Hadoop on host lookup failure. Looks for cases where we treat IOExceptions out of Hadoop specially, then attempts to replicate for UnknownHostExceptions that have been wrapped in IllegalArgumentExceptions. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/f778dcf5 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/f778dcf5 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/f778dcf5 Branch: refs/heads/1.5.1-SNAPSHOT Commit: f778dcf50cd7163722f63471d1585333704e639c Parents: f42ead0 Author: Sean Busbey Authored: Tue Jan 21 08:26:16 2014 -0600 Committer: Sean Busbey Committed: Wed Jan 22 23:12:29 2014 -0600 -- .../accumulo/core/client/ZooKeeperInstance.java | 10 ++- .../org/apache/accumulo/server/Accumulo.java| 28 +--- .../server/master/tableOps/DeleteTable.java | 8 ++ .../accumulo/server/tabletserver/Compactor.java | 2 ++ .../randomwalk/security/SecurityHelper.java | 11 ++-- .../accumulo/server/util/TabletOperations.java | 10 ++- 6 files changed, 62 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/f778dcf5/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java -- diff --git a/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java b/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java index e02c197..05982e4 100644 --- a/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java +++ b/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java @@ -18,6 +18,7 @@ package org.apache.accumulo.core.client; import java.io.FileNotFoundException; import java.io.IOException; +import java.net.UnknownHostException; import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; @@ -276,7 +277,14 @@ public class ZooKeeperInstance implements Instance { return result; } } catch (IOException e) { - throw new RuntimeException("Accumulo not initialized, there is no instance id at " + instanceDirectory, e); + log.error("Problem reading instance id out of hdfs at " + instanceDirectory, e); + throw new RuntimeException("Can't tell if Accumulo is initialized; can't read instance id at " + instanceDirectory, e); +} catch (IllegalArgumentException exception) { + /* HDFS throws this when there's a UnknownHostException due to DNS troubles. */ + if (exception.getCause() instanceof UnknownHostException) { +log.error("Problem reading instance id out of hdfs at " + instanceDirectory, exception); + } + throw exception; } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/f778dcf5/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java -- diff --git a/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java b/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java index 253962b..184692c 100644 --- a/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java +++ b/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java @@ -30,6 +30,7 @@ import java.util.TreeMap; import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.file.FileUtil; import org.apache.accumulo.core.trace.DistributedTrace; +import org.apache.accumulo.core.util.AddressUtil; import org.apache.accumulo.core.util.CachedConfiguration; import org.apache.accumulo.core.util.UtilWaitThread; import org.apache.accumulo.core.util.Version; @@ -209,6 +210,7 @@ public class Accumulo { } log.info("Zookeeper connected and initialized, attemping to talk to HDFS"); long sleep = 1000; +int unknownHostTries = 3; while (true) { try { FileSystem fs = FileSystem.get(CachedConfiguration.getInstance()); @@ -216,10 +218,26 @@ public class Accumulo { break; log.warn("Waiting for the NameNode to leave safemode"); } catch (IOException ex) { -log.warn("Unable to connect to HDFS"); +log.warn("Unable to connect to HDFS", ex); + } catch (IllegalArgumentException exception) { +/* Unwrap the UnknownHostException so we can deal with it directly */ +if (exception.getCause() instanceof UnknownHostException) { + if (unknownHostTries > 0) { +log.warn("Unable to connect to HDFS, will retry. cause: " + exception.getCause()); +/* We need to make sure our sleep period
[01/23] git commit: ACCUMULO-2213 Make writer recovery in the Tracer more robust.
Updated Branches: refs/heads/1.4.5-SNAPSHOT 57f9b6cfd -> f778dcf50 refs/heads/1.5.1-SNAPSHOT 707e74e05 -> c4cd3b1bc refs/heads/1.6.0-SNAPSHOT b353a24e1 -> e7e5c0097 refs/heads/master bfe39f611 -> 353badb05 ACCUMULO-2213 Make writer recovery in the Tracer more robust. * Cleans up writer reseting in the TraceServer, avoids overly broad catching. * tones down log levels in TraceServer to WARN because trace information is transient and we retry everything. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/dfafd9c1 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/dfafd9c1 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/dfafd9c1 Branch: refs/heads/1.4.5-SNAPSHOT Commit: dfafd9c1104d8359ca1eabe345661c541766d57f Parents: 57f9b6c Author: Sean Busbey Authored: Tue Jan 21 14:05:56 2014 -0600 Committer: Sean Busbey Committed: Wed Jan 22 16:36:34 2014 -0600 -- .../accumulo/server/trace/TraceServer.java | 68 ++-- 1 file changed, 49 insertions(+), 19 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/dfafd9c1/src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java -- diff --git a/src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java b/src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java index 4d89e9c..0e2010a 100644 --- a/src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java +++ b/src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java @@ -19,6 +19,7 @@ package org.apache.accumulo.server.trace; import java.net.InetSocketAddress; import java.net.ServerSocket; import java.nio.channels.ServerSocketChannel; +import java.util.concurrent.atomic.AtomicReference; import java.util.TimerTask; import org.apache.accumulo.cloudtrace.instrument.Span; @@ -27,6 +28,7 @@ import org.apache.accumulo.cloudtrace.thrift.SpanReceiver; import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.BatchWriter; import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.client.MutationsRejectedException; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.Mutation; @@ -64,8 +66,8 @@ public class TraceServer implements Watcher { final private static Logger log = Logger.getLogger(TraceServer.class); final private AccumuloConfiguration conf; final private TServer server; - private BatchWriter writer = null; - private Connector connector; + final private AtomicReference writer; + final private Connector connector; final String table; private static void put(Mutation m, String cf, String cq, byte[] bytes, int len) { @@ -128,16 +130,27 @@ public class TraceServer implements Watcher { put(timeMutation, "id", idString, transport.get(), transport.len()); } try { -if (writer == null) - resetWriter(); -if (writer == null) +final BatchWriter writer = TraceServer.this.writer.get(); +/* Check for null, because we expect spans to come in much faster than flush calls. + In the case of failure, we'd rather avoid logging tons of NPEs. + */ +if (null == writer) { + log.warn("writer is not ready; discarding span."); return; +} writer.addMutation(spanMutation); writer.addMutation(indexMutation); if (timeMutation != null) writer.addMutation(timeMutation); - } catch (Exception ex) { -log.error("Unable to write mutation to table: " + spanMutation, ex); + } catch (MutationsRejectedException exception) { +log.warn("Unable to write mutation to table; discarding span. set log level to DEBUG for span information and stacktrace. cause: " + exception); +if (log.isDebugEnabled()) { + log.debug("discarded span due to rejection of mutation: " + spanMutation, exception); +} + /* XXX this could be e.g. an IllegalArgumentExceptoion if we're trying to write this mutation to a writer that has been closed since we retrieved it */ + } catch (RuntimeException exception) { +log.warn("Unable to write mutation to table; discarding span. set log level to DEBUG for stacktrace. cause: " + exception); +log.debug("unable to write mutation to table due to exception.", exception); } } @@ -147,6 +160,7 @@ public class TraceServer implements Watcher { Accumulo.init("tracer"); conf = ServerConfiguration.
[21/23] Merge branch '1.5.1-SNAPSHOT' into 1.6.0-SNAPSHOT
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e7e5c009/server/tserver/src/main/java/org/apache/accumulo/tserver/Compactor.java -- diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/Compactor.java index 0843c13,000..df476ec mode 100644,00..100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/Compactor.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/Compactor.java @@@ -1,546 -1,0 +1,548 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.tserver; + +import java.io.IOException; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.accumulo.core.client.IteratorSetting; +import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.data.ByteSequence; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.KeyExtent; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.data.thrift.IterInfo; +import org.apache.accumulo.core.file.FileOperations; +import org.apache.accumulo.core.file.FileSKVIterator; +import org.apache.accumulo.core.file.FileSKVWriter; +import org.apache.accumulo.core.iterators.IteratorEnvironment; +import org.apache.accumulo.core.iterators.IteratorUtil; +import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; +import org.apache.accumulo.core.iterators.SortedKeyValueIterator; +import org.apache.accumulo.core.iterators.WrappingIterator; +import org.apache.accumulo.core.iterators.system.ColumnFamilySkippingIterator; +import org.apache.accumulo.core.iterators.system.DeletingIterator; +import org.apache.accumulo.core.iterators.system.MultiIterator; +import org.apache.accumulo.core.iterators.system.TimeSettingIterator; +import org.apache.accumulo.core.metadata.schema.DataFileValue; +import org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction; +import org.apache.accumulo.core.tabletserver.thrift.CompactionReason; +import org.apache.accumulo.core.tabletserver.thrift.CompactionType; +import org.apache.accumulo.core.util.LocalityGroupUtil; +import org.apache.accumulo.core.util.LocalityGroupUtil.LocalityGroupConfigurationError; +import org.apache.accumulo.server.fs.FileRef; +import org.apache.accumulo.server.fs.VolumeManager; +import org.apache.accumulo.server.problems.ProblemReport; +import org.apache.accumulo.server.problems.ProblemReportingIterator; +import org.apache.accumulo.server.problems.ProblemReports; +import org.apache.accumulo.server.problems.ProblemType; +import org.apache.accumulo.trace.instrument.Span; +import org.apache.accumulo.trace.instrument.Trace; +import org.apache.accumulo.tserver.Tablet.MinorCompactionReason; +import org.apache.accumulo.tserver.compaction.MajorCompactionReason; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.log4j.Logger; + +public class Compactor implements Callable { + + public static class CountingIterator extends WrappingIterator { + +private long count; +private ArrayList deepCopies; +private AtomicLong entriesRead; + +@Override +public CountingIterator deepCopy(IteratorEnvironment env) { + return new CountingIterator(this, env); +} + +private CountingIterator(CountingIterator other, IteratorEnvironment env) { + setSource(other.getSource().deepCopy(env)); + count = 0; + this.deepCopies = other.deepCopies; + this.entriesRead = other.entriesRead; + deepCopies.add(this); +} + +public CountingIterator(SortedKeyValueIterator source, AtomicLong entriesRead) { + deepCopies = new ArrayList(); + this.setSource(source); + count = 0; + this.entriesRead = entriesR
[11/23] git commit: ACCUMULO-2225 handle IllegalArgumentExceptions from Hadoop on host lookup failure.
ACCUMULO-2225 handle IllegalArgumentExceptions from Hadoop on host lookup failure. Looks for cases where we treat IOExceptions out of Hadoop specially, then attempts to replicate for UnknownHostExceptions that have been wrapped in IllegalArgumentExceptions. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/f778dcf5 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/f778dcf5 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/f778dcf5 Branch: refs/heads/1.6.0-SNAPSHOT Commit: f778dcf50cd7163722f63471d1585333704e639c Parents: f42ead0 Author: Sean Busbey Authored: Tue Jan 21 08:26:16 2014 -0600 Committer: Sean Busbey Committed: Wed Jan 22 23:12:29 2014 -0600 -- .../accumulo/core/client/ZooKeeperInstance.java | 10 ++- .../org/apache/accumulo/server/Accumulo.java| 28 +--- .../server/master/tableOps/DeleteTable.java | 8 ++ .../accumulo/server/tabletserver/Compactor.java | 2 ++ .../randomwalk/security/SecurityHelper.java | 11 ++-- .../accumulo/server/util/TabletOperations.java | 10 ++- 6 files changed, 62 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/f778dcf5/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java -- diff --git a/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java b/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java index e02c197..05982e4 100644 --- a/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java +++ b/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java @@ -18,6 +18,7 @@ package org.apache.accumulo.core.client; import java.io.FileNotFoundException; import java.io.IOException; +import java.net.UnknownHostException; import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; @@ -276,7 +277,14 @@ public class ZooKeeperInstance implements Instance { return result; } } catch (IOException e) { - throw new RuntimeException("Accumulo not initialized, there is no instance id at " + instanceDirectory, e); + log.error("Problem reading instance id out of hdfs at " + instanceDirectory, e); + throw new RuntimeException("Can't tell if Accumulo is initialized; can't read instance id at " + instanceDirectory, e); +} catch (IllegalArgumentException exception) { + /* HDFS throws this when there's a UnknownHostException due to DNS troubles. */ + if (exception.getCause() instanceof UnknownHostException) { +log.error("Problem reading instance id out of hdfs at " + instanceDirectory, exception); + } + throw exception; } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/f778dcf5/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java -- diff --git a/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java b/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java index 253962b..184692c 100644 --- a/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java +++ b/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java @@ -30,6 +30,7 @@ import java.util.TreeMap; import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.file.FileUtil; import org.apache.accumulo.core.trace.DistributedTrace; +import org.apache.accumulo.core.util.AddressUtil; import org.apache.accumulo.core.util.CachedConfiguration; import org.apache.accumulo.core.util.UtilWaitThread; import org.apache.accumulo.core.util.Version; @@ -209,6 +210,7 @@ public class Accumulo { } log.info("Zookeeper connected and initialized, attemping to talk to HDFS"); long sleep = 1000; +int unknownHostTries = 3; while (true) { try { FileSystem fs = FileSystem.get(CachedConfiguration.getInstance()); @@ -216,10 +218,26 @@ public class Accumulo { break; log.warn("Waiting for the NameNode to leave safemode"); } catch (IOException ex) { -log.warn("Unable to connect to HDFS"); +log.warn("Unable to connect to HDFS", ex); + } catch (IllegalArgumentException exception) { +/* Unwrap the UnknownHostException so we can deal with it directly */ +if (exception.getCause() instanceof UnknownHostException) { + if (unknownHostTries > 0) { +log.warn("Unable to connect to HDFS, will retry. cause: " + exception.getCause()); +/* We need to make sure our sleep period
[04/23] git commit: ACCUMULO-2213 Make writer recovery in the Tracer more robust.
ACCUMULO-2213 Make writer recovery in the Tracer more robust. * Cleans up writer reseting in the TraceServer, avoids overly broad catching. * tones down log levels in TraceServer to WARN because trace information is transient and we retry everything. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/dfafd9c1 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/dfafd9c1 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/dfafd9c1 Branch: refs/heads/master Commit: dfafd9c1104d8359ca1eabe345661c541766d57f Parents: 57f9b6c Author: Sean Busbey Authored: Tue Jan 21 14:05:56 2014 -0600 Committer: Sean Busbey Committed: Wed Jan 22 16:36:34 2014 -0600 -- .../accumulo/server/trace/TraceServer.java | 68 ++-- 1 file changed, 49 insertions(+), 19 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/dfafd9c1/src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java -- diff --git a/src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java b/src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java index 4d89e9c..0e2010a 100644 --- a/src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java +++ b/src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java @@ -19,6 +19,7 @@ package org.apache.accumulo.server.trace; import java.net.InetSocketAddress; import java.net.ServerSocket; import java.nio.channels.ServerSocketChannel; +import java.util.concurrent.atomic.AtomicReference; import java.util.TimerTask; import org.apache.accumulo.cloudtrace.instrument.Span; @@ -27,6 +28,7 @@ import org.apache.accumulo.cloudtrace.thrift.SpanReceiver; import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.BatchWriter; import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.client.MutationsRejectedException; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.Mutation; @@ -64,8 +66,8 @@ public class TraceServer implements Watcher { final private static Logger log = Logger.getLogger(TraceServer.class); final private AccumuloConfiguration conf; final private TServer server; - private BatchWriter writer = null; - private Connector connector; + final private AtomicReference writer; + final private Connector connector; final String table; private static void put(Mutation m, String cf, String cq, byte[] bytes, int len) { @@ -128,16 +130,27 @@ public class TraceServer implements Watcher { put(timeMutation, "id", idString, transport.get(), transport.len()); } try { -if (writer == null) - resetWriter(); -if (writer == null) +final BatchWriter writer = TraceServer.this.writer.get(); +/* Check for null, because we expect spans to come in much faster than flush calls. + In the case of failure, we'd rather avoid logging tons of NPEs. + */ +if (null == writer) { + log.warn("writer is not ready; discarding span."); return; +} writer.addMutation(spanMutation); writer.addMutation(indexMutation); if (timeMutation != null) writer.addMutation(timeMutation); - } catch (Exception ex) { -log.error("Unable to write mutation to table: " + spanMutation, ex); + } catch (MutationsRejectedException exception) { +log.warn("Unable to write mutation to table; discarding span. set log level to DEBUG for span information and stacktrace. cause: " + exception); +if (log.isDebugEnabled()) { + log.debug("discarded span due to rejection of mutation: " + spanMutation, exception); +} + /* XXX this could be e.g. an IllegalArgumentExceptoion if we're trying to write this mutation to a writer that has been closed since we retrieved it */ + } catch (RuntimeException exception) { +log.warn("Unable to write mutation to table; discarding span. set log level to DEBUG for stacktrace. cause: " + exception); +log.debug("unable to write mutation to table due to exception.", exception); } } @@ -147,6 +160,7 @@ public class TraceServer implements Watcher { Accumulo.init("tracer"); conf = ServerConfiguration.getSystemConfiguration(); table = conf.get(Property.TRACE_TABLE); +Connector connector = null; while (true) { try { connector = HdfsZooInstance.getInstance().getConnector(conf.get(Property.TRACE_USER), c
[5/6] git commit: Merge branch '1.5.2-SNAPSHOT' into 1.6.1-SNAPSHOT
Merge branch '1.5.2-SNAPSHOT' into 1.6.1-SNAPSHOT -s ours, only change was README for test that don't exist. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/73ca9453 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/73ca9453 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/73ca9453 Branch: refs/heads/1.6.1-SNAPSHOT Commit: 73ca94532f013abe17368069d385f42f6cb18407 Parents: 4e49ef9 9aafd48 Author: Sean Busbey Authored: Sun Jun 29 19:41:13 2014 -0500 Committer: Sean Busbey Committed: Sun Jun 29 19:41:13 2014 -0500 -- --
[6/6] git commit: Merge branch '1.6.1-SNAPSHOT'
Merge branch '1.6.1-SNAPSHOT' Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/ef78dc4b Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/ef78dc4b Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/ef78dc4b Branch: refs/heads/master Commit: ef78dc4bbc625f70d84526eed7a69304f9820841 Parents: 0d4ff12 73ca945 Author: Sean Busbey Authored: Sun Jun 29 19:42:08 2014 -0500 Committer: Sean Busbey Committed: Sun Jun 29 19:42:08 2014 -0500 -- --
[3/6] git commit: ACCUMULO-2443: Change README of auto test to say tool.sh instead of accumulo
ACCUMULO-2443: Change README of auto test to say tool.sh instead of accumulo Signed-off-by: Sean Busbey Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9aafd48b Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9aafd48b Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9aafd48b Branch: refs/heads/master Commit: 9aafd48bee87fdb6d88bf8d3f64f7812c357a3e1 Parents: 727bbf7 Author: Hung Pham Authored: Sun Jun 29 17:58:41 2014 -0400 Committer: Sean Busbey Committed: Sun Jun 29 19:31:43 2014 -0500 -- test/system/auto/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/9aafd48b/test/system/auto/README.md -- diff --git a/test/system/auto/README.md b/test/system/auto/README.md index e147620..cc28e59 100644 --- a/test/system/auto/README.md +++ b/test/system/auto/README.md @@ -93,7 +93,7 @@ only the tests you wish to run. You may specify a 'timeout factor' via an optional integer argument: -> `$ ./bin/accumulo org.apache.accumulo.test.functional.RunTests --tests \ +> `$ ./bin/tool.sh lib/accumulo-test.jar org.apache.accumulo.test.functional.RunTests --tests \ /user/hadoop/tests --output /user/hadoop/results --timeoutFactor timeout_factor` Where `timeout_factor` indicates how much we should scale up timeouts. It will
[4/6] git commit: Merge branch '1.5.2-SNAPSHOT' into 1.6.1-SNAPSHOT
Merge branch '1.5.2-SNAPSHOT' into 1.6.1-SNAPSHOT -s ours, only change was README for test that don't exist. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/73ca9453 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/73ca9453 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/73ca9453 Branch: refs/heads/master Commit: 73ca94532f013abe17368069d385f42f6cb18407 Parents: 4e49ef9 9aafd48 Author: Sean Busbey Authored: Sun Jun 29 19:41:13 2014 -0500 Committer: Sean Busbey Committed: Sun Jun 29 19:41:13 2014 -0500 -- --
[1/6] git commit: ACCUMULO-2443: Change README of auto test to say tool.sh instead of accumulo
Repository: accumulo Updated Branches: refs/heads/1.5.2-SNAPSHOT 727bbf713 -> 9aafd48be refs/heads/1.6.1-SNAPSHOT 4e49ef98a -> 73ca94532 refs/heads/master 0d4ff12a9 -> ef78dc4bb ACCUMULO-2443: Change README of auto test to say tool.sh instead of accumulo Signed-off-by: Sean Busbey Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9aafd48b Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9aafd48b Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9aafd48b Branch: refs/heads/1.5.2-SNAPSHOT Commit: 9aafd48bee87fdb6d88bf8d3f64f7812c357a3e1 Parents: 727bbf7 Author: Hung Pham Authored: Sun Jun 29 17:58:41 2014 -0400 Committer: Sean Busbey Committed: Sun Jun 29 19:31:43 2014 -0500 -- test/system/auto/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/9aafd48b/test/system/auto/README.md -- diff --git a/test/system/auto/README.md b/test/system/auto/README.md index e147620..cc28e59 100644 --- a/test/system/auto/README.md +++ b/test/system/auto/README.md @@ -93,7 +93,7 @@ only the tests you wish to run. You may specify a 'timeout factor' via an optional integer argument: -> `$ ./bin/accumulo org.apache.accumulo.test.functional.RunTests --tests \ +> `$ ./bin/tool.sh lib/accumulo-test.jar org.apache.accumulo.test.functional.RunTests --tests \ /user/hadoop/tests --output /user/hadoop/results --timeoutFactor timeout_factor` Where `timeout_factor` indicates how much we should scale up timeouts. It will
[2/6] git commit: ACCUMULO-2443: Change README of auto test to say tool.sh instead of accumulo
ACCUMULO-2443: Change README of auto test to say tool.sh instead of accumulo Signed-off-by: Sean Busbey Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9aafd48b Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9aafd48b Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9aafd48b Branch: refs/heads/1.6.1-SNAPSHOT Commit: 9aafd48bee87fdb6d88bf8d3f64f7812c357a3e1 Parents: 727bbf7 Author: Hung Pham Authored: Sun Jun 29 17:58:41 2014 -0400 Committer: Sean Busbey Committed: Sun Jun 29 19:31:43 2014 -0500 -- test/system/auto/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/9aafd48b/test/system/auto/README.md -- diff --git a/test/system/auto/README.md b/test/system/auto/README.md index e147620..cc28e59 100644 --- a/test/system/auto/README.md +++ b/test/system/auto/README.md @@ -93,7 +93,7 @@ only the tests you wish to run. You may specify a 'timeout factor' via an optional integer argument: -> `$ ./bin/accumulo org.apache.accumulo.test.functional.RunTests --tests \ +> `$ ./bin/tool.sh lib/accumulo-test.jar org.apache.accumulo.test.functional.RunTests --tests \ /user/hadoop/tests --output /user/hadoop/results --timeoutFactor timeout_factor` Where `timeout_factor` indicates how much we should scale up timeouts. It will
[1/3] git commit: ACCUMULO-2942 Fixes ShardedTableDistributionFormatterTest on all JVMs
Repository: accumulo Updated Branches: refs/heads/1.6.1-SNAPSHOT 73ca94532 -> 985a69922 refs/heads/master ef78dc4bb -> dbb9cc4a2 ACCUMULO-2942 Fixes ShardedTableDistributionFormatterTest on all JVMs ShardedTableDistributionFormatterTest.testAggregate() was failing due to inconsistency in order to stats that were returned in the getStats() method. This is due to the fact that the structure that holds the stats is a HashMap. The test was changed so that it does not assert that elements exists at certain location in results, but rather it now checks that elements exist in the list. The test was also slightly broadened so that it checks that tuples of (day, count) exist in list. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/985a6992 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/985a6992 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/985a6992 Branch: refs/heads/1.6.1-SNAPSHOT Commit: 985a699224f1590fc4073df7eca3b1cb03c86526 Parents: 73ca945 Author: haydenmarchant Authored: Wed Jun 25 06:19:31 2014 -0700 Committer: Sean Busbey Committed: Mon Jun 30 09:25:56 2014 -0500 -- .../ShardedTableDistributionFormatterTest.java | 30 ++-- 1 file changed, 27 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/985a6992/core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java -- diff --git a/core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java b/core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java index 1707dda..8fc68fd 100644 --- a/core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java +++ b/core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java @@ -19,12 +19,16 @@ package org.apache.accumulo.core.util.format; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import java.util.Arrays; +import java.util.List; import java.util.Map; import java.util.TreeMap; import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.Predicate; import org.junit.Before; import org.junit.Test; @@ -58,9 +62,29 @@ public class ShardedTableDistributionFormatterTest { formatter.initialize(data.entrySet(), false); -String[] result = formatter.next().split("\n"); -assertTrue(result[2].endsWith("\t1")); -assertTrue(result[3].endsWith("\t2")); +String[] resultLines = formatter.next().split("\n"); +List results = Arrays.asList(resultLines).subList(2, 4); + +assertTrue(CollectionUtils.exists(results, new AggregateReportChecker("NULL", 1))); +assertTrue(CollectionUtils.exists(results, new AggregateReportChecker("19700101", 2))); + assertFalse(formatter.hasNext()); } + + private static class AggregateReportChecker implements Predicate { +private String day; +private int count; + +AggregateReportChecker(String day, int count) { + this.day = day; + this.count = count; +} + +@Override +public boolean evaluate(Object arg) { + String resLine = (String) arg; + return resLine.startsWith(this.day) && resLine.endsWith("" + this.count); +} + + } }
[2/3] git commit: ACCUMULO-2942 Fixes ShardedTableDistributionFormatterTest on all JVMs
ACCUMULO-2942 Fixes ShardedTableDistributionFormatterTest on all JVMs ShardedTableDistributionFormatterTest.testAggregate() was failing due to inconsistency in order to stats that were returned in the getStats() method. This is due to the fact that the structure that holds the stats is a HashMap. The test was changed so that it does not assert that elements exists at certain location in results, but rather it now checks that elements exist in the list. The test was also slightly broadened so that it checks that tuples of (day, count) exist in list. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/985a6992 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/985a6992 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/985a6992 Branch: refs/heads/master Commit: 985a699224f1590fc4073df7eca3b1cb03c86526 Parents: 73ca945 Author: haydenmarchant Authored: Wed Jun 25 06:19:31 2014 -0700 Committer: Sean Busbey Committed: Mon Jun 30 09:25:56 2014 -0500 -- .../ShardedTableDistributionFormatterTest.java | 30 ++-- 1 file changed, 27 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/985a6992/core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java -- diff --git a/core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java b/core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java index 1707dda..8fc68fd 100644 --- a/core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java +++ b/core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java @@ -19,12 +19,16 @@ package org.apache.accumulo.core.util.format; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import java.util.Arrays; +import java.util.List; import java.util.Map; import java.util.TreeMap; import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.Predicate; import org.junit.Before; import org.junit.Test; @@ -58,9 +62,29 @@ public class ShardedTableDistributionFormatterTest { formatter.initialize(data.entrySet(), false); -String[] result = formatter.next().split("\n"); -assertTrue(result[2].endsWith("\t1")); -assertTrue(result[3].endsWith("\t2")); +String[] resultLines = formatter.next().split("\n"); +List results = Arrays.asList(resultLines).subList(2, 4); + +assertTrue(CollectionUtils.exists(results, new AggregateReportChecker("NULL", 1))); +assertTrue(CollectionUtils.exists(results, new AggregateReportChecker("19700101", 2))); + assertFalse(formatter.hasNext()); } + + private static class AggregateReportChecker implements Predicate { +private String day; +private int count; + +AggregateReportChecker(String day, int count) { + this.day = day; + this.count = count; +} + +@Override +public boolean evaluate(Object arg) { + String resLine = (String) arg; + return resLine.startsWith(this.day) && resLine.endsWith("" + this.count); +} + + } }
[3/3] git commit: Merge branch '1.6.1-SNAPSHOT'
Merge branch '1.6.1-SNAPSHOT' Conflicts: core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/dbb9cc4a Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/dbb9cc4a Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/dbb9cc4a Branch: refs/heads/master Commit: dbb9cc4a229754382e3f977b591f4035df96accc Parents: ef78dc4 985a699 Author: Sean Busbey Authored: Mon Jun 30 09:48:54 2014 -0500 Committer: Sean Busbey Committed: Mon Jun 30 09:48:54 2014 -0500 -- .../ShardedTableDistributionFormatterTest.java | 30 ++-- 1 file changed, 27 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/dbb9cc4a/core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java -- diff --cc core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java index c4a7107,8fc68fd..8f46c89 --- a/core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java +++ b/core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java @@@ -19,12 -19,16 +19,16 @@@ package org.apache.accumulo.core.util.f import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import java.nio.charset.StandardCharsets; + import java.util.Arrays; + import java.util.List; import java.util.Map; import java.util.TreeMap; -import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; + import org.apache.commons.collections.CollectionUtils; + import org.apache.commons.collections.Predicate; import org.junit.Before; import org.junit.Test;
[1/6] git commit: ACCUMULO-2967 Unknown Host should result in timeout.
Repository: accumulo Updated Branches: refs/heads/1.5.2-SNAPSHOT 9aafd48be -> d4d455f1f refs/heads/1.6.1-SNAPSHOT 985a69922 -> 41b72b5b7 refs/heads/master 7c4d620a9 -> d1b3403db ACCUMULO-2967 Unknown Host should result in timeout. - Tests that we get an exception within a reasonable bound - Changes timeout condition to check prior to place where an exception can be thrown. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/d4d455f1 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/d4d455f1 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/d4d455f1 Branch: refs/heads/1.5.2-SNAPSHOT Commit: d4d455f1f3beded6f20b6c3946a7b867d6b3e2d4 Parents: 9aafd48 Author: Sean Busbey Authored: Tue Jul 1 10:48:43 2014 -0500 Committer: Sean Busbey Committed: Tue Jul 1 11:54:39 2014 -0500 -- .../accumulo/fate/zookeeper/ZooSession.java | 14 +++-- .../accumulo/fate/zookeeper/ZooSessionTest.java | 32 2 files changed, 43 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/d4d455f1/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java -- diff --git a/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java b/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java index 205ff01..a9f630e 100644 --- a/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java +++ b/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java @@ -66,6 +66,13 @@ public class ZooSession { } + /** + * @param host comma separated list of zk servers + * @param timeout in milliseconds + * @param scheme authentication type, e.g. 'digest', may be null + * @param auth authentication-scheme-specific token, may be null + * @param watcher ZK notifications, may be null + */ public static ZooKeeper connect(String host, int timeout, String scheme, byte[] auth, Watcher watcher) { final int TIME_BETWEEN_CONNECT_CHECKS_MS = 100; final int TOTAL_CONNECT_TIME_WAIT_MS = 10 * 1000; @@ -88,9 +95,6 @@ public class ZooSession { UtilWaitThread.sleep(TIME_BETWEEN_CONNECT_CHECKS_MS); } -if (System.currentTimeMillis() - startTime > 2 * timeout) - throw new RuntimeException("Failed to connect to zookeeper (" + host + ") within 2x zookeeper timeout period " + timeout); - } catch (IOException e) { if (e instanceof UnknownHostException) { /* @@ -108,6 +112,10 @@ public class ZooSession { log.warn("interrupted", e); } } + + if (System.currentTimeMillis() - startTime > 2 * timeout) { +throw new RuntimeException("Failed to connect to zookeeper (" + host + ") within 2x zookeeper timeout period " + timeout); + } if (tryAgain) { UtilWaitThread.sleep(sleepTime); http://git-wip-us.apache.org/repos/asf/accumulo/blob/d4d455f1/fate/src/test/java/org/apache/accumulo/fate/zookeeper/ZooSessionTest.java -- diff --git a/fate/src/test/java/org/apache/accumulo/fate/zookeeper/ZooSessionTest.java b/fate/src/test/java/org/apache/accumulo/fate/zookeeper/ZooSessionTest.java new file mode 100644 index 000..6f7928d --- /dev/null +++ b/fate/src/test/java/org/apache/accumulo/fate/zookeeper/ZooSessionTest.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.fate.zookeeper; + +import org.apache.zookeeper.ZooKeeper; +import org.junit.Test; + +public class ZooSessionTest { + + private static final int MINIMUM_TIMEOUT=1; + private static final String UNKNOWN_HOST = "hostname.that.should.not.exist.example.com:2181"; + + @Test(expected=RuntimeException.class, timeout=MINIMUM_TIMEOUT*4) + public void testUnknownHost() { +ZooKeeper session = ZooSession.connect(UNKNOWN_HOST, MINIMUM_TIMEOUT, null, null, null); + } + +}
[6/6] git commit: Merge branch '1.6.1-SNAPSHOT'
Merge branch '1.6.1-SNAPSHOT' Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/d1b3403d Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/d1b3403d Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/d1b3403d Branch: refs/heads/master Commit: d1b3403db8a41ef4059c3d17bd759527984543ce Parents: 7c4d620 41b72b5 Author: Sean Busbey Authored: Tue Jul 1 12:48:38 2014 -0500 Committer: Sean Busbey Committed: Tue Jul 1 12:48:38 2014 -0500 -- .../accumulo/fate/zookeeper/ZooSession.java | 14 +++-- .../accumulo/fate/zookeeper/ZooSessionTest.java | 32 2 files changed, 43 insertions(+), 3 deletions(-) --
[3/6] git commit: ACCUMULO-2967 Unknown Host should result in timeout.
ACCUMULO-2967 Unknown Host should result in timeout. - Tests that we get an exception within a reasonable bound - Changes timeout condition to check prior to place where an exception can be thrown. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/d4d455f1 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/d4d455f1 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/d4d455f1 Branch: refs/heads/master Commit: d4d455f1f3beded6f20b6c3946a7b867d6b3e2d4 Parents: 9aafd48 Author: Sean Busbey Authored: Tue Jul 1 10:48:43 2014 -0500 Committer: Sean Busbey Committed: Tue Jul 1 11:54:39 2014 -0500 -- .../accumulo/fate/zookeeper/ZooSession.java | 14 +++-- .../accumulo/fate/zookeeper/ZooSessionTest.java | 32 2 files changed, 43 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/d4d455f1/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java -- diff --git a/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java b/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java index 205ff01..a9f630e 100644 --- a/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java +++ b/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java @@ -66,6 +66,13 @@ public class ZooSession { } + /** + * @param host comma separated list of zk servers + * @param timeout in milliseconds + * @param scheme authentication type, e.g. 'digest', may be null + * @param auth authentication-scheme-specific token, may be null + * @param watcher ZK notifications, may be null + */ public static ZooKeeper connect(String host, int timeout, String scheme, byte[] auth, Watcher watcher) { final int TIME_BETWEEN_CONNECT_CHECKS_MS = 100; final int TOTAL_CONNECT_TIME_WAIT_MS = 10 * 1000; @@ -88,9 +95,6 @@ public class ZooSession { UtilWaitThread.sleep(TIME_BETWEEN_CONNECT_CHECKS_MS); } -if (System.currentTimeMillis() - startTime > 2 * timeout) - throw new RuntimeException("Failed to connect to zookeeper (" + host + ") within 2x zookeeper timeout period " + timeout); - } catch (IOException e) { if (e instanceof UnknownHostException) { /* @@ -108,6 +112,10 @@ public class ZooSession { log.warn("interrupted", e); } } + + if (System.currentTimeMillis() - startTime > 2 * timeout) { +throw new RuntimeException("Failed to connect to zookeeper (" + host + ") within 2x zookeeper timeout period " + timeout); + } if (tryAgain) { UtilWaitThread.sleep(sleepTime); http://git-wip-us.apache.org/repos/asf/accumulo/blob/d4d455f1/fate/src/test/java/org/apache/accumulo/fate/zookeeper/ZooSessionTest.java -- diff --git a/fate/src/test/java/org/apache/accumulo/fate/zookeeper/ZooSessionTest.java b/fate/src/test/java/org/apache/accumulo/fate/zookeeper/ZooSessionTest.java new file mode 100644 index 000..6f7928d --- /dev/null +++ b/fate/src/test/java/org/apache/accumulo/fate/zookeeper/ZooSessionTest.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.fate.zookeeper; + +import org.apache.zookeeper.ZooKeeper; +import org.junit.Test; + +public class ZooSessionTest { + + private static final int MINIMUM_TIMEOUT=1; + private static final String UNKNOWN_HOST = "hostname.that.should.not.exist.example.com:2181"; + + @Test(expected=RuntimeException.class, timeout=MINIMUM_TIMEOUT*4) + public void testUnknownHost() { +ZooKeeper session = ZooSession.connect(UNKNOWN_HOST, MINIMUM_TIMEOUT, null, null, null); + } + +}
[4/6] git commit: Merge branch '1.5.2-SNAPSHOT' into 1.6.1-SNAPSHOT
Merge branch '1.5.2-SNAPSHOT' into 1.6.1-SNAPSHOT Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/41b72b5b Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/41b72b5b Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/41b72b5b Branch: refs/heads/master Commit: 41b72b5b7ea6af31c6cb78dd522c6505229fc135 Parents: 985a699 d4d455f Author: Sean Busbey Authored: Tue Jul 1 12:45:43 2014 -0500 Committer: Sean Busbey Committed: Tue Jul 1 12:45:43 2014 -0500 -- .../accumulo/fate/zookeeper/ZooSession.java | 14 +++-- .../accumulo/fate/zookeeper/ZooSessionTest.java | 32 2 files changed, 43 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/41b72b5b/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java -- diff --cc fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java index c0140ef,a9f630e..33bd77b --- a/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java +++ b/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java @@@ -66,11 -66,18 +66,18 @@@ public class ZooSession } + /** +* @param host comma separated list of zk servers +* @param timeout in milliseconds +* @param scheme authentication type, e.g. 'digest', may be null +* @param auth authentication-scheme-specific token, may be null +* @param watcher ZK notifications, may be null +*/ public static ZooKeeper connect(String host, int timeout, String scheme, byte[] auth, Watcher watcher) { final int TIME_BETWEEN_CONNECT_CHECKS_MS = 100; -final int TOTAL_CONNECT_TIME_WAIT_MS = 10 * 1000; +int connectTimeWait = Math.min(10 * 1000, timeout); boolean tryAgain = true; -int sleepTime = 100; +long sleepTime = 100; ZooKeeper zooKeeper = null; long startTime = System.currentTimeMillis(); @@@ -108,18 -112,15 +112,22 @@@ log.warn("interrupted", e); } } + + if (System.currentTimeMillis() - startTime > 2 * timeout) { + throw new RuntimeException("Failed to connect to zookeeper (" + host + ") within 2x zookeeper timeout period " + timeout); + } if (tryAgain) { +if (startTime + 2 * timeout < System.currentTimeMillis() + sleepTime + connectTimeWait) + sleepTime = startTime + 2 * timeout - System.currentTimeMillis() - connectTimeWait; +if (sleepTime < 0) +{ + connectTimeWait -= sleepTime; + sleepTime = 0; +} UtilWaitThread.sleep(sleepTime); if (sleepTime < 1) - sleepTime = (int) (sleepTime + sleepTime * Math.random()); + sleepTime = sleepTime + (long)(sleepTime * Math.random()); } }
[5/6] git commit: Merge branch '1.5.2-SNAPSHOT' into 1.6.1-SNAPSHOT
Merge branch '1.5.2-SNAPSHOT' into 1.6.1-SNAPSHOT Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/41b72b5b Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/41b72b5b Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/41b72b5b Branch: refs/heads/1.6.1-SNAPSHOT Commit: 41b72b5b7ea6af31c6cb78dd522c6505229fc135 Parents: 985a699 d4d455f Author: Sean Busbey Authored: Tue Jul 1 12:45:43 2014 -0500 Committer: Sean Busbey Committed: Tue Jul 1 12:45:43 2014 -0500 -- .../accumulo/fate/zookeeper/ZooSession.java | 14 +++-- .../accumulo/fate/zookeeper/ZooSessionTest.java | 32 2 files changed, 43 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/41b72b5b/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java -- diff --cc fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java index c0140ef,a9f630e..33bd77b --- a/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java +++ b/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java @@@ -66,11 -66,18 +66,18 @@@ public class ZooSession } + /** +* @param host comma separated list of zk servers +* @param timeout in milliseconds +* @param scheme authentication type, e.g. 'digest', may be null +* @param auth authentication-scheme-specific token, may be null +* @param watcher ZK notifications, may be null +*/ public static ZooKeeper connect(String host, int timeout, String scheme, byte[] auth, Watcher watcher) { final int TIME_BETWEEN_CONNECT_CHECKS_MS = 100; -final int TOTAL_CONNECT_TIME_WAIT_MS = 10 * 1000; +int connectTimeWait = Math.min(10 * 1000, timeout); boolean tryAgain = true; -int sleepTime = 100; +long sleepTime = 100; ZooKeeper zooKeeper = null; long startTime = System.currentTimeMillis(); @@@ -108,18 -112,15 +112,22 @@@ log.warn("interrupted", e); } } + + if (System.currentTimeMillis() - startTime > 2 * timeout) { + throw new RuntimeException("Failed to connect to zookeeper (" + host + ") within 2x zookeeper timeout period " + timeout); + } if (tryAgain) { +if (startTime + 2 * timeout < System.currentTimeMillis() + sleepTime + connectTimeWait) + sleepTime = startTime + 2 * timeout - System.currentTimeMillis() - connectTimeWait; +if (sleepTime < 0) +{ + connectTimeWait -= sleepTime; + sleepTime = 0; +} UtilWaitThread.sleep(sleepTime); if (sleepTime < 1) - sleepTime = (int) (sleepTime + sleepTime * Math.random()); + sleepTime = sleepTime + (long)(sleepTime * Math.random()); } }
[2/6] git commit: ACCUMULO-2967 Unknown Host should result in timeout.
ACCUMULO-2967 Unknown Host should result in timeout. - Tests that we get an exception within a reasonable bound - Changes timeout condition to check prior to place where an exception can be thrown. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/d4d455f1 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/d4d455f1 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/d4d455f1 Branch: refs/heads/1.6.1-SNAPSHOT Commit: d4d455f1f3beded6f20b6c3946a7b867d6b3e2d4 Parents: 9aafd48 Author: Sean Busbey Authored: Tue Jul 1 10:48:43 2014 -0500 Committer: Sean Busbey Committed: Tue Jul 1 11:54:39 2014 -0500 -- .../accumulo/fate/zookeeper/ZooSession.java | 14 +++-- .../accumulo/fate/zookeeper/ZooSessionTest.java | 32 2 files changed, 43 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/d4d455f1/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java -- diff --git a/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java b/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java index 205ff01..a9f630e 100644 --- a/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java +++ b/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java @@ -66,6 +66,13 @@ public class ZooSession { } + /** + * @param host comma separated list of zk servers + * @param timeout in milliseconds + * @param scheme authentication type, e.g. 'digest', may be null + * @param auth authentication-scheme-specific token, may be null + * @param watcher ZK notifications, may be null + */ public static ZooKeeper connect(String host, int timeout, String scheme, byte[] auth, Watcher watcher) { final int TIME_BETWEEN_CONNECT_CHECKS_MS = 100; final int TOTAL_CONNECT_TIME_WAIT_MS = 10 * 1000; @@ -88,9 +95,6 @@ public class ZooSession { UtilWaitThread.sleep(TIME_BETWEEN_CONNECT_CHECKS_MS); } -if (System.currentTimeMillis() - startTime > 2 * timeout) - throw new RuntimeException("Failed to connect to zookeeper (" + host + ") within 2x zookeeper timeout period " + timeout); - } catch (IOException e) { if (e instanceof UnknownHostException) { /* @@ -108,6 +112,10 @@ public class ZooSession { log.warn("interrupted", e); } } + + if (System.currentTimeMillis() - startTime > 2 * timeout) { +throw new RuntimeException("Failed to connect to zookeeper (" + host + ") within 2x zookeeper timeout period " + timeout); + } if (tryAgain) { UtilWaitThread.sleep(sleepTime); http://git-wip-us.apache.org/repos/asf/accumulo/blob/d4d455f1/fate/src/test/java/org/apache/accumulo/fate/zookeeper/ZooSessionTest.java -- diff --git a/fate/src/test/java/org/apache/accumulo/fate/zookeeper/ZooSessionTest.java b/fate/src/test/java/org/apache/accumulo/fate/zookeeper/ZooSessionTest.java new file mode 100644 index 000..6f7928d --- /dev/null +++ b/fate/src/test/java/org/apache/accumulo/fate/zookeeper/ZooSessionTest.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.fate.zookeeper; + +import org.apache.zookeeper.ZooKeeper; +import org.junit.Test; + +public class ZooSessionTest { + + private static final int MINIMUM_TIMEOUT=1; + private static final String UNKNOWN_HOST = "hostname.that.should.not.exist.example.com:2181"; + + @Test(expected=RuntimeException.class, timeout=MINIMUM_TIMEOUT*4) + public void testUnknownHost() { +ZooKeeper session = ZooSession.connect(UNKNOWN_HOST, MINIMUM_TIMEOUT, null, null, null); + } + +}
svn commit: r1609270 - /accumulo/site/trunk/content/source.mdtext
Author: busbey Date: Wed Jul 9 19:15:56 2014 New Revision: 1609270 URL: http://svn.apache.org/r1609270 Log: docs changes for ACCUMULO-2927 Modified: accumulo/site/trunk/content/source.mdtext Modified: accumulo/site/trunk/content/source.mdtext URL: http://svn.apache.org/viewvc/accumulo/site/trunk/content/source.mdtext?rev=1609270&r1=1609269&r2=1609270&view=diff == --- accumulo/site/trunk/content/source.mdtext (original) +++ accumulo/site/trunk/content/source.mdtext Wed Jul 9 19:15:56 2014 @@ -80,11 +80,33 @@ If you regularly switch between major de The easiest fix is to ensure all of your current changes are stored in git and then cleaning your workspace. -git add path/to/file/that/has/changed -git add path/to/other/file -git clean -xdf - -Note that this git clean command will delete any files unknown to git in a way that is irreversible. This includes files and directories created by IDEs in the workspace, even if git is normally configured to ignore them. +$> git add path/to/file/that/has/changed +$> git add path/to/other/file +$> git clean -df + +Note that this git clean command will delete any files unknown to git in a way that is irreversible. You should check that no important files will be included by first looking at the "untracked files" section in a ```git status``` command. + +$> git status +# On branch master +nothing to commit (working directory clean) +$> mvn package +{ maven output elided } +$> git checkout 1.6.1-SNAPSHOT +Switched to branch '1.6.1-SNAPSHOT' +$> git status +# On branch 1.6.1-SNAPSHOT +# Untracked files: +# (use "git add ..." to include in what will be committed) +# +# mapreduce/ +# shell/ +nothing added to commit but untracked files present (use "git add" to track) +$> git clean -df +Removing mapreduce/ +Removing shell/ +$> git status +# On branch 1.6.1-SNAPSHOT +nothing to commit (working directory clean) ### Continuous Integration
svn commit: r915599 - in /websites/production/accumulo/content: ./ 1.4/apidocs/ 1.5/apidocs/ 1.6/apidocs/
Author: busbey Date: Wed Jul 9 19:17:18 2014 New Revision: 915599 Log: docs update for ACCUMULO-2927 Added: websites/production/accumulo/content/ - copied from r915598, websites/staging/accumulo/trunk/content/ websites/production/accumulo/content/1.4/apidocs/ - copied from r915598, websites/production/accumulo/content/1.4/apidocs/ websites/production/accumulo/content/1.5/apidocs/ - copied from r915598, websites/production/accumulo/content/1.5/apidocs/ websites/production/accumulo/content/1.6/apidocs/ - copied from r915598, websites/production/accumulo/content/1.6/apidocs/
[11/11] git commit: Merge branch '1.6.1-SNAPSHOT'
Merge branch '1.6.1-SNAPSHOT' Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/8abe4df6 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/8abe4df6 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/8abe4df6 Branch: refs/heads/master Commit: 8abe4df6ce69f3d86a3609cbbc35dae882d3b85a Parents: c8b3e99 3a93bad Author: Sean Busbey Authored: Thu Jul 31 11:31:37 2014 -0500 Committer: Sean Busbey Committed: Thu Jul 31 11:31:37 2014 -0500 -- README | 41 ++- .../org/apache/accumulo/server/Accumulo.java| 23 +++-- .../apache/accumulo/server/ServerConstants.java | 1 + .../accumulo/server/util/MetadataTableUtil.java | 52 +++- .../java/org/apache/accumulo/master/Master.java | 38 +++--- 5 files changed, 131 insertions(+), 24 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/8abe4df6/README -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/8abe4df6/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java -- diff --cc server/base/src/main/java/org/apache/accumulo/server/Accumulo.java index 3235559,ca70efe..516121d --- a/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java +++ b/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java @@@ -191,8 -189,23 +191,23 @@@ public class Accumulo log.info(key + " = " + (Property.isSensitive(key) ? "" : entry.getValue())); } -monitorSwappiness(); +monitorSwappiness(config.getConfiguration()); } + + /** +* Sanity check that the current persistent version is allowed to upgrade to the version of Accumulo running. +* @param dataVersion the version that is persisted in the backing Volumes +*/ + public static boolean canUpgradeFromDataVersion(final int dataVersion) { + return dataVersion == ServerConstants.DATA_VERSION || dataVersion == ServerConstants.PREV_DATA_VERSION || dataVersion == ServerConstants.TWO_DATA_VERSIONS_AGO; + } + + /** +* Does the data version number stored in the backing Volumes indicate we need to upgrade something? +*/ + public static boolean persistentVersionNeedsUpgrade(final int accumuloPersistentVersion) { + return accumuloPersistentVersion == ServerConstants.TWO_DATA_VERSIONS_AGO || accumuloPersistentVersion == ServerConstants.PREV_DATA_VERSION; + } /** * http://git-wip-us.apache.org/repos/asf/accumulo/blob/8abe4df6/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/8abe4df6/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java -- diff --cc server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java index 88e9ed5,e702af4..c49ccda --- a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java @@@ -17,8 -17,8 +17,9 @@@ package org.apache.accumulo.server.util; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; + import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; http://git-wip-us.apache.org/repos/asf/accumulo/blob/8abe4df6/server/master/src/main/java/org/apache/accumulo/master/Master.java --
[05/11] git commit: ACCUMULO-1264 move !METADATA file delete markers
ACCUMULO-1264 move !METADATA file delete markers Description: Move the !METADATA file delete markers out of the !METADATA table and to the root table on upgrade. Author: Eric Newton Ref: ACCUMULO-2988 Modified from commit 5416eef Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/53fcb526 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/53fcb526 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/53fcb526 Branch: refs/heads/1.6.1-SNAPSHOT Commit: 53fcb5262acaac244ef38a1e20c60c2750fef07c Parents: 3beaa5a Author: Mike Drob Authored: Fri May 2 12:32:20 2014 -0400 Committer: Sean Busbey Committed: Thu Jul 31 11:30:58 2014 -0500 -- .../accumulo/server/util/MetadataTableUtil.java | 51 +++- .../java/org/apache/accumulo/master/Master.java | 6 ++- 2 files changed, 43 insertions(+), 14 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/53fcb526/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java -- diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java index 374017d..827eaa9 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java @@ -94,6 +94,7 @@ import org.apache.zookeeper.KeeperException; public class MetadataTableUtil { private static final Text EMPTY_TEXT = new Text(); + private static final byte[] EMPTY_BYTES = new byte[0]; private static Map root_tables = new HashMap(); private static Map metadata_tables = new HashMap(); private static final Logger log = Logger.getLogger(MetadataTableUtil.class); @@ -965,30 +966,54 @@ public class MetadataTableUtil { * During an upgrade we need to move deletion requests for files under the !METADATA table to the root tablet. */ public static void moveMetaDeleteMarkers(Instance instance, Credentials creds) { -// move old delete markers to new location, to standardize table schema between all metadata tables -byte[] EMPTY_BYTES = new byte[0]; -Scanner scanner = new ScannerImpl(instance, creds, RootTable.ID, Authorizations.EMPTY); String oldDeletesPrefix = "!!~del"; Range oldDeletesRange = new Range(oldDeletesPrefix, true, "!!~dem", false); + +// move old delete markers to new location, to standardize table schema between all metadata tables +Scanner scanner = new ScannerImpl(instance, creds, RootTable.ID, Authorizations.EMPTY); scanner.setRange(oldDeletesRange); for (Entry entry : scanner) { String row = entry.getKey().getRow().toString(); if (row.startsWith(oldDeletesPrefix)) { -String filename = row.substring(oldDeletesPrefix.length()); -// add the new entry first -log.info("Moving " + filename + " marker in " + RootTable.NAME); -Mutation m = new Mutation(MetadataSchema.DeletesSection.getRowPrefix() + filename); -m.put(EMPTY_BYTES, EMPTY_BYTES, EMPTY_BYTES); -update(creds, m, RootTable.EXTENT); -// remove the old entry -m = new Mutation(entry.getKey().getRow()); -m.putDelete(EMPTY_BYTES, EMPTY_BYTES); -update(creds, m, RootTable.OLD_EXTENT); +moveDeleteEntry(creds, RootTable.OLD_EXTENT, entry, row, oldDeletesPrefix); + } else { +break; + } +} + + } + + + public static void moveMetaDeleteMarkersFrom14(Instance instance, Credentials creds) { +// new KeyExtent is only added to force update to write to the metadata table, not the root table +KeyExtent notMetadata = new KeyExtent(new Text("anythingNotMetadata"), null, null); + +// move delete markers from the normal delete keyspace to the root tablet delete keyspace if the files are for the !METADATA table +Scanner scanner = new ScannerImpl(instance, creds, MetadataTable.ID, Authorizations.EMPTY); +scanner.setRange(MetadataSchema.DeletesSection.getRange()); +for (Entry entry : scanner) { + String row = entry.getKey().getRow().toString(); + if (row.startsWith(MetadataSchema.DeletesSection.getRowPrefix() + "/" + MetadataTable.ID)) { +moveDeleteEntry(creds, notMetadata, entry, row, MetadataSchema.DeletesSection.getRowPrefix()); } else { break; } } + } + + private static void moveDeleteEntry(Credentials creds, KeyExtent oldExtent, Entry entry, String rowID, String prefix) { +String filename = rowID.substring(prefix.length()); + +// add the new entry first +lo
[01/11] git commit: ACCUMULO-1010 Support upgrades from 1.4 to 1.5
Repository: accumulo Updated Branches: refs/heads/1.6.1-SNAPSHOT ceb1a74d0 -> 3a93bad37 refs/heads/master c8b3e995e -> 8abe4df6c ACCUMULO-1010 Support upgrades from 1.4 to 1.5 Description: Add steps necessary to move a zk from 1.4 state to 1.5. This leaves it in a good state to be picked up by the rest of the 1.6 upgrade path. Author: kturner Ref: ACCUMULO-2988 Cherry picking caused a huge mess and it was simpler to manually select the parts of the upgrade code that we needed based on the 1.5.1 tag. Gathered from commits 317c669, aa6b4309, and 8811859. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/3beaa5a2 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/3beaa5a2 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/3beaa5a2 Branch: refs/heads/1.6.1-SNAPSHOT Commit: 3beaa5a2bab83e4ff2e189db03b038c32e78a85c Parents: 35b524d Author: Mike Drob Authored: Fri May 2 12:01:38 2014 -0400 Committer: Sean Busbey Committed: Thu Jul 31 11:30:57 2014 -0500 -- .../java/org/apache/accumulo/master/Master.java | 16 +++- 1 file changed, 15 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/3beaa5a2/server/master/src/main/java/org/apache/accumulo/master/Master.java -- diff --git a/server/master/src/main/java/org/apache/accumulo/master/Master.java b/server/master/src/main/java/org/apache/accumulo/master/Master.java index b7d7784..a2048b5 100644 --- a/server/master/src/main/java/org/apache/accumulo/master/Master.java +++ b/server/master/src/main/java/org/apache/accumulo/master/Master.java @@ -274,7 +274,8 @@ public class Master implements LiveTServerSet.Listener, TableObserver, CurrentSt // introduce unnecessary complexity to try to make the master do it), but be aware // that the master is not the only thing that may alter zookeeper before starting. -if (Accumulo.getAccumuloPersistentVersion(fs) == ServerConstants.PREV_DATA_VERSION) { +final int accumuloPersistentVersion = Accumulo.getAccumuloPersistentVersion(fs); +if (accumuloPersistentVersion == ServerConstants.TWO_VERSIONS_AGO || accumuloPersistentVersion == ServerConstants.PREV_DATA_VERSION) { // This Master hasn't started Fate yet, so any outstanding transactions must be from before the upgrade. // Change to Guava's Verify once we use Guava 17. if (null != fate) { @@ -285,6 +286,19 @@ public class Master implements LiveTServerSet.Listener, TableObserver, CurrentSt log.info("Upgrading zookeeper"); IZooReaderWriter zoo = ZooReaderWriter.getInstance(); +final String zooRoot = ZooUtil.getRoot(instance); + +if (accumuloPersistentVersion == ServerConstants.TWO_VERSIONS_AGO) { + zoo.recursiveDelete(zooRoot + "/loggers", NodeMissingPolicy.SKIP); + zoo.recursiveDelete(zooRoot + "/dead/loggers", NodeMissingPolicy.SKIP); + + final byte[] zero = new byte[] {'0'}; + zoo.putPersistentData(zooRoot + Constants.ZRECOVERY, zero, NodeExistsPolicy.SKIP); + + for (String id : Tables.getIdToNameMap(instance).keySet()) { +zoo.putPersistentData(zooRoot + Constants.ZTABLES + "/" + id + Constants.ZTABLE_COMPACT_CANCEL_ID, zero, NodeExistsPolicy.SKIP); + } +} // create initial namespaces String namespaces = ZooUtil.getRoot(instance) + Constants.ZNAMESPACES;
[09/11] git commit: ACCUMULO-2988 Upgrade from 1.4 -> 1.6.
ACCUMULO-2988 Upgrade from 1.4 -> 1.6. * add a README section * add log messages around upgrade actions * change 1.4 -> 1.5 upgrade action around compaction cancellation to use 1.6 method of enumerating table ids. Author: Sean Busbey Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/81e3d277 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/81e3d277 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/81e3d277 Branch: refs/heads/1.6.1-SNAPSHOT Commit: 81e3d277e2cc09e6aef4d8f7d07f8f22fccd23f4 Parents: 53fcb52 Author: Sean Busbey Authored: Wed May 14 15:57:27 2014 -0400 Committer: Sean Busbey Committed: Thu Jul 31 11:30:58 2014 -0500 -- README | 41 +++- .../accumulo/server/util/MetadataTableUtil.java | 1 + .../java/org/apache/accumulo/master/Master.java | 9 - 3 files changed, 49 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/81e3d277/README -- diff --git a/README b/README index c2c2446..ec13201 100644 --- a/README +++ b/README @@ -125,7 +125,9 @@ a profile to maven's settings.xml file. Below is an example of $HOME/.m2/setting ** -3. Upgrading from 1.5 to 1.6 +3. Upgrading + +3.1. From 1.5 to 1.6 This happens automatically the first time Accumulo 1.6 is started. @@ -157,6 +159,43 @@ a profile to maven's settings.xml file. Below is an example of $HOME/.m2/setting These messages are safe to ignore. If you need detail on the upgrade's progress you should view the local logs on the Tablet Servers and active Master. +3.2. From 1.4 to 1.6 + + To upgrade from 1.4 to 1.6 you must perform a manual initial step. + + Prior to upgrading you must: + * Verify that there are no outstanding FATE operations + - Under 1.4 you can list what's in FATE by running + $ACCUMULO_HOME/bin/accumulo org.apache.accumulo.server.fate.Admin print + - Note that operations in any state will prevent an upgrade. It is safe + to delete operations with status SUCCESSFUL. For others, you should restart + your 1.4 cluster and allow them to finish. + * Stop the 1.4 instance. + * Configure 1.6 to use the hdfs directory, walog directories, and zookeepers + that 1.4 was using. + * Copy other 1.4 configuration options as needed. + + Prior to starting the 1.6 instance you will need to run the LocalWALRecovery tool + on each node that previously ran an instance of the Logger role. + +$ACCUMULO_HOME/bin/accumulo org.apache.accumulo.tserver.log.LocalWALRecovery + + The recovery tool will rewrite the 1.4 write ahead logs into a format that 1.6 can read. + After this step has completed on all nodes, start the 1.6 cluster to continue the upgrade. + + The upgrade process must make changes to Accumulo's internal state in both ZooKeeper and + the table metadata. This process may take some time if Tablet Servers have to go through + recovery. During this time, the Monitor will claim that the Master is down and some + services may send the Monitor log messages about failure to communicate with each other. + While the upgrade is in progress, the Garbage Collector may complain about invalid paths. + The Master may also complain about failure to create the trace table because it already + exists. These messages are safe to ignore. If other error messages occur, you should seek + out support before continuing to use Accumulo. If you need detail on the upgrade's progress + you should view the local logs on the Tablet Servers and active Master. + + Note that the LocalWALRecovery tool does not delete the local files. Once you confirm that + 1.6 is successfully running, you should delete these files on the local filesystem. + ** 4. Configuring http://git-wip-us.apache.org/repos/asf/accumulo/blob/81e3d277/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java -- diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java index 827eaa9..e702af4 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java @@ -18,6 +18,7 @@ package org.apache.accumulo.server.util; import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; i