accumulo git commit: ACCUMULO-4399 WIP, print makefile actions instead of running.
Repository: accumulo
Updated Branches:
refs/heads/ACCUMULO-4399 [created] 700c4680b
ACCUMULO-4399 WIP, print makefile actions instead of running.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/700c4680
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/700c4680
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/700c4680
Branch: refs/heads/ACCUMULO-4399
Commit: 700c4680b41ddc8506369da9618b8dc81c206aa7
Parents: 71fde9e
Author: Sean Busbey
Authored: Wed Aug 10 16:25:19 2016 -0500
Committer: Sean Busbey
Committed: Wed Aug 10 16:25:19 2016 -0500
--
server/native/pom.xml | 1 +
1 file changed, 1 insertion(+)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/700c4680/server/native/pom.xml
--
diff --git a/server/native/pom.xml b/server/native/pom.xml
index 8a1ff4c..7a0ed1c 100644
--- a/server/native/pom.xml
+++ b/server/native/pom.xml
@@ -121,6 +121,7 @@
make
${project.build.directory}/${project.artifactId}-${project.version}/${project.artifactId}-${project.version}
+--just-print
test
CXX=${CXX}
OUTPUT_DIR=${project.build.directory}
[1/3] accumulo git commit: ACCUMULO-4422 Initialize logging as first step in server initialization.
Repository: accumulo
Updated Branches:
refs/heads/1.6 82f173496 -> 679a1c3eb
refs/heads/1.7 ffd50280b -> 371db68cf
ACCUMULO-4422 Initialize logging as first step in server initialization.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/679a1c3e
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/679a1c3e
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/679a1c3e
Branch: refs/heads/1.6
Commit: 679a1c3eb9013d8bb0640a617392f921f096b158
Parents: 82f1734
Author: Sean Busbey
Authored: Wed Aug 24 10:04:01 2016 -0500
Committer: Sean Busbey
Committed: Wed Aug 24 10:04:01 2016 -0500
--
.../main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java | 2 +-
.../master/src/main/java/org/apache/accumulo/master/Master.java | 4 ++--
.../src/main/java/org/apache/accumulo/monitor/Monitor.java | 4 ++--
.../src/main/java/org/apache/accumulo/tracer/TraceServer.java| 4 ++--
.../src/main/java/org/apache/accumulo/tserver/TabletServer.java | 4 ++--
5 files changed, 9 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/679a1c3e/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
--
diff --git
a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
index 6d50e94..0191f38 100644
--- a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
@@ -136,9 +136,9 @@ public class SimpleGarbageCollector implements Iface {
private Instance instance;
public static void main(String[] args) throws UnknownHostException,
IOException {
-SecurityUtil.serverLogin(ServerConfiguration.getSiteConfiguration());
final String app = "gc";
Accumulo.setupLogging(app);
+SecurityUtil.serverLogin(ServerConfiguration.getSiteConfiguration());
Instance instance = HdfsZooInstance.getInstance();
log.info("Version " + Constants.VERSION);
log.info("Instance " + instance.getInstanceID());
http://git-wip-us.apache.org/repos/asf/accumulo/blob/679a1c3e/server/master/src/main/java/org/apache/accumulo/master/Master.java
--
diff --git a/server/master/src/main/java/org/apache/accumulo/master/Master.java
b/server/master/src/main/java/org/apache/accumulo/master/Master.java
index cc4a02f..14bdd6b 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/Master.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/Master.java
@@ -1156,13 +1156,13 @@ public class Master implements LiveTServerSet.Listener,
TableObserver, CurrentSt
public static void main(String[] args) throws Exception {
try {
+ final String app = "master";
+ Accumulo.setupLogging(app);
SecurityUtil.serverLogin(ServerConfiguration.getSiteConfiguration());
ServerOpts opts = new ServerOpts();
- final String app = "master";
opts.parseArgs(app, args);
String hostname = opts.getAddress();
- Accumulo.setupLogging(app);
Instance instance = HdfsZooInstance.getInstance();
ServerConfiguration conf = new ServerConfiguration(instance);
VolumeManager fs = VolumeManagerImpl.get();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/679a1c3e/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
--
diff --git
a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
index 4e162c3..e5ba9e4 100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
@@ -422,14 +422,14 @@ public class Monitor {
}
public static void main(String[] args) throws Exception {
+final String app = "monitor";
+Accumulo.setupLogging(app);
SecurityUtil.serverLogin(ServerConfiguration.getSiteConfiguration());
ServerOpts opts = new ServerOpts();
-final String app = "monitor";
opts.parseArgs(app, args);
String hostname = opts.getAddress();
-Accumulo.setupLogging(app);
VolumeManager fs = VolumeManagerImpl.get();
instance = HdfsZooInstance.getInstance();
config = new ServerConfiguration(instance);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/679a1c3e/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
[3/3] accumulo git commit: Merge branch '1.6' into 1.7
Merge branch '1.6' into 1.7
Conflicts:
server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
server/master/src/main/java/org/apache/accumulo/master/Master.java
server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/371db68c
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/371db68c
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/371db68c
Branch: refs/heads/1.7
Commit: 371db68cfc4a82921a0b47c886be3e88c4058cdf
Parents: ffd5028 679a1c3
Author: Sean Busbey
Authored: Wed Aug 24 10:22:25 2016 -0500
Committer: Sean Busbey
Committed: Wed Aug 24 10:22:25 2016 -0500
--
.../main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java | 2 +-
.../master/src/main/java/org/apache/accumulo/master/Master.java | 4 ++--
.../src/main/java/org/apache/accumulo/monitor/Monitor.java | 4 ++--
.../src/main/java/org/apache/accumulo/tracer/TraceServer.java| 4 ++--
.../src/main/java/org/apache/accumulo/tserver/TabletServer.java | 4 ++--
5 files changed, 9 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/371db68c/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
--
diff --cc
server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
index da25d55,0191f38..98acf9a
--- a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
@@@ -139,14 -133,16 +139,14 @@@ public class SimpleGarbageCollector ext
private GCStatus status = new GCStatus(new GcCycleStats(), new
GcCycleStats(), new GcCycleStats(), new GcCycleStats());
- private Instance instance;
-
public static void main(String[] args) throws UnknownHostException,
IOException {
- SecurityUtil.serverLogin(SiteConfiguration.getInstance());
final String app = "gc";
Accumulo.setupLogging(app);
-SecurityUtil.serverLogin(ServerConfiguration.getSiteConfiguration());
++SecurityUtil.serverLogin(SiteConfiguration.getInstance());
Instance instance = HdfsZooInstance.getInstance();
+ServerConfigurationFactory conf = new
ServerConfigurationFactory(instance);
log.info("Version " + Constants.VERSION);
log.info("Instance " + instance.getInstanceID());
-ServerConfiguration conf = new ServerConfiguration(instance);
final VolumeManager fs = VolumeManagerImpl.get();
Accumulo.init(fs, conf, app);
Opts opts = new Opts();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/371db68c/server/master/src/main/java/org/apache/accumulo/master/Master.java
--
diff --cc server/master/src/main/java/org/apache/accumulo/master/Master.java
index 5a2a346,14bdd6b..dc26a11
--- a/server/master/src/main/java/org/apache/accumulo/master/Master.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/Master.java
@@@ -1345,14 -1156,15 +1345,14 @@@ public class Master extends AccumuloSer
public static void main(String[] args) throws Exception {
try {
+ final String app = "master";
+ Accumulo.setupLogging(app);
- SecurityUtil.serverLogin(ServerConfiguration.getSiteConfiguration());
+ SecurityUtil.serverLogin(SiteConfiguration.getInstance());
ServerOpts opts = new ServerOpts();
- final String app = "master";
opts.parseArgs(app, args);
String hostname = opts.getAddress();
- Accumulo.setupLogging(app);
- Instance instance = HdfsZooInstance.getInstance();
- ServerConfiguration conf = new ServerConfiguration(instance);
+ ServerConfigurationFactory conf = new
ServerConfigurationFactory(HdfsZooInstance.getInstance());
VolumeManager fs = VolumeManagerImpl.get();
Accumulo.init(fs, conf, app);
Master master = new Master(conf, fs, hostname);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/371db68c/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
--
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
index c3dd773,e5ba9e4..58cf921
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
@@@ -417,18 -422,17 +417,18 @@@ publi
[2/3] accumulo git commit: ACCUMULO-4422 Initialize logging as first step in server initialization.
ACCUMULO-4422 Initialize logging as first step in server initialization.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/679a1c3e
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/679a1c3e
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/679a1c3e
Branch: refs/heads/1.7
Commit: 679a1c3eb9013d8bb0640a617392f921f096b158
Parents: 82f1734
Author: Sean Busbey
Authored: Wed Aug 24 10:04:01 2016 -0500
Committer: Sean Busbey
Committed: Wed Aug 24 10:04:01 2016 -0500
--
.../main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java | 2 +-
.../master/src/main/java/org/apache/accumulo/master/Master.java | 4 ++--
.../src/main/java/org/apache/accumulo/monitor/Monitor.java | 4 ++--
.../src/main/java/org/apache/accumulo/tracer/TraceServer.java| 4 ++--
.../src/main/java/org/apache/accumulo/tserver/TabletServer.java | 4 ++--
5 files changed, 9 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/679a1c3e/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
--
diff --git
a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
index 6d50e94..0191f38 100644
--- a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
@@ -136,9 +136,9 @@ public class SimpleGarbageCollector implements Iface {
private Instance instance;
public static void main(String[] args) throws UnknownHostException,
IOException {
-SecurityUtil.serverLogin(ServerConfiguration.getSiteConfiguration());
final String app = "gc";
Accumulo.setupLogging(app);
+SecurityUtil.serverLogin(ServerConfiguration.getSiteConfiguration());
Instance instance = HdfsZooInstance.getInstance();
log.info("Version " + Constants.VERSION);
log.info("Instance " + instance.getInstanceID());
http://git-wip-us.apache.org/repos/asf/accumulo/blob/679a1c3e/server/master/src/main/java/org/apache/accumulo/master/Master.java
--
diff --git a/server/master/src/main/java/org/apache/accumulo/master/Master.java
b/server/master/src/main/java/org/apache/accumulo/master/Master.java
index cc4a02f..14bdd6b 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/Master.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/Master.java
@@ -1156,13 +1156,13 @@ public class Master implements LiveTServerSet.Listener,
TableObserver, CurrentSt
public static void main(String[] args) throws Exception {
try {
+ final String app = "master";
+ Accumulo.setupLogging(app);
SecurityUtil.serverLogin(ServerConfiguration.getSiteConfiguration());
ServerOpts opts = new ServerOpts();
- final String app = "master";
opts.parseArgs(app, args);
String hostname = opts.getAddress();
- Accumulo.setupLogging(app);
Instance instance = HdfsZooInstance.getInstance();
ServerConfiguration conf = new ServerConfiguration(instance);
VolumeManager fs = VolumeManagerImpl.get();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/679a1c3e/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
--
diff --git
a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
index 4e162c3..e5ba9e4 100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
@@ -422,14 +422,14 @@ public class Monitor {
}
public static void main(String[] args) throws Exception {
+final String app = "monitor";
+Accumulo.setupLogging(app);
SecurityUtil.serverLogin(ServerConfiguration.getSiteConfiguration());
ServerOpts opts = new ServerOpts();
-final String app = "monitor";
opts.parseArgs(app, args);
String hostname = opts.getAddress();
-Accumulo.setupLogging(app);
VolumeManager fs = VolumeManagerImpl.get();
instance = HdfsZooInstance.getInstance();
config = new ServerConfiguration(instance);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/679a1c3e/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
--
diff --git
a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
b/server/
[1/2] accumulo git commit: ACCUMULO-4421 Check if the Trace User is expected to use Kerberos before attempting to login to Kerberos as the trace user.
Repository: accumulo
Updated Branches:
refs/heads/1.7 40d5a722b -> 2be85ade3
ACCUMULO-4421 Check if the Trace User is expected to use Kerberos before
attempting to login to Kerberos as the trace user.
Signed-off-by: Josh Elser
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/d66a8d08
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/d66a8d08
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/d66a8d08
Branch: refs/heads/1.7
Commit: d66a8d08627e98e9bbdd2bd0b1ab4f4658a84d9f
Parents: 40d5a72
Author: Sean Busbey
Authored: Thu Aug 25 14:47:38 2016 -0500
Committer: Sean Busbey
Committed: Fri Aug 26 19:08:30 2016 -0500
--
.../org/apache/accumulo/tracer/TraceServer.java | 61 +---
1 file changed, 41 insertions(+), 20 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d66a8d08/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
--
diff --git
a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
index 4b07dcc..2a06dc3 100644
--- a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
+++ b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
@@ -36,6 +36,7 @@ import org.apache.accumulo.core.client.IteratorSetting;
import org.apache.accumulo.core.client.MutationsRejectedException;
import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
import
org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Properties;
+import org.apache.accumulo.core.client.security.tokens.KerberosToken;
import org.apache.accumulo.core.client.security.tokens.PasswordToken;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
@@ -306,30 +307,50 @@ public class TraceServer implements Watcher {
}
private static void loginTracer(AccumuloConfiguration acuConf) {
-Map loginMap =
acuConf.getAllPropertiesWithPrefix(Property.TRACE_TOKEN_PROPERTY_PREFIX);
-String keyTab = loginMap.get(Property.TRACE_TOKEN_PROPERTY_PREFIX.getKey()
+ "keytab");
-if (keyTab == null || keyTab.length() == 0) {
- keyTab = acuConf.getPath(Property.GENERAL_KERBEROS_KEYTAB);
-}
-if (keyTab == null || keyTab.length() == 0)
- return;
+try {
+ Class traceTokenType =
AccumuloVFSClassLoader.getClassLoader().loadClass(acuConf.get(Property.TRACE_TOKEN_TYPE))
+ .asSubclass(AuthenticationToken.class);
+
+ if (!(KerberosToken.class.isAssignableFrom(traceTokenType))) {
+// We're not using Kerberos to talk to Accumulo, but we might still
need it for talking to HDFS/ZK for
+// instance information.
+log.info("Handling login under the assumption that Accumulo users are
not using Kerberos.");
+SecurityUtil.serverLogin(acuConf);
+ } else {
+// We're using Kerberos to talk to Accumulo, so check for trace user
specific auth details.
+// We presume this same user will have the needed access for the
service to interact with HDFS/ZK for
+// instance information.
+log.info("Handling login under the assumption that Accumulo users are
using Kerberos.");
+Map loginMap =
acuConf.getAllPropertiesWithPrefix(Property.TRACE_TOKEN_PROPERTY_PREFIX);
+String keyTab =
loginMap.get(Property.TRACE_TOKEN_PROPERTY_PREFIX.getKey() + "keytab");
+if (keyTab == null || keyTab.length() == 0) {
+ keyTab = acuConf.getPath(Property.GENERAL_KERBEROS_KEYTAB);
+}
+if (keyTab == null || keyTab.length() == 0)
+ return;
-String principalConfig = acuConf.get(Property.TRACE_USER);
-if (principalConfig == null || principalConfig.length() == 0)
- return;
+String principalConfig = acuConf.get(Property.TRACE_USER);
+if (principalConfig == null || principalConfig.length() == 0)
+ return;
-log.info("Attempting to login as {} with {}", principalConfig, keyTab);
-if (SecurityUtil.login(principalConfig, keyTab)) {
- try {
-// This spawns a thread to periodically renew the logged in (trace)
user
-UserGroupInformation.getLoginUser();
-return;
- } catch (IOException io) {
-log.error("Error starting up renewal thread. This shouldn't be
happening.", io);
+log.info("Attempting to login as {} with {}", principalConfig, keyTab);
+if (SecurityUtil.login(principalConfig, keyTab)) {
+ try {
+// This spawns a thread to periodically renew the logge
[2/2] accumulo git commit: ACCUMULO-4421 Ensure that TraceServer launches the Kerberos ticket renewal thread
ACCUMULO-4421 Ensure that TraceServer launches the Kerberos ticket renewal
thread
The previous method that TraceServer was using on SecurityUtil
only performed the login and was relying on incorrect functionality
in Hadoop's UGI to launch a renewal thread. This logic is incorrect.
Refactored SecurityUtil a little to prevent other callers from
making the same mistake in the future.
Signed-off-by: Sean Busbey
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/2be85ade
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/2be85ade
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/2be85ade
Branch: refs/heads/1.7
Commit: 2be85ade34c413cc32db838e6125e582b829ef03
Parents: d66a8d0
Author: Josh Elser
Authored: Fri Aug 26 17:33:42 2016 -0400
Committer: Sean Busbey
Committed: Fri Aug 26 19:08:56 2016 -0500
--
.../accumulo/server/security/SecurityUtil.java | 30 ++--
.../org/apache/accumulo/tracer/TraceServer.java | 13 +
2 files changed, 23 insertions(+), 20 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/2be85ade/server/base/src/main/java/org/apache/accumulo/server/security/SecurityUtil.java
--
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/security/SecurityUtil.java
b/server/base/src/main/java/org/apache/accumulo/server/security/SecurityUtil.java
index 73f671b..38afa31 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/security/SecurityUtil.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/security/SecurityUtil.java
@@ -40,17 +40,31 @@ public class SecurityUtil {
* {@link #login(String, String)}
*/
public static void serverLogin(AccumuloConfiguration acuConf) {
-String keyTab = acuConf.getPath(Property.GENERAL_KERBEROS_KEYTAB);
+serverLogin(acuConf, acuConf.getPath(Property.GENERAL_KERBEROS_KEYTAB),
acuConf.get(Property.GENERAL_KERBEROS_PRINCIPAL));
+ }
+
+ /**
+ * Performs a Kerberos login using the given Kerberos principal and keytab
if they are non-null and positive length Strings. This method automaticallys
spawns
+ * a thread to renew the given ticket upon successful login using {@link
Property#GENERAL_KERBEROS_RENEWAL_PERIOD} as the renewal period. This method
does
+ * nothing if either {@code keyTab} or {@code principal} are null or of zero
length.
+ *
+ * @param acuConf
+ * The Accumulo configuration
+ * @param keyTab
+ * The path to the Kerberos keytab file
+ * @param principal
+ * The Kerberos principal
+ */
+ public static void serverLogin(AccumuloConfiguration acuConf, String keyTab,
String principal) {
if (keyTab == null || keyTab.length() == 0)
return;
-usingKerberos = true;
-
-String principalConfig = acuConf.get(Property.GENERAL_KERBEROS_PRINCIPAL);
-if (principalConfig == null || principalConfig.length() == 0)
+if (principal == null || principal.length() == 0)
return;
-if (login(principalConfig, keyTab)) {
+usingKerberos = true;
+
+if (login(principal, keyTab)) {
try {
startTicketRenewalThread(UserGroupInformation.getCurrentUser(),
acuConf.getTimeInMillis(Property.GENERAL_KERBEROS_RENEWAL_PERIOD));
return;
@@ -59,7 +73,7 @@ public class SecurityUtil {
}
}
-throw new RuntimeException("Failed to perform Kerberos login for " +
principalConfig + " using " + keyTab);
+throw new RuntimeException("Failed to perform Kerberos login for " +
principal + " using " + keyTab);
}
/**
@@ -70,7 +84,7 @@ public class SecurityUtil {
* replaced by the systems host name.
* @return true if login succeeded, otherwise false
*/
- public static boolean login(String principalConfig, String keyTabPath) {
+ static boolean login(String principalConfig, String keyTabPath) {
try {
String principalName = getServerPrincipal(principalConfig);
if (keyTabPath != null && principalName != null && keyTabPath.length()
!= 0 && principalName.length() != 0) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/2be85ade/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
--
diff --git
a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
index 2a06dc3..10601ba 100644
--- a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
+++ b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
@@ -61,7
[3/3] accumulo git commit: Merge branch '1.7' into 1.8
Merge branch '1.7' into 1.8 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/e68b8dbc Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/e68b8dbc Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/e68b8dbc Branch: refs/heads/1.8 Commit: e68b8dbc61a204e73452f254a5f130ac813c9b78 Parents: c2900a3 2be85ad Author: Sean Busbey Authored: Fri Aug 26 19:23:13 2016 -0500 Committer: Sean Busbey Committed: Fri Aug 26 19:23:13 2016 -0500 -- .../accumulo/server/security/SecurityUtil.java | 30 --- .../org/apache/accumulo/tracer/TraceServer.java | 52 2 files changed, 53 insertions(+), 29 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/e68b8dbc/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java --
[1/3] accumulo git commit: ACCUMULO-4421 Check if the Trace User is expected to use Kerberos before attempting to login to Kerberos as the trace user.
Repository: accumulo
Updated Branches:
refs/heads/1.8 c2900a380 -> e68b8dbc6
ACCUMULO-4421 Check if the Trace User is expected to use Kerberos before
attempting to login to Kerberos as the trace user.
Signed-off-by: Josh Elser
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/d66a8d08
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/d66a8d08
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/d66a8d08
Branch: refs/heads/1.8
Commit: d66a8d08627e98e9bbdd2bd0b1ab4f4658a84d9f
Parents: 40d5a72
Author: Sean Busbey
Authored: Thu Aug 25 14:47:38 2016 -0500
Committer: Sean Busbey
Committed: Fri Aug 26 19:08:30 2016 -0500
--
.../org/apache/accumulo/tracer/TraceServer.java | 61 +---
1 file changed, 41 insertions(+), 20 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d66a8d08/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
--
diff --git
a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
index 4b07dcc..2a06dc3 100644
--- a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
+++ b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
@@ -36,6 +36,7 @@ import org.apache.accumulo.core.client.IteratorSetting;
import org.apache.accumulo.core.client.MutationsRejectedException;
import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
import
org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Properties;
+import org.apache.accumulo.core.client.security.tokens.KerberosToken;
import org.apache.accumulo.core.client.security.tokens.PasswordToken;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
@@ -306,30 +307,50 @@ public class TraceServer implements Watcher {
}
private static void loginTracer(AccumuloConfiguration acuConf) {
-Map loginMap =
acuConf.getAllPropertiesWithPrefix(Property.TRACE_TOKEN_PROPERTY_PREFIX);
-String keyTab = loginMap.get(Property.TRACE_TOKEN_PROPERTY_PREFIX.getKey()
+ "keytab");
-if (keyTab == null || keyTab.length() == 0) {
- keyTab = acuConf.getPath(Property.GENERAL_KERBEROS_KEYTAB);
-}
-if (keyTab == null || keyTab.length() == 0)
- return;
+try {
+ Class traceTokenType =
AccumuloVFSClassLoader.getClassLoader().loadClass(acuConf.get(Property.TRACE_TOKEN_TYPE))
+ .asSubclass(AuthenticationToken.class);
+
+ if (!(KerberosToken.class.isAssignableFrom(traceTokenType))) {
+// We're not using Kerberos to talk to Accumulo, but we might still
need it for talking to HDFS/ZK for
+// instance information.
+log.info("Handling login under the assumption that Accumulo users are
not using Kerberos.");
+SecurityUtil.serverLogin(acuConf);
+ } else {
+// We're using Kerberos to talk to Accumulo, so check for trace user
specific auth details.
+// We presume this same user will have the needed access for the
service to interact with HDFS/ZK for
+// instance information.
+log.info("Handling login under the assumption that Accumulo users are
using Kerberos.");
+Map loginMap =
acuConf.getAllPropertiesWithPrefix(Property.TRACE_TOKEN_PROPERTY_PREFIX);
+String keyTab =
loginMap.get(Property.TRACE_TOKEN_PROPERTY_PREFIX.getKey() + "keytab");
+if (keyTab == null || keyTab.length() == 0) {
+ keyTab = acuConf.getPath(Property.GENERAL_KERBEROS_KEYTAB);
+}
+if (keyTab == null || keyTab.length() == 0)
+ return;
-String principalConfig = acuConf.get(Property.TRACE_USER);
-if (principalConfig == null || principalConfig.length() == 0)
- return;
+String principalConfig = acuConf.get(Property.TRACE_USER);
+if (principalConfig == null || principalConfig.length() == 0)
+ return;
-log.info("Attempting to login as {} with {}", principalConfig, keyTab);
-if (SecurityUtil.login(principalConfig, keyTab)) {
- try {
-// This spawns a thread to periodically renew the logged in (trace)
user
-UserGroupInformation.getLoginUser();
-return;
- } catch (IOException io) {
-log.error("Error starting up renewal thread. This shouldn't be
happening.", io);
+log.info("Attempting to login as {} with {}", principalConfig, keyTab);
+if (SecurityUtil.login(principalConfig, keyTab)) {
+ try {
+// This spawns a thread to periodically renew the logge
[2/3] accumulo git commit: ACCUMULO-4421 Ensure that TraceServer launches the Kerberos ticket renewal thread
ACCUMULO-4421 Ensure that TraceServer launches the Kerberos ticket renewal
thread
The previous method that TraceServer was using on SecurityUtil
only performed the login and was relying on incorrect functionality
in Hadoop's UGI to launch a renewal thread. This logic is incorrect.
Refactored SecurityUtil a little to prevent other callers from
making the same mistake in the future.
Signed-off-by: Sean Busbey
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/2be85ade
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/2be85ade
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/2be85ade
Branch: refs/heads/1.8
Commit: 2be85ade34c413cc32db838e6125e582b829ef03
Parents: d66a8d0
Author: Josh Elser
Authored: Fri Aug 26 17:33:42 2016 -0400
Committer: Sean Busbey
Committed: Fri Aug 26 19:08:56 2016 -0500
--
.../accumulo/server/security/SecurityUtil.java | 30 ++--
.../org/apache/accumulo/tracer/TraceServer.java | 13 +
2 files changed, 23 insertions(+), 20 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/2be85ade/server/base/src/main/java/org/apache/accumulo/server/security/SecurityUtil.java
--
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/security/SecurityUtil.java
b/server/base/src/main/java/org/apache/accumulo/server/security/SecurityUtil.java
index 73f671b..38afa31 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/security/SecurityUtil.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/security/SecurityUtil.java
@@ -40,17 +40,31 @@ public class SecurityUtil {
* {@link #login(String, String)}
*/
public static void serverLogin(AccumuloConfiguration acuConf) {
-String keyTab = acuConf.getPath(Property.GENERAL_KERBEROS_KEYTAB);
+serverLogin(acuConf, acuConf.getPath(Property.GENERAL_KERBEROS_KEYTAB),
acuConf.get(Property.GENERAL_KERBEROS_PRINCIPAL));
+ }
+
+ /**
+ * Performs a Kerberos login using the given Kerberos principal and keytab
if they are non-null and positive length Strings. This method automaticallys
spawns
+ * a thread to renew the given ticket upon successful login using {@link
Property#GENERAL_KERBEROS_RENEWAL_PERIOD} as the renewal period. This method
does
+ * nothing if either {@code keyTab} or {@code principal} are null or of zero
length.
+ *
+ * @param acuConf
+ * The Accumulo configuration
+ * @param keyTab
+ * The path to the Kerberos keytab file
+ * @param principal
+ * The Kerberos principal
+ */
+ public static void serverLogin(AccumuloConfiguration acuConf, String keyTab,
String principal) {
if (keyTab == null || keyTab.length() == 0)
return;
-usingKerberos = true;
-
-String principalConfig = acuConf.get(Property.GENERAL_KERBEROS_PRINCIPAL);
-if (principalConfig == null || principalConfig.length() == 0)
+if (principal == null || principal.length() == 0)
return;
-if (login(principalConfig, keyTab)) {
+usingKerberos = true;
+
+if (login(principal, keyTab)) {
try {
startTicketRenewalThread(UserGroupInformation.getCurrentUser(),
acuConf.getTimeInMillis(Property.GENERAL_KERBEROS_RENEWAL_PERIOD));
return;
@@ -59,7 +73,7 @@ public class SecurityUtil {
}
}
-throw new RuntimeException("Failed to perform Kerberos login for " +
principalConfig + " using " + keyTab);
+throw new RuntimeException("Failed to perform Kerberos login for " +
principal + " using " + keyTab);
}
/**
@@ -70,7 +84,7 @@ public class SecurityUtil {
* replaced by the systems host name.
* @return true if login succeeded, otherwise false
*/
- public static boolean login(String principalConfig, String keyTabPath) {
+ static boolean login(String principalConfig, String keyTabPath) {
try {
String principalName = getServerPrincipal(principalConfig);
if (keyTabPath != null && principalName != null && keyTabPath.length()
!= 0 && principalName.length() != 0) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/2be85ade/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
--
diff --git
a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
index 2a06dc3..10601ba 100644
--- a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
+++ b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
@@ -61,7
[1/4] accumulo git commit: ACCUMULO-4421 Check if the Trace User is expected to use Kerberos before attempting to login to Kerberos as the trace user.
Repository: accumulo
Updated Branches:
refs/heads/master 97ff25db2 -> 59c6a8e19
ACCUMULO-4421 Check if the Trace User is expected to use Kerberos before
attempting to login to Kerberos as the trace user.
Signed-off-by: Josh Elser
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/d66a8d08
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/d66a8d08
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/d66a8d08
Branch: refs/heads/master
Commit: d66a8d08627e98e9bbdd2bd0b1ab4f4658a84d9f
Parents: 40d5a72
Author: Sean Busbey
Authored: Thu Aug 25 14:47:38 2016 -0500
Committer: Sean Busbey
Committed: Fri Aug 26 19:08:30 2016 -0500
--
.../org/apache/accumulo/tracer/TraceServer.java | 61 +---
1 file changed, 41 insertions(+), 20 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d66a8d08/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
--
diff --git
a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
index 4b07dcc..2a06dc3 100644
--- a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
+++ b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
@@ -36,6 +36,7 @@ import org.apache.accumulo.core.client.IteratorSetting;
import org.apache.accumulo.core.client.MutationsRejectedException;
import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
import
org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Properties;
+import org.apache.accumulo.core.client.security.tokens.KerberosToken;
import org.apache.accumulo.core.client.security.tokens.PasswordToken;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
@@ -306,30 +307,50 @@ public class TraceServer implements Watcher {
}
private static void loginTracer(AccumuloConfiguration acuConf) {
-Map loginMap =
acuConf.getAllPropertiesWithPrefix(Property.TRACE_TOKEN_PROPERTY_PREFIX);
-String keyTab = loginMap.get(Property.TRACE_TOKEN_PROPERTY_PREFIX.getKey()
+ "keytab");
-if (keyTab == null || keyTab.length() == 0) {
- keyTab = acuConf.getPath(Property.GENERAL_KERBEROS_KEYTAB);
-}
-if (keyTab == null || keyTab.length() == 0)
- return;
+try {
+ Class traceTokenType =
AccumuloVFSClassLoader.getClassLoader().loadClass(acuConf.get(Property.TRACE_TOKEN_TYPE))
+ .asSubclass(AuthenticationToken.class);
+
+ if (!(KerberosToken.class.isAssignableFrom(traceTokenType))) {
+// We're not using Kerberos to talk to Accumulo, but we might still
need it for talking to HDFS/ZK for
+// instance information.
+log.info("Handling login under the assumption that Accumulo users are
not using Kerberos.");
+SecurityUtil.serverLogin(acuConf);
+ } else {
+// We're using Kerberos to talk to Accumulo, so check for trace user
specific auth details.
+// We presume this same user will have the needed access for the
service to interact with HDFS/ZK for
+// instance information.
+log.info("Handling login under the assumption that Accumulo users are
using Kerberos.");
+Map loginMap =
acuConf.getAllPropertiesWithPrefix(Property.TRACE_TOKEN_PROPERTY_PREFIX);
+String keyTab =
loginMap.get(Property.TRACE_TOKEN_PROPERTY_PREFIX.getKey() + "keytab");
+if (keyTab == null || keyTab.length() == 0) {
+ keyTab = acuConf.getPath(Property.GENERAL_KERBEROS_KEYTAB);
+}
+if (keyTab == null || keyTab.length() == 0)
+ return;
-String principalConfig = acuConf.get(Property.TRACE_USER);
-if (principalConfig == null || principalConfig.length() == 0)
- return;
+String principalConfig = acuConf.get(Property.TRACE_USER);
+if (principalConfig == null || principalConfig.length() == 0)
+ return;
-log.info("Attempting to login as {} with {}", principalConfig, keyTab);
-if (SecurityUtil.login(principalConfig, keyTab)) {
- try {
-// This spawns a thread to periodically renew the logged in (trace)
user
-UserGroupInformation.getLoginUser();
-return;
- } catch (IOException io) {
-log.error("Error starting up renewal thread. This shouldn't be
happening.", io);
+log.info("Attempting to login as {} with {}", principalConfig, keyTab);
+if (SecurityUtil.login(principalConfig, keyTab)) {
+ try {
+// This spawns a thread to periodically renew the logge
[2/4] accumulo git commit: ACCUMULO-4421 Ensure that TraceServer launches the Kerberos ticket renewal thread
ACCUMULO-4421 Ensure that TraceServer launches the Kerberos ticket renewal
thread
The previous method that TraceServer was using on SecurityUtil
only performed the login and was relying on incorrect functionality
in Hadoop's UGI to launch a renewal thread. This logic is incorrect.
Refactored SecurityUtil a little to prevent other callers from
making the same mistake in the future.
Signed-off-by: Sean Busbey
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/2be85ade
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/2be85ade
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/2be85ade
Branch: refs/heads/master
Commit: 2be85ade34c413cc32db838e6125e582b829ef03
Parents: d66a8d0
Author: Josh Elser
Authored: Fri Aug 26 17:33:42 2016 -0400
Committer: Sean Busbey
Committed: Fri Aug 26 19:08:56 2016 -0500
--
.../accumulo/server/security/SecurityUtil.java | 30 ++--
.../org/apache/accumulo/tracer/TraceServer.java | 13 +
2 files changed, 23 insertions(+), 20 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/2be85ade/server/base/src/main/java/org/apache/accumulo/server/security/SecurityUtil.java
--
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/security/SecurityUtil.java
b/server/base/src/main/java/org/apache/accumulo/server/security/SecurityUtil.java
index 73f671b..38afa31 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/security/SecurityUtil.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/security/SecurityUtil.java
@@ -40,17 +40,31 @@ public class SecurityUtil {
* {@link #login(String, String)}
*/
public static void serverLogin(AccumuloConfiguration acuConf) {
-String keyTab = acuConf.getPath(Property.GENERAL_KERBEROS_KEYTAB);
+serverLogin(acuConf, acuConf.getPath(Property.GENERAL_KERBEROS_KEYTAB),
acuConf.get(Property.GENERAL_KERBEROS_PRINCIPAL));
+ }
+
+ /**
+ * Performs a Kerberos login using the given Kerberos principal and keytab
if they are non-null and positive length Strings. This method automaticallys
spawns
+ * a thread to renew the given ticket upon successful login using {@link
Property#GENERAL_KERBEROS_RENEWAL_PERIOD} as the renewal period. This method
does
+ * nothing if either {@code keyTab} or {@code principal} are null or of zero
length.
+ *
+ * @param acuConf
+ * The Accumulo configuration
+ * @param keyTab
+ * The path to the Kerberos keytab file
+ * @param principal
+ * The Kerberos principal
+ */
+ public static void serverLogin(AccumuloConfiguration acuConf, String keyTab,
String principal) {
if (keyTab == null || keyTab.length() == 0)
return;
-usingKerberos = true;
-
-String principalConfig = acuConf.get(Property.GENERAL_KERBEROS_PRINCIPAL);
-if (principalConfig == null || principalConfig.length() == 0)
+if (principal == null || principal.length() == 0)
return;
-if (login(principalConfig, keyTab)) {
+usingKerberos = true;
+
+if (login(principal, keyTab)) {
try {
startTicketRenewalThread(UserGroupInformation.getCurrentUser(),
acuConf.getTimeInMillis(Property.GENERAL_KERBEROS_RENEWAL_PERIOD));
return;
@@ -59,7 +73,7 @@ public class SecurityUtil {
}
}
-throw new RuntimeException("Failed to perform Kerberos login for " +
principalConfig + " using " + keyTab);
+throw new RuntimeException("Failed to perform Kerberos login for " +
principal + " using " + keyTab);
}
/**
@@ -70,7 +84,7 @@ public class SecurityUtil {
* replaced by the systems host name.
* @return true if login succeeded, otherwise false
*/
- public static boolean login(String principalConfig, String keyTabPath) {
+ static boolean login(String principalConfig, String keyTabPath) {
try {
String principalName = getServerPrincipal(principalConfig);
if (keyTabPath != null && principalName != null && keyTabPath.length()
!= 0 && principalName.length() != 0) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/2be85ade/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
--
diff --git
a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
index 2a06dc3..10601ba 100644
--- a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
+++ b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
@@ -61,7
[3/4] accumulo git commit: Merge branch '1.7' into 1.8
Merge branch '1.7' into 1.8 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/e68b8dbc Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/e68b8dbc Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/e68b8dbc Branch: refs/heads/master Commit: e68b8dbc61a204e73452f254a5f130ac813c9b78 Parents: c2900a3 2be85ad Author: Sean Busbey Authored: Fri Aug 26 19:23:13 2016 -0500 Committer: Sean Busbey Committed: Fri Aug 26 19:23:13 2016 -0500 -- .../accumulo/server/security/SecurityUtil.java | 30 --- .../org/apache/accumulo/tracer/TraceServer.java | 52 2 files changed, 53 insertions(+), 29 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/e68b8dbc/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java --
[4/4] accumulo git commit: Merge branch '1.8'
Merge branch '1.8' Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/59c6a8e1 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/59c6a8e1 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/59c6a8e1 Branch: refs/heads/master Commit: 59c6a8e1f11cd3fc2db1416f8eb82c7de480 Parents: 97ff25d e68b8db Author: Sean Busbey Authored: Fri Aug 26 21:46:04 2016 -0500 Committer: Sean Busbey Committed: Fri Aug 26 21:46:04 2016 -0500 -- .../accumulo/server/security/SecurityUtil.java | 30 --- .../org/apache/accumulo/tracer/TraceServer.java | 52 2 files changed, 53 insertions(+), 29 deletions(-) --
accumulo git commit: ACCUMULO-4458 reduce contention on config retrievals.
Repository: accumulo
Updated Branches:
refs/heads/1.7 4136bfafc -> 48733ce8c
ACCUMULO-4458 reduce contention on config retrievals.
We retrieve configuration values in a few hot code paths, so contention
on getting the site configuration and values that are only read from the
filesystem
once causes a perf impact. We try to reduce that here in a few ways:
* Since SiteConfiguration is a singleton, we only retrieve it once when
HdfsZooInstance is made instead of on each lookup.
* Since HdfsZooInstance is a singleton, we create it once on initialization and
avoid synchronizing on retrieval
* Since the Hadoop Configuration object uses a single lock for all gets against
the shared instance, we copy out the values
it reads when the SiteConfiguration singleton is created. This has the
side-effect of causing our for-test-only set method
to only work on values not set in the Configuration (essentially we treat all
configs as final).
* Additionally, for those configurations that are read within any of the
processing loops of the TabletServer, we have the
SiteConfiguration singleton cache the fact that they weren't in the Hadoop
Configuration object so that we will default to
the parent AccumuloConfiguration without contending for the Hadoop
Configuration object lock.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/48733ce8
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/48733ce8
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/48733ce8
Branch: refs/heads/1.7
Commit: 48733ce8cb9e89f1243e5e46d792010900086c7b
Parents: 4136bfa
Author: Sean Busbey
Authored: Tue Sep 6 11:52:37 2016 -0500
Committer: Sean Busbey
Committed: Wed Sep 14 16:35:31 2016 -0700
--
.../org/apache/accumulo/core/conf/Property.java | 7
.../accumulo/core/conf/SiteConfiguration.java | 24 -
.../accumulo/server/client/HdfsZooInstance.java | 36 +---
3 files changed, 53 insertions(+), 14 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/48733ce8/core/src/main/java/org/apache/accumulo/core/conf/Property.java
--
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 9d145e1..96adaeb 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -803,6 +803,13 @@ public enum Property {
|| key.startsWith(Property.TABLE_ARBITRARY_PROP_PREFIX.getKey());
}
+ /**
+ * Properties we check the value of within the TabletServer request handling
or maintenance processing loops.
+ */
+ public static final EnumSet HOT_PATH_PROPERTIES =
EnumSet.of(Property.TSERV_CLIENT_TIMEOUT,
Property.TSERV_TOTAL_MUTATION_QUEUE_MAX,
+ Property.TSERV_ARCHIVE_WALOGS, Property.GC_TRASH_IGNORE,
Property.TSERV_MAJC_DELAY, Property.TABLE_MINC_LOGS_MAX,
Property.TSERV_MAJC_MAXCONCURRENT,
+ Property.REPLICATION_WORKER_THREADS, Property.TABLE_DURABILITY,
Property.INSTANCE_ZK_TIMEOUT, Property.TABLE_CLASSPATH);
+
private static final EnumSet fixedProperties =
EnumSet.of(Property.TSERV_CLIENTPORT, Property.TSERV_NATIVEMAP_ENABLED,
Property.TSERV_SCAN_MAX_OPENFILES, Property.MASTER_CLIENTPORT,
Property.GC_PORT);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/48733ce8/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java
--
diff --git
a/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java
b/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java
index b2f5a18..07aaf9f 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java
@@ -17,6 +17,8 @@
package org.apache.accumulo.core.conf;
import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
@@ -45,12 +47,31 @@ public class SiteConfiguration extends
AccumuloConfiguration {
private static SiteConfiguration instance = null;
private static Configuration xmlConfig;
+ private final Map staticConfigs;
/**
* Not for consumers. Call {@link
SiteConfiguration#getInstance(AccumuloConfiguration)} instead
*/
SiteConfiguration(AccumuloConfiguration parent) {
this.parent = parent;
+/*
+ * Make a read-only copy of static configs so we can avoid lock contention
on the Hadoop Configuration object
+ */
+final Configuration conf = getXmlConfig();
+Map temp = new HashMap<>((int) (Math.
[1/3] accumulo git commit: ACCUMULO-4458 reduce contention on config retrievals.
Repository: accumulo
Updated Branches:
refs/heads/1.8 8c85c32af -> 923c87e2f
ACCUMULO-4458 reduce contention on config retrievals.
We retrieve configuration values in a few hot code paths, so contention
on getting the site configuration and values that are only read from the
filesystem
once causes a perf impact. We try to reduce that here in a few ways:
* Since SiteConfiguration is a singleton, we only retrieve it once when
HdfsZooInstance is made instead of on each lookup.
* Since HdfsZooInstance is a singleton, we create it once on initialization and
avoid synchronizing on retrieval
* Since the Hadoop Configuration object uses a single lock for all gets against
the shared instance, we copy out the values
it reads when the SiteConfiguration singleton is created. This has the
side-effect of causing our for-test-only set method
to only work on values not set in the Configuration (essentially we treat all
configs as final).
* Additionally, for those configurations that are read within any of the
processing loops of the TabletServer, we have the
SiteConfiguration singleton cache the fact that they weren't in the Hadoop
Configuration object so that we will default to
the parent AccumuloConfiguration without contending for the Hadoop
Configuration object lock.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/48733ce8
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/48733ce8
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/48733ce8
Branch: refs/heads/1.8
Commit: 48733ce8cb9e89f1243e5e46d792010900086c7b
Parents: 4136bfa
Author: Sean Busbey
Authored: Tue Sep 6 11:52:37 2016 -0500
Committer: Sean Busbey
Committed: Wed Sep 14 16:35:31 2016 -0700
--
.../org/apache/accumulo/core/conf/Property.java | 7
.../accumulo/core/conf/SiteConfiguration.java | 24 -
.../accumulo/server/client/HdfsZooInstance.java | 36 +---
3 files changed, 53 insertions(+), 14 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/48733ce8/core/src/main/java/org/apache/accumulo/core/conf/Property.java
--
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 9d145e1..96adaeb 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -803,6 +803,13 @@ public enum Property {
|| key.startsWith(Property.TABLE_ARBITRARY_PROP_PREFIX.getKey());
}
+ /**
+ * Properties we check the value of within the TabletServer request handling
or maintenance processing loops.
+ */
+ public static final EnumSet HOT_PATH_PROPERTIES =
EnumSet.of(Property.TSERV_CLIENT_TIMEOUT,
Property.TSERV_TOTAL_MUTATION_QUEUE_MAX,
+ Property.TSERV_ARCHIVE_WALOGS, Property.GC_TRASH_IGNORE,
Property.TSERV_MAJC_DELAY, Property.TABLE_MINC_LOGS_MAX,
Property.TSERV_MAJC_MAXCONCURRENT,
+ Property.REPLICATION_WORKER_THREADS, Property.TABLE_DURABILITY,
Property.INSTANCE_ZK_TIMEOUT, Property.TABLE_CLASSPATH);
+
private static final EnumSet fixedProperties =
EnumSet.of(Property.TSERV_CLIENTPORT, Property.TSERV_NATIVEMAP_ENABLED,
Property.TSERV_SCAN_MAX_OPENFILES, Property.MASTER_CLIENTPORT,
Property.GC_PORT);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/48733ce8/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java
--
diff --git
a/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java
b/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java
index b2f5a18..07aaf9f 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java
@@ -17,6 +17,8 @@
package org.apache.accumulo.core.conf;
import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
@@ -45,12 +47,31 @@ public class SiteConfiguration extends
AccumuloConfiguration {
private static SiteConfiguration instance = null;
private static Configuration xmlConfig;
+ private final Map staticConfigs;
/**
* Not for consumers. Call {@link
SiteConfiguration#getInstance(AccumuloConfiguration)} instead
*/
SiteConfiguration(AccumuloConfiguration parent) {
this.parent = parent;
+/*
+ * Make a read-only copy of static configs so we can avoid lock contention
on the Hadoop Configuration object
+ */
+final Configuration conf = getXmlConfig();
+Map temp = new HashMap<>((int) (Math.
[3/3] accumulo git commit: ACCUMULO-4458 addendum for 1.8
ACCUMULO-4458 addendum for 1.8
- Adds an additional property to HOT_PATH_PROPERTIES based on use in 1.8 tablet
server.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/923c87e2
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/923c87e2
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/923c87e2
Branch: refs/heads/1.8
Commit: 923c87e2fbdfd49409e8099293cc87371da8f6fd
Parents: 1bf8a63
Author: Sean Busbey
Authored: Thu Sep 15 08:47:24 2016 -0700
Committer: Sean Busbey
Committed: Thu Sep 15 08:47:24 2016 -0700
--
core/src/main/java/org/apache/accumulo/core/conf/Property.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/923c87e2/core/src/main/java/org/apache/accumulo/core/conf/Property.java
--
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 34ef953..ede1c6f 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -830,7 +830,8 @@ public enum Property {
*/
public static final EnumSet HOT_PATH_PROPERTIES =
EnumSet.of(Property.TSERV_CLIENT_TIMEOUT,
Property.TSERV_TOTAL_MUTATION_QUEUE_MAX,
Property.TSERV_ARCHIVE_WALOGS, Property.GC_TRASH_IGNORE,
Property.TSERV_MAJC_DELAY, Property.TABLE_MINC_LOGS_MAX,
Property.TSERV_MAJC_MAXCONCURRENT,
- Property.REPLICATION_WORKER_THREADS, Property.TABLE_DURABILITY,
Property.INSTANCE_ZK_TIMEOUT, Property.TABLE_CLASSPATH);
+ Property.REPLICATION_WORKER_THREADS, Property.TABLE_DURABILITY,
Property.INSTANCE_ZK_TIMEOUT, Property.TABLE_CLASSPATH,
+ Property.MASTER_METADATA_SUSPENDABLE);
private static final EnumSet fixedProperties =
EnumSet.of(Property.TSERV_CLIENTPORT, Property.TSERV_NATIVEMAP_ENABLED,
Property.TSERV_SCAN_MAX_OPENFILES, Property.MASTER_CLIENTPORT,
Property.GC_PORT);
[2/3] accumulo git commit: Merge branch '1.7' into 1.8
Merge branch '1.7' into 1.8
Conflicts:
server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/1bf8a636
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/1bf8a636
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/1bf8a636
Branch: refs/heads/1.8
Commit: 1bf8a6362c685f316d04facb26faa29939cad9ef
Parents: 8c85c32 48733ce
Author: Sean Busbey
Authored: Thu Sep 15 08:41:13 2016 -0700
Committer: Sean Busbey
Committed: Thu Sep 15 08:41:13 2016 -0700
--
.../org/apache/accumulo/core/conf/Property.java | 7
.../accumulo/core/conf/SiteConfiguration.java | 24 -
.../accumulo/server/client/HdfsZooInstance.java | 36 +---
3 files changed, 53 insertions(+), 14 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/1bf8a636/core/src/main/java/org/apache/accumulo/core/conf/Property.java
--
diff --cc core/src/main/java/org/apache/accumulo/core/conf/Property.java
index cdaebef,96adaeb..34ef953
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@@ -822,9 -800,16 +822,16 @@@ public enum Property
return validTableProperties.contains(key) ||
key.startsWith(Property.TABLE_CONSTRAINT_PREFIX.getKey())
|| key.startsWith(Property.TABLE_ITERATOR_PREFIX.getKey()) ||
key.startsWith(Property.TABLE_LOCALITY_GROUP_PREFIX.getKey())
|| key.startsWith(Property.TABLE_COMPACTION_STRATEGY_PREFIX.getKey())
|| key.startsWith(Property.TABLE_REPLICATION_TARGET.getKey())
-|| key.startsWith(Property.TABLE_ARBITRARY_PROP_PREFIX.getKey());
+|| key.startsWith(Property.TABLE_ARBITRARY_PROP_PREFIX.getKey()) ||
key.startsWith(TABLE_SAMPLER_OPTS.getKey());
}
+ /**
+* Properties we check the value of within the TabletServer request
handling or maintenance processing loops.
+*/
+ public static final EnumSet HOT_PATH_PROPERTIES =
EnumSet.of(Property.TSERV_CLIENT_TIMEOUT,
Property.TSERV_TOTAL_MUTATION_QUEUE_MAX,
+ Property.TSERV_ARCHIVE_WALOGS, Property.GC_TRASH_IGNORE,
Property.TSERV_MAJC_DELAY, Property.TABLE_MINC_LOGS_MAX,
Property.TSERV_MAJC_MAXCONCURRENT,
+ Property.REPLICATION_WORKER_THREADS, Property.TABLE_DURABILITY,
Property.INSTANCE_ZK_TIMEOUT, Property.TABLE_CLASSPATH);
+
private static final EnumSet fixedProperties =
EnumSet.of(Property.TSERV_CLIENTPORT, Property.TSERV_NATIVEMAP_ENABLED,
Property.TSERV_SCAN_MAX_OPENFILES, Property.MASTER_CLIENTPORT,
Property.GC_PORT);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/1bf8a636/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
--
diff --cc
server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
index e4e73d2,2dacf61..49bf1f4
---
a/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/client/HdfsZooInstance.java
@@@ -23,7 -23,7 +23,8 @@@ import java.nio.ByteBuffer
import java.util.Collections;
import java.util.List;
import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+ import java.util.concurrent.atomic.AtomicReference;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.AccumuloException;
@@@ -76,9 -76,9 +76,9 @@@ public class HdfsZooInstance implement
return cachedHdfsZooInstance;
}
- private static ZooCache zooCache;
+ private final ZooCache zooCache;
private static String instanceId = null;
- private static final Logger log = Logger.getLogger(HdfsZooInstance.class);
+ private static final Logger log =
LoggerFactory.getLogger(HdfsZooInstance.class);
@Override
public String getRootTabletLocation() {
[1/8] accumulo git commit: ACCUMULO-4467 Ensure commons-math is on the classpath for RandomWalk MR jobs.
Repository: accumulo
Updated Branches:
refs/heads/1.7 dc53d51dc -> 985651b48
refs/heads/1.8 080a3ba5b -> 6a677adaf
refs/heads/master b919a5cfe -> 0bf8f8e1c
ACCUMULO-4467 Ensure commons-math is on the classpath for RandomWalk MR jobs.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/985651b4
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/985651b4
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/985651b4
Branch: refs/heads/1.7
Commit: 985651b48451025db8a1bd00e79730f9d10ded6f
Parents: dc53d51
Author: Sean Busbey
Authored: Wed Sep 21 14:44:56 2016 -0500
Committer: Sean Busbey
Committed: Thu Sep 22 12:58:26 2016 -0500
--
test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/985651b4/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java
--
diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java
b/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java
index 6df5aed..f8cb0b9 100644
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java
@@ -91,7 +91,7 @@ public abstract class Node {
}
File libdir = new File(acuHome + "/lib");
-for (String jar : "accumulo-core accumulo-server-base accumulo-fate
accumulo-trace libthrift htrace-core".split(" ")) {
+for (String jar : "accumulo-core accumulo-server-base accumulo-fate
accumulo-trace commons-math libthrift htrace-core".split(" ")) {
retval += String.format(",%s/%s.jar", libdir.getAbsolutePath(), jar);
}
[4/8] accumulo git commit: Merge branch '1.7' into 1.8
Merge branch '1.7' into 1.8 -sours merge, since 1.7 and 1.8 have different classpath needs for ACCUMULO-4467 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/7ab49808 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/7ab49808 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/7ab49808 Branch: refs/heads/1.8 Commit: 7ab498087b5cf6ed032e8d9d72f31830198ee37a Parents: 080a3ba 985651b Author: Sean Busbey Authored: Thu Sep 22 12:59:14 2016 -0500 Committer: Sean Busbey Committed: Thu Sep 22 12:59:20 2016 -0500 -- --
[5/8] accumulo git commit: Merge branch '1.7' into 1.8
Merge branch '1.7' into 1.8 -sours merge, since 1.7 and 1.8 have different classpath needs for ACCUMULO-4467 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/7ab49808 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/7ab49808 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/7ab49808 Branch: refs/heads/master Commit: 7ab498087b5cf6ed032e8d9d72f31830198ee37a Parents: 080a3ba 985651b Author: Sean Busbey Authored: Thu Sep 22 12:59:14 2016 -0500 Committer: Sean Busbey Committed: Thu Sep 22 12:59:20 2016 -0500 -- --
[8/8] accumulo git commit: Merge branch '1.8'
Merge branch '1.8' Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/0bf8f8e1 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/0bf8f8e1 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/0bf8f8e1 Branch: refs/heads/master Commit: 0bf8f8e1c42e1cc0f1487ad96398999c8796a4a3 Parents: b919a5c 6a677ad Author: Sean Busbey Authored: Thu Sep 22 13:02:30 2016 -0500 Committer: Sean Busbey Committed: Thu Sep 22 13:02:30 2016 -0500 -- test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) --
[2/8] accumulo git commit: ACCUMULO-4467 Ensure commons-math is on the classpath for RandomWalk MR jobs.
ACCUMULO-4467 Ensure commons-math is on the classpath for RandomWalk MR jobs.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/985651b4
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/985651b4
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/985651b4
Branch: refs/heads/1.8
Commit: 985651b48451025db8a1bd00e79730f9d10ded6f
Parents: dc53d51
Author: Sean Busbey
Authored: Wed Sep 21 14:44:56 2016 -0500
Committer: Sean Busbey
Committed: Thu Sep 22 12:58:26 2016 -0500
--
test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/985651b4/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java
--
diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java
b/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java
index 6df5aed..f8cb0b9 100644
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java
@@ -91,7 +91,7 @@ public abstract class Node {
}
File libdir = new File(acuHome + "/lib");
-for (String jar : "accumulo-core accumulo-server-base accumulo-fate
accumulo-trace libthrift htrace-core".split(" ")) {
+for (String jar : "accumulo-core accumulo-server-base accumulo-fate
accumulo-trace commons-math libthrift htrace-core".split(" ")) {
retval += String.format(",%s/%s.jar", libdir.getAbsolutePath(), jar);
}
[7/8] accumulo git commit: ACCUMULO-4467 Ensure commons-math3 is on the classpath for RandomWalk MR jobs.
ACCUMULO-4467 Ensure commons-math3 is on the classpath for RandomWalk MR jobs.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/6a677ada
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/6a677ada
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/6a677ada
Branch: refs/heads/1.8
Commit: 6a677adafe0fe03a6a331d281c1a27b4854f94e7
Parents: 7ab4980
Author: Sean Busbey
Authored: Thu Sep 22 13:01:52 2016 -0500
Committer: Sean Busbey
Committed: Thu Sep 22 13:02:01 2016 -0500
--
test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6a677ada/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java
--
diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java
b/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java
index 6df5aed..282ceeb 100644
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java
@@ -91,7 +91,7 @@ public abstract class Node {
}
File libdir = new File(acuHome + "/lib");
-for (String jar : "accumulo-core accumulo-server-base accumulo-fate
accumulo-trace libthrift htrace-core".split(" ")) {
+for (String jar : "accumulo-core accumulo-server-base accumulo-fate
accumulo-trace commons-math3 libthrift htrace-core".split(" ")) {
retval += String.format(",%s/%s.jar", libdir.getAbsolutePath(), jar);
}
[3/8] accumulo git commit: ACCUMULO-4467 Ensure commons-math is on the classpath for RandomWalk MR jobs.
ACCUMULO-4467 Ensure commons-math is on the classpath for RandomWalk MR jobs.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/985651b4
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/985651b4
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/985651b4
Branch: refs/heads/master
Commit: 985651b48451025db8a1bd00e79730f9d10ded6f
Parents: dc53d51
Author: Sean Busbey
Authored: Wed Sep 21 14:44:56 2016 -0500
Committer: Sean Busbey
Committed: Thu Sep 22 12:58:26 2016 -0500
--
test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/985651b4/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java
--
diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java
b/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java
index 6df5aed..f8cb0b9 100644
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java
@@ -91,7 +91,7 @@ public abstract class Node {
}
File libdir = new File(acuHome + "/lib");
-for (String jar : "accumulo-core accumulo-server-base accumulo-fate
accumulo-trace libthrift htrace-core".split(" ")) {
+for (String jar : "accumulo-core accumulo-server-base accumulo-fate
accumulo-trace commons-math libthrift htrace-core".split(" ")) {
retval += String.format(",%s/%s.jar", libdir.getAbsolutePath(), jar);
}
[6/8] accumulo git commit: ACCUMULO-4467 Ensure commons-math3 is on the classpath for RandomWalk MR jobs.
ACCUMULO-4467 Ensure commons-math3 is on the classpath for RandomWalk MR jobs.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/6a677ada
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/6a677ada
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/6a677ada
Branch: refs/heads/master
Commit: 6a677adafe0fe03a6a331d281c1a27b4854f94e7
Parents: 7ab4980
Author: Sean Busbey
Authored: Thu Sep 22 13:01:52 2016 -0500
Committer: Sean Busbey
Committed: Thu Sep 22 13:02:01 2016 -0500
--
test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6a677ada/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java
--
diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java
b/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java
index 6df5aed..282ceeb 100644
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/Node.java
@@ -91,7 +91,7 @@ public abstract class Node {
}
File libdir = new File(acuHome + "/lib");
-for (String jar : "accumulo-core accumulo-server-base accumulo-fate
accumulo-trace libthrift htrace-core".split(" ")) {
+for (String jar : "accumulo-core accumulo-server-base accumulo-fate
accumulo-trace commons-math3 libthrift htrace-core".split(" ")) {
retval += String.format(",%s/%s.jar", libdir.getAbsolutePath(), jar);
}
[6/6] accumulo git commit: Merge branch '1.8'
Merge branch '1.8' Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/6823140e Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/6823140e Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/6823140e Branch: refs/heads/master Commit: 6823140eb5f3fded282eb02daf8d3f6f9d80c3b4 Parents: ababa81 38606a8 Author: Sean Busbey Authored: Mon Sep 26 11:05:38 2016 -0500 Committer: Sean Busbey Committed: Mon Sep 26 11:05:38 2016 -0500 -- docs/src/main/asciidoc/chapters/replication.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/6823140e/docs/src/main/asciidoc/chapters/replication.txt --
[5/6] accumulo git commit: Merge branch '1.7' into 1.8
Merge branch '1.7' into 1.8 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/38606a86 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/38606a86 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/38606a86 Branch: refs/heads/1.8 Commit: 38606a86e66830c0f5e8083311c062af4a5a42e4 Parents: c1dda77 e76f3a2 Author: Sean Busbey Authored: Mon Sep 26 10:55:12 2016 -0500 Committer: Sean Busbey Committed: Mon Sep 26 10:55:12 2016 -0500 -- docs/src/main/asciidoc/chapters/replication.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) --
[2/6] accumulo git commit: ACCUMULO-4471 Typo in user manual replication instructions
ACCUMULO-4471 Typo in user manual replication instructions
Signed-off-by: Sean Busbey
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/e76f3a23
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/e76f3a23
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/e76f3a23
Branch: refs/heads/1.8
Commit: e76f3a236a8c91553974a8f1981d838de2933ba2
Parents: 064ae0a
Author: Dima Spivak
Authored: Fri Sep 23 20:29:15 2016 -0700
Committer: Sean Busbey
Committed: Mon Sep 26 10:48:34 2016 -0500
--
docs/src/main/asciidoc/chapters/replication.txt | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e76f3a23/docs/src/main/asciidoc/chapters/replication.txt
--
diff --git a/docs/src/main/asciidoc/chapters/replication.txt
b/docs/src/main/asciidoc/chapters/replication.txt
index ad48cd2..82fcf44 100644
--- a/docs/src/main/asciidoc/chapters/replication.txt
+++ b/docs/src/main/asciidoc/chapters/replication.txt
@@ -121,10 +121,10 @@ cluster, this is a table ID. In this example, we want to
enable replication on
+my_table+ and configure our peer +accumulo_peer+ as a target, sending
the data to the table with an ID of +2+ in +accumulo_peer+.
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
+
root@accumulo_primary> config -t my_table -s table.replication=true
-root@accumulo_primary> config -t my_table -s
table.replication.target.acccumulo_peer=2
-\end{verbatim}\endgroup
+root@accumulo_primary> config -t my_table -s
table.replication.target.accumulo_peer=2
+
To replicate a single table on the primary to multiple peers, the second
command
in the above shell snippet can be issued, for each peer and remote identifier
pair.
[1/6] accumulo git commit: ACCUMULO-4471 Typo in user manual replication instructions
Repository: accumulo
Updated Branches:
refs/heads/1.7 064ae0aa7 -> e76f3a236
refs/heads/1.8 c1dda7786 -> 38606a86e
refs/heads/master ababa81c3 -> 6823140eb
ACCUMULO-4471 Typo in user manual replication instructions
Signed-off-by: Sean Busbey
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/e76f3a23
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/e76f3a23
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/e76f3a23
Branch: refs/heads/1.7
Commit: e76f3a236a8c91553974a8f1981d838de2933ba2
Parents: 064ae0a
Author: Dima Spivak
Authored: Fri Sep 23 20:29:15 2016 -0700
Committer: Sean Busbey
Committed: Mon Sep 26 10:48:34 2016 -0500
--
docs/src/main/asciidoc/chapters/replication.txt | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e76f3a23/docs/src/main/asciidoc/chapters/replication.txt
--
diff --git a/docs/src/main/asciidoc/chapters/replication.txt
b/docs/src/main/asciidoc/chapters/replication.txt
index ad48cd2..82fcf44 100644
--- a/docs/src/main/asciidoc/chapters/replication.txt
+++ b/docs/src/main/asciidoc/chapters/replication.txt
@@ -121,10 +121,10 @@ cluster, this is a table ID. In this example, we want to
enable replication on
+my_table+ and configure our peer +accumulo_peer+ as a target, sending
the data to the table with an ID of +2+ in +accumulo_peer+.
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
+
root@accumulo_primary> config -t my_table -s table.replication=true
-root@accumulo_primary> config -t my_table -s
table.replication.target.acccumulo_peer=2
-\end{verbatim}\endgroup
+root@accumulo_primary> config -t my_table -s
table.replication.target.accumulo_peer=2
+
To replicate a single table on the primary to multiple peers, the second
command
in the above shell snippet can be issued, for each peer and remote identifier
pair.
[3/6] accumulo git commit: ACCUMULO-4471 Typo in user manual replication instructions
ACCUMULO-4471 Typo in user manual replication instructions
Signed-off-by: Sean Busbey
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/e76f3a23
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/e76f3a23
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/e76f3a23
Branch: refs/heads/master
Commit: e76f3a236a8c91553974a8f1981d838de2933ba2
Parents: 064ae0a
Author: Dima Spivak
Authored: Fri Sep 23 20:29:15 2016 -0700
Committer: Sean Busbey
Committed: Mon Sep 26 10:48:34 2016 -0500
--
docs/src/main/asciidoc/chapters/replication.txt | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e76f3a23/docs/src/main/asciidoc/chapters/replication.txt
--
diff --git a/docs/src/main/asciidoc/chapters/replication.txt
b/docs/src/main/asciidoc/chapters/replication.txt
index ad48cd2..82fcf44 100644
--- a/docs/src/main/asciidoc/chapters/replication.txt
+++ b/docs/src/main/asciidoc/chapters/replication.txt
@@ -121,10 +121,10 @@ cluster, this is a table ID. In this example, we want to
enable replication on
+my_table+ and configure our peer +accumulo_peer+ as a target, sending
the data to the table with an ID of +2+ in +accumulo_peer+.
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
+
root@accumulo_primary> config -t my_table -s table.replication=true
-root@accumulo_primary> config -t my_table -s
table.replication.target.acccumulo_peer=2
-\end{verbatim}\endgroup
+root@accumulo_primary> config -t my_table -s
table.replication.target.accumulo_peer=2
+
To replicate a single table on the primary to multiple peers, the second
command
in the above shell snippet can be issued, for each peer and remote identifier
pair.
[4/6] accumulo git commit: Merge branch '1.7' into 1.8
Merge branch '1.7' into 1.8 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/38606a86 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/38606a86 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/38606a86 Branch: refs/heads/master Commit: 38606a86e66830c0f5e8083311c062af4a5a42e4 Parents: c1dda77 e76f3a2 Author: Sean Busbey Authored: Mon Sep 26 10:55:12 2016 -0500 Committer: Sean Busbey Committed: Mon Sep 26 10:55:12 2016 -0500 -- docs/src/main/asciidoc/chapters/replication.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) --
[3/6] accumulo git commit: ACCUMULO-4488 update kerberos section in the user manual to fill some gaps.
ACCUMULO-4488 update kerberos section in the user manual to fill some gaps. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/a17390db Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/a17390db Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/a17390db Branch: refs/heads/master Commit: a17390db11c69dea0c7f1fbf75c9b28f8e1d561c Parents: 52f53f5 Author: Sean Busbey Authored: Thu Oct 6 11:43:50 2016 -0500 Committer: Sean Busbey Committed: Fri Oct 7 00:22:11 2016 -0500 -- docs/src/main/asciidoc/chapters/kerberos.txt | 104 +- 1 file changed, 101 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/a17390db/docs/src/main/asciidoc/chapters/kerberos.txt -- diff --git a/docs/src/main/asciidoc/chapters/kerberos.txt b/docs/src/main/asciidoc/chapters/kerberos.txt index a165221..bfb7cb7 100644 --- a/docs/src/main/asciidoc/chapters/kerberos.txt +++ b/docs/src/main/asciidoc/chapters/kerberos.txt @@ -102,6 +102,10 @@ numerous guidelines already exist on the subject of configuring Hadoop and ZooKe use with Kerberos and won't be covered here. It is assumed that you have functional Hadoop and ZooKeeper already installed. +Note that on an existing cluster the server side changes will require a full cluster shutdown and restart. You should +wait to restart the TraceServers until after you've completed the rest of the cluster set up and provisioned +a trace user with appropriate permissions. + Servers The first step is to obtain a Kerberos identity for the Accumulo server processes. @@ -190,6 +194,10 @@ creating a SystemToken from the provided keytab and principal, we can still use keytab/principal to serialize traces. Like non-Kerberized instances, the table must be created and permissions granted to the trace.user. The same +_HOST+ replacement is performed on this value, substituted the FQDN for +_HOST+. +|trace.token.property.keytab || +You can optionally specify the path to a keytab file for the principal given in the +trace.user+ property. If you don't +set this path, it will default to the value given in +general.kerberos.principal+. + |general.delegation.token.lifetime |7d | The length of time that the server-side secret used to create delegation tokens is valid. After a server-side secret expires, a delegation token created with that secret is no longer valid. @@ -233,7 +241,7 @@ The KDC is still the authoritative entity for user management. The previously me are provided as they simplify management of users within Accumulo, especially with respect to granting Authorizations and Permissions to new users. -= Accumulo Initialization += Administrative User Out of the box (without Kerberos enabled), Accumulo has a single user with administrative permissions "root". This users is used to "bootstrap" other users, creating less-privileged users for applications using @@ -246,6 +254,24 @@ enabled, Accumulo will prompt for the name of a user to grant the same permissio user would normally have. The name of the Accumulo user to grant administrative permissions to can also be given by the `-u` or `--user` options. +If you are enabling Kerberos on an existing cluster, you will need to reinitialize the security system in +order to replace the existing "root" user with one that can be used with Kerberos. These steps should be +completed after you have done the previously described configuration changes and will require access to +a complete +accumulo-site.xml+, including the instance secret. Note that this process will delete all +existing users in the system; you will need to reassign user permissions based on Kerberos principals. + +1. Ensure Accumulo is not running. +2. Given the path to a +accumulo-site.xml+ with the instance secret, run the security reset tool. If you are +prompted for a password you can just hit return, since it won't be used. + +$ ACCUMULO_CONF_DIR=/path/to/server/conf/ accumulo init --reset-security +Running against secured HDFS +Principal (user) to grant administrative privileges to : acculumo_ad...@example.com +Enter initial password for accumulo_ad...@example.com (this may not be applicable for your security setup): +Confirm initial password for accumulo_ad...@example.com: + +3. Start the Accumulo cluster + = Verifying secure access To verify that servers have correctly started with Kerberos enabled, ensure that the processes @@ -369,18 +395,55 @@ Valid starting Expires Service principal = Configuration The second thing clients need to do is to set up their client config
[1/6] accumulo git commit: ACCUMULO-4488 update kerberos section in the user manual to fill some gaps.
Repository: accumulo Updated Branches: refs/heads/1.7 52f53f56e -> a17390db1 refs/heads/1.8 f7989c678 -> 91269bf31 refs/heads/master 1c218de97 -> 9dddeed88 ACCUMULO-4488 update kerberos section in the user manual to fill some gaps. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/a17390db Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/a17390db Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/a17390db Branch: refs/heads/1.7 Commit: a17390db11c69dea0c7f1fbf75c9b28f8e1d561c Parents: 52f53f5 Author: Sean Busbey Authored: Thu Oct 6 11:43:50 2016 -0500 Committer: Sean Busbey Committed: Fri Oct 7 00:22:11 2016 -0500 -- docs/src/main/asciidoc/chapters/kerberos.txt | 104 +- 1 file changed, 101 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/a17390db/docs/src/main/asciidoc/chapters/kerberos.txt -- diff --git a/docs/src/main/asciidoc/chapters/kerberos.txt b/docs/src/main/asciidoc/chapters/kerberos.txt index a165221..bfb7cb7 100644 --- a/docs/src/main/asciidoc/chapters/kerberos.txt +++ b/docs/src/main/asciidoc/chapters/kerberos.txt @@ -102,6 +102,10 @@ numerous guidelines already exist on the subject of configuring Hadoop and ZooKe use with Kerberos and won't be covered here. It is assumed that you have functional Hadoop and ZooKeeper already installed. +Note that on an existing cluster the server side changes will require a full cluster shutdown and restart. You should +wait to restart the TraceServers until after you've completed the rest of the cluster set up and provisioned +a trace user with appropriate permissions. + Servers The first step is to obtain a Kerberos identity for the Accumulo server processes. @@ -190,6 +194,10 @@ creating a SystemToken from the provided keytab and principal, we can still use keytab/principal to serialize traces. Like non-Kerberized instances, the table must be created and permissions granted to the trace.user. The same +_HOST+ replacement is performed on this value, substituted the FQDN for +_HOST+. +|trace.token.property.keytab || +You can optionally specify the path to a keytab file for the principal given in the +trace.user+ property. If you don't +set this path, it will default to the value given in +general.kerberos.principal+. + |general.delegation.token.lifetime |7d | The length of time that the server-side secret used to create delegation tokens is valid. After a server-side secret expires, a delegation token created with that secret is no longer valid. @@ -233,7 +241,7 @@ The KDC is still the authoritative entity for user management. The previously me are provided as they simplify management of users within Accumulo, especially with respect to granting Authorizations and Permissions to new users. -= Accumulo Initialization += Administrative User Out of the box (without Kerberos enabled), Accumulo has a single user with administrative permissions "root". This users is used to "bootstrap" other users, creating less-privileged users for applications using @@ -246,6 +254,24 @@ enabled, Accumulo will prompt for the name of a user to grant the same permissio user would normally have. The name of the Accumulo user to grant administrative permissions to can also be given by the `-u` or `--user` options. +If you are enabling Kerberos on an existing cluster, you will need to reinitialize the security system in +order to replace the existing "root" user with one that can be used with Kerberos. These steps should be +completed after you have done the previously described configuration changes and will require access to +a complete +accumulo-site.xml+, including the instance secret. Note that this process will delete all +existing users in the system; you will need to reassign user permissions based on Kerberos principals. + +1. Ensure Accumulo is not running. +2. Given the path to a +accumulo-site.xml+ with the instance secret, run the security reset tool. If you are +prompted for a password you can just hit return, since it won't be used. + +$ ACCUMULO_CONF_DIR=/path/to/server/conf/ accumulo init --reset-security +Running against secured HDFS +Principal (user) to grant administrative privileges to : acculumo_ad...@example.com +Enter initial password for accumulo_ad...@example.com (this may not be applicable for your security setup): +Confirm initial password for accumulo_ad...@example.com: + +3. Start the Accumulo cluster + = Verifying secure access To verify that servers have correctly started with Kerberos enabled, ensure that the processes @@ -369,18 +
[4/6] accumulo git commit: Merge branch '1.7' into 1.8
Merge branch '1.7' into 1.8 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/91269bf3 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/91269bf3 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/91269bf3 Branch: refs/heads/master Commit: 91269bf318586d31756fc3aecc1d42a20e602f76 Parents: f7989c6 a17390d Author: Sean Busbey Authored: Fri Oct 7 00:26:10 2016 -0500 Committer: Sean Busbey Committed: Fri Oct 7 00:26:10 2016 -0500 -- docs/src/main/asciidoc/chapters/kerberos.txt | 104 +- 1 file changed, 101 insertions(+), 3 deletions(-) --
[5/6] accumulo git commit: Merge branch '1.7' into 1.8
Merge branch '1.7' into 1.8 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/91269bf3 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/91269bf3 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/91269bf3 Branch: refs/heads/1.8 Commit: 91269bf318586d31756fc3aecc1d42a20e602f76 Parents: f7989c6 a17390d Author: Sean Busbey Authored: Fri Oct 7 00:26:10 2016 -0500 Committer: Sean Busbey Committed: Fri Oct 7 00:26:10 2016 -0500 -- docs/src/main/asciidoc/chapters/kerberos.txt | 104 +- 1 file changed, 101 insertions(+), 3 deletions(-) --
[6/6] accumulo git commit: Merge branch '1.8'
Merge branch '1.8' Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9dddeed8 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9dddeed8 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9dddeed8 Branch: refs/heads/master Commit: 9dddeed885d8ae4c6ba34eca561dcdb747131d68 Parents: 1c218de 91269bf Author: Sean Busbey Authored: Fri Oct 7 00:28:21 2016 -0500 Committer: Sean Busbey Committed: Fri Oct 7 00:28:21 2016 -0500 -- docs/src/main/asciidoc/chapters/kerberos.txt | 104 +- 1 file changed, 101 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/9dddeed8/docs/src/main/asciidoc/chapters/kerberos.txt -- diff --cc docs/src/main/asciidoc/chapters/kerberos.txt index 7817c8d,bfb7cb7..48553b7 --- a/docs/src/main/asciidoc/chapters/kerberos.txt +++ b/docs/src/main/asciidoc/chapters/kerberos.txt @@@ -558,5 -621,40 +621,40 @@@ Caused by: org.apache.thrift.transport. on, is extremely important when negotiating an SASL connection. This problem commonly arises when the Accumulo servers are not configured to listen on the address denoted by their FQDN. -The values in the Accumulo "hosts" files (In +$ACCUMULO_CONF_DIR+: +masters+, +monitors+, +slaves+, +tracers+, +The values in the Accumulo "hosts" files (In +$ACCUMULO_CONF_DIR+: +masters+, +monitors+, +tservers+, +tracers+, and +gc+) should match the instance componentof the Kerberos server principal (e.g. +host+ in +accumulo/h...@example.com+). + + *Q*: After configuring my system for Kerberos, server processes come up normally and I can interact with the system. However, + when I attempt to use the "Recent Traces" page on the Monitor UI I get a stacktrace similar to: + + + java.lang.AssertionError: AuthenticationToken should not be null +at org.apache.accumulo.monitor.servlets.trace.Basic.getScanner(Basic.java:139) + at org.apache.accumulo.monitor.servlets.trace.Summary.pageBody(Summary.java:164) + at org.apache.accumulo.monitor.servlets.BasicServlet.doGet(BasicServlet.java:63) +at javax.servlet.http.HttpServlet.service(HttpServlet.java:687) +at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) + at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:738) + at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:551) + at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143) +at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:568) + at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:221) + at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:) + at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:478) + at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:183) + at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1045) + at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) + at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97) + at org.eclipse.jetty.server.Server.handle(Server.java:462) + at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:279) +at org.eclipse.jetty.server.HttpConne
[2/6] accumulo git commit: ACCUMULO-4488 update kerberos section in the user manual to fill some gaps.
ACCUMULO-4488 update kerberos section in the user manual to fill some gaps. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/a17390db Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/a17390db Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/a17390db Branch: refs/heads/1.8 Commit: a17390db11c69dea0c7f1fbf75c9b28f8e1d561c Parents: 52f53f5 Author: Sean Busbey Authored: Thu Oct 6 11:43:50 2016 -0500 Committer: Sean Busbey Committed: Fri Oct 7 00:22:11 2016 -0500 -- docs/src/main/asciidoc/chapters/kerberos.txt | 104 +- 1 file changed, 101 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/a17390db/docs/src/main/asciidoc/chapters/kerberos.txt -- diff --git a/docs/src/main/asciidoc/chapters/kerberos.txt b/docs/src/main/asciidoc/chapters/kerberos.txt index a165221..bfb7cb7 100644 --- a/docs/src/main/asciidoc/chapters/kerberos.txt +++ b/docs/src/main/asciidoc/chapters/kerberos.txt @@ -102,6 +102,10 @@ numerous guidelines already exist on the subject of configuring Hadoop and ZooKe use with Kerberos and won't be covered here. It is assumed that you have functional Hadoop and ZooKeeper already installed. +Note that on an existing cluster the server side changes will require a full cluster shutdown and restart. You should +wait to restart the TraceServers until after you've completed the rest of the cluster set up and provisioned +a trace user with appropriate permissions. + Servers The first step is to obtain a Kerberos identity for the Accumulo server processes. @@ -190,6 +194,10 @@ creating a SystemToken from the provided keytab and principal, we can still use keytab/principal to serialize traces. Like non-Kerberized instances, the table must be created and permissions granted to the trace.user. The same +_HOST+ replacement is performed on this value, substituted the FQDN for +_HOST+. +|trace.token.property.keytab || +You can optionally specify the path to a keytab file for the principal given in the +trace.user+ property. If you don't +set this path, it will default to the value given in +general.kerberos.principal+. + |general.delegation.token.lifetime |7d | The length of time that the server-side secret used to create delegation tokens is valid. After a server-side secret expires, a delegation token created with that secret is no longer valid. @@ -233,7 +241,7 @@ The KDC is still the authoritative entity for user management. The previously me are provided as they simplify management of users within Accumulo, especially with respect to granting Authorizations and Permissions to new users. -= Accumulo Initialization += Administrative User Out of the box (without Kerberos enabled), Accumulo has a single user with administrative permissions "root". This users is used to "bootstrap" other users, creating less-privileged users for applications using @@ -246,6 +254,24 @@ enabled, Accumulo will prompt for the name of a user to grant the same permissio user would normally have. The name of the Accumulo user to grant administrative permissions to can also be given by the `-u` or `--user` options. +If you are enabling Kerberos on an existing cluster, you will need to reinitialize the security system in +order to replace the existing "root" user with one that can be used with Kerberos. These steps should be +completed after you have done the previously described configuration changes and will require access to +a complete +accumulo-site.xml+, including the instance secret. Note that this process will delete all +existing users in the system; you will need to reassign user permissions based on Kerberos principals. + +1. Ensure Accumulo is not running. +2. Given the path to a +accumulo-site.xml+ with the instance secret, run the security reset tool. If you are +prompted for a password you can just hit return, since it won't be used. + +$ ACCUMULO_CONF_DIR=/path/to/server/conf/ accumulo init --reset-security +Running against secured HDFS +Principal (user) to grant administrative privileges to : acculumo_ad...@example.com +Enter initial password for accumulo_ad...@example.com (this may not be applicable for your security setup): +Confirm initial password for accumulo_ad...@example.com: + +3. Start the Accumulo cluster + = Verifying secure access To verify that servers have correctly started with Kerberos enabled, ensure that the processes @@ -369,18 +395,55 @@ Valid starting Expires Service principal = Configuration The second thing clients need to do is to set up their client config
[2/2] accumulo git commit: Jekyll build from gh-pages:af8b0e5
Jekyll build from gh-pages:af8b0e5 update 1.7 and 1.8 user manuals with recent changes Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/fc21741f Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/fc21741f Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/fc21741f Branch: refs/heads/asf-site Commit: fc21741fa75d4d2fe2b56b01ad58d6ce1e4bd4aa Parents: a26a875 Author: Sean Busbey Authored: Fri Oct 7 01:18:09 2016 -0500 Committer: Sean Busbey Committed: Fri Oct 7 01:42:53 2016 -0500 -- 1.7/accumulo_user_manual.html | 229 +++-- 1.8/accumulo_user_manual.html | 137 +- feed.xml | 4 +- 3 files changed, 329 insertions(+), 41 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/fc21741f/1.7/accumulo_user_manual.html -- diff --git a/1.7/accumulo_user_manual.html b/1.7/accumulo_user_manual.html index 9f34918..a4f4213 100644 --- a/1.7/accumulo_user_manual.html +++ b/1.7/accumulo_user_manual.html @@ -6,7 +6,7 @@ -Apache Accumulo User Manual Version 1.7 +Apache Accumulo® User Manual Version 1.7 https://fonts.googleapis.com/css?family=Open+Sans:300,300italic,400,400italic,600,600italic%7CNoto+Serif:400,400italic,700,700italic%7CDroid+Sans+Mono:400";>
[1/2] accumulo git commit: update 1.7 and 1.8 user manuals with recent changes
Repository: accumulo Updated Branches: refs/heads/asf-site a26a875ef -> fc21741fa refs/heads/gh-pages d1a74a8de -> af8b0e58a update 1.7 and 1.8 user manuals with recent changes Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/af8b0e58 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/af8b0e58 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/af8b0e58 Branch: refs/heads/gh-pages Commit: af8b0e58a9ef146c5d522b5ebf09d60a627de507 Parents: d1a74a8 Author: Sean Busbey Authored: Fri Oct 7 01:18:09 2016 -0500 Committer: Sean Busbey Committed: Fri Oct 7 01:42:36 2016 -0500 -- 1.7/accumulo_user_manual.html | 229 +++-- 1.8/accumulo_user_manual.html | 137 +- 2 files changed, 327 insertions(+), 39 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/af8b0e58/1.7/accumulo_user_manual.html -- diff --git a/1.7/accumulo_user_manual.html b/1.7/accumulo_user_manual.html index 9f34918..a4f4213 100644 --- a/1.7/accumulo_user_manual.html +++ b/1.7/accumulo_user_manual.html @@ -6,7 +6,7 @@ -Apache Accumulo User Manual Version 1.7 +Apache Accumulo® User Manual Version 1.7 https://fonts.googleapis.com/css?family=Open+Sans:300,300italic,400,400italic,600,600italic%7CNoto+Serif:400,400italic,700,700italic%7CDroid+Sans+Mono:400";>
[3/6] accumulo git commit: ACCUMULO-4489 Monitor login for access to trace table should fall back to general kerberos keytab when trace keytab is not set.
ACCUMULO-4489 Monitor login for access to trace table should fall back to
general kerberos keytab when trace keytab is not set.
Signed-off-by: Josh Elser
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/0d41ec6e
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/0d41ec6e
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/0d41ec6e
Branch: refs/heads/master
Commit: 0d41ec6e64f8c661b476ba74e1670aabd21d85b2
Parents: 9e236f2
Author: Sean Busbey
Authored: Thu Oct 6 09:21:59 2016 -0500
Committer: Sean Busbey
Committed: Mon Oct 17 14:41:32 2016 -0700
--
.../java/org/apache/accumulo/monitor/servlets/trace/Basic.java| 3 +++
1 file changed, 3 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/0d41ec6e/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java
--
diff --git
a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java
b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java
index 2143766..1839abe 100644
---
a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java
+++
b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java
@@ -83,6 +83,9 @@ abstract class Basic extends BasicServlet {
Map loginMap =
conf.getAllPropertiesWithPrefix(Property.TRACE_TOKEN_PROPERTY_PREFIX);
// May be null
String keytab = loginMap.get(Property.TRACE_TOKEN_PROPERTY_PREFIX.getKey()
+ "keytab");
+if (keytab == null || keytab.length() == 0) {
+ keytab = conf.getPath(Property.GENERAL_KERBEROS_KEYTAB);
+}
if (saslEnabled && null != keytab) {
principal =
SecurityUtil.getServerPrincipal(conf.get(Property.TRACE_USER));
[5/6] accumulo git commit: Merge branch '1.7' into 1.8
Merge branch '1.7' into 1.8 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/c7431092 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/c7431092 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/c7431092 Branch: refs/heads/1.8 Commit: c7431092e132e070c27e464b473100552255ba45 Parents: 9c11f72 0d41ec6 Author: Sean Busbey Authored: Mon Oct 17 14:41:50 2016 -0700 Committer: Sean Busbey Committed: Mon Oct 17 14:41:50 2016 -0700 -- .../java/org/apache/accumulo/monitor/servlets/trace/Basic.java| 3 +++ 1 file changed, 3 insertions(+) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/c7431092/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java --
[1/6] accumulo git commit: ACCUMULO-4489 Monitor login for access to trace table should fall back to general kerberos keytab when trace keytab is not set.
Repository: accumulo
Updated Branches:
refs/heads/1.7 9e236f289 -> 0d41ec6e6
refs/heads/1.8 9c11f72e3 -> c7431092e
refs/heads/master 3d11cab03 -> c4749ac35
ACCUMULO-4489 Monitor login for access to trace table should fall back to
general kerberos keytab when trace keytab is not set.
Signed-off-by: Josh Elser
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/0d41ec6e
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/0d41ec6e
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/0d41ec6e
Branch: refs/heads/1.7
Commit: 0d41ec6e64f8c661b476ba74e1670aabd21d85b2
Parents: 9e236f2
Author: Sean Busbey
Authored: Thu Oct 6 09:21:59 2016 -0500
Committer: Sean Busbey
Committed: Mon Oct 17 14:41:32 2016 -0700
--
.../java/org/apache/accumulo/monitor/servlets/trace/Basic.java| 3 +++
1 file changed, 3 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/0d41ec6e/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java
--
diff --git
a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java
b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java
index 2143766..1839abe 100644
---
a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java
+++
b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java
@@ -83,6 +83,9 @@ abstract class Basic extends BasicServlet {
Map loginMap =
conf.getAllPropertiesWithPrefix(Property.TRACE_TOKEN_PROPERTY_PREFIX);
// May be null
String keytab = loginMap.get(Property.TRACE_TOKEN_PROPERTY_PREFIX.getKey()
+ "keytab");
+if (keytab == null || keytab.length() == 0) {
+ keytab = conf.getPath(Property.GENERAL_KERBEROS_KEYTAB);
+}
if (saslEnabled && null != keytab) {
principal =
SecurityUtil.getServerPrincipal(conf.get(Property.TRACE_USER));
[4/6] accumulo git commit: Merge branch '1.7' into 1.8
Merge branch '1.7' into 1.8 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/c7431092 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/c7431092 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/c7431092 Branch: refs/heads/master Commit: c7431092e132e070c27e464b473100552255ba45 Parents: 9c11f72 0d41ec6 Author: Sean Busbey Authored: Mon Oct 17 14:41:50 2016 -0700 Committer: Sean Busbey Committed: Mon Oct 17 14:41:50 2016 -0700 -- .../java/org/apache/accumulo/monitor/servlets/trace/Basic.java| 3 +++ 1 file changed, 3 insertions(+) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/c7431092/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java --
[6/6] accumulo git commit: Merge branch '1.8'
Merge branch '1.8' Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/c4749ac3 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/c4749ac3 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/c4749ac3 Branch: refs/heads/master Commit: c4749ac35840813b11d3c386136b9ee257f19bd1 Parents: 3d11cab c743109 Author: Sean Busbey Authored: Mon Oct 17 14:42:22 2016 -0700 Committer: Sean Busbey Committed: Mon Oct 17 14:42:22 2016 -0700 -- .../java/org/apache/accumulo/monitor/servlets/trace/Basic.java| 3 +++ 1 file changed, 3 insertions(+) --
[2/6] accumulo git commit: ACCUMULO-4489 Monitor login for access to trace table should fall back to general kerberos keytab when trace keytab is not set.
ACCUMULO-4489 Monitor login for access to trace table should fall back to
general kerberos keytab when trace keytab is not set.
Signed-off-by: Josh Elser
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/0d41ec6e
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/0d41ec6e
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/0d41ec6e
Branch: refs/heads/1.8
Commit: 0d41ec6e64f8c661b476ba74e1670aabd21d85b2
Parents: 9e236f2
Author: Sean Busbey
Authored: Thu Oct 6 09:21:59 2016 -0500
Committer: Sean Busbey
Committed: Mon Oct 17 14:41:32 2016 -0700
--
.../java/org/apache/accumulo/monitor/servlets/trace/Basic.java| 3 +++
1 file changed, 3 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/0d41ec6e/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java
--
diff --git
a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java
b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java
index 2143766..1839abe 100644
---
a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java
+++
b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/trace/Basic.java
@@ -83,6 +83,9 @@ abstract class Basic extends BasicServlet {
Map loginMap =
conf.getAllPropertiesWithPrefix(Property.TRACE_TOKEN_PROPERTY_PREFIX);
// May be null
String keytab = loginMap.get(Property.TRACE_TOKEN_PROPERTY_PREFIX.getKey()
+ "keytab");
+if (keytab == null || keytab.length() == 0) {
+ keytab = conf.getPath(Property.GENERAL_KERBEROS_KEYTAB);
+}
if (saslEnabled && null != keytab) {
principal =
SecurityUtil.getServerPrincipal(conf.get(Property.TRACE_USER));
[3/6] accumulo git commit: ACCUMULO-4533 TraceServer shouldn't abort given problems with trace table checks.
ACCUMULO-4533 TraceServer shouldn't abort given problems with trace table
checks.
Signed-off-by: Mike Drob
Signed-off-by: Josh Elser
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/47b57f73
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/47b57f73
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/47b57f73
Branch: refs/heads/master
Commit: 47b57f7300ae5d1df9536cf07d99c83bfb8e2af6
Parents: 3c45441
Author: Sean Busbey
Authored: Wed Dec 7 10:45:51 2016 -0600
Committer: Sean Busbey
Committed: Fri Dec 9 15:21:54 2016 -0600
--
.../org/apache/accumulo/tracer/TraceServer.java | 50 ++--
1 file changed, 35 insertions(+), 15 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/47b57f73/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
--
diff --git
a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
index dbb593d..67bd9d5 100644
--- a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
+++ b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
@@ -28,12 +28,16 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.BatchWriterConfig;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.client.IteratorSetting;
import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
import
org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Properties;
import org.apache.accumulo.core.client.security.tokens.KerberosToken;
@@ -182,6 +186,35 @@ public class TraceServer implements Watcher {
log.info("Instance " + serverConfiguration.getInstance().getInstanceID());
AccumuloConfiguration conf = serverConfiguration.getConfiguration();
table = conf.get(Property.TRACE_TABLE);
+connector = ensureTraceTableExists(conf);
+
+int port = conf.getPort(Property.TRACE_PORT);
+final ServerSocket sock = ServerSocketChannel.open().socket();
+sock.setReuseAddress(true);
+sock.bind(new InetSocketAddress(hostname, port));
+final TServerTransport transport = new TServerSocket(sock);
+TThreadPoolServer.Args options = new TThreadPoolServer.Args(transport);
+options.processor(new Processor(new Receiver()));
+server = new TThreadPoolServer(options);
+registerInZooKeeper(sock.getInetAddress().getHostAddress() + ":" +
sock.getLocalPort(), conf.get(Property.TRACE_ZK_PATH));
+writer = new AtomicReference<>(this.connector.createBatchWriter(table, new
BatchWriterConfig().setMaxLatency(BATCH_WRITER_MAX_LATENCY, TimeUnit.SECONDS)));
+ }
+
+ /**
+ * Exceptions thrown out of here should be things that cause service failure
(e.g. misconfigurations that aren't likely to change on retry).
+ *
+ * @return a working Connection that can be reused
+ * @throws ClassNotFoundException
+ * if TRACE_TOKEN_TYPE is set to a class that we can't load.
+ * @throws InstantiationException
+ * if we fail to create an instance of TRACE_TOKEN_TYPE.
+ * @throws IllegalAccessException
+ * if the class pointed to by TRACE_TOKEN_TYPE is private.
+ * @throws AccumuloSecurityException
+ * if the trace user has the wrong permissions
+ */
+ private Connector ensureTraceTableExists(final AccumuloConfiguration conf)
throws AccumuloSecurityException, ClassNotFoundException,
InstantiationException,
+ IllegalAccessException {
Connector connector = null;
while (true) {
try {
@@ -221,25 +254,12 @@ public class TraceServer implements Watcher {
}
connector.tableOperations().setProperty(table,
Property.TABLE_FORMATTER_CLASS.getKey(), TraceFormatter.class.getName());
break;
- } catch (RuntimeException ex) {
+ } catch (AccumuloException | TableExistsException |
TableNotFoundException | IOException | RuntimeException ex) {
log.info("Waiting to checking/create the trace table.", ex);
UtilWaitThread.sleep(1000);
}
}
-this.connector = connecto
[5/6] accumulo git commit: Merge branch '1.7' into 1.8
Merge branch '1.7' into 1.8
Conflicts:
server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/037c1384
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/037c1384
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/037c1384
Branch: refs/heads/1.8
Commit: 037c1384ac861be95d8c9d1c5d1acbf61e40e2fe
Parents: 7c7bbab 47b57f7
Author: Sean Busbey
Authored: Fri Dec 9 15:33:37 2016 -0600
Committer: Sean Busbey
Committed: Fri Dec 9 15:33:37 2016 -0600
--
.../org/apache/accumulo/tracer/TraceServer.java | 76
1 file changed, 48 insertions(+), 28 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/037c1384/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
--
diff --cc
server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
index 2b6bcbf,67bd9d5..7c0d9b2
--- a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
+++ b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
@@@ -183,7 -185,36 +187,49 @@@ public class TraceServer implements Wat
log.info("Version " + Constants.VERSION);
log.info("Instance " + serverConfiguration.getInstance().getInstanceID());
AccumuloConfiguration conf = serverConfiguration.getConfiguration();
-table = conf.get(Property.TRACE_TABLE);
+tableName = conf.get(Property.TRACE_TABLE);
+ connector = ensureTraceTableExists(conf);
+
-int port = conf.getPort(Property.TRACE_PORT);
-final ServerSocket sock = ServerSocketChannel.open().socket();
-sock.setReuseAddress(true);
-sock.bind(new InetSocketAddress(hostname, port));
++int ports[] = conf.getPort(Property.TRACE_PORT);
++ServerSocket sock = null;
++for (int port : ports) {
++ ServerSocket s = ServerSocketChannel.open().socket();
++ s.setReuseAddress(true);
++ try {
++s.bind(new InetSocketAddress(hostname, port));
++sock = s;
++break;
++ } catch (Exception e) {
++log.warn("Unable to start trace server on port {}", port);
++ }
++}
++if (null == sock) {
++ throw new RuntimeException("Unable to start trace server on configured
ports: " + Arrays.toString(ports));
++}
+ final TServerTransport transport = new TServerSocket(sock);
+ TThreadPoolServer.Args options = new TThreadPoolServer.Args(transport);
+ options.processor(new Processor(new Receiver()));
+ server = new TThreadPoolServer(options);
+ registerInZooKeeper(sock.getInetAddress().getHostAddress() + ":" +
sock.getLocalPort(), conf.get(Property.TRACE_ZK_PATH));
-writer = new AtomicReference<>(this.connector.createBatchWriter(table,
new BatchWriterConfig().setMaxLatency(BATCH_WRITER_MAX_LATENCY,
TimeUnit.SECONDS)));
++writer = new AtomicReference<>(this.connector.createBatchWriter(tableName,
++new BatchWriterConfig().setMaxLatency(BATCH_WRITER_MAX_LATENCY,
TimeUnit.SECONDS)));
+ }
+
+ /**
+* Exceptions thrown out of here should be things that cause service
failure (e.g. misconfigurations that aren't likely to change on retry).
+*
+* @return a working Connection that can be reused
+* @throws ClassNotFoundException
+* if TRACE_TOKEN_TYPE is set to a class that we can't load.
+* @throws InstantiationException
+* if we fail to create an instance of TRACE_TOKEN_TYPE.
+* @throws IllegalAccessException
+* if the class pointed to by TRACE_TOKEN_TYPE is private.
+* @throws AccumuloSecurityException
+* if the trace user has the wrong permissions
+*/
+ private Connector ensureTraceTableExists(final AccumuloConfiguration conf)
throws AccumuloSecurityException, ClassNotFoundException,
InstantiationException,
+ IllegalAccessException {
Connector connector = null;
while (true) {
try {
@@@ -215,46 -246,20 +261,20 @@@
}
connector = serverConfiguration.getInstance().getConnector(principal,
at);
-if (!connector.tableOperations().exists(table)) {
- connector.tableOperations().create(table);
+if (!connector.tableOperations().exists(tableName)) {
+ connector.tableOperations().create(tableName);
IteratorSetting setting = new IteratorSetting(10, "ageoff",
AgeOffFilter.class.getName());
AgeOffFilter.setTTL(setting, 7 * 24 * 60 * 60 * 1000l);
- connector.tableOperations().attachIterator(table, setting);
+ connector.tableOperations().attachItera
[1/6] accumulo git commit: ACCUMULO-4533 TraceServer shouldn't abort given problems with trace table checks.
Repository: accumulo
Updated Branches:
refs/heads/1.7 3c4544144 -> 47b57f730
refs/heads/1.8 7c7bbabb8 -> 037c1384a
refs/heads/master 8e0f19a1c -> 3dc9b373d
ACCUMULO-4533 TraceServer shouldn't abort given problems with trace table
checks.
Signed-off-by: Mike Drob
Signed-off-by: Josh Elser
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/47b57f73
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/47b57f73
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/47b57f73
Branch: refs/heads/1.7
Commit: 47b57f7300ae5d1df9536cf07d99c83bfb8e2af6
Parents: 3c45441
Author: Sean Busbey
Authored: Wed Dec 7 10:45:51 2016 -0600
Committer: Sean Busbey
Committed: Fri Dec 9 15:21:54 2016 -0600
--
.../org/apache/accumulo/tracer/TraceServer.java | 50 ++--
1 file changed, 35 insertions(+), 15 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/47b57f73/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
--
diff --git
a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
index dbb593d..67bd9d5 100644
--- a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
+++ b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
@@ -28,12 +28,16 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.BatchWriterConfig;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.client.IteratorSetting;
import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
import
org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Properties;
import org.apache.accumulo.core.client.security.tokens.KerberosToken;
@@ -182,6 +186,35 @@ public class TraceServer implements Watcher {
log.info("Instance " + serverConfiguration.getInstance().getInstanceID());
AccumuloConfiguration conf = serverConfiguration.getConfiguration();
table = conf.get(Property.TRACE_TABLE);
+connector = ensureTraceTableExists(conf);
+
+int port = conf.getPort(Property.TRACE_PORT);
+final ServerSocket sock = ServerSocketChannel.open().socket();
+sock.setReuseAddress(true);
+sock.bind(new InetSocketAddress(hostname, port));
+final TServerTransport transport = new TServerSocket(sock);
+TThreadPoolServer.Args options = new TThreadPoolServer.Args(transport);
+options.processor(new Processor(new Receiver()));
+server = new TThreadPoolServer(options);
+registerInZooKeeper(sock.getInetAddress().getHostAddress() + ":" +
sock.getLocalPort(), conf.get(Property.TRACE_ZK_PATH));
+writer = new AtomicReference<>(this.connector.createBatchWriter(table, new
BatchWriterConfig().setMaxLatency(BATCH_WRITER_MAX_LATENCY, TimeUnit.SECONDS)));
+ }
+
+ /**
+ * Exceptions thrown out of here should be things that cause service failure
(e.g. misconfigurations that aren't likely to change on retry).
+ *
+ * @return a working Connection that can be reused
+ * @throws ClassNotFoundException
+ * if TRACE_TOKEN_TYPE is set to a class that we can't load.
+ * @throws InstantiationException
+ * if we fail to create an instance of TRACE_TOKEN_TYPE.
+ * @throws IllegalAccessException
+ * if the class pointed to by TRACE_TOKEN_TYPE is private.
+ * @throws AccumuloSecurityException
+ * if the trace user has the wrong permissions
+ */
+ private Connector ensureTraceTableExists(final AccumuloConfiguration conf)
throws AccumuloSecurityException, ClassNotFoundException,
InstantiationException,
+ IllegalAccessException {
Connector connector = null;
while (true) {
try {
@@ -221,25 +254,12 @@ public class TraceServer implements Watcher {
}
connector.tableOperations().setProperty(table,
Property.TABLE_FORMATTER_CLASS.getKey(), TraceFormatter.class.getName());
break;
- } catch (RuntimeException ex) {
+ } catch (AccumuloException | TableExistsException |
TableNotFoundException | IOException | RuntimeException e
[6/6] accumulo git commit: Merge branch '1.8'
Merge branch '1.8' Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/3dc9b373 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/3dc9b373 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/3dc9b373 Branch: refs/heads/master Commit: 3dc9b373d0fb9a78efefefc640e0e395e42ce1c7 Parents: 8e0f19a 037c138 Author: Sean Busbey Authored: Fri Dec 9 15:39:14 2016 -0600 Committer: Sean Busbey Committed: Fri Dec 9 15:39:14 2016 -0600 -- .../org/apache/accumulo/tracer/TraceServer.java | 76 1 file changed, 48 insertions(+), 28 deletions(-) --
[4/6] accumulo git commit: Merge branch '1.7' into 1.8
Merge branch '1.7' into 1.8
Conflicts:
server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/037c1384
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/037c1384
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/037c1384
Branch: refs/heads/master
Commit: 037c1384ac861be95d8c9d1c5d1acbf61e40e2fe
Parents: 7c7bbab 47b57f7
Author: Sean Busbey
Authored: Fri Dec 9 15:33:37 2016 -0600
Committer: Sean Busbey
Committed: Fri Dec 9 15:33:37 2016 -0600
--
.../org/apache/accumulo/tracer/TraceServer.java | 76
1 file changed, 48 insertions(+), 28 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/037c1384/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
--
diff --cc
server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
index 2b6bcbf,67bd9d5..7c0d9b2
--- a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
+++ b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
@@@ -183,7 -185,36 +187,49 @@@ public class TraceServer implements Wat
log.info("Version " + Constants.VERSION);
log.info("Instance " + serverConfiguration.getInstance().getInstanceID());
AccumuloConfiguration conf = serverConfiguration.getConfiguration();
-table = conf.get(Property.TRACE_TABLE);
+tableName = conf.get(Property.TRACE_TABLE);
+ connector = ensureTraceTableExists(conf);
+
-int port = conf.getPort(Property.TRACE_PORT);
-final ServerSocket sock = ServerSocketChannel.open().socket();
-sock.setReuseAddress(true);
-sock.bind(new InetSocketAddress(hostname, port));
++int ports[] = conf.getPort(Property.TRACE_PORT);
++ServerSocket sock = null;
++for (int port : ports) {
++ ServerSocket s = ServerSocketChannel.open().socket();
++ s.setReuseAddress(true);
++ try {
++s.bind(new InetSocketAddress(hostname, port));
++sock = s;
++break;
++ } catch (Exception e) {
++log.warn("Unable to start trace server on port {}", port);
++ }
++}
++if (null == sock) {
++ throw new RuntimeException("Unable to start trace server on configured
ports: " + Arrays.toString(ports));
++}
+ final TServerTransport transport = new TServerSocket(sock);
+ TThreadPoolServer.Args options = new TThreadPoolServer.Args(transport);
+ options.processor(new Processor(new Receiver()));
+ server = new TThreadPoolServer(options);
+ registerInZooKeeper(sock.getInetAddress().getHostAddress() + ":" +
sock.getLocalPort(), conf.get(Property.TRACE_ZK_PATH));
-writer = new AtomicReference<>(this.connector.createBatchWriter(table,
new BatchWriterConfig().setMaxLatency(BATCH_WRITER_MAX_LATENCY,
TimeUnit.SECONDS)));
++writer = new AtomicReference<>(this.connector.createBatchWriter(tableName,
++new BatchWriterConfig().setMaxLatency(BATCH_WRITER_MAX_LATENCY,
TimeUnit.SECONDS)));
+ }
+
+ /**
+* Exceptions thrown out of here should be things that cause service
failure (e.g. misconfigurations that aren't likely to change on retry).
+*
+* @return a working Connection that can be reused
+* @throws ClassNotFoundException
+* if TRACE_TOKEN_TYPE is set to a class that we can't load.
+* @throws InstantiationException
+* if we fail to create an instance of TRACE_TOKEN_TYPE.
+* @throws IllegalAccessException
+* if the class pointed to by TRACE_TOKEN_TYPE is private.
+* @throws AccumuloSecurityException
+* if the trace user has the wrong permissions
+*/
+ private Connector ensureTraceTableExists(final AccumuloConfiguration conf)
throws AccumuloSecurityException, ClassNotFoundException,
InstantiationException,
+ IllegalAccessException {
Connector connector = null;
while (true) {
try {
@@@ -215,46 -246,20 +261,20 @@@
}
connector = serverConfiguration.getInstance().getConnector(principal,
at);
-if (!connector.tableOperations().exists(table)) {
- connector.tableOperations().create(table);
+if (!connector.tableOperations().exists(tableName)) {
+ connector.tableOperations().create(tableName);
IteratorSetting setting = new IteratorSetting(10, "ageoff",
AgeOffFilter.class.getName());
AgeOffFilter.setTTL(setting, 7 * 24 * 60 * 60 * 1000l);
- connector.tableOperations().attachIterator(table, setting);
+ connector.tableOperations().attachItera
[2/6] accumulo git commit: ACCUMULO-4533 TraceServer shouldn't abort given problems with trace table checks.
ACCUMULO-4533 TraceServer shouldn't abort given problems with trace table
checks.
Signed-off-by: Mike Drob
Signed-off-by: Josh Elser
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/47b57f73
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/47b57f73
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/47b57f73
Branch: refs/heads/1.8
Commit: 47b57f7300ae5d1df9536cf07d99c83bfb8e2af6
Parents: 3c45441
Author: Sean Busbey
Authored: Wed Dec 7 10:45:51 2016 -0600
Committer: Sean Busbey
Committed: Fri Dec 9 15:21:54 2016 -0600
--
.../org/apache/accumulo/tracer/TraceServer.java | 50 ++--
1 file changed, 35 insertions(+), 15 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/47b57f73/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
--
diff --git
a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
index dbb593d..67bd9d5 100644
--- a/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
+++ b/server/tracer/src/main/java/org/apache/accumulo/tracer/TraceServer.java
@@ -28,12 +28,16 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.BatchWriterConfig;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.client.IteratorSetting;
import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
import
org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Properties;
import org.apache.accumulo.core.client.security.tokens.KerberosToken;
@@ -182,6 +186,35 @@ public class TraceServer implements Watcher {
log.info("Instance " + serverConfiguration.getInstance().getInstanceID());
AccumuloConfiguration conf = serverConfiguration.getConfiguration();
table = conf.get(Property.TRACE_TABLE);
+connector = ensureTraceTableExists(conf);
+
+int port = conf.getPort(Property.TRACE_PORT);
+final ServerSocket sock = ServerSocketChannel.open().socket();
+sock.setReuseAddress(true);
+sock.bind(new InetSocketAddress(hostname, port));
+final TServerTransport transport = new TServerSocket(sock);
+TThreadPoolServer.Args options = new TThreadPoolServer.Args(transport);
+options.processor(new Processor(new Receiver()));
+server = new TThreadPoolServer(options);
+registerInZooKeeper(sock.getInetAddress().getHostAddress() + ":" +
sock.getLocalPort(), conf.get(Property.TRACE_ZK_PATH));
+writer = new AtomicReference<>(this.connector.createBatchWriter(table, new
BatchWriterConfig().setMaxLatency(BATCH_WRITER_MAX_LATENCY, TimeUnit.SECONDS)));
+ }
+
+ /**
+ * Exceptions thrown out of here should be things that cause service failure
(e.g. misconfigurations that aren't likely to change on retry).
+ *
+ * @return a working Connection that can be reused
+ * @throws ClassNotFoundException
+ * if TRACE_TOKEN_TYPE is set to a class that we can't load.
+ * @throws InstantiationException
+ * if we fail to create an instance of TRACE_TOKEN_TYPE.
+ * @throws IllegalAccessException
+ * if the class pointed to by TRACE_TOKEN_TYPE is private.
+ * @throws AccumuloSecurityException
+ * if the trace user has the wrong permissions
+ */
+ private Connector ensureTraceTableExists(final AccumuloConfiguration conf)
throws AccumuloSecurityException, ClassNotFoundException,
InstantiationException,
+ IllegalAccessException {
Connector connector = null;
while (true) {
try {
@@ -221,25 +254,12 @@ public class TraceServer implements Watcher {
}
connector.tableOperations().setProperty(table,
Property.TABLE_FORMATTER_CLASS.getKey(), TraceFormatter.class.getName());
break;
- } catch (RuntimeException ex) {
+ } catch (AccumuloException | TableExistsException |
TableNotFoundException | IOException | RuntimeException ex) {
log.info("Waiting to checking/create the trace table.", ex);
UtilWaitThread.sleep(1000);
}
}
-this.connector = connector;
-
svn commit: r1558294 - /accumulo/site/trunk/content/people.mdtext
Author: busbey Date: Wed Jan 15 05:10:16 2014 New Revision: 1558294 URL: http://svn.apache.org/r1558294 Log: Moved Sean Busbey from contributors to committers Modified: accumulo/site/trunk/content/people.mdtext Modified: accumulo/site/trunk/content/people.mdtext URL: http://svn.apache.org/viewvc/accumulo/site/trunk/content/people.mdtext?rev=1558294&r1=1558293&r2=1558294&view=diff == --- accumulo/site/trunk/content/people.mdtext (original) +++ accumulo/site/trunk/content/people.mdtext Wed Jan 15 05:10:16 2014 @@ -65,6 +65,7 @@ PMC and Committers elserjJosh Elserhttp://hortonworks.com/>HortonworksET (http://www.timeanddate.com/library/abbreviations/timezones/na/est.html>-5 / http://www.timeanddate.com/library/abbreviations/timezones/na/edt.html>-4) kturnerKeith Turnerhttp://www.ptech-llc.com/>Peterson TechnologiesET (http://www.timeanddate.com/library/abbreviations/timezones/na/est.html>-5 / http://www.timeanddate.com/library/abbreviations/timezones/na/edt.html>-4) mdrobMike Drobhttp://www.cloudera.com";>ClouderaET (http://www.timeanddate.com/library/abbreviations/timezones/na/est.html>-5 / http://www.timeanddate.com/library/abbreviations/timezones/na/edt.html>-4) + busbeySean Busbeyhttp://www.cloudera.com";>ClouderaCT (http://www.timeanddate.com/library/abbreviations/timezones/na/cst.html>-6 / http://www.timeanddate.com/library/abbreviations/timezones/na/cdt.html>-5) ujustgotbilldWilliam Slacumhttp://www.koverse.com";>KoverseET (http://www.timeanddate.com/library/abbreviations/timezones/na/est.html>-5 / http://www.timeanddate.com/library/abbreviations/timezones/na/edt.html>-4) @@ -113,8 +114,7 @@ Contributors Ryan Fishelhttp://www.cloudera.com";>Cloudera Ryan Leary Sapah Shah - Scott Kuehn - Sean Busbeyhttp://www.cloudera.com";>ClouderaCT (http://www.timeanddate.com/library/abbreviations/timezones/na/cst.html>-6 / http://www.timeanddate.com/library/abbreviations/timezones/na/cdt.html>-5) + Scott Kuehn Sean Hickey stevelSteve Loughranhttp://hortonworks.com/>Hortonworks(http://www.timeanddate.com/library/abbreviations/timezones/eu/gmt.html>0 / http://www.timeanddate.com/library/abbreviations/timezones/eu/bst.html>+1) supunSupun Kamburugamuva
svn commit: r894217 - in /websites/production/accumulo/content: ./ 1.4/apidocs/ 1.5/apidocs/
Author: busbey Date: Wed Jan 15 05:13:46 2014 New Revision: 894217 Log: Moved Sean Busbey from contributors to committers Added: websites/production/accumulo/content/ - copied from r894216, websites/staging/accumulo/trunk/content/ websites/production/accumulo/content/1.4/apidocs/ - copied from r894216, websites/production/accumulo/content/1.4/apidocs/ websites/production/accumulo/content/1.5/apidocs/ - copied from r894216, websites/production/accumulo/content/1.5/apidocs/
svn commit: r1558295 - /accumulo/site/trunk/content/governance/releasing.mdtext
Author: busbey Date: Wed Jan 15 05:27:48 2014 New Revision: 1558295 URL: http://svn.apache.org/r1558295 Log: ACCUMULO-1797 Updates release guide to include collecting information about testing. clarifies some test details. Modified: accumulo/site/trunk/content/governance/releasing.mdtext Modified: accumulo/site/trunk/content/governance/releasing.mdtext URL: http://svn.apache.org/viewvc/accumulo/site/trunk/content/governance/releasing.mdtext?rev=1558295&r1=1558294&r2=1558295&view=diff == --- accumulo/site/trunk/content/governance/releasing.mdtext (original) +++ accumulo/site/trunk/content/governance/releasing.mdtext Wed Jan 15 05:27:48 2014 @@ -47,12 +47,30 @@ Note: There will be times where API comp ## Testing +Testing for an Accumulo release includes a few steps that a developer may take without a Hadoop cluster and several that require a working cluster. + +### Stand alone +The following steps can be taken without having an underlying cluster. They SHOULD be handled with each Hadoop profile available for a given release version. To activate an alternative profile specify e.g. "-Dhadoop.profile=2" for the Hadoop 2 profile on the Maven commandline. Some older versions of Accumulo referred to Hadoop profiles diferently; see the README that came with said versions for details on building against different Hadoop versions. + 1. All JUnit tests must pass. This should be a requirement of any patch so it should never be an issue of the codebase. - - All functional tests must complete successfully. +- Use "mvn package" to run against the default profile of a particular release +- Use "mvn -Dhadoop.profile=2 package" to test against the Hadoop 2 profile on e.g. 1.4 or 1.5 +- Use "mvn -Dhadoop.profile=1 package" to test against the Hadoop 1 profile on e.g. 1.6 or later - Analyze output of static analysis tools like Findbugs and PMD. - - Two 24-hour periods of the randomwalk LongClean test with and without agitation need to be run successfully on a cluster. - - Two 24-hour periods of continuous ingest with and without agitation need to be validated successfully on a cluster. - - Two 72-hour periods of continuous ingest with and without agitation on a cluster. No validation is necessary but the cluster should be checked to ensure it is still functional. + - For versions 1.6 and later, all functional tests must pass via the Maven failsafe plugin. +- Use "mvn verify" to run against the default profile of a particular release +- Use "mvn -Dhadoop.profile=1 verify" to run the functional tests against the Hadoop 1 profile + +### Cluster based +The following tests require a Hadoop cluster running a minimum of HDFS, MapReduce, and ZooKeeper. The cluster MAY have any number of worker nodes; it can even be a single node in psuedo-distributed mode. A cluster with multiple tablet servers SHOULD be used so that more of the code base will be exercised. For the purposes of release testing, you should note the number of nodes and versions used. See the Releasing section for more details. + + 1. For versions prior to 1.6, all functional tests must complete successfully. +- See $ACCUMULO_HOME/test/system/auto/README for details on running the functional tests. + - Two 24-hour periods of the randomwalk LongClean test with and without agitation need to be run successfully. +- See $ACCUMULO_HOME/test/system/randomwalk/README for details on running the LongClean module. + - Two 24-hour periods of continuous ingest with and without agitation need to be validated successfully. +- See $ACCUMULO_HOME/test/system/continuous/README for details on running and verifying the continous ingest test. + - Two 72-hour periods of continuous ingest with and without agitation. No validation is necessary but the cluster should be checked to ensure it is still functional. ## Releasing @@ -61,5 +79,6 @@ Note: There will be times where API comp - Be fully built, including a tar.gz of the entire project as well as the documentation. - PGP Signatures of the tarball must be signed to a separate file and made available in the public_html folder of the user creating the release on people.apache.org, along with the tarball and MD5 and SHA512 checksums. - A vote must be made on dev@accumulo. Lazy consensus is not sufficient for a release; at least 3 +1 votes from PMC members are required. All checksums and signatures need to be verified before any voter can +1 it. Voting shall last 72 hours. +- Voters SHOULD include with their vote details on the tests from the testing section they have successfully run. If given, said details for each test MUST include: the number of worker nodes in the cluster, the operating system and version, the Hadoop version, and the Zookeeper version. All such gather
svn commit: r894220 - in /websites/production/accumulo/content: ./ 1.4/apidocs/ 1.5/apidocs/
Author: busbey Date: Wed Jan 15 05:31:44 2014 New Revision: 894220 Log: Publishing svnmucc operation to accumulo site by busbey Added: websites/production/accumulo/content/ - copied from r894219, websites/staging/accumulo/trunk/content/ websites/production/accumulo/content/1.4/apidocs/ - copied from r894219, websites/production/accumulo/content/1.4/apidocs/ websites/production/accumulo/content/1.5/apidocs/ - copied from r894219, websites/production/accumulo/content/1.5/apidocs/
[6/6] git commit: Merge branch '1.6.0-SNAPSHOT'
Merge branch '1.6.0-SNAPSHOT' Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/66cccbe4 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/66cccbe4 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/66cccbe4 Branch: refs/heads/master Commit: 66cccbe48febd25b94cf0e9d6aa73be030139e9f Parents: 7ecb50a 3931cce Author: Sean Busbey Authored: Tue Jan 21 12:17:15 2014 -0600 Committer: Sean Busbey Committed: Tue Jan 21 12:17:15 2014 -0600 -- .../apache/accumulo/test/continuous/ContinuousVerify.java | 10 +- 1 file changed, 5 insertions(+), 5 deletions(-) --
[2/6] git commit: ACCUMULO-2230 Make sure offline scan uses the cloned table.
ACCUMULO-2230 Make sure offline scan uses the cloned table.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/a4384884
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/a4384884
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/a4384884
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: a43848849e78aa9bc6edb5d6d87e2a285d8fccf2
Parents: 78c5f46
Author: Sean Busbey
Authored: Mon Jan 20 23:30:01 2014 -0600
Committer: Sean Busbey
Committed: Tue Jan 21 11:52:14 2014 -0600
--
.../apache/accumulo/test/continuous/ContinuousVerify.java | 10 +-
1 file changed, 5 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a4384884/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
--
diff --git
a/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
b/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
index 1ace774..08d8ff1 100644
---
a/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
+++
b/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
@@ -183,6 +183,9 @@ public class ContinuousVerify extends Configured implements
Tool {
Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" +
System.currentTimeMillis());
job.setJarByClass(this.getClass());
+job.setInputFormatClass(AccumuloInputFormat.class);
+opts.setAccumuloConfigs(job);
+
String clone = opts.getTableName();
Connector conn = null;
if (opts.scanOffline) {
@@ -191,13 +194,10 @@ public class ContinuousVerify extends Configured
implements Tool {
conn = opts.getConnector();
conn.tableOperations().clone(opts.getTableName(), clone, true, new
HashMap(), new HashSet());
conn.tableOperations().offline(clone);
+ AccumuloInputFormat.setInputTableName(job, clone);
+ AccumuloInputFormat.setOfflineTableScan(job, true);
}
-job.setInputFormatClass(AccumuloInputFormat.class);
-
-opts.setAccumuloConfigs(job);
-AccumuloInputFormat.setOfflineTableScan(job, opts.scanOffline);
-
// set up ranges
try {
Set ranges =
opts.getConnector().tableOperations().splitRangeByTablets(opts.getTableName(),
new Range(), opts.maxMaps);
[4/6] git commit: Merge branch '1.5.1-SNAPSHOT' into 1.6.0-SNAPSHOT
Merge branch '1.5.1-SNAPSHOT' into 1.6.0-SNAPSHOT Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/3931cceb Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/3931cceb Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/3931cceb Branch: refs/heads/master Commit: 3931cceb1731c835d16674a5c8fa72374bf16dcd Parents: 8cfdc1f a438488 Author: Sean Busbey Authored: Tue Jan 21 12:16:15 2014 -0600 Committer: Sean Busbey Committed: Tue Jan 21 12:16:15 2014 -0600 -- .../apache/accumulo/test/continuous/ContinuousVerify.java | 10 +- 1 file changed, 5 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/3931cceb/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java --
[5/6] git commit: Merge branch '1.5.1-SNAPSHOT' into 1.6.0-SNAPSHOT
Merge branch '1.5.1-SNAPSHOT' into 1.6.0-SNAPSHOT Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/3931cceb Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/3931cceb Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/3931cceb Branch: refs/heads/1.6.0-SNAPSHOT Commit: 3931cceb1731c835d16674a5c8fa72374bf16dcd Parents: 8cfdc1f a438488 Author: Sean Busbey Authored: Tue Jan 21 12:16:15 2014 -0600 Committer: Sean Busbey Committed: Tue Jan 21 12:16:15 2014 -0600 -- .../apache/accumulo/test/continuous/ContinuousVerify.java | 10 +- 1 file changed, 5 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/3931cceb/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java --
[3/6] git commit: ACCUMULO-2230 Make sure offline scan uses the cloned table.
ACCUMULO-2230 Make sure offline scan uses the cloned table.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/a4384884
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/a4384884
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/a4384884
Branch: refs/heads/master
Commit: a43848849e78aa9bc6edb5d6d87e2a285d8fccf2
Parents: 78c5f46
Author: Sean Busbey
Authored: Mon Jan 20 23:30:01 2014 -0600
Committer: Sean Busbey
Committed: Tue Jan 21 11:52:14 2014 -0600
--
.../apache/accumulo/test/continuous/ContinuousVerify.java | 10 +-
1 file changed, 5 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a4384884/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
--
diff --git
a/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
b/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
index 1ace774..08d8ff1 100644
---
a/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
+++
b/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
@@ -183,6 +183,9 @@ public class ContinuousVerify extends Configured implements
Tool {
Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" +
System.currentTimeMillis());
job.setJarByClass(this.getClass());
+job.setInputFormatClass(AccumuloInputFormat.class);
+opts.setAccumuloConfigs(job);
+
String clone = opts.getTableName();
Connector conn = null;
if (opts.scanOffline) {
@@ -191,13 +194,10 @@ public class ContinuousVerify extends Configured
implements Tool {
conn = opts.getConnector();
conn.tableOperations().clone(opts.getTableName(), clone, true, new
HashMap(), new HashSet());
conn.tableOperations().offline(clone);
+ AccumuloInputFormat.setInputTableName(job, clone);
+ AccumuloInputFormat.setOfflineTableScan(job, true);
}
-job.setInputFormatClass(AccumuloInputFormat.class);
-
-opts.setAccumuloConfigs(job);
-AccumuloInputFormat.setOfflineTableScan(job, opts.scanOffline);
-
// set up ranges
try {
Set ranges =
opts.getConnector().tableOperations().splitRangeByTablets(opts.getTableName(),
new Range(), opts.maxMaps);
[1/6] git commit: ACCUMULO-2230 Make sure offline scan uses the cloned table.
Updated Branches:
refs/heads/1.5.1-SNAPSHOT 78c5f460b -> a43848849
refs/heads/1.6.0-SNAPSHOT 8cfdc1ffc -> 3931cceb1
refs/heads/master 7ecb50a97 -> 66cccbe48
ACCUMULO-2230 Make sure offline scan uses the cloned table.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/a4384884
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/a4384884
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/a4384884
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: a43848849e78aa9bc6edb5d6d87e2a285d8fccf2
Parents: 78c5f46
Author: Sean Busbey
Authored: Mon Jan 20 23:30:01 2014 -0600
Committer: Sean Busbey
Committed: Tue Jan 21 11:52:14 2014 -0600
--
.../apache/accumulo/test/continuous/ContinuousVerify.java | 10 +-
1 file changed, 5 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a4384884/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
--
diff --git
a/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
b/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
index 1ace774..08d8ff1 100644
---
a/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
+++
b/test/src/main/java/org/apache/accumulo/test/continuous/ContinuousVerify.java
@@ -183,6 +183,9 @@ public class ContinuousVerify extends Configured implements
Tool {
Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" +
System.currentTimeMillis());
job.setJarByClass(this.getClass());
+job.setInputFormatClass(AccumuloInputFormat.class);
+opts.setAccumuloConfigs(job);
+
String clone = opts.getTableName();
Connector conn = null;
if (opts.scanOffline) {
@@ -191,13 +194,10 @@ public class ContinuousVerify extends Configured
implements Tool {
conn = opts.getConnector();
conn.tableOperations().clone(opts.getTableName(), clone, true, new
HashMap(), new HashSet());
conn.tableOperations().offline(clone);
+ AccumuloInputFormat.setInputTableName(job, clone);
+ AccumuloInputFormat.setOfflineTableScan(job, true);
}
-job.setInputFormatClass(AccumuloInputFormat.class);
-
-opts.setAccumuloConfigs(job);
-AccumuloInputFormat.setOfflineTableScan(job, opts.scanOffline);
-
// set up ranges
try {
Set ranges =
opts.getConnector().tableOperations().splitRangeByTablets(opts.getTableName(),
new Range(), opts.maxMaps);
[03/23] git commit: ACCUMULO-2213 Make writer recovery in the Tracer more robust.
ACCUMULO-2213 Make writer recovery in the Tracer more robust.
* Cleans up writer reseting in the TraceServer, avoids overly broad
catching.
* tones down log levels in TraceServer to WARN because trace information is
transient and we retry everything.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/dfafd9c1
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/dfafd9c1
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/dfafd9c1
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: dfafd9c1104d8359ca1eabe345661c541766d57f
Parents: 57f9b6c
Author: Sean Busbey
Authored: Tue Jan 21 14:05:56 2014 -0600
Committer: Sean Busbey
Committed: Wed Jan 22 16:36:34 2014 -0600
--
.../accumulo/server/trace/TraceServer.java | 68 ++--
1 file changed, 49 insertions(+), 19 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/dfafd9c1/src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java
--
diff --git
a/src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java
b/src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java
index 4d89e9c..0e2010a 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java
@@ -19,6 +19,7 @@ package org.apache.accumulo.server.trace;
import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.nio.channels.ServerSocketChannel;
+import java.util.concurrent.atomic.AtomicReference;
import java.util.TimerTask;
import org.apache.accumulo.cloudtrace.instrument.Span;
@@ -27,6 +28,7 @@ import org.apache.accumulo.cloudtrace.thrift.SpanReceiver;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.MutationsRejectedException;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Mutation;
@@ -64,8 +66,8 @@ public class TraceServer implements Watcher {
final private static Logger log = Logger.getLogger(TraceServer.class);
final private AccumuloConfiguration conf;
final private TServer server;
- private BatchWriter writer = null;
- private Connector connector;
+ final private AtomicReference writer;
+ final private Connector connector;
final String table;
private static void put(Mutation m, String cf, String cq, byte[] bytes, int
len) {
@@ -128,16 +130,27 @@ public class TraceServer implements Watcher {
put(timeMutation, "id", idString, transport.get(), transport.len());
}
try {
-if (writer == null)
- resetWriter();
-if (writer == null)
+final BatchWriter writer = TraceServer.this.writer.get();
+/* Check for null, because we expect spans to come in much faster than
flush calls.
+ In the case of failure, we'd rather avoid logging tons of NPEs.
+ */
+if (null == writer) {
+ log.warn("writer is not ready; discarding span.");
return;
+}
writer.addMutation(spanMutation);
writer.addMutation(indexMutation);
if (timeMutation != null)
writer.addMutation(timeMutation);
- } catch (Exception ex) {
-log.error("Unable to write mutation to table: " + spanMutation, ex);
+ } catch (MutationsRejectedException exception) {
+log.warn("Unable to write mutation to table; discarding span. set log
level to DEBUG for span information and stacktrace. cause: " + exception);
+if (log.isDebugEnabled()) {
+ log.debug("discarded span due to rejection of mutation: " +
spanMutation, exception);
+}
+ /* XXX this could be e.g. an IllegalArgumentExceptoion if we're trying
to write this mutation to a writer that has been closed since we retrieved it */
+ } catch (RuntimeException exception) {
+log.warn("Unable to write mutation to table; discarding span. set log
level to DEBUG for stacktrace. cause: " + exception);
+log.debug("unable to write mutation to table due to exception.",
exception);
}
}
@@ -147,6 +160,7 @@ public class TraceServer implements Watcher {
Accumulo.init("tracer");
conf = ServerConfiguration.getSystemConfiguration();
table = conf.get(Property.TRACE_TABLE);
+Connector connector = null;
while (true) {
try {
connector =
HdfsZooInstance.getInstance().getConnector(conf.get(Property.TRA
[07/23] git commit: ACCUMULO-2224 Make ZooSession more resiliant in the face of transient DNS issues.
ACCUMULO-2224 Make ZooSession more resiliant in the face of transient DNS
issues.
* retries if host is not found, up to 2xZK timeout (same as other
IOExceptions), rather than bailing on any host name problem.
* adds utility method for getting the max time the JVM will cache host failures
* add test for said method
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/f42ead0d
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/f42ead0d
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/f42ead0d
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: f42ead0d39e34578c6fe9636af4cfbd9d91e47a5
Parents: dfafd9c
Author: Sean Busbey
Authored: Mon Jan 20 14:26:20 2014 -0600
Committer: Sean Busbey
Committed: Wed Jan 22 23:12:24 2014 -0600
--
.../apache/accumulo/core/util/AddressUtil.java | 39 +++
.../accumulo/core/zookeeper/ZooSession.java | 11 ++--
.../accumulo/core/util/AddressUtilTest.java | 69 +++-
src/core/src/test/resources/log4j.properties| 23 +++
4 files changed, 137 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f42ead0d/src/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java
--
diff --git
a/src/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java
b/src/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java
index 0b82128..96c2e18 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java
@@ -16,12 +16,20 @@
*/
package org.apache.accumulo.core.util;
+import java.net.InetAddress; // workaround to enable @see/@link hyperlink
import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
+import java.security.Security;
import org.apache.hadoop.io.Text;
import org.apache.thrift.transport.TSocket;
+import org.apache.log4j.Logger;
+
public class AddressUtil {
+
+ private static final Logger log = Logger.getLogger(AddressUtil.class);
+
static public InetSocketAddress parseAddress(String address, int
defaultPort) throws NumberFormatException {
final String[] parts = address.split(":", 2);
if (parts.length == 2) {
@@ -44,5 +52,36 @@ public class AddressUtil {
static public String toString(InetSocketAddress addr) {
return addr.getAddress().getHostAddress() + ":" + addr.getPort();
}
+
+ /**
+ * Fetch the security value that determines how long DNS failures are cached.
+ * Looks up the security property 'networkaddress.cache.negative.ttl'.
Should that fail returns
+ * the default value used in the Oracle JVM 1.4+, which is 10 seconds.
+ *
+ * @param originalException the host lookup that is the source of needing
this lookup. maybe be null.
+ * @return positive integer number of seconds
+ * @see java.net.InetAddress
+ * @throws IllegalArgumentException if dns failures are cached forever
+ */
+ static public int getAddressCacheNegativeTtl(UnknownHostException
originalException) {
+int negativeTtl = 10;
+try {
+ negativeTtl =
Integer.parseInt(Security.getProperty("networkaddress.cache.negative.ttl"));
+} catch (NumberFormatException exception) {
+ log.warn("Failed to get JVM negative DNS respones cache TTL due to
format problem (e.g. this JVM might not have the " +
+"property). Falling back to default based on Oracle JVM 1.6
(10s)", exception);
+} catch (SecurityException exception) {
+ log.warn("Failed to get JVM negative DNS response cache TTL due to
security manager. Falling back to default based on Oracle JVM 1.6 (10s)",
exception);
+}
+if (-1 == negativeTtl) {
+ log.error("JVM negative DNS repsonse cache TTL is set to 'forever' and
host lookup failed. TTL can be changed with security property " +
+"'networkaddress.cache.negative.ttl', see
java.net.InetAddress.", originalException);
+ throw new IllegalArgumentException(originalException);
+} else if (0 > negativeTtl) {
+ log.warn("JVM specified negative DNS response cache TTL was negative
(and not 'forever'). Falling back to default based on Oracle JVM 1.6 (10s)");
+ negativeTtl = 10;
+}
+return negativeTtl;
+ }
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f42ead0d/src/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooSession.java
--
diff --git
a/src/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooSession.java
b/src/core/src/main/java/org/apache/
[15/23] Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c4cd3b1b/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java
--
diff --cc server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java
index 0777d03,000..116f134
mode 100644,00..100644
--- a/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java
+++ b/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java
@@@ -1,292 -1,0 +1,323 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.trace;
+
+import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+import java.nio.channels.ServerSocketChannel;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.TimeUnit;
++import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import
org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Properties;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
++import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileUtil;
+import org.apache.accumulo.core.iterators.user.AgeOffFilter;
+import org.apache.accumulo.core.security.SecurityUtil;
+import org.apache.accumulo.core.trace.TraceFormatter;
+import org.apache.accumulo.core.util.AddressUtil;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.server.Accumulo;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.util.time.SimpleTimer;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.accumulo.start.classloader.AccumuloClassLoader;
+import org.apache.accumulo.trace.instrument.Span;
+import org.apache.accumulo.trace.thrift.RemoteSpan;
+import org.apache.accumulo.trace.thrift.SpanReceiver.Iface;
+import org.apache.accumulo.trace.thrift.SpanReceiver.Processor;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+import org.apache.thrift.TByteArrayOutputStream;
+import org.apache.thrift.TException;
+import org.apache.thrift.protocol.TCompactProtocol;
+import org.apache.thrift.server.TServer;
+import org.apache.thrift.server.TThreadPoolServer;
+import org.apache.thrift.transport.TServerSocket;
+import org.apache.thrift.transport.TServerTransport;
+import org.apache.thrift.transport.TTransport;
+import org.apache.thrift.transport.TTransportException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.Watcher.Event.EventType;
+import org.apache.zookeeper.Watcher.Event.KeeperState;
+
+public class TraceServer implements Watcher {
+
+ final private static Logger log = Logger.getLogger(TraceServer.class);
+ final private ServerConfiguration serverConfiguration;
+ final private TServer server;
- private BatchWriter writer = null;
- private Connector connector;
++ final private AtomicReference writer;
++ final private Connector connector;
+ final String table;
+
+ private static void put(Mutation m, String cf, String cq, byte[] bytes, int
len) {
+m.put(new Text(cf), new Text(cq), new Value(bytes, 0, len));
+ }
+
+ static class ByteArrayTransport extends TTransport {
+
[23/23] git commit: Merge branch '1.6.0-SNAPSHOT'
Merge branch '1.6.0-SNAPSHOT' Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/353badb0 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/353badb0 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/353badb0 Branch: refs/heads/master Commit: 353badb056681d0ea014810af68128c849070743 Parents: bfe39f6 e7e5c00 Author: Sean Busbey Authored: Thu Jan 23 01:26:44 2014 -0600 Committer: Sean Busbey Committed: Thu Jan 23 01:26:44 2014 -0600 -- .../apache/accumulo/core/util/AddressUtil.java | 2 +- .../apache/accumulo/core/zookeeper/ZooUtil.java | 10 ++- .../apache/accumulo/fate/util/AddressUtil.java | 60 + .../accumulo/fate/zookeeper/ZooSession.java | 14 +-- .../accumulo/fate/util/AddressUtilTest.java | 95 .../org/apache/accumulo/server/Accumulo.java| 22 - .../accumulo/server/fs/VolumeManagerImpl.java | 3 + .../accumulo/server/util/TabletOperations.java | 10 ++- .../accumulo/master/tableOps/DeleteTable.java | 8 ++ .../org/apache/accumulo/tracer/TraceServer.java | 67 ++ .../org/apache/accumulo/tserver/Compactor.java | 2 + 11 files changed, 264 insertions(+), 29 deletions(-) --
[06/23] git commit: ACCUMULO-2224 Make ZooSession more resiliant in the face of transient DNS issues.
ACCUMULO-2224 Make ZooSession more resiliant in the face of transient DNS
issues.
* retries if host is not found, up to 2xZK timeout (same as other
IOExceptions), rather than bailing on any host name problem.
* adds utility method for getting the max time the JVM will cache host failures
* add test for said method
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/f42ead0d
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/f42ead0d
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/f42ead0d
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: f42ead0d39e34578c6fe9636af4cfbd9d91e47a5
Parents: dfafd9c
Author: Sean Busbey
Authored: Mon Jan 20 14:26:20 2014 -0600
Committer: Sean Busbey
Committed: Wed Jan 22 23:12:24 2014 -0600
--
.../apache/accumulo/core/util/AddressUtil.java | 39 +++
.../accumulo/core/zookeeper/ZooSession.java | 11 ++--
.../accumulo/core/util/AddressUtilTest.java | 69 +++-
src/core/src/test/resources/log4j.properties| 23 +++
4 files changed, 137 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f42ead0d/src/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java
--
diff --git
a/src/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java
b/src/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java
index 0b82128..96c2e18 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java
@@ -16,12 +16,20 @@
*/
package org.apache.accumulo.core.util;
+import java.net.InetAddress; // workaround to enable @see/@link hyperlink
import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
+import java.security.Security;
import org.apache.hadoop.io.Text;
import org.apache.thrift.transport.TSocket;
+import org.apache.log4j.Logger;
+
public class AddressUtil {
+
+ private static final Logger log = Logger.getLogger(AddressUtil.class);
+
static public InetSocketAddress parseAddress(String address, int
defaultPort) throws NumberFormatException {
final String[] parts = address.split(":", 2);
if (parts.length == 2) {
@@ -44,5 +52,36 @@ public class AddressUtil {
static public String toString(InetSocketAddress addr) {
return addr.getAddress().getHostAddress() + ":" + addr.getPort();
}
+
+ /**
+ * Fetch the security value that determines how long DNS failures are cached.
+ * Looks up the security property 'networkaddress.cache.negative.ttl'.
Should that fail returns
+ * the default value used in the Oracle JVM 1.4+, which is 10 seconds.
+ *
+ * @param originalException the host lookup that is the source of needing
this lookup. maybe be null.
+ * @return positive integer number of seconds
+ * @see java.net.InetAddress
+ * @throws IllegalArgumentException if dns failures are cached forever
+ */
+ static public int getAddressCacheNegativeTtl(UnknownHostException
originalException) {
+int negativeTtl = 10;
+try {
+ negativeTtl =
Integer.parseInt(Security.getProperty("networkaddress.cache.negative.ttl"));
+} catch (NumberFormatException exception) {
+ log.warn("Failed to get JVM negative DNS respones cache TTL due to
format problem (e.g. this JVM might not have the " +
+"property). Falling back to default based on Oracle JVM 1.6
(10s)", exception);
+} catch (SecurityException exception) {
+ log.warn("Failed to get JVM negative DNS response cache TTL due to
security manager. Falling back to default based on Oracle JVM 1.6 (10s)",
exception);
+}
+if (-1 == negativeTtl) {
+ log.error("JVM negative DNS repsonse cache TTL is set to 'forever' and
host lookup failed. TTL can be changed with security property " +
+"'networkaddress.cache.negative.ttl', see
java.net.InetAddress.", originalException);
+ throw new IllegalArgumentException(originalException);
+} else if (0 > negativeTtl) {
+ log.warn("JVM specified negative DNS response cache TTL was negative
(and not 'forever'). Falling back to default based on Oracle JVM 1.6 (10s)");
+ negativeTtl = 10;
+}
+return negativeTtl;
+ }
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f42ead0d/src/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooSession.java
--
diff --git
a/src/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooSession.java
b/src/core/src/main/java/org/apache/
[09/23] git commit: ACCUMULO-2225 handle IllegalArgumentExceptions from Hadoop on host lookup failure.
ACCUMULO-2225 handle IllegalArgumentExceptions from Hadoop on host lookup
failure.
Looks for cases where we treat IOExceptions out of Hadoop specially, then
attempts to replicate for UnknownHostExceptions that have been wrapped in
IllegalArgumentExceptions.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/f778dcf5
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/f778dcf5
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/f778dcf5
Branch: refs/heads/1.4.5-SNAPSHOT
Commit: f778dcf50cd7163722f63471d1585333704e639c
Parents: f42ead0
Author: Sean Busbey
Authored: Tue Jan 21 08:26:16 2014 -0600
Committer: Sean Busbey
Committed: Wed Jan 22 23:12:29 2014 -0600
--
.../accumulo/core/client/ZooKeeperInstance.java | 10 ++-
.../org/apache/accumulo/server/Accumulo.java| 28 +---
.../server/master/tableOps/DeleteTable.java | 8 ++
.../accumulo/server/tabletserver/Compactor.java | 2 ++
.../randomwalk/security/SecurityHelper.java | 11 ++--
.../accumulo/server/util/TabletOperations.java | 10 ++-
6 files changed, 62 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f778dcf5/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
--
diff --git
a/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
b/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
index e02c197..05982e4 100644
---
a/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
+++
b/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
@@ -18,6 +18,7 @@ package org.apache.accumulo.core.client;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.net.UnknownHostException;
import java.nio.ByteBuffer;
import java.util.Collections;
import java.util.List;
@@ -276,7 +277,14 @@ public class ZooKeeperInstance implements Instance {
return result;
}
} catch (IOException e) {
- throw new RuntimeException("Accumulo not initialized, there is no
instance id at " + instanceDirectory, e);
+ log.error("Problem reading instance id out of hdfs at " +
instanceDirectory, e);
+ throw new RuntimeException("Can't tell if Accumulo is initialized; can't
read instance id at " + instanceDirectory, e);
+} catch (IllegalArgumentException exception) {
+ /* HDFS throws this when there's a UnknownHostException due to DNS
troubles. */
+ if (exception.getCause() instanceof UnknownHostException) {
+log.error("Problem reading instance id out of hdfs at " +
instanceDirectory, exception);
+ }
+ throw exception;
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f778dcf5/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java
--
diff --git a/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java
b/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java
index 253962b..184692c 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java
@@ -30,6 +30,7 @@ import java.util.TreeMap;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.file.FileUtil;
import org.apache.accumulo.core.trace.DistributedTrace;
+import org.apache.accumulo.core.util.AddressUtil;
import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.accumulo.core.util.UtilWaitThread;
import org.apache.accumulo.core.util.Version;
@@ -209,6 +210,7 @@ public class Accumulo {
}
log.info("Zookeeper connected and initialized, attemping to talk to HDFS");
long sleep = 1000;
+int unknownHostTries = 3;
while (true) {
try {
FileSystem fs = FileSystem.get(CachedConfiguration.getInstance());
@@ -216,10 +218,26 @@ public class Accumulo {
break;
log.warn("Waiting for the NameNode to leave safemode");
} catch (IOException ex) {
-log.warn("Unable to connect to HDFS");
+log.warn("Unable to connect to HDFS", ex);
+ } catch (IllegalArgumentException exception) {
+/* Unwrap the UnknownHostException so we can deal with it directly */
+if (exception.getCause() instanceof UnknownHostException) {
+ if (unknownHostTries > 0) {
+log.warn("Unable to connect to HDFS, will retry. cause: " +
exception.getCause());
+/* We need to make sure our sleep period
[13/23] Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c4cd3b1b/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java
--
diff --cc server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java
index 0777d03,000..116f134
mode 100644,00..100644
--- a/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java
+++ b/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java
@@@ -1,292 -1,0 +1,323 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.trace;
+
+import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+import java.nio.channels.ServerSocketChannel;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.TimeUnit;
++import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import
org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Properties;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
++import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileUtil;
+import org.apache.accumulo.core.iterators.user.AgeOffFilter;
+import org.apache.accumulo.core.security.SecurityUtil;
+import org.apache.accumulo.core.trace.TraceFormatter;
+import org.apache.accumulo.core.util.AddressUtil;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.server.Accumulo;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.util.time.SimpleTimer;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.accumulo.start.classloader.AccumuloClassLoader;
+import org.apache.accumulo.trace.instrument.Span;
+import org.apache.accumulo.trace.thrift.RemoteSpan;
+import org.apache.accumulo.trace.thrift.SpanReceiver.Iface;
+import org.apache.accumulo.trace.thrift.SpanReceiver.Processor;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+import org.apache.thrift.TByteArrayOutputStream;
+import org.apache.thrift.TException;
+import org.apache.thrift.protocol.TCompactProtocol;
+import org.apache.thrift.server.TServer;
+import org.apache.thrift.server.TThreadPoolServer;
+import org.apache.thrift.transport.TServerSocket;
+import org.apache.thrift.transport.TServerTransport;
+import org.apache.thrift.transport.TTransport;
+import org.apache.thrift.transport.TTransportException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.Watcher.Event.EventType;
+import org.apache.zookeeper.Watcher.Event.KeeperState;
+
+public class TraceServer implements Watcher {
+
+ final private static Logger log = Logger.getLogger(TraceServer.class);
+ final private ServerConfiguration serverConfiguration;
+ final private TServer server;
- private BatchWriter writer = null;
- private Connector connector;
++ final private AtomicReference writer;
++ final private Connector connector;
+ final String table;
+
+ private static void put(Mutation m, String cf, String cq, byte[] bytes, int
len) {
+m.put(new Text(cf), new Text(cq), new Value(bytes, 0, len));
+ }
+
+ static class ByteArrayTransport extends TTransport {
+
[19/23] Merge branch '1.5.1-SNAPSHOT' into 1.6.0-SNAPSHOT
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e7e5c009/server/tserver/src/main/java/org/apache/accumulo/tserver/Compactor.java
--
diff --cc
server/tserver/src/main/java/org/apache/accumulo/tserver/Compactor.java
index 0843c13,000..df476ec
mode 100644,00..100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/Compactor.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/Compactor.java
@@@ -1,546 -1,0 +1,548 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.tserver;
+
+import java.io.IOException;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.data.thrift.IterInfo;
+import org.apache.accumulo.core.file.FileOperations;
+import org.apache.accumulo.core.file.FileSKVIterator;
+import org.apache.accumulo.core.file.FileSKVWriter;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.IteratorUtil;
+import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.iterators.WrappingIterator;
+import org.apache.accumulo.core.iterators.system.ColumnFamilySkippingIterator;
+import org.apache.accumulo.core.iterators.system.DeletingIterator;
+import org.apache.accumulo.core.iterators.system.MultiIterator;
+import org.apache.accumulo.core.iterators.system.TimeSettingIterator;
+import org.apache.accumulo.core.metadata.schema.DataFileValue;
+import org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction;
+import org.apache.accumulo.core.tabletserver.thrift.CompactionReason;
+import org.apache.accumulo.core.tabletserver.thrift.CompactionType;
+import org.apache.accumulo.core.util.LocalityGroupUtil;
+import
org.apache.accumulo.core.util.LocalityGroupUtil.LocalityGroupConfigurationError;
+import org.apache.accumulo.server.fs.FileRef;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.problems.ProblemReport;
+import org.apache.accumulo.server.problems.ProblemReportingIterator;
+import org.apache.accumulo.server.problems.ProblemReports;
+import org.apache.accumulo.server.problems.ProblemType;
+import org.apache.accumulo.trace.instrument.Span;
+import org.apache.accumulo.trace.instrument.Trace;
+import org.apache.accumulo.tserver.Tablet.MinorCompactionReason;
+import org.apache.accumulo.tserver.compaction.MajorCompactionReason;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.log4j.Logger;
+
+public class Compactor implements Callable {
+
+ public static class CountingIterator extends WrappingIterator {
+
+private long count;
+private ArrayList deepCopies;
+private AtomicLong entriesRead;
+
+@Override
+public CountingIterator deepCopy(IteratorEnvironment env) {
+ return new CountingIterator(this, env);
+}
+
+private CountingIterator(CountingIterator other, IteratorEnvironment env)
{
+ setSource(other.getSource().deepCopy(env));
+ count = 0;
+ this.deepCopies = other.deepCopies;
+ this.entriesRead = other.entriesRead;
+ deepCopies.add(this);
+}
+
+public CountingIterator(SortedKeyValueIterator source,
AtomicLong entriesRead) {
+ deepCopies = new ArrayList();
+ this.setSource(source);
+ count = 0;
+ this.entriesRead = entriesR
[08/23] git commit: ACCUMULO-2224 Make ZooSession more resiliant in the face of transient DNS issues.
ACCUMULO-2224 Make ZooSession more resiliant in the face of transient DNS
issues.
* retries if host is not found, up to 2xZK timeout (same as other
IOExceptions), rather than bailing on any host name problem.
* adds utility method for getting the max time the JVM will cache host failures
* add test for said method
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/f42ead0d
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/f42ead0d
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/f42ead0d
Branch: refs/heads/master
Commit: f42ead0d39e34578c6fe9636af4cfbd9d91e47a5
Parents: dfafd9c
Author: Sean Busbey
Authored: Mon Jan 20 14:26:20 2014 -0600
Committer: Sean Busbey
Committed: Wed Jan 22 23:12:24 2014 -0600
--
.../apache/accumulo/core/util/AddressUtil.java | 39 +++
.../accumulo/core/zookeeper/ZooSession.java | 11 ++--
.../accumulo/core/util/AddressUtilTest.java | 69 +++-
src/core/src/test/resources/log4j.properties| 23 +++
4 files changed, 137 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f42ead0d/src/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java
--
diff --git
a/src/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java
b/src/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java
index 0b82128..96c2e18 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java
@@ -16,12 +16,20 @@
*/
package org.apache.accumulo.core.util;
+import java.net.InetAddress; // workaround to enable @see/@link hyperlink
import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
+import java.security.Security;
import org.apache.hadoop.io.Text;
import org.apache.thrift.transport.TSocket;
+import org.apache.log4j.Logger;
+
public class AddressUtil {
+
+ private static final Logger log = Logger.getLogger(AddressUtil.class);
+
static public InetSocketAddress parseAddress(String address, int
defaultPort) throws NumberFormatException {
final String[] parts = address.split(":", 2);
if (parts.length == 2) {
@@ -44,5 +52,36 @@ public class AddressUtil {
static public String toString(InetSocketAddress addr) {
return addr.getAddress().getHostAddress() + ":" + addr.getPort();
}
+
+ /**
+ * Fetch the security value that determines how long DNS failures are cached.
+ * Looks up the security property 'networkaddress.cache.negative.ttl'.
Should that fail returns
+ * the default value used in the Oracle JVM 1.4+, which is 10 seconds.
+ *
+ * @param originalException the host lookup that is the source of needing
this lookup. maybe be null.
+ * @return positive integer number of seconds
+ * @see java.net.InetAddress
+ * @throws IllegalArgumentException if dns failures are cached forever
+ */
+ static public int getAddressCacheNegativeTtl(UnknownHostException
originalException) {
+int negativeTtl = 10;
+try {
+ negativeTtl =
Integer.parseInt(Security.getProperty("networkaddress.cache.negative.ttl"));
+} catch (NumberFormatException exception) {
+ log.warn("Failed to get JVM negative DNS respones cache TTL due to
format problem (e.g. this JVM might not have the " +
+"property). Falling back to default based on Oracle JVM 1.6
(10s)", exception);
+} catch (SecurityException exception) {
+ log.warn("Failed to get JVM negative DNS response cache TTL due to
security manager. Falling back to default based on Oracle JVM 1.6 (10s)",
exception);
+}
+if (-1 == negativeTtl) {
+ log.error("JVM negative DNS repsonse cache TTL is set to 'forever' and
host lookup failed. TTL can be changed with security property " +
+"'networkaddress.cache.negative.ttl', see
java.net.InetAddress.", originalException);
+ throw new IllegalArgumentException(originalException);
+} else if (0 > negativeTtl) {
+ log.warn("JVM specified negative DNS response cache TTL was negative
(and not 'forever'). Falling back to default based on Oracle JVM 1.6 (10s)");
+ negativeTtl = 10;
+}
+return negativeTtl;
+ }
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f42ead0d/src/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooSession.java
--
diff --git
a/src/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooSession.java
b/src/core/src/main/java/org/apache/
[05/23] git commit: ACCUMULO-2224 Make ZooSession more resiliant in the face of transient DNS issues.
ACCUMULO-2224 Make ZooSession more resiliant in the face of transient DNS
issues.
* retries if host is not found, up to 2xZK timeout (same as other
IOExceptions), rather than bailing on any host name problem.
* adds utility method for getting the max time the JVM will cache host failures
* add test for said method
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/f42ead0d
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/f42ead0d
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/f42ead0d
Branch: refs/heads/1.4.5-SNAPSHOT
Commit: f42ead0d39e34578c6fe9636af4cfbd9d91e47a5
Parents: dfafd9c
Author: Sean Busbey
Authored: Mon Jan 20 14:26:20 2014 -0600
Committer: Sean Busbey
Committed: Wed Jan 22 23:12:24 2014 -0600
--
.../apache/accumulo/core/util/AddressUtil.java | 39 +++
.../accumulo/core/zookeeper/ZooSession.java | 11 ++--
.../accumulo/core/util/AddressUtilTest.java | 69 +++-
src/core/src/test/resources/log4j.properties| 23 +++
4 files changed, 137 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f42ead0d/src/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java
--
diff --git
a/src/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java
b/src/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java
index 0b82128..96c2e18 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/util/AddressUtil.java
@@ -16,12 +16,20 @@
*/
package org.apache.accumulo.core.util;
+import java.net.InetAddress; // workaround to enable @see/@link hyperlink
import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
+import java.security.Security;
import org.apache.hadoop.io.Text;
import org.apache.thrift.transport.TSocket;
+import org.apache.log4j.Logger;
+
public class AddressUtil {
+
+ private static final Logger log = Logger.getLogger(AddressUtil.class);
+
static public InetSocketAddress parseAddress(String address, int
defaultPort) throws NumberFormatException {
final String[] parts = address.split(":", 2);
if (parts.length == 2) {
@@ -44,5 +52,36 @@ public class AddressUtil {
static public String toString(InetSocketAddress addr) {
return addr.getAddress().getHostAddress() + ":" + addr.getPort();
}
+
+ /**
+ * Fetch the security value that determines how long DNS failures are cached.
+ * Looks up the security property 'networkaddress.cache.negative.ttl'.
Should that fail returns
+ * the default value used in the Oracle JVM 1.4+, which is 10 seconds.
+ *
+ * @param originalException the host lookup that is the source of needing
this lookup. maybe be null.
+ * @return positive integer number of seconds
+ * @see java.net.InetAddress
+ * @throws IllegalArgumentException if dns failures are cached forever
+ */
+ static public int getAddressCacheNegativeTtl(UnknownHostException
originalException) {
+int negativeTtl = 10;
+try {
+ negativeTtl =
Integer.parseInt(Security.getProperty("networkaddress.cache.negative.ttl"));
+} catch (NumberFormatException exception) {
+ log.warn("Failed to get JVM negative DNS respones cache TTL due to
format problem (e.g. this JVM might not have the " +
+"property). Falling back to default based on Oracle JVM 1.6
(10s)", exception);
+} catch (SecurityException exception) {
+ log.warn("Failed to get JVM negative DNS response cache TTL due to
security manager. Falling back to default based on Oracle JVM 1.6 (10s)",
exception);
+}
+if (-1 == negativeTtl) {
+ log.error("JVM negative DNS repsonse cache TTL is set to 'forever' and
host lookup failed. TTL can be changed with security property " +
+"'networkaddress.cache.negative.ttl', see
java.net.InetAddress.", originalException);
+ throw new IllegalArgumentException(originalException);
+} else if (0 > negativeTtl) {
+ log.warn("JVM specified negative DNS response cache TTL was negative
(and not 'forever'). Falling back to default based on Oracle JVM 1.6 (10s)");
+ negativeTtl = 10;
+}
+return negativeTtl;
+ }
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f42ead0d/src/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooSession.java
--
diff --git
a/src/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooSession.java
b/src/core/src/main/java/org/apache/
[17/23] Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c4cd3b1b/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java
--
diff --cc server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java
index 0777d03,000..116f134
mode 100644,00..100644
--- a/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java
+++ b/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java
@@@ -1,292 -1,0 +1,323 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.trace;
+
+import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+import java.nio.channels.ServerSocketChannel;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.TimeUnit;
++import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import
org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Properties;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
++import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileUtil;
+import org.apache.accumulo.core.iterators.user.AgeOffFilter;
+import org.apache.accumulo.core.security.SecurityUtil;
+import org.apache.accumulo.core.trace.TraceFormatter;
+import org.apache.accumulo.core.util.AddressUtil;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.server.Accumulo;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.util.time.SimpleTimer;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.accumulo.start.classloader.AccumuloClassLoader;
+import org.apache.accumulo.trace.instrument.Span;
+import org.apache.accumulo.trace.thrift.RemoteSpan;
+import org.apache.accumulo.trace.thrift.SpanReceiver.Iface;
+import org.apache.accumulo.trace.thrift.SpanReceiver.Processor;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+import org.apache.thrift.TByteArrayOutputStream;
+import org.apache.thrift.TException;
+import org.apache.thrift.protocol.TCompactProtocol;
+import org.apache.thrift.server.TServer;
+import org.apache.thrift.server.TThreadPoolServer;
+import org.apache.thrift.transport.TServerSocket;
+import org.apache.thrift.transport.TServerTransport;
+import org.apache.thrift.transport.TTransport;
+import org.apache.thrift.transport.TTransportException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.Watcher.Event.EventType;
+import org.apache.zookeeper.Watcher.Event.KeeperState;
+
+public class TraceServer implements Watcher {
+
+ final private static Logger log = Logger.getLogger(TraceServer.class);
+ final private ServerConfiguration serverConfiguration;
+ final private TServer server;
- private BatchWriter writer = null;
- private Connector connector;
++ final private AtomicReference writer;
++ final private Connector connector;
+ final String table;
+
+ private static void put(Mutation m, String cf, String cq, byte[] bytes, int
len) {
+m.put(new Text(cf), new Text(cq), new Value(bytes, 0, len));
+ }
+
+ static class ByteArrayTransport extends TTransport {
+
[02/23] git commit: ACCUMULO-2213 Make writer recovery in the Tracer more robust.
ACCUMULO-2213 Make writer recovery in the Tracer more robust.
* Cleans up writer reseting in the TraceServer, avoids overly broad
catching.
* tones down log levels in TraceServer to WARN because trace information is
transient and we retry everything.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/dfafd9c1
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/dfafd9c1
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/dfafd9c1
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: dfafd9c1104d8359ca1eabe345661c541766d57f
Parents: 57f9b6c
Author: Sean Busbey
Authored: Tue Jan 21 14:05:56 2014 -0600
Committer: Sean Busbey
Committed: Wed Jan 22 16:36:34 2014 -0600
--
.../accumulo/server/trace/TraceServer.java | 68 ++--
1 file changed, 49 insertions(+), 19 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/dfafd9c1/src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java
--
diff --git
a/src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java
b/src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java
index 4d89e9c..0e2010a 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java
@@ -19,6 +19,7 @@ package org.apache.accumulo.server.trace;
import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.nio.channels.ServerSocketChannel;
+import java.util.concurrent.atomic.AtomicReference;
import java.util.TimerTask;
import org.apache.accumulo.cloudtrace.instrument.Span;
@@ -27,6 +28,7 @@ import org.apache.accumulo.cloudtrace.thrift.SpanReceiver;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.MutationsRejectedException;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Mutation;
@@ -64,8 +66,8 @@ public class TraceServer implements Watcher {
final private static Logger log = Logger.getLogger(TraceServer.class);
final private AccumuloConfiguration conf;
final private TServer server;
- private BatchWriter writer = null;
- private Connector connector;
+ final private AtomicReference writer;
+ final private Connector connector;
final String table;
private static void put(Mutation m, String cf, String cq, byte[] bytes, int
len) {
@@ -128,16 +130,27 @@ public class TraceServer implements Watcher {
put(timeMutation, "id", idString, transport.get(), transport.len());
}
try {
-if (writer == null)
- resetWriter();
-if (writer == null)
+final BatchWriter writer = TraceServer.this.writer.get();
+/* Check for null, because we expect spans to come in much faster than
flush calls.
+ In the case of failure, we'd rather avoid logging tons of NPEs.
+ */
+if (null == writer) {
+ log.warn("writer is not ready; discarding span.");
return;
+}
writer.addMutation(spanMutation);
writer.addMutation(indexMutation);
if (timeMutation != null)
writer.addMutation(timeMutation);
- } catch (Exception ex) {
-log.error("Unable to write mutation to table: " + spanMutation, ex);
+ } catch (MutationsRejectedException exception) {
+log.warn("Unable to write mutation to table; discarding span. set log
level to DEBUG for span information and stacktrace. cause: " + exception);
+if (log.isDebugEnabled()) {
+ log.debug("discarded span due to rejection of mutation: " +
spanMutation, exception);
+}
+ /* XXX this could be e.g. an IllegalArgumentExceptoion if we're trying
to write this mutation to a writer that has been closed since we retrieved it */
+ } catch (RuntimeException exception) {
+log.warn("Unable to write mutation to table; discarding span. set log
level to DEBUG for stacktrace. cause: " + exception);
+log.debug("unable to write mutation to table due to exception.",
exception);
}
}
@@ -147,6 +160,7 @@ public class TraceServer implements Watcher {
Accumulo.init("tracer");
conf = ServerConfiguration.getSystemConfiguration();
table = conf.get(Property.TRACE_TABLE);
+Connector connector = null;
while (true) {
try {
connector =
HdfsZooInstance.getInstance().getConnector(conf.get(Property.TRA
[12/23] git commit: ACCUMULO-2225 handle IllegalArgumentExceptions from Hadoop on host lookup failure.
ACCUMULO-2225 handle IllegalArgumentExceptions from Hadoop on host lookup
failure.
Looks for cases where we treat IOExceptions out of Hadoop specially, then
attempts to replicate for UnknownHostExceptions that have been wrapped in
IllegalArgumentExceptions.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/f778dcf5
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/f778dcf5
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/f778dcf5
Branch: refs/heads/master
Commit: f778dcf50cd7163722f63471d1585333704e639c
Parents: f42ead0
Author: Sean Busbey
Authored: Tue Jan 21 08:26:16 2014 -0600
Committer: Sean Busbey
Committed: Wed Jan 22 23:12:29 2014 -0600
--
.../accumulo/core/client/ZooKeeperInstance.java | 10 ++-
.../org/apache/accumulo/server/Accumulo.java| 28 +---
.../server/master/tableOps/DeleteTable.java | 8 ++
.../accumulo/server/tabletserver/Compactor.java | 2 ++
.../randomwalk/security/SecurityHelper.java | 11 ++--
.../accumulo/server/util/TabletOperations.java | 10 ++-
6 files changed, 62 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f778dcf5/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
--
diff --git
a/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
b/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
index e02c197..05982e4 100644
---
a/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
+++
b/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
@@ -18,6 +18,7 @@ package org.apache.accumulo.core.client;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.net.UnknownHostException;
import java.nio.ByteBuffer;
import java.util.Collections;
import java.util.List;
@@ -276,7 +277,14 @@ public class ZooKeeperInstance implements Instance {
return result;
}
} catch (IOException e) {
- throw new RuntimeException("Accumulo not initialized, there is no
instance id at " + instanceDirectory, e);
+ log.error("Problem reading instance id out of hdfs at " +
instanceDirectory, e);
+ throw new RuntimeException("Can't tell if Accumulo is initialized; can't
read instance id at " + instanceDirectory, e);
+} catch (IllegalArgumentException exception) {
+ /* HDFS throws this when there's a UnknownHostException due to DNS
troubles. */
+ if (exception.getCause() instanceof UnknownHostException) {
+log.error("Problem reading instance id out of hdfs at " +
instanceDirectory, exception);
+ }
+ throw exception;
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f778dcf5/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java
--
diff --git a/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java
b/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java
index 253962b..184692c 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java
@@ -30,6 +30,7 @@ import java.util.TreeMap;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.file.FileUtil;
import org.apache.accumulo.core.trace.DistributedTrace;
+import org.apache.accumulo.core.util.AddressUtil;
import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.accumulo.core.util.UtilWaitThread;
import org.apache.accumulo.core.util.Version;
@@ -209,6 +210,7 @@ public class Accumulo {
}
log.info("Zookeeper connected and initialized, attemping to talk to HDFS");
long sleep = 1000;
+int unknownHostTries = 3;
while (true) {
try {
FileSystem fs = FileSystem.get(CachedConfiguration.getInstance());
@@ -216,10 +218,26 @@ public class Accumulo {
break;
log.warn("Waiting for the NameNode to leave safemode");
} catch (IOException ex) {
-log.warn("Unable to connect to HDFS");
+log.warn("Unable to connect to HDFS", ex);
+ } catch (IllegalArgumentException exception) {
+/* Unwrap the UnknownHostException so we can deal with it directly */
+if (exception.getCause() instanceof UnknownHostException) {
+ if (unknownHostTries > 0) {
+log.warn("Unable to connect to HDFS, will retry. cause: " +
exception.getCause());
+/* We need to make sure our sleep period
[10/23] git commit: ACCUMULO-2225 handle IllegalArgumentExceptions from Hadoop on host lookup failure.
ACCUMULO-2225 handle IllegalArgumentExceptions from Hadoop on host lookup
failure.
Looks for cases where we treat IOExceptions out of Hadoop specially, then
attempts to replicate for UnknownHostExceptions that have been wrapped in
IllegalArgumentExceptions.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/f778dcf5
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/f778dcf5
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/f778dcf5
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: f778dcf50cd7163722f63471d1585333704e639c
Parents: f42ead0
Author: Sean Busbey
Authored: Tue Jan 21 08:26:16 2014 -0600
Committer: Sean Busbey
Committed: Wed Jan 22 23:12:29 2014 -0600
--
.../accumulo/core/client/ZooKeeperInstance.java | 10 ++-
.../org/apache/accumulo/server/Accumulo.java| 28 +---
.../server/master/tableOps/DeleteTable.java | 8 ++
.../accumulo/server/tabletserver/Compactor.java | 2 ++
.../randomwalk/security/SecurityHelper.java | 11 ++--
.../accumulo/server/util/TabletOperations.java | 10 ++-
6 files changed, 62 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f778dcf5/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
--
diff --git
a/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
b/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
index e02c197..05982e4 100644
---
a/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
+++
b/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
@@ -18,6 +18,7 @@ package org.apache.accumulo.core.client;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.net.UnknownHostException;
import java.nio.ByteBuffer;
import java.util.Collections;
import java.util.List;
@@ -276,7 +277,14 @@ public class ZooKeeperInstance implements Instance {
return result;
}
} catch (IOException e) {
- throw new RuntimeException("Accumulo not initialized, there is no
instance id at " + instanceDirectory, e);
+ log.error("Problem reading instance id out of hdfs at " +
instanceDirectory, e);
+ throw new RuntimeException("Can't tell if Accumulo is initialized; can't
read instance id at " + instanceDirectory, e);
+} catch (IllegalArgumentException exception) {
+ /* HDFS throws this when there's a UnknownHostException due to DNS
troubles. */
+ if (exception.getCause() instanceof UnknownHostException) {
+log.error("Problem reading instance id out of hdfs at " +
instanceDirectory, exception);
+ }
+ throw exception;
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f778dcf5/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java
--
diff --git a/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java
b/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java
index 253962b..184692c 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java
@@ -30,6 +30,7 @@ import java.util.TreeMap;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.file.FileUtil;
import org.apache.accumulo.core.trace.DistributedTrace;
+import org.apache.accumulo.core.util.AddressUtil;
import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.accumulo.core.util.UtilWaitThread;
import org.apache.accumulo.core.util.Version;
@@ -209,6 +210,7 @@ public class Accumulo {
}
log.info("Zookeeper connected and initialized, attemping to talk to HDFS");
long sleep = 1000;
+int unknownHostTries = 3;
while (true) {
try {
FileSystem fs = FileSystem.get(CachedConfiguration.getInstance());
@@ -216,10 +218,26 @@ public class Accumulo {
break;
log.warn("Waiting for the NameNode to leave safemode");
} catch (IOException ex) {
-log.warn("Unable to connect to HDFS");
+log.warn("Unable to connect to HDFS", ex);
+ } catch (IllegalArgumentException exception) {
+/* Unwrap the UnknownHostException so we can deal with it directly */
+if (exception.getCause() instanceof UnknownHostException) {
+ if (unknownHostTries > 0) {
+log.warn("Unable to connect to HDFS, will retry. cause: " +
exception.getCause());
+/* We need to make sure our sleep period
[01/23] git commit: ACCUMULO-2213 Make writer recovery in the Tracer more robust.
Updated Branches:
refs/heads/1.4.5-SNAPSHOT 57f9b6cfd -> f778dcf50
refs/heads/1.5.1-SNAPSHOT 707e74e05 -> c4cd3b1bc
refs/heads/1.6.0-SNAPSHOT b353a24e1 -> e7e5c0097
refs/heads/master bfe39f611 -> 353badb05
ACCUMULO-2213 Make writer recovery in the Tracer more robust.
* Cleans up writer reseting in the TraceServer, avoids overly broad
catching.
* tones down log levels in TraceServer to WARN because trace information is
transient and we retry everything.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/dfafd9c1
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/dfafd9c1
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/dfafd9c1
Branch: refs/heads/1.4.5-SNAPSHOT
Commit: dfafd9c1104d8359ca1eabe345661c541766d57f
Parents: 57f9b6c
Author: Sean Busbey
Authored: Tue Jan 21 14:05:56 2014 -0600
Committer: Sean Busbey
Committed: Wed Jan 22 16:36:34 2014 -0600
--
.../accumulo/server/trace/TraceServer.java | 68 ++--
1 file changed, 49 insertions(+), 19 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/dfafd9c1/src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java
--
diff --git
a/src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java
b/src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java
index 4d89e9c..0e2010a 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java
@@ -19,6 +19,7 @@ package org.apache.accumulo.server.trace;
import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.nio.channels.ServerSocketChannel;
+import java.util.concurrent.atomic.AtomicReference;
import java.util.TimerTask;
import org.apache.accumulo.cloudtrace.instrument.Span;
@@ -27,6 +28,7 @@ import org.apache.accumulo.cloudtrace.thrift.SpanReceiver;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.MutationsRejectedException;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Mutation;
@@ -64,8 +66,8 @@ public class TraceServer implements Watcher {
final private static Logger log = Logger.getLogger(TraceServer.class);
final private AccumuloConfiguration conf;
final private TServer server;
- private BatchWriter writer = null;
- private Connector connector;
+ final private AtomicReference writer;
+ final private Connector connector;
final String table;
private static void put(Mutation m, String cf, String cq, byte[] bytes, int
len) {
@@ -128,16 +130,27 @@ public class TraceServer implements Watcher {
put(timeMutation, "id", idString, transport.get(), transport.len());
}
try {
-if (writer == null)
- resetWriter();
-if (writer == null)
+final BatchWriter writer = TraceServer.this.writer.get();
+/* Check for null, because we expect spans to come in much faster than
flush calls.
+ In the case of failure, we'd rather avoid logging tons of NPEs.
+ */
+if (null == writer) {
+ log.warn("writer is not ready; discarding span.");
return;
+}
writer.addMutation(spanMutation);
writer.addMutation(indexMutation);
if (timeMutation != null)
writer.addMutation(timeMutation);
- } catch (Exception ex) {
-log.error("Unable to write mutation to table: " + spanMutation, ex);
+ } catch (MutationsRejectedException exception) {
+log.warn("Unable to write mutation to table; discarding span. set log
level to DEBUG for span information and stacktrace. cause: " + exception);
+if (log.isDebugEnabled()) {
+ log.debug("discarded span due to rejection of mutation: " +
spanMutation, exception);
+}
+ /* XXX this could be e.g. an IllegalArgumentExceptoion if we're trying
to write this mutation to a writer that has been closed since we retrieved it */
+ } catch (RuntimeException exception) {
+log.warn("Unable to write mutation to table; discarding span. set log
level to DEBUG for stacktrace. cause: " + exception);
+log.debug("unable to write mutation to table due to exception.",
exception);
}
}
@@ -147,6 +160,7 @@ public class TraceServer implements Watcher {
Accumulo.init("tracer");
conf = ServerConfiguration.
[21/23] Merge branch '1.5.1-SNAPSHOT' into 1.6.0-SNAPSHOT
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e7e5c009/server/tserver/src/main/java/org/apache/accumulo/tserver/Compactor.java
--
diff --cc
server/tserver/src/main/java/org/apache/accumulo/tserver/Compactor.java
index 0843c13,000..df476ec
mode 100644,00..100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/Compactor.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/Compactor.java
@@@ -1,546 -1,0 +1,548 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.tserver;
+
+import java.io.IOException;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.data.thrift.IterInfo;
+import org.apache.accumulo.core.file.FileOperations;
+import org.apache.accumulo.core.file.FileSKVIterator;
+import org.apache.accumulo.core.file.FileSKVWriter;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.IteratorUtil;
+import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.iterators.WrappingIterator;
+import org.apache.accumulo.core.iterators.system.ColumnFamilySkippingIterator;
+import org.apache.accumulo.core.iterators.system.DeletingIterator;
+import org.apache.accumulo.core.iterators.system.MultiIterator;
+import org.apache.accumulo.core.iterators.system.TimeSettingIterator;
+import org.apache.accumulo.core.metadata.schema.DataFileValue;
+import org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction;
+import org.apache.accumulo.core.tabletserver.thrift.CompactionReason;
+import org.apache.accumulo.core.tabletserver.thrift.CompactionType;
+import org.apache.accumulo.core.util.LocalityGroupUtil;
+import
org.apache.accumulo.core.util.LocalityGroupUtil.LocalityGroupConfigurationError;
+import org.apache.accumulo.server.fs.FileRef;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.problems.ProblemReport;
+import org.apache.accumulo.server.problems.ProblemReportingIterator;
+import org.apache.accumulo.server.problems.ProblemReports;
+import org.apache.accumulo.server.problems.ProblemType;
+import org.apache.accumulo.trace.instrument.Span;
+import org.apache.accumulo.trace.instrument.Trace;
+import org.apache.accumulo.tserver.Tablet.MinorCompactionReason;
+import org.apache.accumulo.tserver.compaction.MajorCompactionReason;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.log4j.Logger;
+
+public class Compactor implements Callable {
+
+ public static class CountingIterator extends WrappingIterator {
+
+private long count;
+private ArrayList deepCopies;
+private AtomicLong entriesRead;
+
+@Override
+public CountingIterator deepCopy(IteratorEnvironment env) {
+ return new CountingIterator(this, env);
+}
+
+private CountingIterator(CountingIterator other, IteratorEnvironment env)
{
+ setSource(other.getSource().deepCopy(env));
+ count = 0;
+ this.deepCopies = other.deepCopies;
+ this.entriesRead = other.entriesRead;
+ deepCopies.add(this);
+}
+
+public CountingIterator(SortedKeyValueIterator source,
AtomicLong entriesRead) {
+ deepCopies = new ArrayList();
+ this.setSource(source);
+ count = 0;
+ this.entriesRead = entriesR
[11/23] git commit: ACCUMULO-2225 handle IllegalArgumentExceptions from Hadoop on host lookup failure.
ACCUMULO-2225 handle IllegalArgumentExceptions from Hadoop on host lookup
failure.
Looks for cases where we treat IOExceptions out of Hadoop specially, then
attempts to replicate for UnknownHostExceptions that have been wrapped in
IllegalArgumentExceptions.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/f778dcf5
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/f778dcf5
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/f778dcf5
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: f778dcf50cd7163722f63471d1585333704e639c
Parents: f42ead0
Author: Sean Busbey
Authored: Tue Jan 21 08:26:16 2014 -0600
Committer: Sean Busbey
Committed: Wed Jan 22 23:12:29 2014 -0600
--
.../accumulo/core/client/ZooKeeperInstance.java | 10 ++-
.../org/apache/accumulo/server/Accumulo.java| 28 +---
.../server/master/tableOps/DeleteTable.java | 8 ++
.../accumulo/server/tabletserver/Compactor.java | 2 ++
.../randomwalk/security/SecurityHelper.java | 11 ++--
.../accumulo/server/util/TabletOperations.java | 10 ++-
6 files changed, 62 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f778dcf5/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
--
diff --git
a/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
b/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
index e02c197..05982e4 100644
---
a/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
+++
b/src/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
@@ -18,6 +18,7 @@ package org.apache.accumulo.core.client;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.net.UnknownHostException;
import java.nio.ByteBuffer;
import java.util.Collections;
import java.util.List;
@@ -276,7 +277,14 @@ public class ZooKeeperInstance implements Instance {
return result;
}
} catch (IOException e) {
- throw new RuntimeException("Accumulo not initialized, there is no
instance id at " + instanceDirectory, e);
+ log.error("Problem reading instance id out of hdfs at " +
instanceDirectory, e);
+ throw new RuntimeException("Can't tell if Accumulo is initialized; can't
read instance id at " + instanceDirectory, e);
+} catch (IllegalArgumentException exception) {
+ /* HDFS throws this when there's a UnknownHostException due to DNS
troubles. */
+ if (exception.getCause() instanceof UnknownHostException) {
+log.error("Problem reading instance id out of hdfs at " +
instanceDirectory, exception);
+ }
+ throw exception;
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f778dcf5/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java
--
diff --git a/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java
b/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java
index 253962b..184692c 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/Accumulo.java
@@ -30,6 +30,7 @@ import java.util.TreeMap;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.file.FileUtil;
import org.apache.accumulo.core.trace.DistributedTrace;
+import org.apache.accumulo.core.util.AddressUtil;
import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.accumulo.core.util.UtilWaitThread;
import org.apache.accumulo.core.util.Version;
@@ -209,6 +210,7 @@ public class Accumulo {
}
log.info("Zookeeper connected and initialized, attemping to talk to HDFS");
long sleep = 1000;
+int unknownHostTries = 3;
while (true) {
try {
FileSystem fs = FileSystem.get(CachedConfiguration.getInstance());
@@ -216,10 +218,26 @@ public class Accumulo {
break;
log.warn("Waiting for the NameNode to leave safemode");
} catch (IOException ex) {
-log.warn("Unable to connect to HDFS");
+log.warn("Unable to connect to HDFS", ex);
+ } catch (IllegalArgumentException exception) {
+/* Unwrap the UnknownHostException so we can deal with it directly */
+if (exception.getCause() instanceof UnknownHostException) {
+ if (unknownHostTries > 0) {
+log.warn("Unable to connect to HDFS, will retry. cause: " +
exception.getCause());
+/* We need to make sure our sleep period
[04/23] git commit: ACCUMULO-2213 Make writer recovery in the Tracer more robust.
ACCUMULO-2213 Make writer recovery in the Tracer more robust.
* Cleans up writer reseting in the TraceServer, avoids overly broad
catching.
* tones down log levels in TraceServer to WARN because trace information is
transient and we retry everything.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/dfafd9c1
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/dfafd9c1
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/dfafd9c1
Branch: refs/heads/master
Commit: dfafd9c1104d8359ca1eabe345661c541766d57f
Parents: 57f9b6c
Author: Sean Busbey
Authored: Tue Jan 21 14:05:56 2014 -0600
Committer: Sean Busbey
Committed: Wed Jan 22 16:36:34 2014 -0600
--
.../accumulo/server/trace/TraceServer.java | 68 ++--
1 file changed, 49 insertions(+), 19 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/dfafd9c1/src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java
--
diff --git
a/src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java
b/src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java
index 4d89e9c..0e2010a 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java
@@ -19,6 +19,7 @@ package org.apache.accumulo.server.trace;
import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.nio.channels.ServerSocketChannel;
+import java.util.concurrent.atomic.AtomicReference;
import java.util.TimerTask;
import org.apache.accumulo.cloudtrace.instrument.Span;
@@ -27,6 +28,7 @@ import org.apache.accumulo.cloudtrace.thrift.SpanReceiver;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.MutationsRejectedException;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Mutation;
@@ -64,8 +66,8 @@ public class TraceServer implements Watcher {
final private static Logger log = Logger.getLogger(TraceServer.class);
final private AccumuloConfiguration conf;
final private TServer server;
- private BatchWriter writer = null;
- private Connector connector;
+ final private AtomicReference writer;
+ final private Connector connector;
final String table;
private static void put(Mutation m, String cf, String cq, byte[] bytes, int
len) {
@@ -128,16 +130,27 @@ public class TraceServer implements Watcher {
put(timeMutation, "id", idString, transport.get(), transport.len());
}
try {
-if (writer == null)
- resetWriter();
-if (writer == null)
+final BatchWriter writer = TraceServer.this.writer.get();
+/* Check for null, because we expect spans to come in much faster than
flush calls.
+ In the case of failure, we'd rather avoid logging tons of NPEs.
+ */
+if (null == writer) {
+ log.warn("writer is not ready; discarding span.");
return;
+}
writer.addMutation(spanMutation);
writer.addMutation(indexMutation);
if (timeMutation != null)
writer.addMutation(timeMutation);
- } catch (Exception ex) {
-log.error("Unable to write mutation to table: " + spanMutation, ex);
+ } catch (MutationsRejectedException exception) {
+log.warn("Unable to write mutation to table; discarding span. set log
level to DEBUG for span information and stacktrace. cause: " + exception);
+if (log.isDebugEnabled()) {
+ log.debug("discarded span due to rejection of mutation: " +
spanMutation, exception);
+}
+ /* XXX this could be e.g. an IllegalArgumentExceptoion if we're trying
to write this mutation to a writer that has been closed since we retrieved it */
+ } catch (RuntimeException exception) {
+log.warn("Unable to write mutation to table; discarding span. set log
level to DEBUG for stacktrace. cause: " + exception);
+log.debug("unable to write mutation to table due to exception.",
exception);
}
}
@@ -147,6 +160,7 @@ public class TraceServer implements Watcher {
Accumulo.init("tracer");
conf = ServerConfiguration.getSystemConfiguration();
table = conf.get(Property.TRACE_TABLE);
+Connector connector = null;
while (true) {
try {
connector =
HdfsZooInstance.getInstance().getConnector(conf.get(Property.TRACE_USER),
c
[5/6] git commit: Merge branch '1.5.2-SNAPSHOT' into 1.6.1-SNAPSHOT
Merge branch '1.5.2-SNAPSHOT' into 1.6.1-SNAPSHOT -s ours, only change was README for test that don't exist. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/73ca9453 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/73ca9453 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/73ca9453 Branch: refs/heads/1.6.1-SNAPSHOT Commit: 73ca94532f013abe17368069d385f42f6cb18407 Parents: 4e49ef9 9aafd48 Author: Sean Busbey Authored: Sun Jun 29 19:41:13 2014 -0500 Committer: Sean Busbey Committed: Sun Jun 29 19:41:13 2014 -0500 -- --
[6/6] git commit: Merge branch '1.6.1-SNAPSHOT'
Merge branch '1.6.1-SNAPSHOT' Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/ef78dc4b Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/ef78dc4b Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/ef78dc4b Branch: refs/heads/master Commit: ef78dc4bbc625f70d84526eed7a69304f9820841 Parents: 0d4ff12 73ca945 Author: Sean Busbey Authored: Sun Jun 29 19:42:08 2014 -0500 Committer: Sean Busbey Committed: Sun Jun 29 19:42:08 2014 -0500 -- --
[3/6] git commit: ACCUMULO-2443: Change README of auto test to say tool.sh instead of accumulo
ACCUMULO-2443: Change README of auto test to say tool.sh instead of accumulo Signed-off-by: Sean Busbey Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9aafd48b Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9aafd48b Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9aafd48b Branch: refs/heads/master Commit: 9aafd48bee87fdb6d88bf8d3f64f7812c357a3e1 Parents: 727bbf7 Author: Hung Pham Authored: Sun Jun 29 17:58:41 2014 -0400 Committer: Sean Busbey Committed: Sun Jun 29 19:31:43 2014 -0500 -- test/system/auto/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/9aafd48b/test/system/auto/README.md -- diff --git a/test/system/auto/README.md b/test/system/auto/README.md index e147620..cc28e59 100644 --- a/test/system/auto/README.md +++ b/test/system/auto/README.md @@ -93,7 +93,7 @@ only the tests you wish to run. You may specify a 'timeout factor' via an optional integer argument: -> `$ ./bin/accumulo org.apache.accumulo.test.functional.RunTests --tests \ +> `$ ./bin/tool.sh lib/accumulo-test.jar org.apache.accumulo.test.functional.RunTests --tests \ /user/hadoop/tests --output /user/hadoop/results --timeoutFactor timeout_factor` Where `timeout_factor` indicates how much we should scale up timeouts. It will
[4/6] git commit: Merge branch '1.5.2-SNAPSHOT' into 1.6.1-SNAPSHOT
Merge branch '1.5.2-SNAPSHOT' into 1.6.1-SNAPSHOT -s ours, only change was README for test that don't exist. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/73ca9453 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/73ca9453 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/73ca9453 Branch: refs/heads/master Commit: 73ca94532f013abe17368069d385f42f6cb18407 Parents: 4e49ef9 9aafd48 Author: Sean Busbey Authored: Sun Jun 29 19:41:13 2014 -0500 Committer: Sean Busbey Committed: Sun Jun 29 19:41:13 2014 -0500 -- --
[1/6] git commit: ACCUMULO-2443: Change README of auto test to say tool.sh instead of accumulo
Repository: accumulo Updated Branches: refs/heads/1.5.2-SNAPSHOT 727bbf713 -> 9aafd48be refs/heads/1.6.1-SNAPSHOT 4e49ef98a -> 73ca94532 refs/heads/master 0d4ff12a9 -> ef78dc4bb ACCUMULO-2443: Change README of auto test to say tool.sh instead of accumulo Signed-off-by: Sean Busbey Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9aafd48b Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9aafd48b Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9aafd48b Branch: refs/heads/1.5.2-SNAPSHOT Commit: 9aafd48bee87fdb6d88bf8d3f64f7812c357a3e1 Parents: 727bbf7 Author: Hung Pham Authored: Sun Jun 29 17:58:41 2014 -0400 Committer: Sean Busbey Committed: Sun Jun 29 19:31:43 2014 -0500 -- test/system/auto/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/9aafd48b/test/system/auto/README.md -- diff --git a/test/system/auto/README.md b/test/system/auto/README.md index e147620..cc28e59 100644 --- a/test/system/auto/README.md +++ b/test/system/auto/README.md @@ -93,7 +93,7 @@ only the tests you wish to run. You may specify a 'timeout factor' via an optional integer argument: -> `$ ./bin/accumulo org.apache.accumulo.test.functional.RunTests --tests \ +> `$ ./bin/tool.sh lib/accumulo-test.jar org.apache.accumulo.test.functional.RunTests --tests \ /user/hadoop/tests --output /user/hadoop/results --timeoutFactor timeout_factor` Where `timeout_factor` indicates how much we should scale up timeouts. It will
[2/6] git commit: ACCUMULO-2443: Change README of auto test to say tool.sh instead of accumulo
ACCUMULO-2443: Change README of auto test to say tool.sh instead of accumulo Signed-off-by: Sean Busbey Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9aafd48b Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9aafd48b Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9aafd48b Branch: refs/heads/1.6.1-SNAPSHOT Commit: 9aafd48bee87fdb6d88bf8d3f64f7812c357a3e1 Parents: 727bbf7 Author: Hung Pham Authored: Sun Jun 29 17:58:41 2014 -0400 Committer: Sean Busbey Committed: Sun Jun 29 19:31:43 2014 -0500 -- test/system/auto/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/9aafd48b/test/system/auto/README.md -- diff --git a/test/system/auto/README.md b/test/system/auto/README.md index e147620..cc28e59 100644 --- a/test/system/auto/README.md +++ b/test/system/auto/README.md @@ -93,7 +93,7 @@ only the tests you wish to run. You may specify a 'timeout factor' via an optional integer argument: -> `$ ./bin/accumulo org.apache.accumulo.test.functional.RunTests --tests \ +> `$ ./bin/tool.sh lib/accumulo-test.jar org.apache.accumulo.test.functional.RunTests --tests \ /user/hadoop/tests --output /user/hadoop/results --timeoutFactor timeout_factor` Where `timeout_factor` indicates how much we should scale up timeouts. It will
[1/3] git commit: ACCUMULO-2942 Fixes ShardedTableDistributionFormatterTest on all JVMs
Repository: accumulo
Updated Branches:
refs/heads/1.6.1-SNAPSHOT 73ca94532 -> 985a69922
refs/heads/master ef78dc4bb -> dbb9cc4a2
ACCUMULO-2942 Fixes ShardedTableDistributionFormatterTest on all JVMs
ShardedTableDistributionFormatterTest.testAggregate() was failing due
to inconsistency in order to stats that were returned in the getStats()
method. This is due to the fact that the structure that holds the stats is
a HashMap.
The test was changed so that it does not assert that elements exists
at certain location in results, but rather it now checks that elements
exist in the list. The test was also slightly broadened so that it
checks that tuples of (day, count) exist in list.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/985a6992
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/985a6992
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/985a6992
Branch: refs/heads/1.6.1-SNAPSHOT
Commit: 985a699224f1590fc4073df7eca3b1cb03c86526
Parents: 73ca945
Author: haydenmarchant
Authored: Wed Jun 25 06:19:31 2014 -0700
Committer: Sean Busbey
Committed: Mon Jun 30 09:25:56 2014 -0500
--
.../ShardedTableDistributionFormatterTest.java | 30 ++--
1 file changed, 27 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/985a6992/core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java
--
diff --git
a/core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java
b/core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java
index 1707dda..8fc68fd 100644
---
a/core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java
+++
b/core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java
@@ -19,12 +19,16 @@ package org.apache.accumulo.core.util.format;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import java.util.Arrays;
+import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Value;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections.Predicate;
import org.junit.Before;
import org.junit.Test;
@@ -58,9 +62,29 @@ public class ShardedTableDistributionFormatterTest {
formatter.initialize(data.entrySet(), false);
-String[] result = formatter.next().split("\n");
-assertTrue(result[2].endsWith("\t1"));
-assertTrue(result[3].endsWith("\t2"));
+String[] resultLines = formatter.next().split("\n");
+List results = Arrays.asList(resultLines).subList(2, 4);
+
+assertTrue(CollectionUtils.exists(results, new
AggregateReportChecker("NULL", 1)));
+assertTrue(CollectionUtils.exists(results, new
AggregateReportChecker("19700101", 2)));
+
assertFalse(formatter.hasNext());
}
+
+ private static class AggregateReportChecker implements Predicate {
+private String day;
+private int count;
+
+AggregateReportChecker(String day, int count) {
+ this.day = day;
+ this.count = count;
+}
+
+@Override
+public boolean evaluate(Object arg) {
+ String resLine = (String) arg;
+ return resLine.startsWith(this.day) && resLine.endsWith("" + this.count);
+}
+
+ }
}
[2/3] git commit: ACCUMULO-2942 Fixes ShardedTableDistributionFormatterTest on all JVMs
ACCUMULO-2942 Fixes ShardedTableDistributionFormatterTest on all JVMs
ShardedTableDistributionFormatterTest.testAggregate() was failing due
to inconsistency in order to stats that were returned in the getStats()
method. This is due to the fact that the structure that holds the stats is
a HashMap.
The test was changed so that it does not assert that elements exists
at certain location in results, but rather it now checks that elements
exist in the list. The test was also slightly broadened so that it
checks that tuples of (day, count) exist in list.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/985a6992
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/985a6992
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/985a6992
Branch: refs/heads/master
Commit: 985a699224f1590fc4073df7eca3b1cb03c86526
Parents: 73ca945
Author: haydenmarchant
Authored: Wed Jun 25 06:19:31 2014 -0700
Committer: Sean Busbey
Committed: Mon Jun 30 09:25:56 2014 -0500
--
.../ShardedTableDistributionFormatterTest.java | 30 ++--
1 file changed, 27 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/985a6992/core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java
--
diff --git
a/core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java
b/core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java
index 1707dda..8fc68fd 100644
---
a/core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java
+++
b/core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java
@@ -19,12 +19,16 @@ package org.apache.accumulo.core.util.format;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import java.util.Arrays;
+import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Value;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections.Predicate;
import org.junit.Before;
import org.junit.Test;
@@ -58,9 +62,29 @@ public class ShardedTableDistributionFormatterTest {
formatter.initialize(data.entrySet(), false);
-String[] result = formatter.next().split("\n");
-assertTrue(result[2].endsWith("\t1"));
-assertTrue(result[3].endsWith("\t2"));
+String[] resultLines = formatter.next().split("\n");
+List results = Arrays.asList(resultLines).subList(2, 4);
+
+assertTrue(CollectionUtils.exists(results, new
AggregateReportChecker("NULL", 1)));
+assertTrue(CollectionUtils.exists(results, new
AggregateReportChecker("19700101", 2)));
+
assertFalse(formatter.hasNext());
}
+
+ private static class AggregateReportChecker implements Predicate {
+private String day;
+private int count;
+
+AggregateReportChecker(String day, int count) {
+ this.day = day;
+ this.count = count;
+}
+
+@Override
+public boolean evaluate(Object arg) {
+ String resLine = (String) arg;
+ return resLine.startsWith(this.day) && resLine.endsWith("" + this.count);
+}
+
+ }
}
[3/3] git commit: Merge branch '1.6.1-SNAPSHOT'
Merge branch '1.6.1-SNAPSHOT' Conflicts: core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/dbb9cc4a Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/dbb9cc4a Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/dbb9cc4a Branch: refs/heads/master Commit: dbb9cc4a229754382e3f977b591f4035df96accc Parents: ef78dc4 985a699 Author: Sean Busbey Authored: Mon Jun 30 09:48:54 2014 -0500 Committer: Sean Busbey Committed: Mon Jun 30 09:48:54 2014 -0500 -- .../ShardedTableDistributionFormatterTest.java | 30 ++-- 1 file changed, 27 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/dbb9cc4a/core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java -- diff --cc core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java index c4a7107,8fc68fd..8f46c89 --- a/core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java +++ b/core/src/test/java/org/apache/accumulo/core/util/format/ShardedTableDistributionFormatterTest.java @@@ -19,12 -19,16 +19,16 @@@ package org.apache.accumulo.core.util.f import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import java.nio.charset.StandardCharsets; + import java.util.Arrays; + import java.util.List; import java.util.Map; import java.util.TreeMap; -import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; + import org.apache.commons.collections.CollectionUtils; + import org.apache.commons.collections.Predicate; import org.junit.Before; import org.junit.Test;
[1/6] git commit: ACCUMULO-2967 Unknown Host should result in timeout.
Repository: accumulo
Updated Branches:
refs/heads/1.5.2-SNAPSHOT 9aafd48be -> d4d455f1f
refs/heads/1.6.1-SNAPSHOT 985a69922 -> 41b72b5b7
refs/heads/master 7c4d620a9 -> d1b3403db
ACCUMULO-2967 Unknown Host should result in timeout.
- Tests that we get an exception within a reasonable bound
- Changes timeout condition to check prior to place where an exception can be
thrown.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/d4d455f1
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/d4d455f1
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/d4d455f1
Branch: refs/heads/1.5.2-SNAPSHOT
Commit: d4d455f1f3beded6f20b6c3946a7b867d6b3e2d4
Parents: 9aafd48
Author: Sean Busbey
Authored: Tue Jul 1 10:48:43 2014 -0500
Committer: Sean Busbey
Committed: Tue Jul 1 11:54:39 2014 -0500
--
.../accumulo/fate/zookeeper/ZooSession.java | 14 +++--
.../accumulo/fate/zookeeper/ZooSessionTest.java | 32
2 files changed, 43 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d4d455f1/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
--
diff --git
a/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
b/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
index 205ff01..a9f630e 100644
--- a/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
+++ b/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
@@ -66,6 +66,13 @@ public class ZooSession {
}
+ /**
+ * @param host comma separated list of zk servers
+ * @param timeout in milliseconds
+ * @param scheme authentication type, e.g. 'digest', may be null
+ * @param auth authentication-scheme-specific token, may be null
+ * @param watcher ZK notifications, may be null
+ */
public static ZooKeeper connect(String host, int timeout, String scheme,
byte[] auth, Watcher watcher) {
final int TIME_BETWEEN_CONNECT_CHECKS_MS = 100;
final int TOTAL_CONNECT_TIME_WAIT_MS = 10 * 1000;
@@ -88,9 +95,6 @@ public class ZooSession {
UtilWaitThread.sleep(TIME_BETWEEN_CONNECT_CHECKS_MS);
}
-if (System.currentTimeMillis() - startTime > 2 * timeout)
- throw new RuntimeException("Failed to connect to zookeeper (" + host
+ ") within 2x zookeeper timeout period " + timeout);
-
} catch (IOException e) {
if (e instanceof UnknownHostException) {
/*
@@ -108,6 +112,10 @@ public class ZooSession {
log.warn("interrupted", e);
}
}
+
+ if (System.currentTimeMillis() - startTime > 2 * timeout) {
+throw new RuntimeException("Failed to connect to zookeeper (" + host +
") within 2x zookeeper timeout period " + timeout);
+ }
if (tryAgain) {
UtilWaitThread.sleep(sleepTime);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d4d455f1/fate/src/test/java/org/apache/accumulo/fate/zookeeper/ZooSessionTest.java
--
diff --git
a/fate/src/test/java/org/apache/accumulo/fate/zookeeper/ZooSessionTest.java
b/fate/src/test/java/org/apache/accumulo/fate/zookeeper/ZooSessionTest.java
new file mode 100644
index 000..6f7928d
--- /dev/null
+++ b/fate/src/test/java/org/apache/accumulo/fate/zookeeper/ZooSessionTest.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.fate.zookeeper;
+
+import org.apache.zookeeper.ZooKeeper;
+import org.junit.Test;
+
+public class ZooSessionTest {
+
+ private static final int MINIMUM_TIMEOUT=1;
+ private static final String UNKNOWN_HOST =
"hostname.that.should.not.exist.example.com:2181";
+
+ @Test(expected=RuntimeException.class, timeout=MINIMUM_TIMEOUT*4)
+ public void testUnknownHost() {
+ZooKeeper session = ZooSession.connect(UNKNOWN_HOST, MINIMUM_TIMEOUT,
null, null, null);
+ }
+
+}
[6/6] git commit: Merge branch '1.6.1-SNAPSHOT'
Merge branch '1.6.1-SNAPSHOT' Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/d1b3403d Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/d1b3403d Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/d1b3403d Branch: refs/heads/master Commit: d1b3403db8a41ef4059c3d17bd759527984543ce Parents: 7c4d620 41b72b5 Author: Sean Busbey Authored: Tue Jul 1 12:48:38 2014 -0500 Committer: Sean Busbey Committed: Tue Jul 1 12:48:38 2014 -0500 -- .../accumulo/fate/zookeeper/ZooSession.java | 14 +++-- .../accumulo/fate/zookeeper/ZooSessionTest.java | 32 2 files changed, 43 insertions(+), 3 deletions(-) --
[3/6] git commit: ACCUMULO-2967 Unknown Host should result in timeout.
ACCUMULO-2967 Unknown Host should result in timeout.
- Tests that we get an exception within a reasonable bound
- Changes timeout condition to check prior to place where an exception can be
thrown.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/d4d455f1
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/d4d455f1
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/d4d455f1
Branch: refs/heads/master
Commit: d4d455f1f3beded6f20b6c3946a7b867d6b3e2d4
Parents: 9aafd48
Author: Sean Busbey
Authored: Tue Jul 1 10:48:43 2014 -0500
Committer: Sean Busbey
Committed: Tue Jul 1 11:54:39 2014 -0500
--
.../accumulo/fate/zookeeper/ZooSession.java | 14 +++--
.../accumulo/fate/zookeeper/ZooSessionTest.java | 32
2 files changed, 43 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d4d455f1/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
--
diff --git
a/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
b/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
index 205ff01..a9f630e 100644
--- a/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
+++ b/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
@@ -66,6 +66,13 @@ public class ZooSession {
}
+ /**
+ * @param host comma separated list of zk servers
+ * @param timeout in milliseconds
+ * @param scheme authentication type, e.g. 'digest', may be null
+ * @param auth authentication-scheme-specific token, may be null
+ * @param watcher ZK notifications, may be null
+ */
public static ZooKeeper connect(String host, int timeout, String scheme,
byte[] auth, Watcher watcher) {
final int TIME_BETWEEN_CONNECT_CHECKS_MS = 100;
final int TOTAL_CONNECT_TIME_WAIT_MS = 10 * 1000;
@@ -88,9 +95,6 @@ public class ZooSession {
UtilWaitThread.sleep(TIME_BETWEEN_CONNECT_CHECKS_MS);
}
-if (System.currentTimeMillis() - startTime > 2 * timeout)
- throw new RuntimeException("Failed to connect to zookeeper (" + host
+ ") within 2x zookeeper timeout period " + timeout);
-
} catch (IOException e) {
if (e instanceof UnknownHostException) {
/*
@@ -108,6 +112,10 @@ public class ZooSession {
log.warn("interrupted", e);
}
}
+
+ if (System.currentTimeMillis() - startTime > 2 * timeout) {
+throw new RuntimeException("Failed to connect to zookeeper (" + host +
") within 2x zookeeper timeout period " + timeout);
+ }
if (tryAgain) {
UtilWaitThread.sleep(sleepTime);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d4d455f1/fate/src/test/java/org/apache/accumulo/fate/zookeeper/ZooSessionTest.java
--
diff --git
a/fate/src/test/java/org/apache/accumulo/fate/zookeeper/ZooSessionTest.java
b/fate/src/test/java/org/apache/accumulo/fate/zookeeper/ZooSessionTest.java
new file mode 100644
index 000..6f7928d
--- /dev/null
+++ b/fate/src/test/java/org/apache/accumulo/fate/zookeeper/ZooSessionTest.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.fate.zookeeper;
+
+import org.apache.zookeeper.ZooKeeper;
+import org.junit.Test;
+
+public class ZooSessionTest {
+
+ private static final int MINIMUM_TIMEOUT=1;
+ private static final String UNKNOWN_HOST =
"hostname.that.should.not.exist.example.com:2181";
+
+ @Test(expected=RuntimeException.class, timeout=MINIMUM_TIMEOUT*4)
+ public void testUnknownHost() {
+ZooKeeper session = ZooSession.connect(UNKNOWN_HOST, MINIMUM_TIMEOUT,
null, null, null);
+ }
+
+}
[4/6] git commit: Merge branch '1.5.2-SNAPSHOT' into 1.6.1-SNAPSHOT
Merge branch '1.5.2-SNAPSHOT' into 1.6.1-SNAPSHOT
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/41b72b5b
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/41b72b5b
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/41b72b5b
Branch: refs/heads/master
Commit: 41b72b5b7ea6af31c6cb78dd522c6505229fc135
Parents: 985a699 d4d455f
Author: Sean Busbey
Authored: Tue Jul 1 12:45:43 2014 -0500
Committer: Sean Busbey
Committed: Tue Jul 1 12:45:43 2014 -0500
--
.../accumulo/fate/zookeeper/ZooSession.java | 14 +++--
.../accumulo/fate/zookeeper/ZooSessionTest.java | 32
2 files changed, 43 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/41b72b5b/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
--
diff --cc fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
index c0140ef,a9f630e..33bd77b
--- a/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
+++ b/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
@@@ -66,11 -66,18 +66,18 @@@ public class ZooSession
}
+ /**
+* @param host comma separated list of zk servers
+* @param timeout in milliseconds
+* @param scheme authentication type, e.g. 'digest', may be null
+* @param auth authentication-scheme-specific token, may be null
+* @param watcher ZK notifications, may be null
+*/
public static ZooKeeper connect(String host, int timeout, String scheme,
byte[] auth, Watcher watcher) {
final int TIME_BETWEEN_CONNECT_CHECKS_MS = 100;
-final int TOTAL_CONNECT_TIME_WAIT_MS = 10 * 1000;
+int connectTimeWait = Math.min(10 * 1000, timeout);
boolean tryAgain = true;
-int sleepTime = 100;
+long sleepTime = 100;
ZooKeeper zooKeeper = null;
long startTime = System.currentTimeMillis();
@@@ -108,18 -112,15 +112,22 @@@
log.warn("interrupted", e);
}
}
+
+ if (System.currentTimeMillis() - startTime > 2 * timeout) {
+ throw new RuntimeException("Failed to connect to zookeeper (" + host
+ ") within 2x zookeeper timeout period " + timeout);
+ }
if (tryAgain) {
+if (startTime + 2 * timeout < System.currentTimeMillis() + sleepTime
+ connectTimeWait)
+ sleepTime = startTime + 2 * timeout - System.currentTimeMillis() -
connectTimeWait;
+if (sleepTime < 0)
+{
+ connectTimeWait -= sleepTime;
+ sleepTime = 0;
+}
UtilWaitThread.sleep(sleepTime);
if (sleepTime < 1)
- sleepTime = (int) (sleepTime + sleepTime * Math.random());
+ sleepTime = sleepTime + (long)(sleepTime * Math.random());
}
}
[5/6] git commit: Merge branch '1.5.2-SNAPSHOT' into 1.6.1-SNAPSHOT
Merge branch '1.5.2-SNAPSHOT' into 1.6.1-SNAPSHOT
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/41b72b5b
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/41b72b5b
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/41b72b5b
Branch: refs/heads/1.6.1-SNAPSHOT
Commit: 41b72b5b7ea6af31c6cb78dd522c6505229fc135
Parents: 985a699 d4d455f
Author: Sean Busbey
Authored: Tue Jul 1 12:45:43 2014 -0500
Committer: Sean Busbey
Committed: Tue Jul 1 12:45:43 2014 -0500
--
.../accumulo/fate/zookeeper/ZooSession.java | 14 +++--
.../accumulo/fate/zookeeper/ZooSessionTest.java | 32
2 files changed, 43 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/41b72b5b/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
--
diff --cc fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
index c0140ef,a9f630e..33bd77b
--- a/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
+++ b/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
@@@ -66,11 -66,18 +66,18 @@@ public class ZooSession
}
+ /**
+* @param host comma separated list of zk servers
+* @param timeout in milliseconds
+* @param scheme authentication type, e.g. 'digest', may be null
+* @param auth authentication-scheme-specific token, may be null
+* @param watcher ZK notifications, may be null
+*/
public static ZooKeeper connect(String host, int timeout, String scheme,
byte[] auth, Watcher watcher) {
final int TIME_BETWEEN_CONNECT_CHECKS_MS = 100;
-final int TOTAL_CONNECT_TIME_WAIT_MS = 10 * 1000;
+int connectTimeWait = Math.min(10 * 1000, timeout);
boolean tryAgain = true;
-int sleepTime = 100;
+long sleepTime = 100;
ZooKeeper zooKeeper = null;
long startTime = System.currentTimeMillis();
@@@ -108,18 -112,15 +112,22 @@@
log.warn("interrupted", e);
}
}
+
+ if (System.currentTimeMillis() - startTime > 2 * timeout) {
+ throw new RuntimeException("Failed to connect to zookeeper (" + host
+ ") within 2x zookeeper timeout period " + timeout);
+ }
if (tryAgain) {
+if (startTime + 2 * timeout < System.currentTimeMillis() + sleepTime
+ connectTimeWait)
+ sleepTime = startTime + 2 * timeout - System.currentTimeMillis() -
connectTimeWait;
+if (sleepTime < 0)
+{
+ connectTimeWait -= sleepTime;
+ sleepTime = 0;
+}
UtilWaitThread.sleep(sleepTime);
if (sleepTime < 1)
- sleepTime = (int) (sleepTime + sleepTime * Math.random());
+ sleepTime = sleepTime + (long)(sleepTime * Math.random());
}
}
[2/6] git commit: ACCUMULO-2967 Unknown Host should result in timeout.
ACCUMULO-2967 Unknown Host should result in timeout.
- Tests that we get an exception within a reasonable bound
- Changes timeout condition to check prior to place where an exception can be
thrown.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/d4d455f1
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/d4d455f1
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/d4d455f1
Branch: refs/heads/1.6.1-SNAPSHOT
Commit: d4d455f1f3beded6f20b6c3946a7b867d6b3e2d4
Parents: 9aafd48
Author: Sean Busbey
Authored: Tue Jul 1 10:48:43 2014 -0500
Committer: Sean Busbey
Committed: Tue Jul 1 11:54:39 2014 -0500
--
.../accumulo/fate/zookeeper/ZooSession.java | 14 +++--
.../accumulo/fate/zookeeper/ZooSessionTest.java | 32
2 files changed, 43 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d4d455f1/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
--
diff --git
a/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
b/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
index 205ff01..a9f630e 100644
--- a/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
+++ b/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
@@ -66,6 +66,13 @@ public class ZooSession {
}
+ /**
+ * @param host comma separated list of zk servers
+ * @param timeout in milliseconds
+ * @param scheme authentication type, e.g. 'digest', may be null
+ * @param auth authentication-scheme-specific token, may be null
+ * @param watcher ZK notifications, may be null
+ */
public static ZooKeeper connect(String host, int timeout, String scheme,
byte[] auth, Watcher watcher) {
final int TIME_BETWEEN_CONNECT_CHECKS_MS = 100;
final int TOTAL_CONNECT_TIME_WAIT_MS = 10 * 1000;
@@ -88,9 +95,6 @@ public class ZooSession {
UtilWaitThread.sleep(TIME_BETWEEN_CONNECT_CHECKS_MS);
}
-if (System.currentTimeMillis() - startTime > 2 * timeout)
- throw new RuntimeException("Failed to connect to zookeeper (" + host
+ ") within 2x zookeeper timeout period " + timeout);
-
} catch (IOException e) {
if (e instanceof UnknownHostException) {
/*
@@ -108,6 +112,10 @@ public class ZooSession {
log.warn("interrupted", e);
}
}
+
+ if (System.currentTimeMillis() - startTime > 2 * timeout) {
+throw new RuntimeException("Failed to connect to zookeeper (" + host +
") within 2x zookeeper timeout period " + timeout);
+ }
if (tryAgain) {
UtilWaitThread.sleep(sleepTime);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d4d455f1/fate/src/test/java/org/apache/accumulo/fate/zookeeper/ZooSessionTest.java
--
diff --git
a/fate/src/test/java/org/apache/accumulo/fate/zookeeper/ZooSessionTest.java
b/fate/src/test/java/org/apache/accumulo/fate/zookeeper/ZooSessionTest.java
new file mode 100644
index 000..6f7928d
--- /dev/null
+++ b/fate/src/test/java/org/apache/accumulo/fate/zookeeper/ZooSessionTest.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.fate.zookeeper;
+
+import org.apache.zookeeper.ZooKeeper;
+import org.junit.Test;
+
+public class ZooSessionTest {
+
+ private static final int MINIMUM_TIMEOUT=1;
+ private static final String UNKNOWN_HOST =
"hostname.that.should.not.exist.example.com:2181";
+
+ @Test(expected=RuntimeException.class, timeout=MINIMUM_TIMEOUT*4)
+ public void testUnknownHost() {
+ZooKeeper session = ZooSession.connect(UNKNOWN_HOST, MINIMUM_TIMEOUT,
null, null, null);
+ }
+
+}
svn commit: r1609270 - /accumulo/site/trunk/content/source.mdtext
Author: busbey
Date: Wed Jul 9 19:15:56 2014
New Revision: 1609270
URL: http://svn.apache.org/r1609270
Log:
docs changes for ACCUMULO-2927
Modified:
accumulo/site/trunk/content/source.mdtext
Modified: accumulo/site/trunk/content/source.mdtext
URL:
http://svn.apache.org/viewvc/accumulo/site/trunk/content/source.mdtext?rev=1609270&r1=1609269&r2=1609270&view=diff
==
--- accumulo/site/trunk/content/source.mdtext (original)
+++ accumulo/site/trunk/content/source.mdtext Wed Jul 9 19:15:56 2014
@@ -80,11 +80,33 @@ If you regularly switch between major de
The easiest fix is to ensure all of your current changes are stored in git and
then cleaning your workspace.
-git add path/to/file/that/has/changed
-git add path/to/other/file
-git clean -xdf
-
-Note that this git clean command will delete any files unknown to git in a way
that is irreversible. This includes files and directories created by IDEs in
the workspace, even if git is normally configured to ignore them.
+$> git add path/to/file/that/has/changed
+$> git add path/to/other/file
+$> git clean -df
+
+Note that this git clean command will delete any files unknown to git in a way
that is irreversible. You should check that no important files will be included
by first looking at the "untracked files" section in a ```git status``` command.
+
+$> git status
+# On branch master
+nothing to commit (working directory clean)
+$> mvn package
+{ maven output elided }
+$> git checkout 1.6.1-SNAPSHOT
+Switched to branch '1.6.1-SNAPSHOT'
+$> git status
+# On branch 1.6.1-SNAPSHOT
+# Untracked files:
+# (use "git add ..." to include in what will be committed)
+#
+# mapreduce/
+# shell/
+nothing added to commit but untracked files present (use "git add" to
track)
+$> git clean -df
+Removing mapreduce/
+Removing shell/
+$> git status
+# On branch 1.6.1-SNAPSHOT
+nothing to commit (working directory clean)
### Continuous Integration
svn commit: r915599 - in /websites/production/accumulo/content: ./ 1.4/apidocs/ 1.5/apidocs/ 1.6/apidocs/
Author: busbey Date: Wed Jul 9 19:17:18 2014 New Revision: 915599 Log: docs update for ACCUMULO-2927 Added: websites/production/accumulo/content/ - copied from r915598, websites/staging/accumulo/trunk/content/ websites/production/accumulo/content/1.4/apidocs/ - copied from r915598, websites/production/accumulo/content/1.4/apidocs/ websites/production/accumulo/content/1.5/apidocs/ - copied from r915598, websites/production/accumulo/content/1.5/apidocs/ websites/production/accumulo/content/1.6/apidocs/ - copied from r915598, websites/production/accumulo/content/1.6/apidocs/
[11/11] git commit: Merge branch '1.6.1-SNAPSHOT'
Merge branch '1.6.1-SNAPSHOT'
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/8abe4df6
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/8abe4df6
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/8abe4df6
Branch: refs/heads/master
Commit: 8abe4df6ce69f3d86a3609cbbc35dae882d3b85a
Parents: c8b3e99 3a93bad
Author: Sean Busbey
Authored: Thu Jul 31 11:31:37 2014 -0500
Committer: Sean Busbey
Committed: Thu Jul 31 11:31:37 2014 -0500
--
README | 41 ++-
.../org/apache/accumulo/server/Accumulo.java| 23 +++--
.../apache/accumulo/server/ServerConstants.java | 1 +
.../accumulo/server/util/MetadataTableUtil.java | 52 +++-
.../java/org/apache/accumulo/master/Master.java | 38 +++---
5 files changed, 131 insertions(+), 24 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8abe4df6/README
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8abe4df6/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
--
diff --cc server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
index 3235559,ca70efe..516121d
--- a/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
@@@ -191,8 -189,23 +191,23 @@@ public class Accumulo
log.info(key + " = " + (Property.isSensitive(key) ? "" :
entry.getValue()));
}
-monitorSwappiness();
+monitorSwappiness(config.getConfiguration());
}
+
+ /**
+* Sanity check that the current persistent version is allowed to upgrade
to the version of Accumulo running.
+* @param dataVersion the version that is persisted in the backing Volumes
+*/
+ public static boolean canUpgradeFromDataVersion(final int dataVersion) {
+ return dataVersion == ServerConstants.DATA_VERSION || dataVersion ==
ServerConstants.PREV_DATA_VERSION || dataVersion ==
ServerConstants.TWO_DATA_VERSIONS_AGO;
+ }
+
+ /**
+* Does the data version number stored in the backing Volumes indicate we
need to upgrade something?
+*/
+ public static boolean persistentVersionNeedsUpgrade(final int
accumuloPersistentVersion) {
+ return accumuloPersistentVersion == ServerConstants.TWO_DATA_VERSIONS_AGO
|| accumuloPersistentVersion == ServerConstants.PREV_DATA_VERSION;
+ }
/**
*
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8abe4df6/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8abe4df6/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
--
diff --cc
server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
index 88e9ed5,e702af4..c49ccda
---
a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
@@@ -17,8 -17,8 +17,9 @@@
package org.apache.accumulo.server.util;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
+ import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/8abe4df6/server/master/src/main/java/org/apache/accumulo/master/Master.java
--
[05/11] git commit: ACCUMULO-1264 move !METADATA file delete markers
ACCUMULO-1264 move !METADATA file delete markers
Description: Move the !METADATA file delete markers out of the
!METADATA table and to the root table on upgrade.
Author: Eric Newton
Ref: ACCUMULO-2988
Modified from commit 5416eef
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/53fcb526
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/53fcb526
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/53fcb526
Branch: refs/heads/1.6.1-SNAPSHOT
Commit: 53fcb5262acaac244ef38a1e20c60c2750fef07c
Parents: 3beaa5a
Author: Mike Drob
Authored: Fri May 2 12:32:20 2014 -0400
Committer: Sean Busbey
Committed: Thu Jul 31 11:30:58 2014 -0500
--
.../accumulo/server/util/MetadataTableUtil.java | 51 +++-
.../java/org/apache/accumulo/master/Master.java | 6 ++-
2 files changed, 43 insertions(+), 14 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/53fcb526/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
--
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
index 374017d..827eaa9 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
@@ -94,6 +94,7 @@ import org.apache.zookeeper.KeeperException;
public class MetadataTableUtil {
private static final Text EMPTY_TEXT = new Text();
+ private static final byte[] EMPTY_BYTES = new byte[0];
private static Map root_tables = new
HashMap();
private static Map metadata_tables = new
HashMap();
private static final Logger log = Logger.getLogger(MetadataTableUtil.class);
@@ -965,30 +966,54 @@ public class MetadataTableUtil {
* During an upgrade we need to move deletion requests for files under the
!METADATA table to the root tablet.
*/
public static void moveMetaDeleteMarkers(Instance instance, Credentials
creds) {
-// move old delete markers to new location, to standardize table schema
between all metadata tables
-byte[] EMPTY_BYTES = new byte[0];
-Scanner scanner = new ScannerImpl(instance, creds, RootTable.ID,
Authorizations.EMPTY);
String oldDeletesPrefix = "!!~del";
Range oldDeletesRange = new Range(oldDeletesPrefix, true, "!!~dem", false);
+
+// move old delete markers to new location, to standardize table schema
between all metadata tables
+Scanner scanner = new ScannerImpl(instance, creds, RootTable.ID,
Authorizations.EMPTY);
scanner.setRange(oldDeletesRange);
for (Entry entry : scanner) {
String row = entry.getKey().getRow().toString();
if (row.startsWith(oldDeletesPrefix)) {
-String filename = row.substring(oldDeletesPrefix.length());
-// add the new entry first
-log.info("Moving " + filename + " marker in " + RootTable.NAME);
-Mutation m = new Mutation(MetadataSchema.DeletesSection.getRowPrefix()
+ filename);
-m.put(EMPTY_BYTES, EMPTY_BYTES, EMPTY_BYTES);
-update(creds, m, RootTable.EXTENT);
-// remove the old entry
-m = new Mutation(entry.getKey().getRow());
-m.putDelete(EMPTY_BYTES, EMPTY_BYTES);
-update(creds, m, RootTable.OLD_EXTENT);
+moveDeleteEntry(creds, RootTable.OLD_EXTENT, entry, row,
oldDeletesPrefix);
+ } else {
+break;
+ }
+}
+
+ }
+
+
+ public static void moveMetaDeleteMarkersFrom14(Instance instance,
Credentials creds) {
+// new KeyExtent is only added to force update to write to the metadata
table, not the root table
+KeyExtent notMetadata = new KeyExtent(new Text("anythingNotMetadata"),
null, null);
+
+// move delete markers from the normal delete keyspace to the root tablet
delete keyspace if the files are for the !METADATA table
+Scanner scanner = new ScannerImpl(instance, creds, MetadataTable.ID,
Authorizations.EMPTY);
+scanner.setRange(MetadataSchema.DeletesSection.getRange());
+for (Entry entry : scanner) {
+ String row = entry.getKey().getRow().toString();
+ if (row.startsWith(MetadataSchema.DeletesSection.getRowPrefix() + "/" +
MetadataTable.ID)) {
+moveDeleteEntry(creds, notMetadata, entry, row,
MetadataSchema.DeletesSection.getRowPrefix());
} else {
break;
}
}
+ }
+
+ private static void moveDeleteEntry(Credentials creds, KeyExtent oldExtent,
Entry entry, String rowID, String prefix) {
+String filename = rowID.substring(prefix.length());
+
+// add the new entry first
+lo
[01/11] git commit: ACCUMULO-1010 Support upgrades from 1.4 to 1.5
Repository: accumulo
Updated Branches:
refs/heads/1.6.1-SNAPSHOT ceb1a74d0 -> 3a93bad37
refs/heads/master c8b3e995e -> 8abe4df6c
ACCUMULO-1010 Support upgrades from 1.4 to 1.5
Description: Add steps necessary to move a zk from 1.4 state to 1.5.
This leaves it in a good state to be picked up by the rest of the
1.6 upgrade path.
Author: kturner
Ref: ACCUMULO-2988
Cherry picking caused a huge mess and it was simpler to manually select
the parts of the upgrade code that we needed based on the 1.5.1 tag.
Gathered from commits 317c669, aa6b4309, and 8811859.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/3beaa5a2
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/3beaa5a2
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/3beaa5a2
Branch: refs/heads/1.6.1-SNAPSHOT
Commit: 3beaa5a2bab83e4ff2e189db03b038c32e78a85c
Parents: 35b524d
Author: Mike Drob
Authored: Fri May 2 12:01:38 2014 -0400
Committer: Sean Busbey
Committed: Thu Jul 31 11:30:57 2014 -0500
--
.../java/org/apache/accumulo/master/Master.java | 16 +++-
1 file changed, 15 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3beaa5a2/server/master/src/main/java/org/apache/accumulo/master/Master.java
--
diff --git a/server/master/src/main/java/org/apache/accumulo/master/Master.java
b/server/master/src/main/java/org/apache/accumulo/master/Master.java
index b7d7784..a2048b5 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/Master.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/Master.java
@@ -274,7 +274,8 @@ public class Master implements LiveTServerSet.Listener,
TableObserver, CurrentSt
// introduce unnecessary complexity to try to make the master do it), but
be aware
// that the master is not the only thing that may alter zookeeper before
starting.
-if (Accumulo.getAccumuloPersistentVersion(fs) ==
ServerConstants.PREV_DATA_VERSION) {
+final int accumuloPersistentVersion =
Accumulo.getAccumuloPersistentVersion(fs);
+if (accumuloPersistentVersion == ServerConstants.TWO_VERSIONS_AGO ||
accumuloPersistentVersion == ServerConstants.PREV_DATA_VERSION) {
// This Master hasn't started Fate yet, so any outstanding transactions
must be from before the upgrade.
// Change to Guava's Verify once we use Guava 17.
if (null != fate) {
@@ -285,6 +286,19 @@ public class Master implements LiveTServerSet.Listener,
TableObserver, CurrentSt
log.info("Upgrading zookeeper");
IZooReaderWriter zoo = ZooReaderWriter.getInstance();
+final String zooRoot = ZooUtil.getRoot(instance);
+
+if (accumuloPersistentVersion == ServerConstants.TWO_VERSIONS_AGO) {
+ zoo.recursiveDelete(zooRoot + "/loggers", NodeMissingPolicy.SKIP);
+ zoo.recursiveDelete(zooRoot + "/dead/loggers",
NodeMissingPolicy.SKIP);
+
+ final byte[] zero = new byte[] {'0'};
+ zoo.putPersistentData(zooRoot + Constants.ZRECOVERY, zero,
NodeExistsPolicy.SKIP);
+
+ for (String id : Tables.getIdToNameMap(instance).keySet()) {
+zoo.putPersistentData(zooRoot + Constants.ZTABLES + "/" + id +
Constants.ZTABLE_COMPACT_CANCEL_ID, zero, NodeExistsPolicy.SKIP);
+ }
+}
// create initial namespaces
String namespaces = ZooUtil.getRoot(instance) + Constants.ZNAMESPACES;
[09/11] git commit: ACCUMULO-2988 Upgrade from 1.4 -> 1.6.
ACCUMULO-2988 Upgrade from 1.4 -> 1.6. * add a README section * add log messages around upgrade actions * change 1.4 -> 1.5 upgrade action around compaction cancellation to use 1.6 method of enumerating table ids. Author: Sean Busbey Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/81e3d277 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/81e3d277 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/81e3d277 Branch: refs/heads/1.6.1-SNAPSHOT Commit: 81e3d277e2cc09e6aef4d8f7d07f8f22fccd23f4 Parents: 53fcb52 Author: Sean Busbey Authored: Wed May 14 15:57:27 2014 -0400 Committer: Sean Busbey Committed: Thu Jul 31 11:30:58 2014 -0500 -- README | 41 +++- .../accumulo/server/util/MetadataTableUtil.java | 1 + .../java/org/apache/accumulo/master/Master.java | 9 - 3 files changed, 49 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/81e3d277/README -- diff --git a/README b/README index c2c2446..ec13201 100644 --- a/README +++ b/README @@ -125,7 +125,9 @@ a profile to maven's settings.xml file. Below is an example of $HOME/.m2/setting ** -3. Upgrading from 1.5 to 1.6 +3. Upgrading + +3.1. From 1.5 to 1.6 This happens automatically the first time Accumulo 1.6 is started. @@ -157,6 +159,43 @@ a profile to maven's settings.xml file. Below is an example of $HOME/.m2/setting These messages are safe to ignore. If you need detail on the upgrade's progress you should view the local logs on the Tablet Servers and active Master. +3.2. From 1.4 to 1.6 + + To upgrade from 1.4 to 1.6 you must perform a manual initial step. + + Prior to upgrading you must: + * Verify that there are no outstanding FATE operations + - Under 1.4 you can list what's in FATE by running + $ACCUMULO_HOME/bin/accumulo org.apache.accumulo.server.fate.Admin print + - Note that operations in any state will prevent an upgrade. It is safe + to delete operations with status SUCCESSFUL. For others, you should restart + your 1.4 cluster and allow them to finish. + * Stop the 1.4 instance. + * Configure 1.6 to use the hdfs directory, walog directories, and zookeepers + that 1.4 was using. + * Copy other 1.4 configuration options as needed. + + Prior to starting the 1.6 instance you will need to run the LocalWALRecovery tool + on each node that previously ran an instance of the Logger role. + +$ACCUMULO_HOME/bin/accumulo org.apache.accumulo.tserver.log.LocalWALRecovery + + The recovery tool will rewrite the 1.4 write ahead logs into a format that 1.6 can read. + After this step has completed on all nodes, start the 1.6 cluster to continue the upgrade. + + The upgrade process must make changes to Accumulo's internal state in both ZooKeeper and + the table metadata. This process may take some time if Tablet Servers have to go through + recovery. During this time, the Monitor will claim that the Master is down and some + services may send the Monitor log messages about failure to communicate with each other. + While the upgrade is in progress, the Garbage Collector may complain about invalid paths. + The Master may also complain about failure to create the trace table because it already + exists. These messages are safe to ignore. If other error messages occur, you should seek + out support before continuing to use Accumulo. If you need detail on the upgrade's progress + you should view the local logs on the Tablet Servers and active Master. + + Note that the LocalWALRecovery tool does not delete the local files. Once you confirm that + 1.6 is successfully running, you should delete these files on the local filesystem. + ** 4. Configuring http://git-wip-us.apache.org/repos/asf/accumulo/blob/81e3d277/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java -- diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java index 827eaa9..e702af4 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java @@ -18,6 +18,7 @@ package org.apache.accumulo.server.util; import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; i
