[4/4] git commit: ACCUMULO-378 Set a more realistic maximum size on the distributed work queue for work assignment.
ACCUMULO-378 Set a more realistic maximum size on the distributed work queue
for work assignment.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/1f0ee9c5
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/1f0ee9c5
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/1f0ee9c5
Branch: refs/heads/ACCUMULO-378
Commit: 1f0ee9c5027714997a04c5cacf4ef92fff27ecc1
Parents: 2790542
Author: Josh Elser els...@apache.org
Authored: Wed May 28 11:57:45 2014 -0400
Committer: Josh Elser els...@apache.org
Committed: Wed May 28 11:57:45 2014 -0400
--
core/src/main/java/org/apache/accumulo/core/conf/Property.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/1f0ee9c5/core/src/main/java/org/apache/accumulo/core/conf/Property.java
--
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 8409781..2c7e27b 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -457,7 +457,7 @@ public enum Property {
@Sensitive
REPLICATION_PEER_PASSWORD(replication.peer.password., null,
PropertyType.PREFIX, The password to provide when authenticating with the
given peer),
REPLICATION_NAME(replication.name, , PropertyType.STRING, Name of this
cluster with respect to replication. Used to identify this instance from other
peers),
- REPLICATION_MAX_WORK_QUEUE(replication.max.work.queue, 2000,
PropertyType.COUNT, Upper bound of the number of files queued for
replication),
+ REPLICATION_MAX_WORK_QUEUE(replication.max.work.queue, 1000,
PropertyType.COUNT, Upper bound of the number of files queued for
replication),
REPLICATION_WORK_ASSIGNMENT_SLEEP(replication.work.assignment.sleep,
30s, PropertyType.TIMEDURATION, Amount of time to sleep between replication
work assignment),
REPLICATION_WORKER_THREADS(replication.worker.threads, 4,
PropertyType.COUNT, Size of the threadpool that each tabletserver devotes to
replicating data),
REPLICATION_RECEIPT_SERVICE_PORT(replication.receipt.service.port,
10002, PropertyType.PORT, Listen port used by thrift service in tserver
listening for replication),
[2/4] git commit: ACCUMULO-378 Removing experimental annotation.
ACCUMULO-378 Removing experimental annotation.
In talking to [~ctubbsii] on the subject, he stated that the intent of
Experimental
was for features that are incomplete or not expected to fully work. Replication
is
not one of those features (it is tested and expected to work with known
functionality),
therefore I'm removing the experimental annotation.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/5e8d6d2c
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/5e8d6d2c
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/5e8d6d2c
Branch: refs/heads/ACCUMULO-378
Commit: 5e8d6d2c7fdfd41a99ab812500eaf92661ba8481
Parents: 03d5752
Author: Josh Elser els...@apache.org
Authored: Wed May 28 11:44:49 2014 -0400
Committer: Josh Elser els...@apache.org
Committed: Wed May 28 11:44:49 2014 -0400
--
.../org/apache/accumulo/core/conf/Property.java | 26 +---
1 file changed, 1 insertion(+), 25 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/5e8d6d2c/core/src/main/java/org/apache/accumulo/core/conf/Property.java
--
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 6afa956..8409781 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -179,14 +179,10 @@ public enum Property {
A class that implements a mechansim to steal write access to a file),
MASTER_FATE_THREADPOOL_SIZE(master.fate.threadpool.size, 4,
PropertyType.COUNT,
The number of threads used to run FAult-Tolerant Executions. These are
primarily table operations like merge.),
- @Experimental
MASTER_REPLICATION_SCAN_INTERVAL(master.replication.status.scan.interval,
30s, PropertyType.TIMEDURATION,
Amount of time to sleep before scanning the status section of the
replication table for new data),
- @Experimental
MASTER_REPLICATION_COORDINATOR_PORT(master.replication.coordinator.port,
10001, PropertyType.PORT, Port for the replication coordinator service),
- @Experimental
MASTER_REPLICATION_COORDINATOR_MINTHREADS(master.replication.coordinator.minthreads,
4, PropertyType.COUNT, Minimum number of threads dedicated to answering
coordinator requests),
- @Experimental
MASTER_REPLICATION_COORDINATOR_THREADCHECK(master.replication.coordinator.threadcheck.time,
5s, PropertyType.TIMEDURATION, The time between adjustments of the
coordinator thread pool),
// properties that are specific to tablet server behavior
@@ -276,12 +272,9 @@ public enum Property {
The number of threads for the distributed work queue. These threads are
used for copying failed bulk files.),
TSERV_WAL_SYNC(tserver.wal.sync, true, PropertyType.BOOLEAN,
Use the SYNC_BLOCK create flag to sync WAL writes to disk. Prevents
problems recovering from sudden system resets.),
- @Experimental
TSERV_REPLICATION_REPLAYERS(tserver.replication.replayer., null,
PropertyType.PREFIX, Allows configuration of implementation used to apply
replicated data),
- @Experimental
TSERV_REPLICATION_DEFAULT_HANDLER(tserver.replication.default.replayer,
org.apache.accumulo.tserver.replication.BatchWriterReplicationReplayer,
PropertyType.CLASSNAME, Default AccumuloReplicationReplayer
implementation),
- @Experimental
TSERV_REPLICATION_BW_REPLAYER_MEMORY(tserver.replication.batchwriter.replayer.memory,
50M, PropertyType.MEMORY, Memory to provide to batchwriter to replay
mutations for replication),
// properties that are specific to logger server behavior
@@ -432,9 +425,7 @@ public enum Property {
A customizable major compaction strategy.),
TABLE_COMPACTION_STRATEGY_PREFIX(table.majc.compaction.strategy.opts.,
null, PropertyType.PREFIX,
Properties in this category are used to configure the compaction
strategy.),
- @Experimental
TABLE_REPLICATION(table.replication, false, PropertyType.BOOLEAN, Is
replication enabled for the given table),
- @Experimental
TABLE_REPLICATION_TARGETS(table.replication.target., null,
PropertyType.PREFIX, Enumerate a mapping of other systems which this table
should +
replicate their data to. The key suffix is the identifying cluster name
and the value is an identifier for a location on the target system, +
e.g. the ID of the table on the target to replicate to),
@@ -459,39 +450,24 @@ public enum Property {
GENERAL_MAVEN_PROJECT_BASEDIR(AccumuloClassLoader.MAVEN_PROJECT_BASEDIR_PROPERTY_NAME,
AccumuloClassLoader.DEFAULT_MAVEN_PROJECT_BASEDIR_VALUE,
PropertyType.ABSOLUTEPATH, Set this to automatically add maven
target/classes
[1/4] git commit: ACCUMULO-378 Jenkins fix -- retry if we get a security exception because the grant on the replication table didn't happen yet
Repository: accumulo
Updated Branches:
refs/heads/ACCUMULO-378 070ceb1da - 1f0ee9c50
ACCUMULO-378 Jenkins fix -- retry if we get a security exception because the
grant on the replication table didn't happen yet
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/03d57520
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/03d57520
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/03d57520
Branch: refs/heads/ACCUMULO-378
Commit: 03d57520a9668fb0d82b62708096ce6b595b0cdc
Parents: 070ceb1
Author: Josh Elser els...@apache.org
Authored: Wed May 28 11:31:34 2014 -0400
Committer: Josh Elser els...@apache.org
Committed: Wed May 28 11:31:34 2014 -0400
--
.../test/replication/ReplicationTest.java | 85
1 file changed, 50 insertions(+), 35 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/03d57520/test/src/test/java/org/apache/accumulo/test/replication/ReplicationTest.java
--
diff --git
a/test/src/test/java/org/apache/accumulo/test/replication/ReplicationTest.java
b/test/src/test/java/org/apache/accumulo/test/replication/ReplicationTest.java
index b59f8da..51e4e46 100644
---
a/test/src/test/java/org/apache/accumulo/test/replication/ReplicationTest.java
+++
b/test/src/test/java/org/apache/accumulo/test/replication/ReplicationTest.java
@@ -680,30 +680,44 @@ public class ReplicationTest extends ConfigurableMacIT {
log.info(entry.getKey().toStringNoTruncate() + = + entry.getValue());
}
- s = ReplicationTable.getScanner(conn);
- StatusSection.limit(s);
- Text buff = new Text();
- boolean allReferencedLogsClosed = true;
- int recordsFound = 0;
- for (EntryKey,Value e : s) {
-recordsFound++;
-allReferencedLogsClosed = true;
-StatusSection.getFile(e.getKey(), buff);
-String file = buff.toString();
-if (wals.contains(file)) {
- Status stat = Status.parseFrom(e.getValue().get());
- if (!stat.getClosed()) {
-log.info({} wasn't closed, file);
-allReferencedLogsClosed = false;
+ try {
+s = ReplicationTable.getScanner(conn);
+StatusSection.limit(s);
+Text buff = new Text();
+boolean allReferencedLogsClosed = true;
+int recordsFound = 0;
+for (EntryKey,Value e : s) {
+ recordsFound++;
+ allReferencedLogsClosed = true;
+ StatusSection.getFile(e.getKey(), buff);
+ String file = buff.toString();
+ if (wals.contains(file)) {
+Status stat = Status.parseFrom(e.getValue().get());
+if (!stat.getClosed()) {
+ log.info({} wasn't closed, file);
+ allReferencedLogsClosed = false;
+}
}
}
- }
- if (recordsFound 0 allReferencedLogsClosed) {
-return;
+if (recordsFound 0 allReferencedLogsClosed) {
+ return;
+}
+Thread.sleep(1000);
+ } catch (RuntimeException e) {
+Throwable cause = e.getCause();
+if (cause instanceof AccumuloSecurityException) {
+ AccumuloSecurityException ase = (AccumuloSecurityException) cause;
+ switch (ase.getSecurityErrorCode()) {
+case PERMISSION_DENIED:
+ // We tried to read the replication table before the GRANT went
through
+ Thread.sleep(1000);
+ break;
+default:
+ throw e;
+ }
+}
}
-
- Thread.sleep(1000);
}
Assert.fail(We had a file that was referenced but didn't get closed);
@@ -817,7 +831,8 @@ public class ReplicationTest extends ConfigurableMacIT {
Assert.assertNotNull(Could not find expected entry in replication table,
entry);
Status actual = Status.parseFrom(entry.getValue().get());
-Assert.assertTrue(Expected to find a replication entry that is open with
infinite length: + ProtobufUtil.toString(actual), !actual.getClosed()
actual.getInfiniteEnd());
+Assert.assertTrue(Expected to find a replication entry that is open with
infinite length: + ProtobufUtil.toString(actual),
+!actual.getClosed() actual.getInfiniteEnd());
// Try a couple of times to watch for the work record to be created
boolean notFound = true;
@@ -1011,7 +1026,7 @@ public class ReplicationTest extends ConfigurableMacIT {
});
t.start();
-
+
String table1 = table1, table2 = table2, table3 = table3;
BatchWriter bw;
@@ -1021,7 +1036,7 @@ public class ReplicationTest extends ConfigurableMacIT {
conn.tableOperations().setProperty(table1,
Property.TABLE_REPLICATION_TARGETS.getKey() + cluster1, 1);
[26/32] git commit: Merge branch '1.5.2-SNAPSHOT' of https://git-wip-us.apache.org/repos/asf/accumulo into 1.5.2-SNAPSHOT
Merge branch '1.5.2-SNAPSHOT' of https://git-wip-us.apache.org/repos/asf/accumulo into 1.5.2-SNAPSHOT Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/cff4c2d1 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/cff4c2d1 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/cff4c2d1 Branch: refs/heads/ACCUMULO-378 Commit: cff4c2d1aa9a7f6e36c6e29cb1ffa22183cf4ea9 Parents: 5d4cf3b 0d4f4b1 Author: Josh Elser els...@apache.org Authored: Fri May 23 20:31:03 2014 -0400 Committer: Josh Elser els...@apache.org Committed: Fri May 23 20:31:03 2014 -0400 -- .../random/RandomByteArrayMakerFunction.java| 60 .../test/stress/random/RandomByteArrays.java| 14 ++--- .../test/stress/random/RandomMutations.java | 11 +++- .../test/stress/random/RandomWithinRange.java | 59 +++ .../accumulo/test/stress/random/Write.java | 33 +++ .../test/stress/random/WriteOptions.java| 19 ++- test/system/stress/writer.sh| 4 +- 7 files changed, 114 insertions(+), 86 deletions(-) --
[30/32] git commit: Merge branch '1.5.2-SNAPSHOT' into 1.6.1-SNAPSHOT
Merge branch '1.5.2-SNAPSHOT' into 1.6.1-SNAPSHOT Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/7c17eb16 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/7c17eb16 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/7c17eb16 Branch: refs/heads/ACCUMULO-378 Commit: 7c17eb169c2ea367a3ce6bd67787600ff8fb3ffd Parents: c9c3fdd 9fbc24a Author: Josh Elser els...@apache.org Authored: Sat May 24 18:22:04 2014 -0400 Committer: Josh Elser els...@apache.org Committed: Sat May 24 18:22:04 2014 -0400 -- .../java/org/apache/accumulo/core/util/LocalityGroupUtil.java | 3 +++ 1 file changed, 3 insertions(+) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c17eb16/core/src/main/java/org/apache/accumulo/core/util/LocalityGroupUtil.java --
[11/32] git commit: ACCUMULO-2842 Mention hflush as an alternative configuration for the property
ACCUMULO-2842 Mention hflush as an alternative configuration for the property
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/2526f0a2
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/2526f0a2
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/2526f0a2
Branch: refs/heads/ACCUMULO-378
Commit: 2526f0a2cc2d1dca63483b614e938fcb1ddd7f35
Parents: f654e7f
Author: Josh Elser els...@apache.org
Authored: Fri May 23 16:25:38 2014 -0400
Committer: Josh Elser els...@apache.org
Committed: Fri May 23 16:25:38 2014 -0400
--
core/src/main/java/org/apache/accumulo/core/conf/Property.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/2526f0a2/core/src/main/java/org/apache/accumulo/core/conf/Property.java
--
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 3d82046..d8f9beb 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -206,7 +206,8 @@ public enum Property {
Use the SYNC_BLOCK create flag to sync WAL writes to disk. Prevents
problems recovering from sudden system resets.),
TSERV_WAL_SYNC_METHOD(tserver.wal.sync.method, hsync,
PropertyType.STRING, The method to invoke when sync'ing WALs. HSync will
provide +
resiliency in the face of unexpected power outages, at the cost of
speed. If method is not available, the legacy 'sync' method +
- will be used to ensure backwards compatibility with older Hadoop
versions),
+ will be used to ensure backwards compatibility with older Hadoop
versions. A value of 'hflush' is the alternative to the default value +
+ of 'hsync' which will result in faster writes, but with less
durability),
// properties that are specific to logger server behavior
LOGGER_PREFIX(logger., null, PropertyType.PREFIX, Properties in this
category affect the behavior of the write-ahead logger servers),
[24/32] git commit: Merge branch '1.5.2-SNAPSHOT' into 1.6.1-SNAPSHOT
Merge branch '1.5.2-SNAPSHOT' into 1.6.1-SNAPSHOT
Conflicts:
server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/ShellServlet.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/eb6b3253
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/eb6b3253
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/eb6b3253
Branch: refs/heads/ACCUMULO-378
Commit: eb6b3253b7df7c7e3c4b8ed7df055d951c0e5621
Parents: eb7aac3 5d4cf3b
Author: Josh Elser els...@apache.org
Authored: Fri May 23 20:24:18 2014 -0400
Committer: Josh Elser els...@apache.org
Committed: Fri May 23 20:24:18 2014 -0400
--
.../accumulo/monitor/servlets/ShellServlet.java | 43 +++-
1 file changed, 33 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/eb6b3253/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/ShellServlet.java
--
diff --cc
server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/ShellServlet.java
index 03f6831,000..665b132
mode 100644,00..100644
---
a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/ShellServlet.java
+++
b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/ShellServlet.java
@@@ -1,333 -1,0 +1,356 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the License); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.monitor.servlets;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.util.HashMap;
+import java.util.Map;
++import java.util.UUID;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import javax.servlet.http.HttpSession;
+
+import jline.console.ConsoleReader;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.util.shell.Shell;
+
+public class ShellServlet extends BasicServlet {
+ private static final long serialVersionUID = 1L;
+ private MapString,ShellExecutionThread userShells = new
HashMapString,ShellExecutionThread();
+ private ExecutorService service = Executors.newCachedThreadPool();
-
++
++ public static final String CSRF_KEY = csrf_token;
++
+ @Override
+ protected String getTitle(HttpServletRequest req) {
+return Shell;
+ }
+
+ @Override
+ protected void pageBody(HttpServletRequest req, HttpServletResponse
response, StringBuilder sb) throws IOException {
+HttpSession session = req.getSession(true);
++final String CSRF_TOKEN;
++if (null == session.getAttribute(CSRF_KEY)) {
++ // No token, make one
++ CSRF_TOKEN = UUID.randomUUID().toString();
++ session.setAttribute(CSRF_KEY, CSRF_TOKEN);
++} else {
++ // Pull the token out of the session
++ CSRF_TOKEN = (String) session.getAttribute(CSRF_KEY);
++ if (null == CSRF_TOKEN) {
++throw new RuntimeException(No valid CSRF token exists in session);
++ }
++}
++
+String user = (String) session.getAttribute(user);
+if (user == null) {
+ // user attribute is null, check to see if username and password are
passed as parameters
+ user = req.getParameter(user);
+ String pass = req.getParameter(pass);
+ String mock = req.getParameter(mock);
+ if (user == null || pass == null) {
+// username or password are null, re-authenticate
- sb.append(authenticationForm(req.getRequestURI()));
++sb.append(authenticationForm(req.getRequestURI(), CSRF_TOKEN));
+return;
+ }
+ try {
+// get a new shell for this user
+ShellExecutionThread shellThread = new ShellExecutionThread(user,
pass, mock);
+service.submit(shellThread);
+userShells.put(session.getId(), shellThread);
+
[16/32] git commit: Merge branch '1.5.2-SNAPSHOT' into 1.6.1-SNAPSHOT
Merge branch '1.5.2-SNAPSHOT' into 1.6.1-SNAPSHOT Conflicts: core/src/main/java/org/apache/accumulo/core/conf/Property.java Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/480634fe Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/480634fe Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/480634fe Branch: refs/heads/ACCUMULO-378 Commit: 480634fe4a8edfeb9f35613f1518bd742012b3d9 Parents: ff3ad73 2526f0a Author: Josh Elser els...@apache.org Authored: Fri May 23 16:32:22 2014 -0400 Committer: Josh Elser els...@apache.org Committed: Fri May 23 16:32:22 2014 -0400 -- core/src/main/java/org/apache/accumulo/core/conf/Property.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/480634fe/core/src/main/java/org/apache/accumulo/core/conf/Property.java -- diff --cc core/src/main/java/org/apache/accumulo/core/conf/Property.java index 2ade366,d8f9beb..87a29fb --- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java +++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java @@@ -267,14 -206,15 +267,15 @@@ public enum Property Use the SYNC_BLOCK create flag to sync WAL writes to disk. Prevents problems recovering from sudden system resets.), TSERV_WAL_SYNC_METHOD(tserver.wal.sync.method, hsync, PropertyType.STRING, The method to invoke when sync'ing WALs. HSync will provide + resiliency in the face of unexpected power outages, at the cost of speed. If method is not available, the legacy 'sync' method + - will be used to ensure backwards compatibility with older Hadoop versions), - + will be used to ensure backwards compatibility with older Hadoop versions. A value of 'hflush' is the alternative to the default value + + of 'hsync' which will result in faster writes, but with less durability), - ++ // properties that are specific to logger server behavior LOGGER_PREFIX(logger., null, PropertyType.PREFIX, Properties in this category affect the behavior of the write-ahead logger servers), - LOGGER_DIR(logger.dir.walog, walogs, PropertyType.PATH, - The property only needs to be set if upgrading from 1.4 which used to store write-ahead logs on the local filesystem. In 1.5 write-ahead logs are - + stored in DFS. When 1.5 is started for the first time it will copy any 1.4 write ahead logs into DFS. It is possible to specify a - + comma-separated list of directories.), + LOGGER_DIR(logger.dir.walog, walogs, PropertyType.PATH, This property is only needed if Accumulo was upgraded from a 1.4 or earlier version. + + In the upgrade to 1.5 this property is used to copy any earlier write ahead logs into DFS. + + In 1.6+, this property is used by the LocalWALRecovery utility in the event that something went wrong with that earlier upgrade. + + It is possible to specify a comma-separated list of directories.), // accumulo garbage collector properties GC_PREFIX(gc., null, PropertyType.PREFIX, Properties in this category affect the behavior of the accumulo garbage collector.),
[05/32] git commit: ACCUMULO-2789 Add stress test with variable cell sizes
ACCUMULO-2789 Add stress test with variable cell sizes
Added a test that users can run to stress their instances. They can supply
variable cell sizes (currently row, column family and column qualifier are
supported), and the write clients will generate random data within those size
constraints. Users can also scan random tablets to add futher load onto the
system. A variable number of write and reader clients can be executed to run
the test under a mixed workload.
This test is an acid test-- it is used to verify that an instance can handle
the workload provided by it. It doesn't assert an actual state of the system,
beyond reporting what errors the write and scan clients will see.
See the included package-info.java and/or README.md files for more information.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/74acc30c
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/74acc30c
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/74acc30c
Branch: refs/heads/ACCUMULO-378
Commit: 74acc30cafb8be0ee5c7cc457c5e09f0ea56a993
Parents: c16f105
Author: Bill Slacum ujustgotbi...@apache.org
Authored: Fri May 16 14:06:15 2014 -0400
Committer: Bill Slacum ujustgotbi...@apache.org
Committed: Thu May 22 07:20:23 2014 -0400
--
.../accumulo/test/stress/random/DataWriter.java | 50 ++
.../test/stress/random/IntArgValidator.java | 32
.../random/RandomByteArrayMakerFunction.java| 60
.../test/stress/random/RandomByteArrays.java| 36 +
.../test/stress/random/RandomMutations.java | 39 +
.../accumulo/test/stress/random/Scan.java | 124 +++
.../accumulo/test/stress/random/ScanOpts.java | 44 ++
.../accumulo/test/stress/random/Stream.java | 41 +
.../accumulo/test/stress/random/Write.java | 78 ++
.../test/stress/random/WriteOptions.java| 152 +++
.../test/stress/random/package-info.java| 36 +
test/system/stress/README.md| 33
test/system/stress/reader.sh| 34 +
test/system/stress/writer.sh| 38 +
14 files changed, 797 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/74acc30c/test/src/main/java/org/apache/accumulo/test/stress/random/DataWriter.java
--
diff --git
a/test/src/main/java/org/apache/accumulo/test/stress/random/DataWriter.java
b/test/src/main/java/org/apache/accumulo/test/stress/random/DataWriter.java
new file mode 100644
index 000..33a3984
--- /dev/null
+++ b/test/src/main/java/org/apache/accumulo/test/stress/random/DataWriter.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the License); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.stress.random;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+
+public class DataWriter extends StreamVoid {
+ private final BatchWriter writer;
+ private final RandomMutations mutations;
+
+ public DataWriter(BatchWriter writer, RandomMutations mutations) {
+this.writer = writer;
+this.mutations = mutations;
+ }
+
+ @Override
+ public Void next() {
+try {
+ writer.addMutation(mutations.next());
+} catch (MutationsRejectedException e) {
+ throw new RuntimeException(e);
+}
+return null;
+ }
+
+ @Override
+ public void finalize() {
+try {
+ this.writer.close();
+} catch (MutationsRejectedException e) {
+ System.err.println(Error closing batch writer.);
+ e.printStackTrace();
+}
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/74acc30c/test/src/main/java/org/apache/accumulo/test/stress/random/IntArgValidator.java
--
diff --git
a/test/src/main/java/org/apache/accumulo/test/stress/random/IntArgValidator.java
b/test/src/main/java/org/apache/accumulo/test/stress/random/IntArgValidator.java
[25/32] git commit: Merge branch '1.6.1-SNAPSHOT'
Merge branch '1.6.1-SNAPSHOT' Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/d37f05f6 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/d37f05f6 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/d37f05f6 Branch: refs/heads/ACCUMULO-378 Commit: d37f05f652cce435fa706f26a0b81a4da5d93015 Parents: 9f357e6 eb6b325 Author: Josh Elser els...@apache.org Authored: Fri May 23 20:30:49 2014 -0400 Committer: Josh Elser els...@apache.org Committed: Fri May 23 20:30:49 2014 -0400 -- .../accumulo/monitor/servlets/ShellServlet.java | 43 +++- 1 file changed, 33 insertions(+), 10 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/d37f05f6/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/ShellServlet.java -- diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/ShellServlet.java index 25e9e33,665b132..bf38064 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/ShellServlet.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/ShellServlet.java @@@ -21,9 -21,9 +21,10 @@@ import java.io.InputStream import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.PrintWriter; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Map; + import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors;
[07/32] git commit: Merge remote-tracking branch 'origin/1.6.1-SNAPSHOT'
Merge remote-tracking branch 'origin/1.6.1-SNAPSHOT' Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/0164991f Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/0164991f Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/0164991f Branch: refs/heads/ACCUMULO-378 Commit: 0164991f4674f43edbf7fb1e09fecc1686f97aec Parents: 58e7665 adaf146 Author: Bill Slacum ujustgotbi...@apache.org Authored: Thu May 22 07:34:09 2014 -0400 Committer: Bill Slacum ujustgotbi...@apache.org Committed: Thu May 22 07:34:09 2014 -0400 -- .../accumulo/test/stress/random/DataWriter.java | 50 ++ .../test/stress/random/IntArgValidator.java | 32 .../random/RandomByteArrayMakerFunction.java| 60 .../test/stress/random/RandomByteArrays.java| 36 + .../test/stress/random/RandomMutations.java | 39 + .../accumulo/test/stress/random/Scan.java | 124 +++ .../accumulo/test/stress/random/ScanOpts.java | 44 ++ .../accumulo/test/stress/random/Stream.java | 41 + .../accumulo/test/stress/random/Write.java | 78 ++ .../test/stress/random/WriteOptions.java| 152 +++ .../test/stress/random/package-info.java| 36 + test/system/stress/README.md| 34 + test/system/stress/reader.sh| 34 + test/system/stress/writer.sh| 38 + 14 files changed, 798 insertions(+) --
[31/32] git commit: Merge branch '1.6.1-SNAPSHOT'
Merge branch '1.6.1-SNAPSHOT' Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/0712ec97 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/0712ec97 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/0712ec97 Branch: refs/heads/ACCUMULO-378 Commit: 0712ec97243d4524d3ca8553a64a0fa1c2e5d02d Parents: d6b1e9a 7c17eb1 Author: Josh Elser els...@apache.org Authored: Sat May 24 18:22:23 2014 -0400 Committer: Josh Elser els...@apache.org Committed: Sat May 24 18:22:23 2014 -0400 -- .../java/org/apache/accumulo/core/util/LocalityGroupUtil.java | 3 +++ 1 file changed, 3 insertions(+) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/0712ec97/core/src/main/java/org/apache/accumulo/core/util/LocalityGroupUtil.java --
[17/32] git commit: Merge branch '1.6.1-SNAPSHOT'
Merge branch '1.6.1-SNAPSHOT' Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/f4611077 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/f4611077 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/f4611077 Branch: refs/heads/ACCUMULO-378 Commit: f4611077021b85d2988e7307de980ab4a441f96a Parents: 3411023 480634f Author: Josh Elser els...@apache.org Authored: Fri May 23 16:32:29 2014 -0400 Committer: Josh Elser els...@apache.org Committed: Fri May 23 16:32:29 2014 -0400 -- core/src/main/java/org/apache/accumulo/core/conf/Property.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/f4611077/core/src/main/java/org/apache/accumulo/core/conf/Property.java --
[27/32] git commit: ACCUMULO-2836: Added context classpath support to BloomFilter, AggregatingIterator, and TableLoadBalancer
ACCUMULO-2836: Added context classpath support to BloomFilter,
AggregatingIterator, and TableLoadBalancer
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/c9c3fdd4
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/c9c3fdd4
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/c9c3fdd4
Branch: refs/heads/ACCUMULO-378
Commit: c9c3fdd4f5fc648ccc5e3a905236a82e88524251
Parents: eb6b325
Author: Dave Marion dlmar...@hotmail.com
Authored: Fri May 23 22:31:23 2014 -0400
Committer: Dave Marion dlmar...@hotmail.com
Committed: Fri May 23 22:31:23 2014 -0400
--
.../accumulo/core/file/BloomFilterLayer.java | 19 ---
.../core/iterators/AggregatingIterator.java | 6 +-
.../aggregation/conf/AggregatorSet.java | 3 ++-
.../iterators/conf/ColumnToClassMapping.java | 16 +---
.../master/balancer/TableLoadBalancer.java | 9 -
.../java/org/apache/accumulo/master/Master.java | 14 ++
6 files changed, 58 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c9c3fdd4/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java
--
diff --git
a/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java
b/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java
index 5829ce6..d0e736c 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java
@@ -114,7 +114,14 @@ public class BloomFilterLayer {
* load KeyFunctor
*/
try {
-Class? extends KeyFunctor clazz =
AccumuloVFSClassLoader.loadClass(acuconf.get(Property.TABLE_BLOOM_KEY_FUNCTOR),
KeyFunctor.class);
+String context = acuconf.get(Property.TABLE_CLASSPATH);
+String classname = acuconf.get(Property.TABLE_BLOOM_KEY_FUNCTOR);
+Class? extends KeyFunctor clazz;
+if (context != null !context.equals())
+ clazz =
AccumuloVFSClassLoader.getContextManager().loadClass(context, classname,
KeyFunctor.class);
+else
+ clazz = AccumuloVFSClassLoader.loadClass(classname,
KeyFunctor.class);
+
transformer = clazz.newInstance();
} catch (Exception e) {
@@ -186,6 +193,8 @@ public class BloomFilterLayer {
loadThreshold = acuconf.getCount(Property.TABLE_BLOOM_LOAD_THRESHOLD);
+ final String context = acuconf.get(Property.TABLE_CLASSPATH);
+
loadTask = new Runnable() {
@Override
public void run() {
@@ -208,8 +217,12 @@ public class BloomFilterLayer {
* Load classname for keyFunctor
*/
ClassName = in.readUTF();
-
-Class? extends KeyFunctor clazz =
AccumuloVFSClassLoader.loadClass(ClassName, KeyFunctor.class);
+
+Class? extends KeyFunctor clazz;
+if (context != null !context.equals())
+ clazz =
AccumuloVFSClassLoader.getContextManager().loadClass(context, ClassName,
KeyFunctor.class);
+else
+ clazz = AccumuloVFSClassLoader.loadClass(ClassName,
KeyFunctor.class);
transformer = clazz.newInstance();
/**
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c9c3fdd4/core/src/main/java/org/apache/accumulo/core/iterators/AggregatingIterator.java
--
diff --git
a/core/src/main/java/org/apache/accumulo/core/iterators/AggregatingIterator.java
b/core/src/main/java/org/apache/accumulo/core/iterators/AggregatingIterator.java
index c5c034e..9b89b47 100644
---
a/core/src/main/java/org/apache/accumulo/core/iterators/AggregatingIterator.java
+++
b/core/src/main/java/org/apache/accumulo/core/iterators/AggregatingIterator.java
@@ -22,6 +22,7 @@ import java.util.Collections;
import java.util.Map;
import java.util.Map.Entry;
+import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.ByteSequence;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.PartialKey;
@@ -167,7 +168,10 @@ public class AggregatingIterator implements
SortedKeyValueIteratorKey,Value, O
this.iterator = source;
try {
- this.aggregators = new ColumnToClassMappingAggregator(options,
Aggregator.class);
+ String context = null;
+ if (null != env)
+context = env.getConfig().get(Property.TABLE_CLASSPATH);
+ this.aggregators = new ColumnToClassMappingAggregator(options,
Aggregator.class, context);
} catch (ClassNotFoundException e) {
log.error(e.toString());
throw new
[18/32] git commit: ACCUMULO-2789 Fix error handling max column qualifier length. It was using the values for the column family size.
ACCUMULO-2789 Fix error handling max column qualifier length. It was using the
values for the column family size.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/6970d73e
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/6970d73e
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/6970d73e
Branch: refs/heads/ACCUMULO-378
Commit: 6970d73ea94fb8ab4bd3e429c25510194c1ff9e3
Parents: 2526f0a
Author: Bill Slacum ujustgotbi...@apache.org
Authored: Fri May 23 18:00:13 2014 -0400
Committer: Bill Slacum ujustgotbi...@apache.org
Committed: Fri May 23 18:00:13 2014 -0400
--
.../java/org/apache/accumulo/test/stress/random/WriteOptions.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/6970d73e/test/src/main/java/org/apache/accumulo/test/stress/random/WriteOptions.java
--
diff --git
a/test/src/main/java/org/apache/accumulo/test/stress/random/WriteOptions.java
b/test/src/main/java/org/apache/accumulo/test/stress/random/WriteOptions.java
index b246ab7..b850e85 100644
---
a/test/src/main/java/org/apache/accumulo/test/stress/random/WriteOptions.java
+++
b/test/src/main/java/org/apache/accumulo/test/stress/random/WriteOptions.java
@@ -139,7 +139,7 @@ class WriteOptions extends ClientOnDefaultTable {
}
public int cqMax() {
-return calculateMax(cf_min, cf_max);
+return calculateMax(cq_min, cq_max);
}
public int valueMin() {
[12/32] git commit: ACCUMULO-2840 disable audit logging in mini
ACCUMULO-2840 disable audit logging in mini
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/ff3ad736
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/ff3ad736
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/ff3ad736
Branch: refs/heads/ACCUMULO-378
Commit: ff3ad736329584a9eda52948029fb00a8d7346d4
Parents: b2dd64d
Author: Keith Turner ktur...@apache.org
Authored: Fri May 23 11:47:23 2014 -0400
Committer: Keith Turner ktur...@apache.org
Committed: Fri May 23 16:27:25 2014 -0400
--
.../impl/MiniAccumuloClusterImpl.java | 8
minicluster/src/main/resources/auditLog.xml | 41
2 files changed, 49 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/ff3ad736/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
--
diff --git
a/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
b/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
index ef5056c..c82fab3 100644
---
a/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
+++
b/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
@@ -74,6 +74,7 @@ import org.apache.accumulo.start.Main;
import org.apache.accumulo.start.classloader.vfs.MiniDFSUtil;
import org.apache.accumulo.tserver.TabletServer;
import org.apache.commons.configuration.MapConfiguration;
+import org.apache.commons.io.FileUtils;
import org.apache.commons.vfs2.FileObject;
import org.apache.commons.vfs2.impl.VFSClassLoader;
import org.apache.hadoop.conf.Configuration;
@@ -400,6 +401,13 @@ public class MiniAccumuloClusterImpl implements
AccumuloCluster {
zooCfg.store(fileWriter, null);
fileWriter.close();
+
+// disable audit logging for mini
+InputStream auditStream =
this.getClass().getResourceAsStream(/auditLog.xml);
+
+if (auditStream != null) {
+ FileUtils.copyInputStreamToFile(auditStream, new
File(config.getConfDir(), auditLog.xml));
+}
}
private void writeConfig(File file, IterableMap.EntryString,String
settings) throws IOException {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/ff3ad736/minicluster/src/main/resources/auditLog.xml
--
diff --git a/minicluster/src/main/resources/auditLog.xml
b/minicluster/src/main/resources/auditLog.xml
new file mode 100644
index 000..9b7987e
--- /dev/null
+++ b/minicluster/src/main/resources/auditLog.xml
@@ -0,0 +1,41 @@
+?xml version=1.0 encoding=UTF-8?
+!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the License); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an AS IS BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+--
+!DOCTYPE log4j:configuration SYSTEM log4j.dtd
+log4j:configuration xmlns:log4j=http://jakarta.apache.org/log4j/;
+
+
+
+!-- Write out Audit info to an Audit file --
+appender name=Audit class=org.apache.log4j.DailyRollingFileAppender
+param name=File
value=${org.apache.accumulo.core.dir.log}/${org.apache.accumulo.core.ip.localhost.hostname}.audit/
+param name=MaxBackupIndex value=10/
+param name=DatePattern value='.'-MM-dd/
+layout class=org.apache.log4j.PatternLayout
+param name=ConversionPattern value=%d{-MM-dd
HH:mm:ss,SSS/Z} [%c{2}] %-5p: %m%n/
+/layout
+/appender
+logger name=Audit additivity=false
+appender-ref ref=Audit /
+level value=OFF/
+/logger
+
+
+
+
+
+/log4j:configuration
[06/32] git commit: Merge remote-tracking branch 'origin/1.5.2-SNAPSHOT' into 1.6.1-SNAPSHOT
Merge remote-tracking branch 'origin/1.5.2-SNAPSHOT' into 1.6.1-SNAPSHOT Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/adaf1469 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/adaf1469 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/adaf1469 Branch: refs/heads/ACCUMULO-378 Commit: adaf14693ed47c3dd744795e33b44a7f705773f5 Parents: db1b63c 74acc30 Author: Bill Slacum ujustgotbi...@apache.org Authored: Thu May 22 07:32:56 2014 -0400 Committer: Bill Slacum ujustgotbi...@apache.org Committed: Thu May 22 07:32:56 2014 -0400 -- .../accumulo/test/stress/random/DataWriter.java | 50 ++ .../test/stress/random/IntArgValidator.java | 32 .../random/RandomByteArrayMakerFunction.java| 60 .../test/stress/random/RandomByteArrays.java| 36 + .../test/stress/random/RandomMutations.java | 39 + .../accumulo/test/stress/random/Scan.java | 124 +++ .../accumulo/test/stress/random/ScanOpts.java | 44 ++ .../accumulo/test/stress/random/Stream.java | 41 + .../accumulo/test/stress/random/Write.java | 78 ++ .../test/stress/random/WriteOptions.java| 152 +++ .../test/stress/random/package-info.java| 36 + test/system/stress/README.md| 34 + test/system/stress/reader.sh| 34 + test/system/stress/writer.sh| 38 + 14 files changed, 798 insertions(+) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/adaf1469/test/system/stress/README.md -- diff --cc test/system/stress/README.md index 000,1a0bbae..9546b21 mode 00,100644..100644 --- a/test/system/stress/README.md +++ b/test/system/stress/README.md @@@ -1,0 -1,33 +1,34 @@@ + !-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the License); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an AS IS BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + -- + + + Accumulo Stress Test + + + This is an attempt to observe the behavior Accumulo displays when compacting and reading these cells. There are two components to this package: `org.apache.accumulo.test.stress.random.Write` and `org.apache.accumulo.test.stress.random.Scan`. + + Since this test is meant to stress the system, there is no pre-defined test period and no success criteria that we can programmatically check. During a successful test period, the Accumulo cluster should remain stable and functional. Possible failure conditions include component failures, such as tablet servers running out of memory. + + The `org.apache.accumulo.test.stress.random.Write` utility provides facilities for writing random sized cells. Users can configure minimum and maximum sized portions of a cell. The portions users can configure are the row, column family, column qualifier and value. Note that the sizes are uniformly distributed between the minimum and maximum values + + The `org.apache.accumulo.test.stress.random.Scan` utility provides users with the ability to query tables generated by the Write. It will pick a tablet at random and scan the entire range. The amount of times this process is performed is user configurable. By default, it happens 1,024 times. Users can also specify whether or not the scan should be isolated or not. + + There is no shared state intended by either of these services. This allows multiple clients to be run in parallel, either on the same host or distributedacross hosts. + + See the provided `test/system/stress/writer.sh` and `test/system/stress/reader.sh` scripts for example configuration options and how to launch the processes. + ++Note that in versions of Accumulo greater than 1.6.0, users should verify that the cell sizes they supply meet the criteria set forth by the key constraints on their tables. By default, tables are configured to reject keys that are are greater than 1MB.
[32/32] git commit: Merge remote-tracking branch 'origin/master' into ACCUMULO-378
Merge remote-tracking branch 'origin/master' into ACCUMULO-378 Conflicts: core/src/main/java/org/apache/accumulo/core/conf/Property.java Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/58fbf143 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/58fbf143 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/58fbf143 Branch: refs/heads/ACCUMULO-378 Commit: 58fbf1438d2cd82133ca0d559dfb22d8b8f55d7f Parents: 1f0ee9c 0712ec9 Author: Josh Elser els...@apache.org Authored: Wed May 28 16:49:48 2014 -0400 Committer: Josh Elser els...@apache.org Committed: Wed May 28 16:49:48 2014 -0400 -- bin/tool.sh | 5 +- .../1GB/native-standalone/log4j.properties | 2 +- conf/examples/1GB/standalone/log4j.properties | 2 +- .../2GB/native-standalone/log4j.properties | 2 +- conf/examples/2GB/standalone/log4j.properties | 2 +- .../3GB/native-standalone/log4j.properties | 2 +- conf/examples/3GB/standalone/log4j.properties | 2 +- .../512MB/native-standalone/log4j.properties| 2 +- conf/examples/512MB/standalone/log4j.properties | 2 +- conf/templates/log4j.properties | 2 +- .../core/conf/DefaultConfiguration.java | 2 +- .../org/apache/accumulo/core/conf/Property.java | 4 + .../accumulo/core/file/BloomFilterLayer.java| 19 ++- .../core/iterators/AggregatingIterator.java | 6 +- .../aggregation/conf/AggregatorSet.java | 3 +- .../iterators/conf/ColumnToClassMapping.java| 16 +- .../accumulo/core/util/LocalityGroupUtil.java | 3 + .../impl/MiniAccumuloClusterImpl.java | 8 + minicluster/src/main/resources/auditLog.xml | 41 + .../master/balancer/TableLoadBalancer.java | 9 +- .../security/AuditedSecurityOperation.java | 3 +- .../java/org/apache/accumulo/master/Master.java | 14 ++ .../accumulo/monitor/servlets/ShellServlet.java | 43 +++-- .../apache/accumulo/tserver/log/DfsLogger.java | 10 +- .../classloader/vfs/AccumuloVFSClassLoader.java | 19 +++ .../accumulo/test/stress/random/DataWriter.java | 50 ++ .../test/stress/random/IntArgValidator.java | 32 .../test/stress/random/RandomByteArrays.java| 32 .../test/stress/random/RandomMutations.java | 44 + .../test/stress/random/RandomWithinRange.java | 59 +++ .../accumulo/test/stress/random/Scan.java | 124 ++ .../accumulo/test/stress/random/ScanOpts.java | 44 + .../accumulo/test/stress/random/Stream.java | 41 + .../accumulo/test/stress/random/Write.java | 87 ++ .../test/stress/random/WriteOptions.java| 169 +++ .../test/stress/random/package-info.java| 36 test/system/stress/README.md| 34 test/system/stress/reader.sh| 34 test/system/stress/writer.sh| 40 + 39 files changed, 1013 insertions(+), 36 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/58fbf143/core/src/main/java/org/apache/accumulo/core/conf/Property.java -- diff --cc core/src/main/java/org/apache/accumulo/core/conf/Property.java index 2c7e27b,1200fd1..8ad849b --- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java +++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java @@@ -272,10 -267,10 +272,14 @@@ public enum Property The number of threads for the distributed work queue. These threads are used for copying failed bulk files.), TSERV_WAL_SYNC(tserver.wal.sync, true, PropertyType.BOOLEAN, Use the SYNC_BLOCK create flag to sync WAL writes to disk. Prevents problems recovering from sudden system resets.), + TSERV_WAL_SYNC_METHOD(tserver.wal.sync.method, hsync, PropertyType.STRING, The method to invoke when sync'ing WALs. HSync will provide + + resiliency in the face of unexpected power outages, at the cost of speed. If method is not available, the legacy 'sync' method + + will be used to ensure backwards compatibility with older Hadoop versions. A value of 'hflush' is the alternative to the default value + + of 'hsync' which will result in faster writes, but with less durability), + TSERV_REPLICATION_REPLAYERS(tserver.replication.replayer., null, PropertyType.PREFIX, Allows configuration of implementation used to apply replicated data), + TSERV_REPLICATION_DEFAULT_HANDLER(tserver.replication.default.replayer, org.apache.accumulo.tserver.replication.BatchWriterReplicationReplayer, + PropertyType.CLASSNAME, Default AccumuloReplicationReplayer implementation), +
[22/32] git commit: Merge remote-tracking branch 'origin/1.6.1-SNAPSHOT'
Merge remote-tracking branch 'origin/1.6.1-SNAPSHOT' Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9f357e66 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9f357e66 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9f357e66 Branch: refs/heads/ACCUMULO-378 Commit: 9f357e66a6fca3ecb17dce41b782993ffa23f1c2 Parents: f461107 eb7aac3 Author: Bill Slacum ujustgotbi...@apache.org Authored: Fri May 23 19:27:07 2014 -0400 Committer: Bill Slacum ujustgotbi...@apache.org Committed: Fri May 23 19:27:07 2014 -0400 -- .../random/RandomByteArrayMakerFunction.java| 60 .../test/stress/random/RandomByteArrays.java| 14 ++--- .../test/stress/random/RandomMutations.java | 11 +++- .../test/stress/random/RandomWithinRange.java | 59 +++ .../accumulo/test/stress/random/Write.java | 33 +++ .../test/stress/random/WriteOptions.java| 19 ++- test/system/stress/writer.sh| 4 +- 7 files changed, 114 insertions(+), 86 deletions(-) --
[14/32] git commit: ACCUMULO-2838 use resolved props
ACCUMULO-2838 use resolved props
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/850f4bc0
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/850f4bc0
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/850f4bc0
Branch: refs/heads/ACCUMULO-378
Commit: 850f4bc04ac910526c7c9cae3d8424ccb55a5d8e
Parents: 2bf3ebb
Author: Keith Turner ktur...@apache.org
Authored: Fri May 23 11:45:58 2014 -0400
Committer: Keith Turner ktur...@apache.org
Committed: Fri May 23 16:27:25 2014 -0400
--
.../java/org/apache/accumulo/core/conf/DefaultConfiguration.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/850f4bc0/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java
--
diff --git
a/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java
b/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java
index 847fd02..44e198a 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java
@@ -39,7 +39,7 @@ public class DefaultConfiguration extends
AccumuloConfiguration {
@Override
public String get(Property property) {
-return property.getDefaultValue();
+return getResolvedProps().get(property.getKey());
}
private synchronized MapString,String getResolvedProps() {
[29/32] git commit: ACCUMULO-2578: Throw exception when offset is non-zero.
ACCUMULO-2578: Throw exception when offset is non-zero.
Signed-off-by: Josh Elser els...@apache.org
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9fbc24ab
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9fbc24ab
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9fbc24ab
Branch: refs/heads/ACCUMULO-378
Commit: 9fbc24abea69aa1da162d48a93ddc09970bdb6e1
Parents: cff4c2d
Author: Gary Singh gary.si...@sabre-eng.com
Authored: Fri May 16 13:13:02 2014 -0400
Committer: Josh Elser els...@apache.org
Committed: Sat May 24 18:15:54 2014 -0400
--
.../java/org/apache/accumulo/core/util/LocalityGroupUtil.java | 3 +++
1 file changed, 3 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9fbc24ab/core/src/main/java/org/apache/accumulo/core/util/LocalityGroupUtil.java
--
diff --git
a/core/src/main/java/org/apache/accumulo/core/util/LocalityGroupUtil.java
b/core/src/main/java/org/apache/accumulo/core/util/LocalityGroupUtil.java
index 6efd098..da3593d 100644
--- a/core/src/main/java/org/apache/accumulo/core/util/LocalityGroupUtil.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/LocalityGroupUtil.java
@@ -155,6 +155,9 @@ public class LocalityGroupUtil {
}
public static String encodeColumnFamily(ByteSequence bs) {
+if (bs.offset() != 0) {
+ throw new IllegalArgumentException(The offset cannot be non-zero.);
+}
return encodeColumnFamily(new StringBuilder(), bs.getBackingArray(),
bs.length());
}
[15/32] git commit: Merge branch '1.6.1-SNAPSHOT'
Merge branch '1.6.1-SNAPSHOT' Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/3411023a Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/3411023a Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/3411023a Branch: refs/heads/ACCUMULO-378 Commit: 3411023a94b0d460b2bf57f0997247cebaf6e9dc Parents: 20a466b ff3ad73 Author: Keith Turner ktur...@apache.org Authored: Fri May 23 16:32:01 2014 -0400 Committer: Keith Turner ktur...@apache.org Committed: Fri May 23 16:32:01 2014 -0400 -- .../core/conf/DefaultConfiguration.java | 2 +- .../impl/MiniAccumuloClusterImpl.java | 8 minicluster/src/main/resources/auditLog.xml | 41 .../security/AuditedSecurityOperation.java | 3 +- 4 files changed, 52 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/3411023a/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/3411023a/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java --
[08/32] git commit: ACCUMULO-2842 Allow configuration of FSDataOutputStream method used to sync WALs.
ACCUMULO-2842 Allow configuration of FSDataOutputStream method used to sync
WALs.
A critical piece of the bigtable design is using write-ahead logs to ensure
recovery after node failure. The append only file structure is meant to provide
a durable log of events so that the correct state can be reconstructed after
a failure when key-values have not been yet serialized to disk.
Hadoop 2 offers two levels of durability in regards to writes to files in HDFS.
HFlush
ensures that all others clients who try to read the file you just wrote data to
will
see that new data that was written. HSync will ensure that the data you wrote
for that
file is actually sent to the underlying disk. As such, HFlush does not provide
durability
when the node crashes without that data being flushed to the drive, while HSync
does. HSync
does, however, increase the amount of time needed for sync/flush to complete
(as hitting disk
consistently is slower than not).
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/f654e7fa
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/f654e7fa
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/f654e7fa
Branch: refs/heads/ACCUMULO-378
Commit: f654e7faadb9e94e2923087af334a707a4d2c81a
Parents: 74acc30
Author: Josh Elser els...@apache.org
Authored: Fri May 23 14:40:37 2014 -0400
Committer: Josh Elser els...@apache.org
Committed: Fri May 23 15:46:55 2014 -0400
--
.../java/org/apache/accumulo/core/conf/Property.java | 3 +++
.../accumulo/server/tabletserver/log/DfsLogger.java | 14 +-
2 files changed, 12 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f654e7fa/core/src/main/java/org/apache/accumulo/core/conf/Property.java
--
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 557ca1f..3d82046 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -204,6 +204,9 @@ public enum Property {
The number of threads for the distributed workq. These threads are
used for copying failed bulk files.),
TSERV_WAL_SYNC(tserver.wal.sync, true, PropertyType.BOOLEAN,
Use the SYNC_BLOCK create flag to sync WAL writes to disk. Prevents
problems recovering from sudden system resets.),
+ TSERV_WAL_SYNC_METHOD(tserver.wal.sync.method, hsync,
PropertyType.STRING, The method to invoke when sync'ing WALs. HSync will
provide +
+ resiliency in the face of unexpected power outages, at the cost of
speed. If method is not available, the legacy 'sync' method +
+ will be used to ensure backwards compatibility with older Hadoop
versions),
// properties that are specific to logger server behavior
LOGGER_PREFIX(logger., null, PropertyType.PREFIX, Properties in this
category affect the behavior of the write-ahead logger servers),
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f654e7fa/server/src/main/java/org/apache/accumulo/server/tabletserver/log/DfsLogger.java
--
diff --git
a/server/src/main/java/org/apache/accumulo/server/tabletserver/log/DfsLogger.java
b/server/src/main/java/org/apache/accumulo/server/tabletserver/log/DfsLogger.java
index 120c844..01c2448 100644
---
a/server/src/main/java/org/apache/accumulo/server/tabletserver/log/DfsLogger.java
+++
b/server/src/main/java/org/apache/accumulo/server/tabletserver/log/DfsLogger.java
@@ -264,13 +264,17 @@ public class DfsLogger {
else
logFile = fs.create(logPath, true,
fs.getConf().getInt(io.file.buffer.size, 4096), replication, blockSize);
+ String syncMethod =
conf.getConfiguration().get(Property.TSERV_WAL_SYNC_METHOD);
try {
-// sync: send data to datanodes
-sync = logFile.getClass().getMethod(sync);
try {
- // hsych: send data to datanodes and sync the data to disk
- sync = logFile.getClass().getMethod(hsync);
-} catch (NoSuchMethodException ex) {}
+ // hsync: send data to datanodes and sync the data to disk
+ sync = logFile.getClass().getMethod(syncMethod);
+} catch (NoSuchMethodException ex) {
+ log.warn(Could not find configured + syncMethod + method,
trying to fall back to old Hadoop sync method, ex);
+
+ // sync: send data to datanodes
+ sync = logFile.getClass().getMethod(sync);
+}
} catch (Exception e) {
throw new RuntimeException(e);
}
[28/32] git commit: Merge branch '1.6.1-SNAPSHOT'
Merge branch '1.6.1-SNAPSHOT' Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/d6b1e9a9 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/d6b1e9a9 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/d6b1e9a9 Branch: refs/heads/ACCUMULO-378 Commit: d6b1e9a959c7e52e571ce9997ae6915780773307 Parents: d37f05f c9c3fdd Author: Dave Marion dlmar...@hotmail.com Authored: Fri May 23 22:48:30 2014 -0400 Committer: Dave Marion dlmar...@hotmail.com Committed: Fri May 23 22:48:30 2014 -0400 -- .../accumulo/core/file/BloomFilterLayer.java | 19 --- .../core/iterators/AggregatingIterator.java | 6 +- .../aggregation/conf/AggregatorSet.java | 3 ++- .../iterators/conf/ColumnToClassMapping.java | 16 +--- .../master/balancer/TableLoadBalancer.java | 9 - .../java/org/apache/accumulo/master/Master.java | 14 ++ 6 files changed, 58 insertions(+), 9 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/d6b1e9a9/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/d6b1e9a9/server/master/src/main/java/org/apache/accumulo/master/Master.java --
[03/32] git commit: ACCUMULO-2831: Reset the classloader on the Hadoop Configuration objects that were created before the VFS ClassLoader was created.
ACCUMULO-2831: Reset the classloader on the Hadoop Configuration objects that
were created before the VFS ClassLoader was created.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/db1b63c4
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/db1b63c4
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/db1b63c4
Branch: refs/heads/ACCUMULO-378
Commit: db1b63c4cb9f2f1bfdb45765b9c79f84105d601a
Parents: 1193f4b
Author: Dave Marion dlmar...@hotmail.com
Authored: Wed May 21 21:11:26 2014 -0400
Committer: Dave Marion dlmar...@hotmail.com
Committed: Wed May 21 21:11:26 2014 -0400
--
.../classloader/vfs/AccumuloVFSClassLoader.java | 19 +++
1 file changed, 19 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/db1b63c4/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
--
diff --git
a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
index fa20725..d4044a5 100644
---
a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
+++
b/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java
@@ -27,6 +27,7 @@ import java.util.Collections;
import java.util.List;
import org.apache.accumulo.start.classloader.AccumuloClassLoader;
+import org.apache.accumulo.start.classloader.vfs.providers.HdfsFileObject;
import org.apache.accumulo.start.classloader.vfs.providers.HdfsFileProvider;
import org.apache.commons.vfs2.CacheStrategy;
import org.apache.commons.vfs2.FileObject;
@@ -37,6 +38,8 @@ import org.apache.commons.vfs2.cache.SoftRefFilesCache;
import org.apache.commons.vfs2.impl.DefaultFileSystemManager;
import org.apache.commons.vfs2.impl.FileContentInfoFilenameFactory;
import org.apache.commons.vfs2.impl.VFSClassLoader;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.log4j.Logger;
/**
@@ -213,6 +216,22 @@ public class AccumuloVFSClassLoader {
// Create the Accumulo Context ClassLoader using the DEFAULT_CONTEXT
localLoader = createDynamicClassloader(new VFSClassLoader(vfsCP,
vfs, parent));
loader = localLoader;
+
+ //An HDFS FileSystem and Configuration object were created for each
unique HDFS namespace in the call to resolve above.
+ //The HDFS Client did us a favor and cached these objects so that
the next time someone calls FileSystem.get(uri), they
+ //get the cached object. However, these objects were created not
with the system VFS classloader, but the classloader above
+ //it. We need to override the classloader on the Configuration
objects. Ran into an issue were log recovery was being attempted
+ //and SequenceFile$Reader was trying to instantiate the key class
via WritableName.getClass(String, Configuration)
+ for (FileObject fo : vfsCP) {
+if (fo instanceof HdfsFileObject) {
+ String uri = fo.getName().getRootURI();
+ Configuration c = new Configuration(true);
+ c.set(FileSystem.FS_DEFAULT_NAME_KEY, uri);
+ FileSystem fs = FileSystem.get(c);
+ fs.getConf().setClassLoader(loader.getClassLoader());
+}
+ }
+
}
}
}
[10/32] git commit: Merge branch '1.6.1-SNAPSHOT'
Merge branch '1.6.1-SNAPSHOT' Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/20a466be Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/20a466be Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/20a466be Branch: refs/heads/ACCUMULO-378 Commit: 20a466be30ec0849b29c862ba2d8f53905837014 Parents: 0164991 2bf3ebb Author: Josh Elser els...@apache.org Authored: Fri May 23 16:07:40 2014 -0400 Committer: Josh Elser els...@apache.org Committed: Fri May 23 16:07:40 2014 -0400 -- .../main/java/org/apache/accumulo/core/conf/Property.java | 5 - .../java/org/apache/accumulo/tserver/log/DfsLogger.java | 10 ++ 2 files changed, 10 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/20a466be/core/src/main/java/org/apache/accumulo/core/conf/Property.java -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/20a466be/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java --
[04/32] git commit: Merge branch '1.6.1-SNAPSHOT'
Merge branch '1.6.1-SNAPSHOT' Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/58e7665a Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/58e7665a Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/58e7665a Branch: refs/heads/ACCUMULO-378 Commit: 58e7665ae9d677904eda5745b7fd0ea24835e7b9 Parents: f985d3c db1b63c Author: Dave Marion dlmar...@hotmail.com Authored: Wed May 21 21:14:52 2014 -0400 Committer: Dave Marion dlmar...@hotmail.com Committed: Wed May 21 21:14:52 2014 -0400 -- .../classloader/vfs/AccumuloVFSClassLoader.java | 19 +++ 1 file changed, 19 insertions(+) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/58e7665a/start/src/main/java/org/apache/accumulo/start/classloader/vfs/AccumuloVFSClassLoader.java --
[1/3] git commit: ACCUMULO-378 Allow for dynamic reconfiguration of the WorkAssigner impl
Repository: accumulo
Updated Branches:
refs/heads/ACCUMULO-378 58fbf1438 - 49fc9855f
ACCUMULO-378 Allow for dynamic reconfiguration of the WorkAssigner impl
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/db10cfe2
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/db10cfe2
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/db10cfe2
Branch: refs/heads/ACCUMULO-378
Commit: db10cfe26823f5a31b516070521f54e5b890fb7b
Parents: 58fbf14
Author: Josh Elser els...@apache.org
Authored: Wed May 28 16:55:39 2014 -0400
Committer: Josh Elser els...@apache.org
Committed: Wed May 28 16:55:39 2014 -0400
--
.../accumulo/master/replication/WorkDriver.java | 33 +---
1 file changed, 22 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/db10cfe2/server/master/src/main/java/org/apache/accumulo/master/replication/WorkDriver.java
--
diff --git
a/server/master/src/main/java/org/apache/accumulo/master/replication/WorkDriver.java
b/server/master/src/main/java/org/apache/accumulo/master/replication/WorkDriver.java
index 00b0480..8c3e3e3 100644
---
a/server/master/src/main/java/org/apache/accumulo/master/replication/WorkDriver.java
+++
b/server/master/src/main/java/org/apache/accumulo/master/replication/WorkDriver.java
@@ -44,20 +44,28 @@ public class WorkDriver extends Daemon {
this.master = master;
this.conn = conn;
this.conf = master.getConfiguration().getConfiguration();
+configureWorkAssigner();
+ }
+ protected void configureWorkAssigner() {
String workAssignerClass = conf.get(Property.REPLICATION_WORK_ASSIGNER);
-try {
- Class? clz = Class.forName(workAssignerClass);
- Class? extends WorkAssigner workAssignerClz =
clz.asSubclass(WorkAssigner.class);
- this.assigner = workAssignerClz.newInstance();
-} catch (ClassNotFoundException | InstantiationException |
IllegalAccessException e) {
- log.error(Could not instantiate configured work assigner {},
workAssignerClass, e);
- throw new RuntimeException(e);
-}
-this.assigner.configure(conf, conn);
-this.assignerImplName = assigner.getClass().getName();
-this.setName(assigner.getName());
+if (null == assigner ||
!assigner.getClass().getName().equals(workAssignerClass)) {
+ log.info(Initializing work assigner implementation of {},
workAssignerClass);
+
+ try {
+Class? clz = Class.forName(workAssignerClass);
+Class? extends WorkAssigner workAssignerClz =
clz.asSubclass(WorkAssigner.class);
+this.assigner = workAssignerClz.newInstance();
+ } catch (ClassNotFoundException | InstantiationException |
IllegalAccessException e) {
+log.error(Could not instantiate configured work assigner {},
workAssignerClass, e);
+throw new RuntimeException(e);
+ }
+
+ this.assigner.configure(conf, conn);
+ this.assignerImplName = assigner.getClass().getName();
+ this.setName(assigner.getName());
+}
}
/*
@@ -90,6 +98,9 @@ public class WorkDriver extends Daemon {
long sleepTime =
conf.getTimeInMillis(Property.REPLICATION_WORK_ASSIGNMENT_SLEEP);
log.debug(Sleeping {} ms before next work assignment, sleepTime);
UtilWaitThread.sleep(sleepTime);
+
+ // After each loop, make sure that the WorkAssigner implementation
didn't change
+ configureWorkAssigner();
}
}
}
[2/3] git commit: ACCUMULO-378 Resize the threadpool used for sending data by checking the configuration periodically
ACCUMULO-378 Resize the threadpool used for sending data by checking the
configuration periodically
Use the SimpleTimer to just schedule a check of the configuration
to see what the value is for the maximum size of the threadpool which
is used with the DistributedWorkQueue and ReplicationProcessor.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/0ff0e021
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/0ff0e021
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/0ff0e021
Branch: refs/heads/ACCUMULO-378
Commit: 0ff0e021d3fc95794137dfeb3f6e1335b61b0a16
Parents: db10cfe
Author: Josh Elser els...@apache.org
Authored: Wed May 28 17:23:51 2014 -0400
Committer: Josh Elser els...@apache.org
Committed: Wed May 28 17:23:51 2014 -0400
--
.../org/apache/accumulo/tserver/TabletServer.java | 16 +++-
1 file changed, 15 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/0ff0e021/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
--
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index 567b2ad..e4c7ef9 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@ -3241,10 +3241,24 @@ public class TabletServer extends AbstractMetricsImpl
implements org.apache.accu
log.info(Started replication service at + replicationAddress);
// Start the pool to handle outgoing replications
-ThreadPoolExecutor replicationThreadPool = new
SimpleThreadPool(getSystemConfiguration().getCount(Property.REPLICATION_WORKER_THREADS),
replication task);
+final ThreadPoolExecutor replicationThreadPool = new
SimpleThreadPool(getSystemConfiguration().getCount(Property.REPLICATION_WORKER_THREADS),
replication task);
replWorker.setExecutor(replicationThreadPool);
replWorker.run();
+// Check the configuration value for the size of the pool and, if changed,
resize the pool, every 5 seconds);
+final AccumuloConfiguration aconf = getSystemConfiguration();
+Runnable replicationWorkThreadPoolResizer = new Runnable() {
+ @Override
+ public void run() {
+int maxPoolSize = aconf.getCount(Property.REPLICATION_WORKER_THREADS);
+if (replicationThreadPool.getMaximumPoolSize() != maxPoolSize) {
+ log.info(Resizing thread pool for sending replication work from +
replicationThreadPool.getMaximumPoolSize() + to + maxPoolSize);
+ replicationThreadPool.setMaximumPoolSize(maxPoolSize);
+}
+ }
+};
+SimpleTimer.getInstance(aconf).schedule(replicationWorkThreadPoolResizer,
1, 3);
+
try {
OBJECT_NAME = new
ObjectName(accumulo.server.metrics:service=TServerInfo,name=TabletServerMBean,instance=
+ Thread.currentThread().getName());
// Do this because interface not in same package.
[3/3] git commit: ACCUMULO-378 Can't use '_' as the row separator for Order records.
ACCUMULO-378 Can't use '_' as the row separator for Order records.
The ULongLexicoder *might* create bytes that actually equal the '_', which
will mess up the splitting logic of the row key. Switch it to a \x00 instead
and find the last instance of it.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/49fc9855
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/49fc9855
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/49fc9855
Branch: refs/heads/ACCUMULO-378
Commit: 49fc9855f996ae0f5b3cc20e03e77ea8f707d640
Parents: 0ff0e02
Author: Josh Elser els...@apache.org
Authored: Wed May 28 20:07:52 2014 -0400
Committer: Josh Elser els...@apache.org
Committed: Wed May 28 20:58:43 2014 -0400
--
.../core/replication/ReplicationSchema.java | 30
.../core/replication/ReplicationSchemaTest.java | 12
.../accumulo/master/replication/WorkDriver.java | 6 +++-
3 files changed, 41 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/49fc9855/core/src/main/java/org/apache/accumulo/core/replication/ReplicationSchema.java
--
diff --git
a/core/src/main/java/org/apache/accumulo/core/replication/ReplicationSchema.java
b/core/src/main/java/org/apache/accumulo/core/replication/ReplicationSchema.java
index 8699bd2..ab350e6 100644
---
a/core/src/main/java/org/apache/accumulo/core/replication/ReplicationSchema.java
+++
b/core/src/main/java/org/apache/accumulo/core/replication/ReplicationSchema.java
@@ -153,11 +153,11 @@ public class ReplicationSchema {
* Holds the order in which files needed for replication were closed. The
intent is to be able to guarantee that files which were closed earlier were
* replicated first and we don't replay data in the wrong order on our peers
* p
- *
codeencodedTimeOfClosure_hdfs://localhost:8020/accumulo/wal/tserver+port/WAL
order:source_table_id [] - Status Protobuf/code
+ *
codeencodedTimeOfClosure\x00hdfs://localhost:8020/accumulo/wal/tserver+port/WAL
order:source_table_id [] - Status Protobuf/code
*/
public static class OrderSection {
public static final Text NAME = new Text(order);
-public static final String ROW_SEPARATOR = _;
+public static final Text ROW_SEPARATOR = new Text(new byte[]{0});
private static final ULongLexicoder longEncoder = new ULongLexicoder();
/**
@@ -218,10 +218,10 @@ public class ReplicationSchema {
Path p = new Path(file);
String pathString = p.toUri().toString();
- log.info(Normalized {} into {}, file, pathString);
+ log.trace(Normalized {} into {}, file, pathString);
// Append the file as a suffix to the row
- row.append((ROW_SEPARATOR + pathString).getBytes(), 0,
pathString.length() + ROW_SEPARATOR.length());
+ row.append((ROW_SEPARATOR + pathString).getBytes(), 0,
pathString.length() + ROW_SEPARATOR.getLength());
// Make the mutation and add the column update
return new Mutation(row);
@@ -249,7 +249,16 @@ public class ReplicationSchema {
public static long getTimeClosed(Key k, Text buff) {
k.getRow(buff);
- int offset = buff.find(ROW_SEPARATOR);
+ int offset = 0;
+ // find the last offset
+ while (true) {
+int nextOffset = buff.find(ROW_SEPARATOR.toString(), offset + 1);
+if (-1 == nextOffset) {
+ break;
+}
+offset = nextOffset;
+ }
+
if (-1 == offset) {
throw new IllegalArgumentException(Row does not contain expected
separator for OrderSection);
}
@@ -266,7 +275,16 @@ public class ReplicationSchema {
public static String getFile(Key k, Text buff) {
k.getRow(buff);
- int offset = buff.find(ROW_SEPARATOR);
+ int offset = 0;
+ // find the last offset
+ while (true) {
+int nextOffset = buff.find(ROW_SEPARATOR.toString(), offset + 1);
+if (-1 == nextOffset) {
+ break;
+}
+offset = nextOffset;
+ }
+
if (-1 == offset) {
throw new IllegalArgumentException(Row does not contain expected
separator for OrderSection);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/49fc9855/core/src/test/java/org/apache/accumulo/core/replication/ReplicationSchemaTest.java
--
diff --git
a/core/src/test/java/org/apache/accumulo/core/replication/ReplicationSchemaTest.java
b/core/src/test/java/org/apache/accumulo/core/replication/ReplicationSchemaTest.java
index d321153..3822641 100644
---
a/core/src/test/java/org/apache/accumulo/core/replication/ReplicationSchemaTest.java
+++
git commit: ACCUMULO-2847 Add some basic documentation to the user manual for replication
Repository: accumulo Updated Branches: refs/heads/ACCUMULO-378 49fc9855f - 3ddefc641 ACCUMULO-2847 Add some basic documentation to the user manual for replication Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/3ddefc64 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/3ddefc64 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/3ddefc64 Branch: refs/heads/ACCUMULO-378 Commit: 3ddefc641ecf69b1130b79e411128665610cf168 Parents: 49fc985 Author: Josh Elser els...@apache.org Authored: Wed May 28 22:37:05 2014 -0400 Committer: Josh Elser els...@apache.org Committed: Wed May 28 22:37:05 2014 -0400 -- .../main/asciidoc/accumulo_user_manual.asciidoc | 2 + docs/src/main/asciidoc/chapters/replication.txt | 162 +++ 2 files changed, 164 insertions(+) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/3ddefc64/docs/src/main/asciidoc/accumulo_user_manual.asciidoc -- diff --git a/docs/src/main/asciidoc/accumulo_user_manual.asciidoc b/docs/src/main/asciidoc/accumulo_user_manual.asciidoc index fec40ca..b958c9d 100644 --- a/docs/src/main/asciidoc/accumulo_user_manual.asciidoc +++ b/docs/src/main/asciidoc/accumulo_user_manual.asciidoc @@ -49,6 +49,8 @@ include::chapters/analytics.txt[] include::chapters/security.txt[] +include::chapters/replication.txt[] + include::chapters/administration.txt[] include::chapters/multivolume.txt[] http://git-wip-us.apache.org/repos/asf/accumulo/blob/3ddefc64/docs/src/main/asciidoc/chapters/replication.txt -- diff --git a/docs/src/main/asciidoc/chapters/replication.txt b/docs/src/main/asciidoc/chapters/replication.txt new file mode 100644 index 000..20843a9 --- /dev/null +++ b/docs/src/main/asciidoc/chapters/replication.txt @@ -0,0 +1,162 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the License); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an AS IS BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +== Replication + +=== Overview + +Replication is a feature of Accumulo which provides a mechanism to automatically +copy data to other systems, typically for the purpose of disaster recovery, +high availability, or geographic locality. It is best to consider this feature +as a framework for automatic replication instead of the ability to copy data +from to another Accumulo instance as copying to another Accumulo cluster is +only an implementation detail. The local Accumulo cluster is hereby referred +to as the +primary+ while systems being replicated to are known as ++peers+. + +This replication framework makes two Accumulo instances, where one instance +replicates to another, eventually consistent between one another, as opposed +to the strong consistency that each single Accumulo instance still holds. That +is to say, attempts to read data from a table on a peer which has pending replication +from the primary will not wait for that data to be replicated before running the scan. +This is desirable for a number of reasons, the most important is that the replication +framework is not limited by network outages or offline peers, but only by the HDFS +space available on the primary system. + +Replication configurations can be considered as a directed graph which allows cycles. +The systems in which data was replicated from is maintained in each Mutation which +allow each system to determine if a peer has already has the data in which +the system wants to send. + +Data is replicated by using the Write-Ahead logs (WAL) that each TabletServer is +already maintaining. TabletServers records which WALs have data that need to be +replicated to the +accumulo.metadata+ table. The Master uses these records, +combined with the local Accumulo table that the WAL was used with, to create records +in the +replication+ table which track which peers the given WAL should be +replicated to. The Master latter uses these work entries to assign the actual +replication task to a local TabletServer using ZooKeeper. A TabletServer will get +a lock in ZooKeeper for the
