[3/3] accumulo git commit: Merge branch '1.7'
Merge branch '1.7' Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/5b7a14eb Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/5b7a14eb Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/5b7a14eb Branch: refs/heads/master Commit: 5b7a14eb51d66b24f1b3d3a8cf6f45f71eb02e4a Parents: 6760c5c f830300 Author: Christopher Tubbs ctubb...@apache.org Authored: Wed May 6 18:55:33 2015 -0400 Committer: Christopher Tubbs ctubb...@apache.org Committed: Wed May 6 18:55:33 2015 -0400 -- core/src/main/java/org/apache/accumulo/core/data/Mutation.java | 3 ++- .../org/apache/accumulo/server/fs/PreferredVolumeChooser.java | 5 - .../java/org/apache/accumulo/server/replication/StatusUtil.java | 2 +- .../master/replication/RemoveCompleteReplicationRecords.java| 2 +- 4 files changed, 8 insertions(+), 4 deletions(-) --
[1/3] accumulo git commit: ACCUMULO-3778 Fix minor javadoc bugs
Repository: accumulo
Updated Branches:
refs/heads/1.7 cd765a862 - f8303008c
refs/heads/master 6760c5c63 - 5b7a14eb5
ACCUMULO-3778 Fix minor javadoc bugs
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/f8303008
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/f8303008
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/f8303008
Branch: refs/heads/1.7
Commit: f8303008c62a8ca49dbefe3ad2499f8fb8f89f5f
Parents: cd765a8
Author: Christopher Tubbs ctubb...@apache.org
Authored: Wed May 6 18:53:55 2015 -0400
Committer: Christopher Tubbs ctubb...@apache.org
Committed: Wed May 6 18:55:25 2015 -0400
--
core/src/main/java/org/apache/accumulo/core/data/Mutation.java | 3 ++-
.../org/apache/accumulo/server/fs/PreferredVolumeChooser.java | 5 -
.../java/org/apache/accumulo/server/replication/StatusUtil.java | 2 +-
.../master/replication/RemoveCompleteReplicationRecords.java| 2 +-
4 files changed, 8 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f8303008/core/src/main/java/org/apache/accumulo/core/data/Mutation.java
--
diff --git a/core/src/main/java/org/apache/accumulo/core/data/Mutation.java
b/core/src/main/java/org/apache/accumulo/core/data/Mutation.java
index 1cc4242..5a49341 100644
--- a/core/src/main/java/org/apache/accumulo/core/data/Mutation.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/Mutation.java
@@ -98,7 +98,8 @@ public class Mutation implements Writable {
}
}
- /* This is so hashCode equals can be called without changing this object.
+ /**
+ * This is so hashCode and equals can be called without changing this object.
*
* It will return a copy of the current data buffer if serialized has not
been
* called previously. Otherwise, this.data will be returned since the buffer
is
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f8303008/server/base/src/main/java/org/apache/accumulo/server/fs/PreferredVolumeChooser.java
--
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/fs/PreferredVolumeChooser.java
b/server/base/src/main/java/org/apache/accumulo/server/fs/PreferredVolumeChooser.java
index 04242a9..6bc225f 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/fs/PreferredVolumeChooser.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/fs/PreferredVolumeChooser.java
@@ -46,7 +46,10 @@ import com.google.common.base.Predicate;
public class PreferredVolumeChooser extends RandomVolumeChooser implements
VolumeChooser {
private static final Logger log =
LoggerFactory.getLogger(PreferredVolumeChooser.class);
- public static final String PREFERRED_VOLUMES_CUSTOM_KEY =
Property.TABLE_ARBITRARY_PROP_PREFIX.getKey() + preferredVolumes;
+ /**
+ * This should match {@link Property#TABLE_ARBITRARY_PROP_PREFIX}
+ */
+ public static final String PREFERRED_VOLUMES_CUSTOM_KEY =
table.custom.preferredVolumes;
// TODO ACCUMULO-3417 replace this with the ability to retrieve by String
key.
private static final PredicateString PREFERRED_VOLUMES_FILTER = new
PredicateString() {
@Override
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f8303008/server/base/src/main/java/org/apache/accumulo/server/replication/StatusUtil.java
--
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/replication/StatusUtil.java
b/server/base/src/main/java/org/apache/accumulo/server/replication/StatusUtil.java
index d72eea2..ad892b8 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/replication/StatusUtil.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/replication/StatusUtil.java
@@ -86,7 +86,7 @@ public class StatusUtil {
* Existing {@link Builder} to use
* @param recordsReplicated
* Offset of records which have been replicated
- * @returnA {@link Status} tracking data that must be replicated
+ * @return A {@link Status} tracking data that must be replicated
*/
public static Status replicated(Status.Builder builder, long
recordsReplicated) {
return replicatedAndIngested(builder, recordsReplicated, 0);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f8303008/server/master/src/main/java/org/apache/accumulo/master/replication/RemoveCompleteReplicationRecords.java
--
diff --git
a/server/master/src/main/java/org/apache/accumulo/master/replication/RemoveCompleteReplicationRecords.java
[2/3] accumulo git commit: ACCUMULO-3778 Fix minor javadoc bugs
ACCUMULO-3778 Fix minor javadoc bugs
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/f8303008
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/f8303008
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/f8303008
Branch: refs/heads/master
Commit: f8303008c62a8ca49dbefe3ad2499f8fb8f89f5f
Parents: cd765a8
Author: Christopher Tubbs ctubb...@apache.org
Authored: Wed May 6 18:53:55 2015 -0400
Committer: Christopher Tubbs ctubb...@apache.org
Committed: Wed May 6 18:55:25 2015 -0400
--
core/src/main/java/org/apache/accumulo/core/data/Mutation.java | 3 ++-
.../org/apache/accumulo/server/fs/PreferredVolumeChooser.java | 5 -
.../java/org/apache/accumulo/server/replication/StatusUtil.java | 2 +-
.../master/replication/RemoveCompleteReplicationRecords.java| 2 +-
4 files changed, 8 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f8303008/core/src/main/java/org/apache/accumulo/core/data/Mutation.java
--
diff --git a/core/src/main/java/org/apache/accumulo/core/data/Mutation.java
b/core/src/main/java/org/apache/accumulo/core/data/Mutation.java
index 1cc4242..5a49341 100644
--- a/core/src/main/java/org/apache/accumulo/core/data/Mutation.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/Mutation.java
@@ -98,7 +98,8 @@ public class Mutation implements Writable {
}
}
- /* This is so hashCode equals can be called without changing this object.
+ /**
+ * This is so hashCode and equals can be called without changing this object.
*
* It will return a copy of the current data buffer if serialized has not
been
* called previously. Otherwise, this.data will be returned since the buffer
is
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f8303008/server/base/src/main/java/org/apache/accumulo/server/fs/PreferredVolumeChooser.java
--
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/fs/PreferredVolumeChooser.java
b/server/base/src/main/java/org/apache/accumulo/server/fs/PreferredVolumeChooser.java
index 04242a9..6bc225f 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/fs/PreferredVolumeChooser.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/fs/PreferredVolumeChooser.java
@@ -46,7 +46,10 @@ import com.google.common.base.Predicate;
public class PreferredVolumeChooser extends RandomVolumeChooser implements
VolumeChooser {
private static final Logger log =
LoggerFactory.getLogger(PreferredVolumeChooser.class);
- public static final String PREFERRED_VOLUMES_CUSTOM_KEY =
Property.TABLE_ARBITRARY_PROP_PREFIX.getKey() + preferredVolumes;
+ /**
+ * This should match {@link Property#TABLE_ARBITRARY_PROP_PREFIX}
+ */
+ public static final String PREFERRED_VOLUMES_CUSTOM_KEY =
table.custom.preferredVolumes;
// TODO ACCUMULO-3417 replace this with the ability to retrieve by String
key.
private static final PredicateString PREFERRED_VOLUMES_FILTER = new
PredicateString() {
@Override
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f8303008/server/base/src/main/java/org/apache/accumulo/server/replication/StatusUtil.java
--
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/replication/StatusUtil.java
b/server/base/src/main/java/org/apache/accumulo/server/replication/StatusUtil.java
index d72eea2..ad892b8 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/replication/StatusUtil.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/replication/StatusUtil.java
@@ -86,7 +86,7 @@ public class StatusUtil {
* Existing {@link Builder} to use
* @param recordsReplicated
* Offset of records which have been replicated
- * @returnA {@link Status} tracking data that must be replicated
+ * @return A {@link Status} tracking data that must be replicated
*/
public static Status replicated(Status.Builder builder, long
recordsReplicated) {
return replicatedAndIngested(builder, recordsReplicated, 0);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f8303008/server/master/src/main/java/org/apache/accumulo/master/replication/RemoveCompleteReplicationRecords.java
--
diff --git
a/server/master/src/main/java/org/apache/accumulo/master/replication/RemoveCompleteReplicationRecords.java
b/server/master/src/main/java/org/apache/accumulo/master/replication/RemoveCompleteReplicationRecords.java
index 9966feb..e286371 100644
---
[3/3] accumulo git commit: Merge branch '1.7'
Merge branch '1.7' Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/adacc2e5 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/adacc2e5 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/adacc2e5 Branch: refs/heads/master Commit: adacc2e5a096d0c463e3027d386b945e5cc0cfaa Parents: 46f7602 dfa5255 Author: Josh Elser els...@apache.org Authored: Wed May 6 17:06:26 2015 -0400 Committer: Josh Elser els...@apache.org Committed: Wed May 6 17:06:26 2015 -0400 -- .../iterators/user/RowEncodingIterator.java | 56 +- .../core/iterators/user/WholeRowIterator.java | 7 - .../iterators/user/RowEncodingIteratorTest.java | 201 +++ 3 files changed, 253 insertions(+), 11 deletions(-) --
[1/3] accumulo git commit: ACCUMULO-3761 RowEncodingIterator should take a maximum buffer size parameter
Repository: accumulo
Updated Branches:
refs/heads/1.7 2938cfad9 - dfa5255ce
refs/heads/master 46f76023b - adacc2e5a
ACCUMULO-3761 RowEncodingIterator should take a maximum buffer size parameter
Added the optional parameter maxBufferSize to the RowEncodingIterator. This
parameter specifies how large the RowEncodingIterator's buffer can grow to as
it encounters key/value pairs to be encoded. The name, encoding and behaviour
of the maxBufferSize parameter match the TransformingIterator.
Discussion is here:
http://www.mail-archive.com/dev%40accumulo.apache.org/msg09821.html
Added maxBufferSize parameter for RowEncodingIterator based on implementation
of TransformingIterator; still needs tests
Signed-off-by: Josh Elser els...@apache.org
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/dfa5255c
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/dfa5255c
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/dfa5255c
Branch: refs/heads/1.7
Commit: dfa5255cee3219cfaf560e949386da23bd9135d6
Parents: 2938cfa
Author: Russ Weeks rwe...@newbrightidea.com
Authored: Fri Apr 10 00:10:32 2015 -0700
Committer: Josh Elser els...@apache.org
Committed: Wed May 6 16:43:14 2015 -0400
--
.../iterators/user/RowEncodingIterator.java | 56 +-
.../core/iterators/user/WholeRowIterator.java | 7 -
.../iterators/user/RowEncodingIteratorTest.java | 201 +++
3 files changed, 253 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/dfa5255c/core/src/main/java/org/apache/accumulo/core/iterators/user/RowEncodingIterator.java
--
diff --git
a/core/src/main/java/org/apache/accumulo/core/iterators/user/RowEncodingIterator.java
b/core/src/main/java/org/apache/accumulo/core/iterators/user/RowEncodingIterator.java
index f776569..8a36bef 100644
---
a/core/src/main/java/org/apache/accumulo/core/iterators/user/RowEncodingIterator.java
+++
b/core/src/main/java/org/apache/accumulo/core/iterators/user/RowEncodingIterator.java
@@ -22,14 +22,18 @@ import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.SortedMap;
+import java.util.HashMap;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.data.ByteSequence;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.PartialKey;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.OptionDescriber;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.commons.collections.BufferOverflowException;
import org.apache.hadoop.io.Text;
/**
@@ -52,11 +56,15 @@ import org.apache.hadoop.io.Text;
*
* @see RowFilter
*/
-public abstract class RowEncodingIterator implements
SortedKeyValueIteratorKey,Value {
+public abstract class RowEncodingIterator implements
SortedKeyValueIteratorKey,Value, OptionDescriber {
+
+ public static final String MAX_BUFFER_SIZE_OPT = maxBufferSize;
+ private static final long DEFAULT_MAX_BUFFER_SIZE = Long.MAX_VALUE;
protected SortedKeyValueIteratorKey,Value sourceIter;
private Key topKey = null;
private Value topValue = null;
+ private long maxBufferSize = DEFAULT_MAX_BUFFER_SIZE;
// decode a bunch of key value pairs that have been encoded into a single
value
/**
@@ -71,12 +79,23 @@ public abstract class RowEncodingIterator implements
SortedKeyValueIteratorKey,
public abstract Value rowEncoder(ListKey keys, ListValue values) throws
IOException;
@Override
- public abstract SortedKeyValueIteratorKey,Value
deepCopy(IteratorEnvironment env);
+ public SortedKeyValueIteratorKey,Value deepCopy(IteratorEnvironment env) {
+RowEncodingIterator newInstance;
+try {
+ newInstance = this.getClass().newInstance();
+} catch (Exception e) {
+ throw new RuntimeException(e);
+}
+newInstance.sourceIter = sourceIter.deepCopy(env);
+newInstance.maxBufferSize = maxBufferSize;
+return newInstance;
+ }
ListKey keys = new ArrayListKey();
ListValue values = new ArrayListValue();
private void prepKeys() throws IOException {
+long kvBufSize = 0;
if (topKey != null)
return;
Text currentRow;
@@ -87,8 +106,14 @@ public abstract class RowEncodingIterator implements
SortedKeyValueIteratorKey,
keys.clear();
values.clear();
while (sourceIter.hasTop()
sourceIter.getTopKey().getRow().equals(currentRow)) {
-keys.add(new Key(sourceIter.getTopKey()));
-values.add(new Value(sourceIter.getTopValue()));
+Key
[1/5] accumulo git commit: ACCUMULO-3749 Sudo and then set ACCUMULO_CONF_DIR
Repository: accumulo
Updated Branches:
refs/heads/1.7 83d1b8388 - 2938cfad9
refs/heads/master 8cbaa0576 - 46f76023b
ACCUMULO-3749 Sudo and then set ACCUMULO_CONF_DIR
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/4749513a
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/4749513a
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/4749513a
Branch: refs/heads/1.7
Commit: 4749513aed63d627c7c204fda7c336ebf1c5c3a7
Parents: 83d1b83
Author: Josh Elser els...@apache.org
Authored: Wed May 6 15:33:24 2015 -0400
Committer: Josh Elser els...@apache.org
Committed: Wed May 6 15:33:24 2015 -0400
--
.../accumulo/cluster/standalone/StandaloneClusterControl.java| 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4749513a/minicluster/src/main/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControl.java
--
diff --git
a/minicluster/src/main/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControl.java
b/minicluster/src/main/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControl.java
index 8779878..89027e7 100644
---
a/minicluster/src/main/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControl.java
+++
b/minicluster/src/main/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControl.java
@@ -140,7 +140,7 @@ public class StandaloneClusterControl implements
ClusterControl {
public void adminStopAll() throws IOException {
File confDir = getConfDir();
String master = getHosts(new File(confDir, masters)).get(0);
-String[] cmd = new String[] {ACCUMULO_CONF_DIR + accumuloConfDir,
SUDO_CMD, -u, user, accumuloPath, Admin.class.getName(), stopAll};
+String[] cmd = new String[] {SUDO_CMD, -u, user, ACCUMULO_CONF_DIR +
accumuloConfDir, accumuloPath, Admin.class.getName(), stopAll};
EntryInteger,String pair = exec(master, cmd);
if (0 != pair.getKey().intValue()) {
throw new IOException(stopAll did not finish successfully, retcode= +
pair.getKey() + , stdout= + pair.getValue());
@@ -193,7 +193,7 @@ public class StandaloneClusterControl implements
ClusterControl {
@Override
public void start(ServerType server, String hostname) throws IOException {
-String[] cmd = new String[] {ACCUMULO_CONF_DIR + accumuloConfDir,
SUDO_CMD, -u, user, startServerPath, hostname, getProcessString(server)};
+String[] cmd = new String[] {SUDO_CMD, -u, user, ACCUMULO_CONF_DIR +
accumuloConfDir, startServerPath, hostname, getProcessString(server)};
EntryInteger,String pair = exec(hostname, cmd);
if (0 != pair.getKey()) {
throw new IOException(Start + server + on + hostname + failed
for execute successfully);
[3/5] accumulo git commit: ACCUMULO-3772 GC must read both current_logs and walogs in ZK
ACCUMULO-3772 GC must read both current_logs and walogs in ZK
If the GC only reads walogs from ZK, it is possible that it will
delete WALs that are not also in the root or metadata table, preventing
the root table from performing recovery.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/2938cfad
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/2938cfad
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/2938cfad
Branch: refs/heads/master
Commit: 2938cfad91949b205e500bc73137e15e713c4262
Parents: 4749513
Author: Josh Elser els...@apache.org
Authored: Wed May 6 00:22:27 2015 -0400
Committer: Josh Elser els...@apache.org
Committed: Wed May 6 15:33:58 2015 -0400
--
.../gc/GarbageCollectWriteAheadLogs.java| 146 +++-
.../gc/GarbageCollectWriteAheadLogsTest.java| 168 +++
2 files changed, 276 insertions(+), 38 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/2938cfad/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
--
diff --git
a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
index a9a3f65..c880fef 100644
---
a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
+++
b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
@@ -32,6 +32,7 @@ import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.data.Key;
@@ -54,6 +55,7 @@ import org.apache.accumulo.core.trace.Span;
import org.apache.accumulo.core.trace.Trace;
import org.apache.accumulo.core.volume.Volume;
import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooReader;
import org.apache.accumulo.server.AccumuloServerContext;
import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.fs.VolumeManager;
@@ -76,6 +78,7 @@ import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Iterables;
import com.google.common.collect.Iterators;
import com.google.common.net.HostAndPort;
@@ -84,6 +87,10 @@ import com.google.protobuf.InvalidProtocolBufferException;
public class GarbageCollectWriteAheadLogs {
private static final Logger log =
LoggerFactory.getLogger(GarbageCollectWriteAheadLogs.class);
+ // The order of these is _very_ important. Must read from current_logs, then
walogs because ZooTabletStateStore writes to
+ // walogs and then removes from current_logs
+ private static final String[] ZK_LOG_SUFFIXES = new String[]
{RootTable.ZROOT_TABLET_CURRENT_LOGS, RootTable.ZROOT_TABLET_WALOGS};
+
private final AccumuloServerContext context;
private final VolumeManager fs;
private final boolean useTrash;
@@ -113,6 +120,26 @@ public class GarbageCollectWriteAheadLogs {
liveServers.startListeningForTabletServerChanges();
}
+ /**
+ * Creates a new GC WAL object. Meant for testing -- allows mocked objects.
+ *
+ * @param context
+ * the collection server's context
+ * @param fs
+ * volume manager to use
+ * @param useTrash
+ * true to move files to trash rather than delete them
+ * @param liveTServerSet
+ * a started LiveTServerSet instance
+ */
+ @VisibleForTesting
+ GarbageCollectWriteAheadLogs(AccumuloServerContext context, VolumeManager
fs, boolean useTrash, LiveTServerSet liveTServerSet) throws IOException {
+this.context = context;
+this.fs = fs;
+this.useTrash = useTrash;
+this.liveServers = liveTServerSet;
+ }
+
public void collect(GCStatus status) {
Span span = Trace.start(getCandidates);
@@ -375,9 +402,6 @@ public class GarbageCollectWriteAheadLogs {
return metaScanner;
}
-
-
-
/**
* Scans log markers. The map passed in is populated with the logs for dead
servers.
*
@@ -386,15 +410,9 @@ public class GarbageCollectWriteAheadLogs {
* @return total number of log files
*/
private long getCurrent(MapTServerInstance, SetPath unusedLogs,
SetTServerInstance currentServers) throws Exception {
-SetPath rootWALs = new HashSet();
-// Get entries in zookeeper:
-String zpath =
[5/5] accumulo git commit: Merge branch '1.7'
Merge branch '1.7' Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/46f76023 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/46f76023 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/46f76023 Branch: refs/heads/master Commit: 46f76023b93b23f27f55d9ac1833b0f7072609b4 Parents: 8cbaa05 2938cfa Author: Josh Elser els...@apache.org Authored: Wed May 6 16:25:34 2015 -0400 Committer: Josh Elser els...@apache.org Committed: Wed May 6 16:25:34 2015 -0400 -- .../standalone/StandaloneClusterControl.java| 4 +- .../gc/GarbageCollectWriteAheadLogs.java| 146 +++- .../gc/GarbageCollectWriteAheadLogsTest.java| 168 +++ 3 files changed, 278 insertions(+), 40 deletions(-) --
[1/3] accumulo git commit: ACCUMULO-3772 All ZK WAL refs contain LogEntry in the node.
Repository: accumulo
Updated Branches:
refs/heads/1.7 f8303008c - f55751bc0
refs/heads/master 5b7a14eb5 - 134300382
ACCUMULO-3772 All ZK WAL refs contain LogEntry in the node.
As bad as having two nodes for tracking WAL for the root
tablet, the data of current_logs and walogs in ZK varied.
Make them all the same value for consistency
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/f55751bc
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/f55751bc
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/f55751bc
Branch: refs/heads/1.7
Commit: f55751bc073d449099794fd4fc5cc7a3ce7ee388
Parents: f830300
Author: Josh Elser els...@apache.org
Authored: Thu May 7 00:20:26 2015 -0400
Committer: Josh Elser els...@apache.org
Committed: Thu May 7 00:42:37 2015 -0400
--
.../apache/accumulo/server/util/MetadataTableUtil.java | 13 +++--
.../apache/accumulo/test/functional/WALSunnyDayIT.java | 4 +++-
2 files changed, 14 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f55751bc/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
--
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
index 4470c55..c85cee7 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
@@ -1056,6 +1056,13 @@ public class MetadataTableUtil {
public static void addNewLogMarker(ClientContext context, ZooLock zooLock,
final TServerInstance tabletSession, final Path filename, TabletLevel level) {
log.debug(Adding log entry + filename);
if (level == TabletLevel.ROOT) {
+ LogEntry log = new LogEntry(RootTable.EXTENT,
System.currentTimeMillis(), tabletSession.hostPort(), filename.toString());
+ final byte[] node;
+ try {
+node = log.toBytes();
+ } catch (IOException e) {
+throw new RuntimeException(Failed to write to byte array, e);
+ }
retryZooKeeperUpdate(context, zooLock, new ZooOperation() {
@Override
public void run(IZooReaderWriter rw) throws KeeperException,
InterruptedException, IOException {
@@ -1066,7 +1073,7 @@ public class MetadataTableUtil {
path.append(CurrentLogsSection.getRowPrefix());
path.append(tabletSession.toString());
path.append(uniqueId);
- rw.putPersistentData(path.toString(),
filename.toString().getBytes(UTF_8), NodeExistsPolicy.OVERWRITE);
+ rw.putPersistentData(path.toString(), node,
NodeExistsPolicy.OVERWRITE);
}
});
} else {
@@ -1153,7 +1160,9 @@ public class MetadataTableUtil {
String root = ZooUtil.getRoot(HdfsZooInstance.getInstance()) +
RootTable.ZROOT_TABLET_CURRENT_LOGS;
logs.clear();
for (String child : rw.getChildren(root)) {
-logs.add(new Path(new String(rw.getData(root + / + child, null),
UTF_8)));
+byte[] data = rw.getData(root + / + child, null);
+LogEntry entry = LogEntry.fromBytes(data);
+logs.add(new Path(entry.filename));
}
}
});
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f55751bc/test/src/test/java/org/apache/accumulo/test/functional/WALSunnyDayIT.java
--
diff --git
a/test/src/test/java/org/apache/accumulo/test/functional/WALSunnyDayIT.java
b/test/src/test/java/org/apache/accumulo/test/functional/WALSunnyDayIT.java
index 9f22466..74c79a1 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/WALSunnyDayIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/WALSunnyDayIT.java
@@ -48,6 +48,7 @@ import org.apache.accumulo.core.metadata.RootTable;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.CurrentLogsSection;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily;
+import org.apache.accumulo.core.tabletserver.log.LogEntry;
import org.apache.accumulo.core.util.UtilWaitThread;
import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.master.state.SetGoalState;
@@ -213,7 +214,8 @@ public class WALSunnyDayIT extends ConfigurableMacIT {
ListString children = zoo.getChildren(zpath, null);
for (String child : children) {
byte[] data = zoo.getData(zpath + / + child, null, null);
- result.put(new String(data), true);
+ LogEntry entry =
