git commit: HBASE-11863 WAL files are not archived and stays in the WAL directory after splitting
Repository: hbase
Updated Branches:
refs/heads/0.98 8ac090c6d -> 7f28fcf42
HBASE-11863 WAL files are not archived and stays in the WAL directory after
splitting
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7f28fcf4
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7f28fcf4
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7f28fcf4
Branch: refs/heads/0.98
Commit: 7f28fcf429242c549219502bfb7da0ad28753f4c
Parents: 8ac090c
Author: Enis Soztutar
Authored: Tue Sep 2 18:24:15 2014 -0700
Committer: Enis Soztutar
Committed: Tue Sep 2 18:24:15 2014 -0700
--
.../hadoop/hbase/master/MasterFileSystem.java | 5 +-
.../hadoop/hbase/master/SplitLogManager.java| 43 --
.../master/TestDistributedLogSplitting.java | 19 ++---
.../hbase/master/TestSplitLogManager.java | 86 ++--
4 files changed, 88 insertions(+), 65 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/7f28fcf4/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
index 7a0a03f..842cd86 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java
@@ -54,7 +54,6 @@ import org.apache.hadoop.hbase.fs.HFileSystem;
import
org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.wal.HLog;
-import org.apache.hadoop.hbase.regionserver.wal.HLogSplitter;
import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
@@ -95,12 +94,14 @@ public class MasterFileSystem {
private final MasterServices services;
final static PathFilter META_FILTER = new PathFilter() {
+@Override
public boolean accept(Path p) {
return HLogUtil.isMetaFile(p);
}
};
final static PathFilter NON_META_FILTER = new PathFilter() {
+@Override
public boolean accept(Path p) {
return !HLogUtil.isMetaFile(p);
}
@@ -487,7 +488,7 @@ public class MasterFileSystem {
org.apache.hadoop.hbase.util.FSTableDescriptorMigrationToSubdir
.migrateFSTableDescriptorsIfNecessary(fs, rd);
}
-
+
// Create tableinfo-s for hbase:meta if not already there.
new FSTableDescriptors(fs,
rd).createTableDescriptor(HTableDescriptor.META_TABLEDESC);
http://git-wip-us.apache.org/repos/asf/hbase/blob/7f28fcf4/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
index 2e98433..192cd9e 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
@@ -78,6 +78,8 @@ import org.apache.zookeeper.KeeperException.NoNodeException;
import org.apache.zookeeper.ZooDefs.Ids;
import org.apache.zookeeper.data.Stat;
+import com.google.common.annotations.VisibleForTesting;
+
/**
* Distributes the task of log splitting to the available region servers.
* Coordination happens via zookeeper. For every log file that has to be split
a
@@ -156,26 +158,6 @@ public class SplitLogManager extends ZooKeeperListener {
* Wrapper around {@link #SplitLogManager(ZooKeeperWatcher zkw,
Configuration conf,
* Stoppable stopper, MasterServices master, ServerName serverName,
* boolean masterRecovery, TaskFinisher tf)}
- * with masterRecovery = false, and tf = null. Used in unit tests.
- *
- * @param zkw the ZK watcher
- * @param conf the HBase configuration
- * @param stopper the stoppable in case anything is wrong
- * @param master the master services
- * @param serverName the master server name
- * @throws KeeperException
- * @throws InterruptedIOException
- */
-public SplitLogManager(ZooKeeperWatcher zkw, final Configuration conf,
- Stoppable stopper, MasterServices master, ServerName serverName)
- throws InterruptedIOException, KeeperException {
-this(zkw, conf, stopper, master, serverName, false, null);
- }
-
- /**
- * Wrapper around {@link #SplitLogManager(ZooKeeperWatcher zkw,
Configuration conf,
- * Stoppable stopper
git commit: HBASE-11863 WAL files are not archived and stays in the WAL directory after splitting
Repository: hbase
Updated Branches:
refs/heads/branch-1 5e656f85d -> 4b2f43faf
HBASE-11863 WAL files are not archived and stays in the WAL directory after
splitting
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4b2f43fa
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4b2f43fa
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4b2f43fa
Branch: refs/heads/branch-1
Commit: 4b2f43fafb3528b05b92651435457cebde4c684d
Parents: 5e656f8
Author: Enis Soztutar
Authored: Tue Sep 2 18:11:38 2014 -0700
Committer: Enis Soztutar
Committed: Tue Sep 2 18:13:12 2014 -0700
--
.../hadoop/hbase/master/SplitLogManager.java| 7
.../master/TestDistributedLogSplitting.java | 4 ++
.../hbase/master/TestSplitLogManager.java | 42 +++-
3 files changed, 52 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/4b2f43fa/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
index a214111..b65b57e 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
@@ -78,6 +78,8 @@ import org.apache.zookeeper.KeeperException.NoNodeException;
import org.apache.zookeeper.ZooDefs.Ids;
import org.apache.zookeeper.data.Stat;
+import com.google.common.annotations.VisibleForTesting;
+
/**
* Distributes the task of log splitting to the available region servers.
* Coordination happens via zookeeper. For every log file that has to be split
a
@@ -437,6 +439,11 @@ public class SplitLogManager extends ZooKeeperListener {
}
}
+ @VisibleForTesting
+ ConcurrentMap getTasks() {
+return tasks;
+ }
+
private int activeTasks(final TaskBatch batch) {
int count = 0;
for (Task t: tasks.values()) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/4b2f43fa/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
index 1df81b7..0802dea 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
@@ -260,6 +260,10 @@ public class TestDistributedLogSplitting {
}
LOG.info(count + " edits in " + files.length + " recovered edits
files.");
}
+
+// check that the log file is moved
+assertFalse(fs.exists(logDir));
+
assertEquals(NUM_LOG_LINES, count);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/4b2f43fa/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
index ceb6ada..c88d2c7 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
@@ -40,6 +40,7 @@ import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.util.List;
+import java.util.Map;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicLong;
@@ -148,7 +149,7 @@ public class TestSplitLogManager {
@After
public void teardown() throws IOException, KeeperException {
stopper.stop("");
-slm.stop();
+if (slm != null) slm.stop();
TEST_UTIL.shutdownMiniZKCluster();
}
@@ -159,6 +160,7 @@ public class TestSplitLogManager {
private void waitForCounter(final AtomicLong ctr, long oldval, long newval,
long timems)
throws Exception {
Expr e = new Expr() {
+ @Override
public long eval() {
return ctr.get();
}
@@ -500,6 +502,44 @@ public class TestSplitLogManager {
assertFalse(fs.exists(emptyLogDirPath));
}
+ @Test (timeout = 6)
+ public void testLogFilesAreArchived() throws Exception {
+LOG.info("testLogFilesAreArchived");
+final SplitLogManager slm = new SplitLogManager(zkw, conf, stopper,
master, DUMMY_MASTER);
+FileSystem fs = TEST_UTIL.getTestFileSystem();
+Path dir = TEST_UTIL.g
git commit: HBASE-11863 WAL files are not archived and stays in the WAL directory after splitting
Repository: hbase
Updated Branches:
refs/heads/master b2d528aac -> bb1d95385
HBASE-11863 WAL files are not archived and stays in the WAL directory after
splitting
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/bb1d9538
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/bb1d9538
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/bb1d9538
Branch: refs/heads/master
Commit: bb1d95385ab6176493d4a3a4555e2b24933c4e3d
Parents: b2d528a
Author: Enis Soztutar
Authored: Tue Sep 2 17:52:18 2014 -0700
Committer: Enis Soztutar
Committed: Tue Sep 2 17:52:18 2014 -0700
--
.../hadoop/hbase/master/SplitLogManager.java| 7
.../master/TestDistributedLogSplitting.java | 4 ++
.../hbase/master/TestSplitLogManager.java | 42 +++-
3 files changed, 52 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/bb1d9538/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
index f684024..3b59509 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
@@ -64,6 +64,8 @@ import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.Threads;
+import com.google.common.annotations.VisibleForTesting;
+
/**
* Distributes the task of log splitting to the available region servers.
* Coordination happens via coordination engine. For every log file that has
to be split a
@@ -355,6 +357,11 @@ public class SplitLogManager {
}
}
+ @VisibleForTesting
+ ConcurrentMap getTasks() {
+return tasks;
+ }
+
private int activeTasks(final TaskBatch batch) {
int count = 0;
for (Task t : tasks.values()) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/bb1d9538/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
index 4506ce9..a78414a 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
@@ -260,6 +260,10 @@ public class TestDistributedLogSplitting {
}
LOG.info(count + " edits in " + files.length + " recovered edits
files.");
}
+
+// check that the log file is moved
+assertFalse(fs.exists(logDir));
+
assertEquals(NUM_LOG_LINES, count);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/bb1d9538/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
index 2aa4434..42a9b6b 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java
@@ -37,6 +37,7 @@ import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.util.List;
+import java.util.Map;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicLong;
@@ -216,7 +217,7 @@ public class TestSplitLogManager {
@After
public void teardown() throws IOException, KeeperException {
stopper.stop("");
-slm.stop();
+if (slm != null) slm.stop();
TEST_UTIL.shutdownMiniZKCluster();
}
@@ -227,6 +228,7 @@ public class TestSplitLogManager {
private void waitForCounter(final AtomicLong ctr, long oldval, long newval,
long timems)
throws Exception {
Expr e = new Expr() {
+ @Override
public long eval() {
return ctr.get();
}
@@ -567,6 +569,44 @@ public class TestSplitLogManager {
assertFalse(fs.exists(emptyLogDirPath));
}
+ @Test (timeout = 6)
+ public void testLogFilesAreArchived() throws Exception {
+LOG.info("testLogFilesAreArchived");
+final SplitLogManager slm = new SplitLogManager(ds, conf, stopper, master,
DUMMY_MASTER);
+FileSystem fs = TEST_UTIL.getTestFileSystem();
+Path dir = TEST_UTIL.getDataTestDirOnTestFS("testLo
