hbase git commit: HBASE-16464 archive folder grows bigger and bigger due to corrupt snapshot under tmp dir
Repository: hbase
Updated Branches:
refs/heads/branch-1.1 8eca263c4 -> bb7a054ec
HBASE-16464 archive folder grows bigger and bigger due to corrupt snapshot
under tmp dir
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/bb7a054e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/bb7a054e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/bb7a054e
Branch: refs/heads/branch-1.1
Commit: bb7a054ec69cc198dc9605928c9cc7ea0aa81418
Parents: 8eca263
Author: chenheng
Authored: Mon Aug 22 16:45:20 2016 +0800
Committer: chenheng
Committed: Wed Aug 24 13:51:08 2016 +0800
--
.../master/snapshot/SnapshotFileCache.java | 17 -
.../master/snapshot/TakeSnapshotHandler.java| 2 +-
.../snapshot/SnapshotDescriptionUtils.java | 14 ++
.../hbase/snapshot/SnapshotManifestV2.java | 18 --
.../snapshot/TestSnapshotHFileCleaner.java | 20
.../hbase/snapshot/SnapshotTestingUtils.java| 12
6 files changed, 75 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/bb7a054e/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
index dfd3cb5..5b367c5 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.snapshot.CorruptedSnapshotException;
import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
import org.apache.hadoop.hbase.util.FSUtils;
@@ -300,7 +301,21 @@ public class SnapshotFileCache implements Stoppable {
FileStatus[] running = FSUtils.listStatus(fs, snapshotTmpDir);
if (running != null) {
for (FileStatus run : running) {
-
snapshotInProgress.addAll(fileInspector.filesUnderSnapshot(run.getPath()));
+try {
+
snapshotInProgress.addAll(fileInspector.filesUnderSnapshot(run.getPath()));
+} catch (CorruptedSnapshotException e) {
+ // See HBASE-16464
+ if (e.getCause() instanceof FileNotFoundException) {
+// If the snapshot is not in progress, we will delete it
+if (!fs.exists(new Path(run.getPath(),
+ SnapshotDescriptionUtils.SNAPSHOT_IN_PROGRESS))) {
+ fs.delete(run.getPath(), true);
+ LOG.warn("delete the " + run.getPath() + " due to exception:",
e.getCause());
+}
+ } else {
+throw e;
+ }
+}
}
}
return snapshotInProgress;
http://git-wip-us.apache.org/repos/asf/hbase/blob/bb7a054e/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
index 5ac9cbc..d590798 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
@@ -162,7 +162,7 @@ public abstract class TakeSnapshotHandler extends
EventHandler implements Snapsh
try {
// If regions move after this meta scan, the region specific snapshot
should fail, triggering
// an external exception that gets captured here.
-
+ SnapshotDescriptionUtils.createInProgressTag(workingDir, fs);
// write down the snapshot info in the working directory
SnapshotDescriptionUtils.writeSnapshotInfo(snapshot, workingDir, fs);
snapshotManifest.addTableDescriptor(this.htd);
http://git-wip-us.apache.org/repos/asf/hbase/blob/bb7a054e/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
index 2211aab..28a9d05 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils
hbase git commit: HBASE-16464 archive folder grows bigger and bigger due to corrupt snapshot under tmp dir
Repository: hbase
Updated Branches:
refs/heads/branch-1.2 71e583df3 -> 09ade3208
HBASE-16464 archive folder grows bigger and bigger due to corrupt snapshot
under tmp dir
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/09ade320
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/09ade320
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/09ade320
Branch: refs/heads/branch-1.2
Commit: 09ade32086cdb3ba76587da773de007c17bdfc5c
Parents: 71e583d
Author: chenheng
Authored: Mon Aug 22 16:45:20 2016 +0800
Committer: chenheng
Committed: Wed Aug 24 13:48:14 2016 +0800
--
.../master/snapshot/SnapshotFileCache.java | 17 -
.../master/snapshot/TakeSnapshotHandler.java| 2 +-
.../snapshot/SnapshotDescriptionUtils.java | 14 ++
.../hbase/snapshot/SnapshotManifestV2.java | 18 --
.../snapshot/TestSnapshotHFileCleaner.java | 20
.../hbase/snapshot/SnapshotTestingUtils.java| 12
6 files changed, 75 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/09ade320/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
index dfd3cb5..5b367c5 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.snapshot.CorruptedSnapshotException;
import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
import org.apache.hadoop.hbase.util.FSUtils;
@@ -300,7 +301,21 @@ public class SnapshotFileCache implements Stoppable {
FileStatus[] running = FSUtils.listStatus(fs, snapshotTmpDir);
if (running != null) {
for (FileStatus run : running) {
-
snapshotInProgress.addAll(fileInspector.filesUnderSnapshot(run.getPath()));
+try {
+
snapshotInProgress.addAll(fileInspector.filesUnderSnapshot(run.getPath()));
+} catch (CorruptedSnapshotException e) {
+ // See HBASE-16464
+ if (e.getCause() instanceof FileNotFoundException) {
+// If the snapshot is not in progress, we will delete it
+if (!fs.exists(new Path(run.getPath(),
+ SnapshotDescriptionUtils.SNAPSHOT_IN_PROGRESS))) {
+ fs.delete(run.getPath(), true);
+ LOG.warn("delete the " + run.getPath() + " due to exception:",
e.getCause());
+}
+ } else {
+throw e;
+ }
+}
}
}
return snapshotInProgress;
http://git-wip-us.apache.org/repos/asf/hbase/blob/09ade320/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
index 7c87ea1..39387cb 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
@@ -164,7 +164,7 @@ public abstract class TakeSnapshotHandler extends
EventHandler implements Snapsh
try {
// If regions move after this meta scan, the region specific snapshot
should fail, triggering
// an external exception that gets captured here.
-
+ SnapshotDescriptionUtils.createInProgressTag(workingDir, fs);
// write down the snapshot info in the working directory
SnapshotDescriptionUtils.writeSnapshotInfo(snapshot, workingDir, fs);
snapshotManifest.addTableDescriptor(this.htd);
http://git-wip-us.apache.org/repos/asf/hbase/blob/09ade320/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
index b6b608c..aebb647 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils
hbase git commit: HBASE-16464 archive folder grows bigger and bigger due to corrupt snapshot under tmp dir
Repository: hbase
Updated Branches:
refs/heads/branch-1.3 cf3230468 -> 58891babc
HBASE-16464 archive folder grows bigger and bigger due to corrupt snapshot
under tmp dir
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/58891bab
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/58891bab
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/58891bab
Branch: refs/heads/branch-1.3
Commit: 58891babc853c75d067ff7e4d74a6801b2f41102
Parents: cf32304
Author: chenheng
Authored: Mon Aug 22 16:45:20 2016 +0800
Committer: chenheng
Committed: Wed Aug 24 13:45:52 2016 +0800
--
.../master/snapshot/SnapshotFileCache.java | 17 -
.../master/snapshot/TakeSnapshotHandler.java| 2 +-
.../snapshot/SnapshotDescriptionUtils.java | 14 ++
.../hbase/snapshot/SnapshotManifestV2.java | 18 --
.../snapshot/TestSnapshotHFileCleaner.java | 20
.../hbase/snapshot/SnapshotTestingUtils.java| 12
6 files changed, 75 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/58891bab/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
index dfd3cb5..5b367c5 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.snapshot.CorruptedSnapshotException;
import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
import org.apache.hadoop.hbase.util.FSUtils;
@@ -300,7 +301,21 @@ public class SnapshotFileCache implements Stoppable {
FileStatus[] running = FSUtils.listStatus(fs, snapshotTmpDir);
if (running != null) {
for (FileStatus run : running) {
-
snapshotInProgress.addAll(fileInspector.filesUnderSnapshot(run.getPath()));
+try {
+
snapshotInProgress.addAll(fileInspector.filesUnderSnapshot(run.getPath()));
+} catch (CorruptedSnapshotException e) {
+ // See HBASE-16464
+ if (e.getCause() instanceof FileNotFoundException) {
+// If the snapshot is not in progress, we will delete it
+if (!fs.exists(new Path(run.getPath(),
+ SnapshotDescriptionUtils.SNAPSHOT_IN_PROGRESS))) {
+ fs.delete(run.getPath(), true);
+ LOG.warn("delete the " + run.getPath() + " due to exception:",
e.getCause());
+}
+ } else {
+throw e;
+ }
+}
}
}
return snapshotInProgress;
http://git-wip-us.apache.org/repos/asf/hbase/blob/58891bab/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
index 7c87ea1..39387cb 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
@@ -164,7 +164,7 @@ public abstract class TakeSnapshotHandler extends
EventHandler implements Snapsh
try {
// If regions move after this meta scan, the region specific snapshot
should fail, triggering
// an external exception that gets captured here.
-
+ SnapshotDescriptionUtils.createInProgressTag(workingDir, fs);
// write down the snapshot info in the working directory
SnapshotDescriptionUtils.writeSnapshotInfo(snapshot, workingDir, fs);
snapshotManifest.addTableDescriptor(this.htd);
http://git-wip-us.apache.org/repos/asf/hbase/blob/58891bab/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
index b6b608c..aebb647 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils
hbase git commit: HBASE-16464 archive folder grows bigger and bigger due to corrupt snapshot under tmp dir
Repository: hbase
Updated Branches:
refs/heads/branch-1 73818646b -> de2f653ca
HBASE-16464 archive folder grows bigger and bigger due to corrupt snapshot
under tmp dir
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/de2f653c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/de2f653c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/de2f653c
Branch: refs/heads/branch-1
Commit: de2f653cafde0264ec0b8eeabafada8c5eef34ea
Parents: 7381864
Author: chenheng
Authored: Mon Aug 22 16:45:20 2016 +0800
Committer: chenheng
Committed: Wed Aug 24 13:43:18 2016 +0800
--
.../master/snapshot/SnapshotFileCache.java | 17 -
.../master/snapshot/TakeSnapshotHandler.java| 2 +-
.../snapshot/SnapshotDescriptionUtils.java | 14 ++
.../hbase/snapshot/SnapshotManifestV2.java | 18 --
.../snapshot/TestSnapshotHFileCleaner.java | 20
.../hbase/snapshot/SnapshotTestingUtils.java| 12
6 files changed, 75 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/de2f653c/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
index dfd3cb5..5b367c5 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.snapshot.CorruptedSnapshotException;
import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
import org.apache.hadoop.hbase.util.FSUtils;
@@ -300,7 +301,21 @@ public class SnapshotFileCache implements Stoppable {
FileStatus[] running = FSUtils.listStatus(fs, snapshotTmpDir);
if (running != null) {
for (FileStatus run : running) {
-
snapshotInProgress.addAll(fileInspector.filesUnderSnapshot(run.getPath()));
+try {
+
snapshotInProgress.addAll(fileInspector.filesUnderSnapshot(run.getPath()));
+} catch (CorruptedSnapshotException e) {
+ // See HBASE-16464
+ if (e.getCause() instanceof FileNotFoundException) {
+// If the snapshot is not in progress, we will delete it
+if (!fs.exists(new Path(run.getPath(),
+ SnapshotDescriptionUtils.SNAPSHOT_IN_PROGRESS))) {
+ fs.delete(run.getPath(), true);
+ LOG.warn("delete the " + run.getPath() + " due to exception:",
e.getCause());
+}
+ } else {
+throw e;
+ }
+}
}
}
return snapshotInProgress;
http://git-wip-us.apache.org/repos/asf/hbase/blob/de2f653c/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
index 7c87ea1..39387cb 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
@@ -164,7 +164,7 @@ public abstract class TakeSnapshotHandler extends
EventHandler implements Snapsh
try {
// If regions move after this meta scan, the region specific snapshot
should fail, triggering
// an external exception that gets captured here.
-
+ SnapshotDescriptionUtils.createInProgressTag(workingDir, fs);
// write down the snapshot info in the working directory
SnapshotDescriptionUtils.writeSnapshotInfo(snapshot, workingDir, fs);
snapshotManifest.addTableDescriptor(this.htd);
http://git-wip-us.apache.org/repos/asf/hbase/blob/de2f653c/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
index b6b608c..aebb647 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.jav
hbase git commit: HBASE-16464 archive folder grows bigger and bigger due to corrupt snapshot under tmp dir
Repository: hbase
Updated Branches:
refs/heads/master 97b164ac3 -> 3909b7c96
HBASE-16464 archive folder grows bigger and bigger due to corrupt snapshot
under tmp dir
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3909b7c9
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3909b7c9
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3909b7c9
Branch: refs/heads/master
Commit: 3909b7c96fc36e9fddef90fa7ade9ed61f19826d
Parents: 97b164a
Author: chenheng
Authored: Mon Aug 22 16:45:20 2016 +0800
Committer: chenheng
Committed: Wed Aug 24 13:38:32 2016 +0800
--
.../master/snapshot/SnapshotFileCache.java | 17 -
.../master/snapshot/TakeSnapshotHandler.java| 2 +-
.../snapshot/SnapshotDescriptionUtils.java | 14 ++
.../hbase/snapshot/SnapshotManifestV2.java | 18 --
.../snapshot/TestSnapshotHFileCleaner.java | 20
.../hbase/snapshot/SnapshotTestingUtils.java| 12
6 files changed, 75 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/3909b7c9/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
index dfd3cb5..5b367c5 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotFileCache.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.snapshot.CorruptedSnapshotException;
import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
import org.apache.hadoop.hbase.util.FSUtils;
@@ -300,7 +301,21 @@ public class SnapshotFileCache implements Stoppable {
FileStatus[] running = FSUtils.listStatus(fs, snapshotTmpDir);
if (running != null) {
for (FileStatus run : running) {
-
snapshotInProgress.addAll(fileInspector.filesUnderSnapshot(run.getPath()));
+try {
+
snapshotInProgress.addAll(fileInspector.filesUnderSnapshot(run.getPath()));
+} catch (CorruptedSnapshotException e) {
+ // See HBASE-16464
+ if (e.getCause() instanceof FileNotFoundException) {
+// If the snapshot is not in progress, we will delete it
+if (!fs.exists(new Path(run.getPath(),
+ SnapshotDescriptionUtils.SNAPSHOT_IN_PROGRESS))) {
+ fs.delete(run.getPath(), true);
+ LOG.warn("delete the " + run.getPath() + " due to exception:",
e.getCause());
+}
+ } else {
+throw e;
+ }
+}
}
}
return snapshotInProgress;
http://git-wip-us.apache.org/repos/asf/hbase/blob/3909b7c9/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
index 9172e06..8967a70 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
@@ -164,7 +164,7 @@ public abstract class TakeSnapshotHandler extends
EventHandler implements Snapsh
try {
// If regions move after this meta scan, the region specific snapshot
should fail, triggering
// an external exception that gets captured here.
-
+ SnapshotDescriptionUtils.createInProgressTag(workingDir, fs);
// write down the snapshot info in the working directory
SnapshotDescriptionUtils.writeSnapshotInfo(snapshot, workingDir, fs);
snapshotManifest.addTableDescriptor(this.htd);
http://git-wip-us.apache.org/repos/asf/hbase/blob/3909b7c9/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
index 560a42b..2fd619d 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java
++
[1/2] hbase git commit: HBASE-16451 Procedure v2 - Test WAL protobuf entry size limit
Repository: hbase
Updated Branches:
refs/heads/branch-1 6e9b49cac -> 73818646b
refs/heads/master 32c21f459 -> 97b164ac3
HBASE-16451 Procedure v2 - Test WAL protobuf entry size limit
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/73818646
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/73818646
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/73818646
Branch: refs/heads/branch-1
Commit: 73818646be6ac39aad1603e4d95766d8491cf0f0
Parents: 6e9b49c
Author: Matteo Bertozzi
Authored: Tue Aug 23 20:43:29 2016 -0700
Committer: Matteo Bertozzi
Committed: Tue Aug 23 20:52:40 2016 -0700
--
.../hadoop/hbase/procedure2/util/ByteSlot.java | 18 ++--
.../procedure2/ProcedureTestingUtility.java | 47 ++--
.../store/wal/TestStressWALProcedureStore.java | 15 +++
.../store/wal/TestWALProcedureStore.java| 14 ++
4 files changed, 77 insertions(+), 17 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/73818646/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/util/ByteSlot.java
--
diff --git
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/util/ByteSlot.java
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/util/ByteSlot.java
index 8904116..c4ed9b7 100644
---
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/util/ByteSlot.java
+++
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/util/ByteSlot.java
@@ -43,7 +43,9 @@ import
org.apache.hadoop.hbase.classification.InterfaceStability;
@InterfaceAudience.Private
@InterfaceStability.Evolving
public class ByteSlot extends OutputStream {
- private static final int DOUBLE_GROW_LIMIT = 1 << 20;
+ private static final int LARGE_GROW_SIZE_THRESHOLD = 8 << 20;
+ private static final int LARGE_GROW_SIZE = 1 << 20;
+ private static final int RESET_THRESHOLD = 64 << 20;
private static final int GROW_ALIGN = 128;
private byte[] buf;
@@ -51,6 +53,9 @@ public class ByteSlot extends OutputStream {
private int size;
public void reset() {
+if (buf != null && buf.length > RESET_THRESHOLD) {
+ buf = null;
+}
head = 0;
size = 0;
}
@@ -101,11 +106,16 @@ public class ByteSlot extends OutputStream {
if (buf == null) {
buf = new byte[minCapacity];
} else if (minCapacity > buf.length) {
- int newCapacity = buf.length << 1;
- if (minCapacity > newCapacity || newCapacity > DOUBLE_GROW_LIMIT) {
+ int newCapacity;
+ if (buf.length <= LARGE_GROW_SIZE_THRESHOLD) {
+newCapacity = buf.length << 1;
+ } else {
+newCapacity = buf.length + LARGE_GROW_SIZE;
+ }
+ if (minCapacity > newCapacity) {
newCapacity = minCapacity;
}
buf = Arrays.copyOf(buf, newCapacity);
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/73818646/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.java
--
diff --git
a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.java
b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.java
index 041461b..8b49e53 100644
---
a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.java
+++
b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.java
@@ -31,15 +31,16 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.ProcedureInfo;
-import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hbase.exceptions.IllegalArgumentIOException;
import org.apache.hadoop.hbase.exceptions.TimeoutIOException;
+import org.apache.hadoop.hbase.io.util.StreamUtils;
import org.apache.hadoop.hbase.procedure2.store.ProcedureStore;
import
org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureIterator;
import org.apache.hadoop.hbase.procedure2.store.NoopProcedureStore;
import org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore;
import
org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage;
import
org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureState;
+import org.apache.hadoop.hbase.util.Threads;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -89,6 +90,24 @@ public class ProcedureTestingUtility {
procExecutor.start(execThreads, failOnCorrupted);
}
+ public static void storeRestart(ProcedureStore procSt
[2/2] hbase git commit: HBASE-16451 Procedure v2 - Test WAL protobuf entry size limit
HBASE-16451 Procedure v2 - Test WAL protobuf entry size limit
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/97b164ac
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/97b164ac
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/97b164ac
Branch: refs/heads/master
Commit: 97b164ac3892991f23fe8a02581ef30336cf5909
Parents: 32c21f4
Author: Matteo Bertozzi
Authored: Tue Aug 23 20:43:29 2016 -0700
Committer: Matteo Bertozzi
Committed: Tue Aug 23 21:02:38 2016 -0700
--
.../hadoop/hbase/procedure2/util/ByteSlot.java | 18 ++--
.../procedure2/ProcedureTestingUtility.java | 47 ++--
.../store/wal/TestStressWALProcedureStore.java | 15 +++
.../store/wal/TestWALProcedureStore.java| 14 ++
4 files changed, 77 insertions(+), 17 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/97b164ac/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/util/ByteSlot.java
--
diff --git
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/util/ByteSlot.java
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/util/ByteSlot.java
index 8904116..c4ed9b7 100644
---
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/util/ByteSlot.java
+++
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/util/ByteSlot.java
@@ -43,7 +43,9 @@ import
org.apache.hadoop.hbase.classification.InterfaceStability;
@InterfaceAudience.Private
@InterfaceStability.Evolving
public class ByteSlot extends OutputStream {
- private static final int DOUBLE_GROW_LIMIT = 1 << 20;
+ private static final int LARGE_GROW_SIZE_THRESHOLD = 8 << 20;
+ private static final int LARGE_GROW_SIZE = 1 << 20;
+ private static final int RESET_THRESHOLD = 64 << 20;
private static final int GROW_ALIGN = 128;
private byte[] buf;
@@ -51,6 +53,9 @@ public class ByteSlot extends OutputStream {
private int size;
public void reset() {
+if (buf != null && buf.length > RESET_THRESHOLD) {
+ buf = null;
+}
head = 0;
size = 0;
}
@@ -101,11 +106,16 @@ public class ByteSlot extends OutputStream {
if (buf == null) {
buf = new byte[minCapacity];
} else if (minCapacity > buf.length) {
- int newCapacity = buf.length << 1;
- if (minCapacity > newCapacity || newCapacity > DOUBLE_GROW_LIMIT) {
+ int newCapacity;
+ if (buf.length <= LARGE_GROW_SIZE_THRESHOLD) {
+newCapacity = buf.length << 1;
+ } else {
+newCapacity = buf.length + LARGE_GROW_SIZE;
+ }
+ if (minCapacity > newCapacity) {
newCapacity = minCapacity;
}
buf = Arrays.copyOf(buf, newCapacity);
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/97b164ac/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.java
--
diff --git
a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.java
b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.java
index ceec855..7365de9 100644
---
a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.java
+++
b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.java
@@ -31,15 +31,16 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.ProcedureInfo;
-import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hbase.exceptions.IllegalArgumentIOException;
import org.apache.hadoop.hbase.exceptions.TimeoutIOException;
+import org.apache.hadoop.hbase.io.util.StreamUtils;
import org.apache.hadoop.hbase.procedure2.store.ProcedureStore;
import
org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureIterator;
import org.apache.hadoop.hbase.procedure2.store.NoopProcedureStore;
import org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore;
import
org.apache.hadoop.hbase.protobuf.generated.ErrorHandlingProtos.ForeignExceptionMessage;
import
org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureState;
+import org.apache.hadoop.hbase.util.Threads;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -89,6 +90,24 @@ public class ProcedureTestingUtility {
procExecutor.start(execThreads, failOnCorrupted);
}
+ public static void storeRestart(ProcedureStore procStore,
ProcedureStore.ProcedureLoader loader)
+ throws Exception {
+procStore.stop(false);
+procStore.start(procStor
[2/2] hbase git commit: HBASE-14921 In-Memory MemStore Flush and Compaction : Memory optimizations. (Anastasia)
HBASE-14921 In-Memory MemStore Flush and Compaction : Memory optimizations.
(Anastasia)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/32c21f45
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/32c21f45
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/32c21f45
Branch: refs/heads/master
Commit: 32c21f45942f9f45e34b28df47180fe6498fc337
Parents: cb02be3
Author: anoopsamjohn
Authored: Wed Aug 24 09:25:21 2016 +0530
Committer: anoopsamjohn
Committed: Wed Aug 24 09:25:21 2016 +0530
--
.../org/apache/hadoop/hbase/util/ClassSize.java | 26 +-
.../hbase/regionserver/AbstractMemStore.java| 8 +-
.../hadoop/hbase/regionserver/CellArrayMap.java | 54 +++
.../hadoop/hbase/regionserver/CellFlatMap.java | 473 +++
.../hadoop/hbase/regionserver/CellSet.java | 6 +-
.../hbase/regionserver/CompactingMemStore.java | 73 ++-
.../hbase/regionserver/CompactionPipeline.java | 47 +-
.../hbase/regionserver/DefaultMemStore.java | 10 +-
.../hbase/regionserver/ImmutableSegment.java| 178 ++-
.../hbase/regionserver/MemStoreCompactor.java | 279 +++
.../regionserver/MemStoreCompactorIterator.java | 160 +++
.../hbase/regionserver/MemStoreScanner.java | 68 ++-
.../hbase/regionserver/MutableSegment.java | 30 +-
.../hadoop/hbase/regionserver/Segment.java | 83 +++-
.../hbase/regionserver/SegmentFactory.java | 28 +-
.../hbase/regionserver/SegmentScanner.java | 8 +-
.../regionserver/VersionedSegmentsList.java | 15 +-
.../apache/hadoop/hbase/io/TestHeapSize.java| 17 +-
.../hbase/regionserver/TestCellFlatSet.java | 143 ++
.../regionserver/TestCompactingMemStore.java| 43 +-
.../TestCompactingToCellArrayMapMemStore.java | 361 ++
.../hadoop/hbase/regionserver/TestHRegion.java | 4 +-
.../TestHRegionWithInMemoryFlush.java | 94
.../TestWalAndCompactingMemStoreFlush.java | 276 ++-
hbase-shell/src/main/ruby/hbase/admin.rb| 3 +-
25 files changed, 2211 insertions(+), 276 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/32c21f45/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java
--
diff --git
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java
index 41c93ea..ff9dbcb 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java
@@ -29,6 +29,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
+
/**
* Class for determining the "size" of a class, an attempt to calculate the
* actual bytes that an object of this class will occupy in memory
@@ -81,6 +82,12 @@ public class ClassSize {
/** Overhead for ConcurrentSkipListMap Entry */
public static final int CONCURRENT_SKIPLISTMAP_ENTRY;
+ /** Overhead for CellArrayMap */
+ public static final int CELL_ARRAY_MAP;
+
+ /** Overhead for Cell Array Entry */
+ public static final int CELL_ARRAY_MAP_ENTRY;
+
/** Overhead for ReentrantReadWriteLock */
public static final int REENTRANT_LOCK;
@@ -106,7 +113,7 @@ public class ClassSize {
public static final int TIMERANGE_TRACKER;
/** Overhead for CellSkipListSet */
- public static final int CELL_SKIPLIST_SET;
+ public static final int CELL_SET;
public static final int STORE_SERVICES;
@@ -262,9 +269,20 @@ public class ClassSize {
// The size changes from jdk7 to jdk8, estimate the size rather than use a
conditional
CONCURRENT_SKIPLISTMAP = (int) estimateBase(ConcurrentSkipListMap.class,
false);
-CONCURRENT_SKIPLISTMAP_ENTRY =
+// CELL_ARRAY_MAP is the size of an instance of CellArrayMap class, which
extends
+// CellFlatMap class. CellArrayMap object containing a ref to an Array, so
+// OBJECT + REFERENCE + ARRAY
+// CellFlatMap object contains two integers, one boolean and one reference
to object, so
+// 2*INT + BOOLEAN + REFERENCE
+CELL_ARRAY_MAP = align(OBJECT + 2*Bytes.SIZEOF_INT + Bytes.SIZEOF_BOOLEAN
++ ARRAY + 2*REFERENCE);
+
+CONCURRENT_SKIPLISTMAP_ENTRY = align(
align(OBJECT + (3 * REFERENCE)) + /* one node per entry */
-align((OBJECT + (3 * REFERENCE))/2); /* one index per two entries */
+align((OBJECT + (3 * REFERENCE))/2)); /* one index per two entries */
+
+// REFERENCE in the CellArrayMap all the rest is counted in
KeyValue.heapSize()
+CELL_ARRAY_MAP_ENTRY = align(REFERENCE);
REENTRANT_LOCK = align(OBJECT + (3 * REF
[1/2] hbase git commit: HBASE-14921 In-Memory MemStore Flush and Compaction : Memory optimizations. (Anastasia)
Repository: hbase
Updated Branches:
refs/heads/master cb02be38a -> 32c21f459
http://git-wip-us.apache.org/repos/asf/hbase/blob/32c21f45/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentScanner.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentScanner.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentScanner.java
index 1191f30..8cf0a7c 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentScanner.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SegmentScanner.java
@@ -305,10 +305,6 @@ public class SegmentScanner implements KeyValueScanner {
// do nothing
}
- protected Segment getSegment(){
-return segment;
- }
-
//debug method
@Override
public String toString() {
@@ -320,6 +316,10 @@ public class SegmentScanner implements KeyValueScanner {
/* Private Methods **/
+ private Segment getSegment(){
+return segment;
+ }
+
/**
* Private internal method for iterating over the segment,
* skipping the cells with irrelevant MVCC
http://git-wip-us.apache.org/repos/asf/hbase/blob/32c21f45/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/VersionedSegmentsList.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/VersionedSegmentsList.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/VersionedSegmentsList.java
index 9d7a723..2e8bead 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/VersionedSegmentsList.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/VersionedSegmentsList.java
@@ -38,8 +38,7 @@ public class VersionedSegmentsList {
private final LinkedList storeSegments;
private final long version;
- public VersionedSegmentsList(
- LinkedList storeSegments, long version) {
+ public VersionedSegmentsList(LinkedList storeSegments,
long version) {
this.storeSegments = storeSegments;
this.version = version;
}
@@ -51,4 +50,16 @@ public class VersionedSegmentsList {
public long getVersion() {
return version;
}
+
+ public int getNumOfCells() {
+int totalCells = 0;
+for (ImmutableSegment s : storeSegments) {
+ totalCells += s.getCellsCount();
+}
+return totalCells;
+ }
+
+ public int getNumOfSegments() {
+return storeSegments.size();
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/32c21f45/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java
index 09e2271..e7d6661 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java
@@ -28,11 +28,7 @@ import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.io.hfile.BlockCacheKey;
import org.apache.hadoop.hbase.io.hfile.LruBlockCache;
import org.apache.hadoop.hbase.io.hfile.LruCachedBlock;
-import org.apache.hadoop.hbase.regionserver.CellSet;
-import org.apache.hadoop.hbase.regionserver.DefaultMemStore;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.HStore;
-import org.apache.hadoop.hbase.regionserver.TimeRangeTracker;
+import org.apache.hadoop.hbase.regionserver.*;
import org.apache.hadoop.hbase.testclassification.IOTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.util.ClassSize;
@@ -174,6 +170,15 @@ public class TestHeapSize {
assertEquals(expected, actual);
}
+// CellArrayMap
+cl = CellArrayMap.class;
+expected = ClassSize.estimateBase(cl, false);
+actual = ClassSize.CELL_ARRAY_MAP;
+if(expected != actual) {
+ ClassSize.estimateBase(cl, true);
+ assertEquals(expected, actual);
+}
+
// ReentrantReadWriteLock
cl = ReentrantReadWriteLock.class;
expected = ClassSize.estimateBase(cl, false);
@@ -240,7 +245,7 @@ public class TestHeapSize {
// CellSet
cl = CellSet.class;
expected = ClassSize.estimateBase(cl, false);
-actual = ClassSize.CELL_SKIPLIST_SET;
+actual = ClassSize.CELL_SET;
if (expected != actual) {
ClassSize.estimateBase(cl, true);
assertEquals(expected, actual);
http://git-wip-us.apache.org/repos/asf/hbase/blob/32c21f45/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellFlatSet.java
--
diff --git
a/hbase-server/src/test/java/org/apache/h
[2/3] hbase git commit: HBASE-16448 Custom metrics for custom replication endpoints
HBASE-16448 Custom metrics for custom replication endpoints
Signed-off-by: Andrew Purtell
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6e9b49ca
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6e9b49ca
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6e9b49ca
Branch: refs/heads/branch-1
Commit: 6e9b49cac7b118732659eef0cebb804be3e16238
Parents: 1e15fa5
Author: Geoffrey
Authored: Tue Aug 23 14:42:07 2016 -0700
Committer: Andrew Purtell
Committed: Tue Aug 23 17:17:42 2016 -0700
--
.../MetricsReplicationSourceSource.java | 4 +-
.../MetricsReplicationGlobalSourceSource.java | 56 ++
.../MetricsReplicationSourceSourceImpl.java | 80 +---
.../replication/regionserver/MetricsSource.java | 78 ++-
.../replication/TestReplicationEndpoint.java| 67 +++-
5 files changed, 269 insertions(+), 16 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/6e9b49ca/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSource.java
--
diff --git
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSource.java
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSource.java
index 271f0ac..c877608 100644
---
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSource.java
+++
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSource.java
@@ -18,7 +18,9 @@
package org.apache.hadoop.hbase.replication.regionserver;
-public interface MetricsReplicationSourceSource {
+import org.apache.hadoop.hbase.metrics.BaseSource;
+
+public interface MetricsReplicationSourceSource extends BaseSource {
public static final String SOURCE_SIZE_OF_LOG_QUEUE =
"source.sizeOfLogQueue";
public static final String SOURCE_AGE_OF_LAST_SHIPPED_OP =
"source.ageOfLastShippedOp";
http://git-wip-us.apache.org/repos/asf/hbase/blob/6e9b49ca/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java
--
diff --git
a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java
b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java
index 476d2f7..d595ca9 100644
---
a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java
+++
b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java
@@ -143,4 +143,60 @@ public class MetricsReplicationGlobalSourceSource
implements MetricsReplicationS
public int getSizeOfLogQueue() {
return (int)sizeOfLogQueueGauge.value();
}
+
+
+ @Override
+ public void init() {
+rms.init();
+ }
+
+ @Override
+ public void setGauge(String gaugeName, long value) {
+rms.setGauge(gaugeName, value);
+ }
+
+ @Override
+ public void incGauge(String gaugeName, long delta) {
+rms.incGauge(gaugeName, delta);
+ }
+
+ @Override
+ public void decGauge(String gaugeName, long delta) {
+rms.decGauge(gaugeName, delta);
+ }
+
+ @Override
+ public void removeMetric(String key) {
+rms.removeMetric(key);
+ }
+
+ @Override
+ public void incCounters(String counterName, long delta) {
+rms.incCounters(counterName, delta);
+ }
+
+ @Override
+ public void updateHistogram(String name, long value) {
+rms.updateHistogram(name, value);
+ }
+
+ @Override
+ public String getMetricsContext() {
+return rms.getMetricsContext();
+ }
+
+ @Override
+ public String getMetricsDescription() {
+return rms.getMetricsDescription();
+ }
+
+ @Override
+ public String getMetricsJmxContext() {
+return rms.getMetricsJmxContext();
+ }
+
+ @Override
+ public String getMetricsName() {
+return rms.getMetricsName();
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/6e9b49ca/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java
--
diff --git
a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java
b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java
[1/3] hbase git commit: HBASE-16448 Custom metrics for custom replication endpoints
Repository: hbase
Updated Branches:
refs/heads/0.98 ce58f5890 -> aac4e0951
refs/heads/branch-1 1e15fa57d -> 6e9b49cac
refs/heads/master 91fee265c -> cb02be38a
HBASE-16448 Custom metrics for custom replication endpoints
Signed-off-by: Andrew Purtell
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/cb02be38
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/cb02be38
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/cb02be38
Branch: refs/heads/master
Commit: cb02be38ab20ec5343fc9a7450bed33461e38f10
Parents: 91fee26
Author: Geoffrey
Authored: Thu Aug 18 14:24:10 2016 -0700
Committer: Andrew Purtell
Committed: Tue Aug 23 17:17:08 2016 -0700
--
.../MetricsReplicationSourceSource.java | 4 +-
.../MetricsReplicationGlobalSourceSource.java | 56 ++
.../MetricsReplicationSourceSourceImpl.java | 80 +---
.../replication/regionserver/MetricsSource.java | 78 ++-
.../replication/TestReplicationEndpoint.java| 67 +++-
5 files changed, 269 insertions(+), 16 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/cb02be38/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSource.java
--
diff --git
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSource.java
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSource.java
index 271f0ac..c877608 100644
---
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSource.java
+++
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSource.java
@@ -18,7 +18,9 @@
package org.apache.hadoop.hbase.replication.regionserver;
-public interface MetricsReplicationSourceSource {
+import org.apache.hadoop.hbase.metrics.BaseSource;
+
+public interface MetricsReplicationSourceSource extends BaseSource {
public static final String SOURCE_SIZE_OF_LOG_QUEUE =
"source.sizeOfLogQueue";
public static final String SOURCE_AGE_OF_LAST_SHIPPED_OP =
"source.ageOfLastShippedOp";
http://git-wip-us.apache.org/repos/asf/hbase/blob/cb02be38/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java
--
diff --git
a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java
b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java
index 476d2f7..d595ca9 100644
---
a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java
+++
b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java
@@ -143,4 +143,60 @@ public class MetricsReplicationGlobalSourceSource
implements MetricsReplicationS
public int getSizeOfLogQueue() {
return (int)sizeOfLogQueueGauge.value();
}
+
+
+ @Override
+ public void init() {
+rms.init();
+ }
+
+ @Override
+ public void setGauge(String gaugeName, long value) {
+rms.setGauge(gaugeName, value);
+ }
+
+ @Override
+ public void incGauge(String gaugeName, long delta) {
+rms.incGauge(gaugeName, delta);
+ }
+
+ @Override
+ public void decGauge(String gaugeName, long delta) {
+rms.decGauge(gaugeName, delta);
+ }
+
+ @Override
+ public void removeMetric(String key) {
+rms.removeMetric(key);
+ }
+
+ @Override
+ public void incCounters(String counterName, long delta) {
+rms.incCounters(counterName, delta);
+ }
+
+ @Override
+ public void updateHistogram(String name, long value) {
+rms.updateHistogram(name, value);
+ }
+
+ @Override
+ public String getMetricsContext() {
+return rms.getMetricsContext();
+ }
+
+ @Override
+ public String getMetricsDescription() {
+return rms.getMetricsDescription();
+ }
+
+ @Override
+ public String getMetricsJmxContext() {
+return rms.getMetricsJmxContext();
+ }
+
+ @Override
+ public String getMetricsName() {
+return rms.getMetricsName();
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/cb02be38/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSourceImpl.java
--
diff --git
a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Me
[3/3] hbase git commit: HBASE-16448 Custom metrics for custom replication endpoints
HBASE-16448 Custom metrics for custom replication endpoints
Signed-off-by: Andrew Purtell
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/aac4e095
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/aac4e095
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/aac4e095
Branch: refs/heads/0.98
Commit: aac4e09514b8b7dcf9d253d5722d0fe813973b99
Parents: ce58f58
Author: Geoffrey
Authored: Tue Aug 23 16:49:24 2016 -0700
Committer: Andrew Purtell
Committed: Tue Aug 23 17:18:00 2016 -0700
--
.../MetricsReplicationSourceSource.java | 4 +-
.../MetricsReplicationGlobalSourceSource.java | 64 ++-
.../MetricsReplicationSourceSourceImpl.java | 81 ---
.../replication/regionserver/MetricsSource.java | 85 +++-
.../replication/TestReplicationEndpoint.java| 76 -
5 files changed, 293 insertions(+), 17 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/aac4e095/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSource.java
--
diff --git
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSource.java
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSource.java
index 8611e15..ea0ae20 100644
---
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSource.java
+++
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSourceSource.java
@@ -18,7 +18,9 @@
package org.apache.hadoop.hbase.replication.regionserver;
-public interface MetricsReplicationSourceSource {
+import org.apache.hadoop.hbase.metrics.BaseSource;
+
+public interface MetricsReplicationSourceSource extends BaseSource {
public static final String SOURCE_SIZE_OF_LOG_QUEUE =
"source.sizeOfLogQueue";
public static final String SOURCE_AGE_OF_LAST_SHIPPED_OP =
"source.ageOfLastShippedOp";
http://git-wip-us.apache.org/repos/asf/hbase/blob/aac4e095/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java
--
diff --git
a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java
b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java
index b3e1766..da1bcf4 100644
---
a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java
+++
b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationGlobalSourceSource.java
@@ -22,7 +22,7 @@ import org.apache.hadoop.metrics2.lib.MutableCounterLong;
import org.apache.hadoop.metrics2.lib.MutableGaugeLong;
public class MetricsReplicationGlobalSourceSource implements
MetricsReplicationSourceSource {
-
+ private final MetricsReplicationSource rms;
private final MutableGaugeLong ageOfLastShippedOpGauge;
private final MutableGaugeLong sizeOfLogQueueGauge;
private final MutableCounterLong logReadInEditsCounter;
@@ -35,7 +35,7 @@ public class MetricsReplicationGlobalSourceSource implements
MetricsReplicationS
private final MutableCounterLong logReadInBytesCounter;
public MetricsReplicationGlobalSourceSource(MetricsReplicationSourceImpl
rms) {
-
+this.rms = rms;
ageOfLastShippedOpGauge =
rms.getMetricsRegistry().getLongGauge(SOURCE_AGE_OF_LAST_SHIPPED_OP, 0L);
sizeOfLogQueueGauge =
rms.getMetricsRegistry().getLongGauge(SOURCE_SIZE_OF_LOG_QUEUE, 0L);
@@ -118,4 +118,64 @@ public class MetricsReplicationGlobalSourceSource
implements MetricsReplicationS
public long getLastShippedAge() {
return ageOfLastShippedOpGauge.value();
}
+
+ @Override
+ public void init() {
+rms.init();
+ }
+
+ @Override
+ public void setGauge(String gaugeName, long value) {
+rms.setGauge(gaugeName, value);
+ }
+
+ @Override
+ public void incGauge(String gaugeName, long delta) {
+rms.incGauge(gaugeName, delta);
+ }
+
+ @Override
+ public void decGauge(String gaugeName, long delta) {
+rms.decGauge(gaugeName, delta);
+ }
+
+ @Override
+ public void removeMetric(String key) {
+rms.removeMetric(key);
+ }
+
+ @Override
+ public void incCounters(String counterName, long delta) {
+rms.incCounters(counterName, delta);
+ }
+
+ @Override
+ public void updateHistogram(String name, long value) {
+rms.updateHistogram(name, value)
[3/3] hbase git commit: HBASE-16485 Procedure v2 - Add support to addChildProcedure() as last "step" in StateMachineProcedure
HBASE-16485 Procedure v2 - Add support to addChildProcedure() as last "step" in
StateMachineProcedure
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1e15fa57
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1e15fa57
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1e15fa57
Branch: refs/heads/branch-1
Commit: 1e15fa57df5ee11ffce3f1e4052ce75f61e2c774
Parents: a1e57d9
Author: Matteo Bertozzi
Authored: Tue Aug 23 16:42:57 2016 -0700
Committer: Matteo Bertozzi
Committed: Tue Aug 23 16:54:22 2016 -0700
--
.../hbase/procedure2/ProcedureExecutor.java | 6 +
.../hbase/procedure2/StateMachineProcedure.java | 25 +---
.../hbase/procedure2/TestProcedureRecovery.java | 14 +++
3 files changed, 32 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/1e15fa57/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
--
diff --git
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
index c4f06f1..c195f65 100644
---
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
+++
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hbase.procedure2;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import java.io.IOException;
@@ -1288,6 +1289,11 @@ public class ProcedureExecutor {
return procId;
}
+ @VisibleForTesting
+ protected long getLastProcId() {
+return lastProcId.get();
+ }
+
private Long getRootProcedureId(Procedure proc) {
return Procedure.getRootProcedureId(procedures, proc);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/1e15fa57/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/StateMachineProcedure.java
--
diff --git
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/StateMachineProcedure.java
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/StateMachineProcedure.java
index c2b4548..47eb09e 100644
---
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/StateMachineProcedure.java
+++
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/StateMachineProcedure.java
@@ -43,6 +43,7 @@ import
org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.StateMachinePr
@InterfaceStability.Evolving
public abstract class StateMachineProcedure
extends Procedure {
+ private Flow stateFlow = Flow.HAS_MORE_STATE;
private int stateCount = 0;
private int[] states = null;
@@ -114,11 +115,13 @@ public abstract class StateMachineProcedure
* Add a child procedure to execute
* @param subProcedure the child procedure
*/
- protected void addChildProcedure(Procedure subProcedure) {
+ protected void addChildProcedure(Procedure... subProcedure) {
if (subProcList == null) {
- subProcList = new ArrayList();
+ subProcList = new ArrayList(subProcedure.length);
+}
+for (int i = 0; i < subProcedure.length; ++i) {
+ subProcList.add(subProcedure[i]);
}
-subProcList.add(subProcedure);
}
@Override
@@ -126,14 +129,14 @@ public abstract class StateMachineProcedure
throws ProcedureYieldException, InterruptedException {
updateTimestamp();
try {
+ if (!hasMoreState()) return null;
+
TState state = getCurrentState();
if (stateCount == 0) {
setNextState(getStateId(state));
}
- if (executeFromState(env, state) == Flow.NO_MORE_STATE) {
-// completed
-return null;
- }
+
+ stateFlow = executeFromState(env, state);
if (subProcList != null && subProcList.size() != 0) {
Procedure[] subProcedures = subProcList.toArray(new
Procedure[subProcList.size()]);
@@ -141,7 +144,7 @@ public abstract class StateMachineProcedure
return subProcedures;
}
- return (isWaiting() || isFailed()) ? null : new Procedure[] {this};
+ return (isWaiting() || isFailed() || !hasMoreState()) ? null : new
Procedure[] {this};
} finally {
updateTimestamp();
}
@@ -164,6 +167,10 @@ public abstract class StateMachineProcedure
return isYieldBeforeExecuteFromState(env, getCurrentState());
}
+ private boolean hasMoreState() {
+return stateFlow != Flow.NO_MORE_STATE;
+ }
+
private TState getCurrentState() {
return stateCount > 0 ? getState(states[stateCount-1]) : getInitialState();
}
@@ -
[2/3] hbase git commit: HBASE-16485 Procedure v2 - Add support to addChildProcedure() as last "step" in StateMachineProcedure
HBASE-16485 Procedure v2 - Add support to addChildProcedure() as last "step" in
StateMachineProcedure
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/cf323046
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/cf323046
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/cf323046
Branch: refs/heads/branch-1.3
Commit: cf323046868cd8534c6b4bc74bc2943239b304dc
Parents: 6c5d0cf
Author: Matteo Bertozzi
Authored: Tue Aug 23 16:42:57 2016 -0700
Committer: Matteo Bertozzi
Committed: Tue Aug 23 16:52:25 2016 -0700
--
.../hbase/procedure2/ProcedureExecutor.java | 10 ++--
.../hbase/procedure2/StateMachineProcedure.java | 25 +---
.../hbase/procedure2/TestProcedureRecovery.java | 14 +++
3 files changed, 34 insertions(+), 15 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/cf323046/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
--
diff --git
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
index 37bcb00..64e3eae 100644
---
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
+++
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
@@ -18,6 +18,9 @@
package org.apache.hadoop.hbase.procedure2;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@@ -55,8 +58,6 @@ import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.NonceKey;
import org.apache.hadoop.hbase.util.Pair;
-import com.google.common.base.Preconditions;
-
/**
* Thread Pool that executes the submitted procedures.
* The executor has a ProcedureStore associated.
@@ -1259,6 +1260,11 @@ public class ProcedureExecutor {
return procId;
}
+ @VisibleForTesting
+ protected long getLastProcId() {
+return lastProcId.get();
+ }
+
private Long getRootProcedureId(Procedure proc) {
return Procedure.getRootProcedureId(procedures, proc);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/cf323046/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/StateMachineProcedure.java
--
diff --git
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/StateMachineProcedure.java
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/StateMachineProcedure.java
index c2b4548..47eb09e 100644
---
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/StateMachineProcedure.java
+++
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/StateMachineProcedure.java
@@ -43,6 +43,7 @@ import
org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.StateMachinePr
@InterfaceStability.Evolving
public abstract class StateMachineProcedure
extends Procedure {
+ private Flow stateFlow = Flow.HAS_MORE_STATE;
private int stateCount = 0;
private int[] states = null;
@@ -114,11 +115,13 @@ public abstract class StateMachineProcedure
* Add a child procedure to execute
* @param subProcedure the child procedure
*/
- protected void addChildProcedure(Procedure subProcedure) {
+ protected void addChildProcedure(Procedure... subProcedure) {
if (subProcList == null) {
- subProcList = new ArrayList();
+ subProcList = new ArrayList(subProcedure.length);
+}
+for (int i = 0; i < subProcedure.length; ++i) {
+ subProcList.add(subProcedure[i]);
}
-subProcList.add(subProcedure);
}
@Override
@@ -126,14 +129,14 @@ public abstract class StateMachineProcedure
throws ProcedureYieldException, InterruptedException {
updateTimestamp();
try {
+ if (!hasMoreState()) return null;
+
TState state = getCurrentState();
if (stateCount == 0) {
setNextState(getStateId(state));
}
- if (executeFromState(env, state) == Flow.NO_MORE_STATE) {
-// completed
-return null;
- }
+
+ stateFlow = executeFromState(env, state);
if (subProcList != null && subProcList.size() != 0) {
Procedure[] subProcedures = subProcList.toArray(new
Procedure[subProcList.size()]);
@@ -141,7 +144,7 @@ public abstract class StateMachineProcedure
return subProcedures;
}
- return (isWaiting() || isFailed()) ? null : new Procedure[] {this};
+ return (isWaiting() || isFailed() || !hasMoreState()) ? null : new
Procedure[] {this};
} final
[1/3] hbase git commit: HBASE-16485 Procedure v2 - Add support to addChildProcedure() as last "step" in StateMachineProcedure
Repository: hbase
Updated Branches:
refs/heads/branch-1 a1e57d942 -> 1e15fa57d
refs/heads/branch-1.3 6c5d0cf23 -> cf3230468
refs/heads/master cb507b8cf -> 91fee265c
HBASE-16485 Procedure v2 - Add support to addChildProcedure() as last "step" in
StateMachineProcedure
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/91fee265
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/91fee265
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/91fee265
Branch: refs/heads/master
Commit: 91fee265cfdd59f49fc0e3132af74bdae58f6f8a
Parents: cb507b8
Author: Matteo Bertozzi
Authored: Tue Aug 23 16:42:57 2016 -0700
Committer: Matteo Bertozzi
Committed: Tue Aug 23 16:42:57 2016 -0700
--
.../hbase/procedure2/ProcedureExecutor.java | 6 +
.../hbase/procedure2/StateMachineProcedure.java | 27
.../hbase/procedure2/TestProcedureRecovery.java | 14 +++---
3 files changed, 33 insertions(+), 14 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/91fee265/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
--
diff --git
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
index 198623d..f714260 100644
---
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
+++
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hbase.procedure2;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import java.io.IOException;
@@ -1290,6 +1291,11 @@ public class ProcedureExecutor {
return procId;
}
+ @VisibleForTesting
+ protected long getLastProcId() {
+return lastProcId.get();
+ }
+
private Long getRootProcedureId(Procedure proc) {
return Procedure.getRootProcedureId(procedures, proc);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/91fee265/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/StateMachineProcedure.java
--
diff --git
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/StateMachineProcedure.java
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/StateMachineProcedure.java
index c2b4548..a403193 100644
---
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/StateMachineProcedure.java
+++
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/StateMachineProcedure.java
@@ -43,6 +43,7 @@ import
org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.StateMachinePr
@InterfaceStability.Evolving
public abstract class StateMachineProcedure
extends Procedure {
+ private Flow stateFlow = Flow.HAS_MORE_STATE;
private int stateCount = 0;
private int[] states = null;
@@ -60,7 +61,7 @@ public abstract class StateMachineProcedure
* Flow.HAS_MORE_STATE if there is another step.
*/
protected abstract Flow executeFromState(TEnvironment env, TState state)
-throws ProcedureYieldException, InterruptedException;
+throws ProcedureSuspendedException, ProcedureYieldException,
InterruptedException;
/**
* called to perform the rollback of the specified state
@@ -114,26 +115,28 @@ public abstract class StateMachineProcedure
* Add a child procedure to execute
* @param subProcedure the child procedure
*/
- protected void addChildProcedure(Procedure subProcedure) {
+ protected void addChildProcedure(Procedure... subProcedure) {
if (subProcList == null) {
- subProcList = new ArrayList();
+ subProcList = new ArrayList(subProcedure.length);
+}
+for (int i = 0; i < subProcedure.length; ++i) {
+ subProcList.add(subProcedure[i]);
}
-subProcList.add(subProcedure);
}
@Override
protected Procedure[] execute(final TEnvironment env)
- throws ProcedureYieldException, InterruptedException {
+ throws ProcedureSuspendedException, ProcedureYieldException,
InterruptedException {
updateTimestamp();
try {
+ if (!hasMoreState()) return null;
+
TState state = getCurrentState();
if (stateCount == 0) {
setNextState(getStateId(state));
}
- if (executeFromState(env, state) == Flow.NO_MORE_STATE) {
-// completed
-return null;
- }
+
+ stateFlow = executeFromState(env, state);
if (subProcList != null && subProcList.size() != 0) {
Procedure[] subProcedures = subProcList.toArray(new
Procedure[subProcList.size()]);
@@ -14
[5/6] hbase git commit: HBASE-16360 TableMapReduceUtil addHBaseDependencyJars has the wrong class name for PrefixTreeCodec (Jing Pu Chen)
HBASE-16360 TableMapReduceUtil addHBaseDependencyJars has the wrong class name
for PrefixTreeCodec (Jing Pu Chen)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8eca263c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8eca263c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8eca263c
Branch: refs/heads/branch-1.1
Commit: 8eca263c43fb920aa073e07612628a5365f18cc6
Parents: 5317c7b
Author: Matteo Bertozzi
Authored: Tue Aug 23 14:22:20 2016 -0700
Committer: Matteo Bertozzi
Committed: Tue Aug 23 14:37:12 2016 -0700
--
.../java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/8eca263c/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
index ce273f5..0749bb7 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
@@ -722,7 +722,7 @@ public class TableMapReduceUtil {
Class prefixTreeCodecClass = null;
try {
prefixTreeCodecClass =
-
Class.forName("org.apache.hadoop.hbase.code.prefixtree.PrefixTreeCodec");
+
Class.forName("org.apache.hadoop.hbase.codec.prefixtree.PrefixTreeCodec");
} catch (ClassNotFoundException e) {
// this will show up in unit tests but should not show in real
deployments
LOG.warn("The hbase-prefix-tree module jar containing PrefixTreeCodec is
not present." +
[4/6] hbase git commit: HBASE-16360 TableMapReduceUtil addHBaseDependencyJars has the wrong class name for PrefixTreeCodec (Jing Pu Chen)
HBASE-16360 TableMapReduceUtil addHBaseDependencyJars has the wrong class name
for PrefixTreeCodec (Jing Pu Chen)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/71e583df
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/71e583df
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/71e583df
Branch: refs/heads/branch-1.2
Commit: 71e583df30840e2b4319d850d4647dec262aaada
Parents: 4ef97c3
Author: Matteo Bertozzi
Authored: Tue Aug 23 14:22:20 2016 -0700
Committer: Matteo Bertozzi
Committed: Tue Aug 23 14:35:57 2016 -0700
--
.../java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/71e583df/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
index 8cad7ab..bbcf99f 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
@@ -771,7 +771,7 @@ public class TableMapReduceUtil {
Class prefixTreeCodecClass = null;
try {
prefixTreeCodecClass =
-
Class.forName("org.apache.hadoop.hbase.code.prefixtree.PrefixTreeCodec");
+
Class.forName("org.apache.hadoop.hbase.codec.prefixtree.PrefixTreeCodec");
} catch (ClassNotFoundException e) {
// this will show up in unit tests but should not show in real
deployments
LOG.warn("The hbase-prefix-tree module jar containing PrefixTreeCodec is
not present." +
[6/6] hbase git commit: HBASE-16360 TableMapReduceUtil addHBaseDependencyJars has the wrong class name for PrefixTreeCodec (Jing Pu Chen)
HBASE-16360 TableMapReduceUtil addHBaseDependencyJars has the wrong class name
for PrefixTreeCodec (Jing Pu Chen)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ce58f589
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ce58f589
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ce58f589
Branch: refs/heads/0.98
Commit: ce58f589025ccaa4386d70e616369c748a264b81
Parents: ac3ec63
Author: Matteo Bertozzi
Authored: Tue Aug 23 14:22:20 2016 -0700
Committer: Matteo Bertozzi
Committed: Tue Aug 23 14:38:58 2016 -0700
--
.../java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/ce58f589/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
index e051ea2..bf4c84e 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
@@ -752,7 +752,7 @@ public class TableMapReduceUtil {
Class prefixTreeCodecClass = null;
try {
prefixTreeCodecClass =
-
Class.forName("org.apache.hadoop.hbase.code.prefixtree.PrefixTreeCodec");
+
Class.forName("org.apache.hadoop.hbase.codec.prefixtree.PrefixTreeCodec");
} catch (ClassNotFoundException e) {
// this will show up in unit tests but should not show in real
deployments
LOG.warn("The hbase-prefix-tree module jar containing PrefixTreeCodec is
not present." +
[2/6] hbase git commit: HBASE-16360 TableMapReduceUtil addHBaseDependencyJars has the wrong class name for PrefixTreeCodec (Jing Pu Chen)
HBASE-16360 TableMapReduceUtil addHBaseDependencyJars has the wrong class name
for PrefixTreeCodec (Jing Pu Chen)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a1e57d94
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a1e57d94
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a1e57d94
Branch: refs/heads/branch-1
Commit: a1e57d942e55378745a5fbd9c1f9ea5fad5e2485
Parents: e7cd7ff
Author: Matteo Bertozzi
Authored: Tue Aug 23 14:22:20 2016 -0700
Committer: Matteo Bertozzi
Committed: Tue Aug 23 14:31:58 2016 -0700
--
.../java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/a1e57d94/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
index a939ef3..9ba1f07 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
@@ -771,7 +771,7 @@ public class TableMapReduceUtil {
Class prefixTreeCodecClass = null;
try {
prefixTreeCodecClass =
-
Class.forName("org.apache.hadoop.hbase.code.prefixtree.PrefixTreeCodec");
+
Class.forName("org.apache.hadoop.hbase.codec.prefixtree.PrefixTreeCodec");
} catch (ClassNotFoundException e) {
// this will show up in unit tests but should not show in real
deployments
LOG.warn("The hbase-prefix-tree module jar containing PrefixTreeCodec is
not present." +
[1/6] hbase git commit: HBASE-16360 TableMapReduceUtil addHBaseDependencyJars has the wrong class name for PrefixTreeCodec (Jing Pu Chen)
Repository: hbase
Updated Branches:
refs/heads/0.98 ac3ec6333 -> ce58f5890
refs/heads/branch-1 e7cd7fffb -> a1e57d942
refs/heads/branch-1.1 5317c7b31 -> 8eca263c4
refs/heads/branch-1.2 4ef97c32b -> 71e583df3
refs/heads/branch-1.3 c3ccbbc8d -> 6c5d0cf23
refs/heads/master 897631f8d -> cb507b8cf
HBASE-16360 TableMapReduceUtil addHBaseDependencyJars has the wrong class name
for PrefixTreeCodec (Jing Pu Chen)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/cb507b8c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/cb507b8c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/cb507b8c
Branch: refs/heads/master
Commit: cb507b8cff091d736edc9d35d1ce08347626
Parents: 897631f
Author: Matteo Bertozzi
Authored: Tue Aug 23 14:22:20 2016 -0700
Committer: Matteo Bertozzi
Committed: Tue Aug 23 14:22:20 2016 -0700
--
.../java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/cb507b8c/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
index 8d8a784..2701059 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
@@ -780,7 +780,7 @@ public class TableMapReduceUtil {
Class prefixTreeCodecClass = null;
try {
prefixTreeCodecClass =
-
Class.forName("org.apache.hadoop.hbase.code.prefixtree.PrefixTreeCodec");
+
Class.forName("org.apache.hadoop.hbase.codec.prefixtree.PrefixTreeCodec");
} catch (ClassNotFoundException e) {
// this will show up in unit tests but should not show in real
deployments
LOG.warn("The hbase-prefix-tree module jar containing PrefixTreeCodec is
not present." +
[3/6] hbase git commit: HBASE-16360 TableMapReduceUtil addHBaseDependencyJars has the wrong class name for PrefixTreeCodec (Jing Pu Chen)
HBASE-16360 TableMapReduceUtil addHBaseDependencyJars has the wrong class name
for PrefixTreeCodec (Jing Pu Chen)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6c5d0cf2
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6c5d0cf2
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6c5d0cf2
Branch: refs/heads/branch-1.3
Commit: 6c5d0cf235e6c94507918df7a005298a8d0e7b9d
Parents: c3ccbbc
Author: Matteo Bertozzi
Authored: Tue Aug 23 14:22:20 2016 -0700
Committer: Matteo Bertozzi
Committed: Tue Aug 23 14:34:24 2016 -0700
--
.../java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/6c5d0cf2/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
index a939ef3..9ba1f07 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
@@ -771,7 +771,7 @@ public class TableMapReduceUtil {
Class prefixTreeCodecClass = null;
try {
prefixTreeCodecClass =
-
Class.forName("org.apache.hadoop.hbase.code.prefixtree.PrefixTreeCodec");
+
Class.forName("org.apache.hadoop.hbase.codec.prefixtree.PrefixTreeCodec");
} catch (ClassNotFoundException e) {
// this will show up in unit tests but should not show in real
deployments
LOG.warn("The hbase-prefix-tree module jar containing PrefixTreeCodec is
not present." +
hbase git commit: HBASE-16440 MemstoreChunkPool might cross its maxCount of chunks to pool.
Repository: hbase
Updated Branches:
refs/heads/master 77a7394f1 -> 897631f8d
HBASE-16440 MemstoreChunkPool might cross its maxCount of chunks to pool.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/897631f8
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/897631f8
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/897631f8
Branch: refs/heads/master
Commit: 897631f8d194911e0004b9c98d4040e9794bb2e9
Parents: 77a7394
Author: anoopsamjohn
Authored: Tue Aug 23 22:28:14 2016 +0530
Committer: anoopsamjohn
Committed: Tue Aug 23 22:28:14 2016 +0530
--
.../apache/hadoop/hbase/regionserver/Chunk.java | 143
.../hbase/regionserver/HeapMemStoreLAB.java | 161 ---
.../hbase/regionserver/MemStoreChunkPool.java | 113 +++--
.../regionserver/TestMemStoreChunkPool.java | 39 +
4 files changed, 273 insertions(+), 183 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/897631f8/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Chunk.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Chunk.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Chunk.java
new file mode 100644
index 000..d968ed9
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Chunk.java
@@ -0,0 +1,143 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+
+/**
+ * A chunk of memory out of which allocations are sliced.
+ */
+@InterfaceAudience.Private
+public class Chunk {
+ /** Actual underlying data */
+ private byte[] data;
+
+ private static final int UNINITIALIZED = -1;
+ private static final int OOM = -2;
+ /**
+ * Offset for the next allocation, or the sentinel value -1 which implies
that the chunk is still
+ * uninitialized.
+ */
+ private AtomicInteger nextFreeOffset = new AtomicInteger(UNINITIALIZED);
+
+ /** Total number of allocations satisfied from this buffer */
+ private AtomicInteger allocCount = new AtomicInteger();
+
+ /** Size of chunk in bytes */
+ private final int size;
+
+ /**
+ * Create an uninitialized chunk. Note that memory is not allocated yet, so
this is cheap.
+ *
+ * @param size in bytes
+ */
+ Chunk(int size) {
+this.size = size;
+ }
+
+ /**
+ * Actually claim the memory for this chunk. This should only be called from
the thread that
+ * constructed the chunk. It is thread-safe against other threads calling
alloc(), who will block
+ * until the allocation is complete.
+ */
+ public void init() {
+assert nextFreeOffset.get() == UNINITIALIZED;
+try {
+ if (data == null) {
+data = new byte[size];
+ }
+} catch (OutOfMemoryError e) {
+ boolean failInit = nextFreeOffset.compareAndSet(UNINITIALIZED, OOM);
+ assert failInit; // should be true.
+ throw e;
+}
+// Mark that it's ready for use
+boolean initted = nextFreeOffset.compareAndSet(UNINITIALIZED, 0);
+// We should always succeed the above CAS since only one thread
+// calls init()!
+Preconditions.checkState(initted, "Multiple threads tried to init same
chunk");
+ }
+
+ /**
+ * Reset the offset to UNINITIALIZED before before reusing an old chunk
+ */
+ void reset() {
+if (nextFreeOffset.get() != UNINITIALIZED) {
+ nextFreeOffset.set(UNINITIALIZED);
+ allocCount.set(0);
+}
+ }
+
+ /**
+ * Try to allocate size bytes from the chunk.
+ *
+ * @return the offset of the successful allocation, or -1 to indicate
not-enough-space
+ */
+ public int alloc(int size) {
+while (true) {
+ int oldOffset = nextFreeOffset.get();
+ if (oldOffset ==
hbase git commit: HBASE-16446 append_peer_tableCFs failed when there already have this table's partial cfs in the peer (Guanghao Zhang)
Repository: hbase
Updated Branches:
refs/heads/master 2f7b9b542 -> 77a7394f1
HBASE-16446 append_peer_tableCFs failed when there already have this table's
partial cfs in the peer (Guanghao Zhang)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/77a7394f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/77a7394f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/77a7394f
Branch: refs/heads/master
Commit: 77a7394f1770249f33d07df6bac6cf16ef34140e
Parents: 2f7b9b5
Author: Ashish Singhi
Authored: Tue Aug 23 15:28:33 2016 +0530
Committer: Ashish Singhi
Committed: Tue Aug 23 15:28:33 2016 +0530
--
.../client/replication/ReplicationAdmin.java| 11 +++---
.../replication/TestReplicationAdmin.java | 38
2 files changed, 43 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/77a7394f/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
index ee26e38..de6cb7f 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
@@ -289,13 +289,12 @@ public class ReplicationAdmin implements Closeable {
Collection appendCfs = entry.getValue();
if (preTableCfs.containsKey(table)) {
List cfs = preTableCfs.get(table);
-if (cfs == null || appendCfs == null) {
+if (cfs == null || appendCfs == null || appendCfs.isEmpty()) {
preTableCfs.put(table, null);
} else {
Set cfSet = new HashSet(cfs);
cfSet.addAll(appendCfs);
preTableCfs.put(table, Lists.newArrayList(cfSet));
-
}
} else {
if (appendCfs == null || appendCfs.isEmpty()) {
@@ -342,9 +341,9 @@ public class ReplicationAdmin implements Closeable {
Collection removeCfs = entry.getValue();
if (preTableCfs.containsKey(table)) {
List cfs = preTableCfs.get(table);
-if (cfs == null && removeCfs == null) {
+if (cfs == null && (removeCfs == null || removeCfs.isEmpty())) {
preTableCfs.remove(table);
-} else if (cfs != null && removeCfs != null) {
+} else if (cfs != null && (removeCfs != null && !removeCfs.isEmpty()))
{
Set cfSet = new HashSet(cfs);
cfSet.removeAll(removeCfs);
if (cfSet.isEmpty()) {
@@ -352,10 +351,10 @@ public class ReplicationAdmin implements Closeable {
} else {
preTableCfs.put(table, Lists.newArrayList(cfSet));
}
-} else if (cfs == null && removeCfs != null) {
+} else if (cfs == null && (removeCfs != null && !removeCfs.isEmpty()))
{
throw new ReplicationException("Cannot remove cf of table: " + table
+ " which doesn't specify cfs from table-cfs config in peer: " +
id);
-} else if (cfs != null && removeCfs == null) {
+} else if (cfs != null && (removeCfs == null || removeCfs.isEmpty())) {
throw new ReplicationException("Cannot remove table: " + table
+ " which has specified cfs from table-cfs config in peer: " +
id);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/77a7394f/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java
index 9c3f23a..85820af 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/replication/TestReplicationAdmin.java
@@ -216,6 +216,8 @@ public class TestReplicationAdmin {
TableName tab2 = TableName.valueOf("t2");
TableName tab3 = TableName.valueOf("t3");
TableName tab4 = TableName.valueOf("t4");
+TableName tab5 = TableName.valueOf("t5");
+TableName tab6 = TableName.valueOf("t6");
// Add a valid peer
admin.addPeer(ID_ONE, rpc1, null);
@@ -275,6 +277,34 @@ public class TestReplicationAdmin {
assertEquals("f1", result.get(tab4).get(0));
assertEquals("f2", result.get(tab4).get(1));
+// append "table5" => [], then append "table5" => ["f1"]
+tableCFs.clear();
+tableCFs.put(tab5, new Arr
hbase git commit: HBASE-16444 - Remove white space
Repository: hbase
Updated Branches:
refs/heads/master ff6182b02 -> 2f7b9b542
HBASE-16444 - Remove white space
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2f7b9b54
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2f7b9b54
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2f7b9b54
Branch: refs/heads/master
Commit: 2f7b9b5423ea7a5783197f049e5c15f4d544abeb
Parents: ff6182b
Author: Ramkrishna
Authored: Tue Aug 23 12:32:26 2016 +0530
Committer: Ramkrishna
Committed: Tue Aug 23 12:32:26 2016 +0530
--
hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/2f7b9b54/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java
--
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java
b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java
index ac55112..ad13e9e 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java
@@ -902,7 +902,7 @@ public final class CellUtil {
}
/**
- * Estimate based on keyvalue's serialization format.
+ * Estimate based on keyvalue's serialization format.
* @param cell
* @return Estimate of the cell size in bytes.
*/
