hbase git commit: Revert "HBASE-17081 Flush the entire CompactingMemStore content to disk (Anastasia"

[zhangduo]

Repository: hbase
Updated Branches:
  refs/heads/master 665fe3eef -> d53f012bb


Revert "HBASE-17081 Flush the entire CompactingMemStore content to disk 
(Anastasia"

Revert to address tests failure.

This reverts commit a2a7618d261bfe121f05821d89242d770cd7b7ec.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d53f012b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d53f012b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d53f012b

Branch: refs/heads/master
Commit: d53f012bb750a9a5970a33a29b685fa078e3568c
Parents: 665fe3e
Author: zhangduo 
Authored: Fri Dec 16 18:49:48 2016 +0800
Committer: zhangduo 
Committed: Fri Dec 16 18:49:48 2016 +0800

--
 .../hbase/regionserver/AbstractMemStore.java|  35 +-
 .../hbase/regionserver/CompactingMemStore.java  |  85 ++---
 .../hbase/regionserver/CompactionPipeline.java  |  34 +-
 .../regionserver/CompositeImmutableSegment.java | 352 ---
 .../hbase/regionserver/DefaultMemStore.java |  23 +-
 .../hadoop/hbase/regionserver/HRegion.java  |   5 +-
 .../hbase/regionserver/ImmutableSegment.java|  23 +-
 .../hbase/regionserver/MemStoreCompactor.java   |   4 +-
 .../hadoop/hbase/regionserver/MemstoreSize.java |  25 +-
 .../hadoop/hbase/regionserver/Segment.java  |  21 +-
 .../hbase/regionserver/SegmentFactory.java  |  11 -
 .../regionserver/TestCompactingMemStore.java|   8 +-
 .../hbase/regionserver/TestDefaultMemStore.java |  12 +-
 .../TestWalAndCompactingMemStoreFlush.java  | 238 ++---
 14 files changed, 175 insertions(+), 701 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/d53f012b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
index 8564045..225dd73 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
@@ -159,12 +159,14 @@ public abstract class AbstractMemStore implements 
MemStore {
   public String toString() {
 StringBuffer buf = new StringBuffer();
 int i = 1;
-
-for (Segment segment : getSegments()) {
-  buf.append("Segment (" + i + ") " + segment.toString() + "; ");
-  i++;
+try {
+  for (Segment segment : getSegments()) {
+buf.append("Segment (" + i + ") " + segment.toString() + "; ");
+i++;
+  }
+} catch (IOException e){
+  return e.toString();
 }
-
 return buf.toString();
   }
 
@@ -230,7 +232,6 @@ public abstract class AbstractMemStore implements MemStore {
* @return Next row or null if none found.  If one found, will be a new
* KeyValue -- can be destroyed by subsequent calls to this method.
*/
-  @VisibleForTesting
   protected Cell getNextRow(final Cell key,
   final NavigableSet set) {
 Cell result = null;
@@ -248,26 +249,6 @@ public abstract class AbstractMemStore implements MemStore 
{
 return result;
   }
 
-  /**
-   * @param cell Find the row that comes after this one.  If null, we return 
the
-   * first.
-   * @return Next row or null if none found.
-   */
-  @VisibleForTesting
-  Cell getNextRow(final Cell cell) {
-Cell lowest = null;
-List segments = getSegments();
-for (Segment segment : segments) {
-  if (lowest == null) {
-//TODO: we may want to move the getNextRow ability to the segment
-lowest = getNextRow(cell, segment.getCellSet());
-  } else {
-lowest = getLowest(lowest, getNextRow(cell, segment.getCellSet()));
-  }
-}
-return lowest;
-  }
-
   private Cell maybeCloneWithAllocator(Cell cell) {
 return active.maybeCloneWithAllocator(cell);
   }
@@ -326,6 +307,6 @@ public abstract class AbstractMemStore implements MemStore {
   /**
* @return an ordered list of segments from most recent to oldest in memstore
*/
-  protected abstract List getSegments();
+  protected abstract List getSegments() throws IOException;
 
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/d53f012b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
index f02feb5..d279f4a 100644
--- 

[hbase] Git Push Summary

[ndimiduk]

Repository: hbase
Updated Tags:  refs/tags/rel/1.1.8 [created] 8614a6b4c

hbase git commit: bump version for next dev cycle

[ndimiduk]

Repository: hbase
Updated Branches:
  refs/heads/branch-1.1 1687ca79b -> aa47da890


bump version for next dev cycle


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/aa47da89
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/aa47da89
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/aa47da89

Branch: refs/heads/branch-1.1
Commit: aa47da8903fa1042cabc2fefee3cfc47db1d82b9
Parents: 1687ca7
Author: Nick Dimiduk 
Authored: Sun Dec 18 12:13:26 2016 -0800
Committer: Nick Dimiduk 
Committed: Sun Dec 18 12:13:26 2016 -0800

--
 hbase-annotations/pom.xml| 2 +-
 hbase-assembly/pom.xml   | 2 +-
 hbase-checkstyle/pom.xml | 4 ++--
 hbase-client/pom.xml | 2 +-
 hbase-common/pom.xml | 2 +-
 hbase-examples/pom.xml   | 2 +-
 hbase-hadoop-compat/pom.xml  | 2 +-
 hbase-hadoop2-compat/pom.xml | 2 +-
 hbase-it/pom.xml | 2 +-
 hbase-prefix-tree/pom.xml| 2 +-
 hbase-procedure/pom.xml  | 2 +-
 hbase-protocol/pom.xml   | 2 +-
 hbase-resource-bundle/pom.xml| 2 +-
 hbase-rest/pom.xml   | 2 +-
 hbase-server/pom.xml | 2 +-
 hbase-shaded/hbase-shaded-client/pom.xml | 2 +-
 hbase-shaded/hbase-shaded-server/pom.xml | 2 +-
 hbase-shaded/pom.xml | 2 +-
 hbase-shell/pom.xml  | 2 +-
 hbase-testing-util/pom.xml   | 2 +-
 hbase-thrift/pom.xml | 2 +-
 pom.xml  | 2 +-
 22 files changed, 23 insertions(+), 23 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/aa47da89/hbase-annotations/pom.xml
--
diff --git a/hbase-annotations/pom.xml b/hbase-annotations/pom.xml
index c0d5243..c5e7b00 100644
--- a/hbase-annotations/pom.xml
+++ b/hbase-annotations/pom.xml
@@ -23,7 +23,7 @@
   
 hbase
 org.apache.hbase
-1.1.8
+1.1.9-SNAPSHOT
 ..
   
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/aa47da89/hbase-assembly/pom.xml
--
diff --git a/hbase-assembly/pom.xml b/hbase-assembly/pom.xml
index 7ec72d2..74f8279 100644
--- a/hbase-assembly/pom.xml
+++ b/hbase-assembly/pom.xml
@@ -23,7 +23,7 @@
   
 hbase
 org.apache.hbase
-1.1.8
+1.1.9-SNAPSHOT
 ..
   
   hbase-assembly

http://git-wip-us.apache.org/repos/asf/hbase/blob/aa47da89/hbase-checkstyle/pom.xml
--
diff --git a/hbase-checkstyle/pom.xml b/hbase-checkstyle/pom.xml
index 0bc3c37..e185d88 100644
--- a/hbase-checkstyle/pom.xml
+++ b/hbase-checkstyle/pom.xml
@@ -24,14 +24,14 @@
 4.0.0
 org.apache.hbase
 hbase-checkstyle
-1.1.8
+1.1.9-SNAPSHOT
 Apache HBase - Checkstyle
 Module to hold Checkstyle properties for HBase.
 
   
 hbase
 org.apache.hbase
-1.1.8
+1.1.9-SNAPSHOT
 ..
   
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/aa47da89/hbase-client/pom.xml
--
diff --git a/hbase-client/pom.xml b/hbase-client/pom.xml
index 6793b4f..aa54598 100644
--- a/hbase-client/pom.xml
+++ b/hbase-client/pom.xml
@@ -24,7 +24,7 @@
   
 hbase
 org.apache.hbase
-1.1.8
+1.1.9-SNAPSHOT
 ..
   
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/aa47da89/hbase-common/pom.xml
--
diff --git a/hbase-common/pom.xml b/hbase-common/pom.xml
index 39b5526..0312a99 100644
--- a/hbase-common/pom.xml
+++ b/hbase-common/pom.xml
@@ -23,7 +23,7 @@
   
 hbase
 org.apache.hbase
-1.1.8
+1.1.9-SNAPSHOT
 ..
   
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/aa47da89/hbase-examples/pom.xml
--
diff --git a/hbase-examples/pom.xml b/hbase-examples/pom.xml
index ee5c63d..40a73d3 100644
--- a/hbase-examples/pom.xml
+++ b/hbase-examples/pom.xml
@@ -23,7 +23,7 @@
   
 hbase
 org.apache.hbase
-1.1.8
+1.1.9-SNAPSHOT
 ..
   
   hbase-examples

http://git-wip-us.apache.org/repos/asf/hbase/blob/aa47da89/hbase-hadoop-compat/pom.xml
--
diff --git a/hbase-hadoop-compat/pom.xml b/hbase-hadoop-compat/pom.xml
index 25f1b8e..422cc50 100644
--- a/hbase-hadoop-compat/pom.xml
+++ b/hbase-hadoop-compat/pom.xml
@@ -23,7 +23,7 @@
 
 hbase
 org.apache.hbase
-1.1.8
+1.1.9-SNAPSHOT
 ..
 
 

svn commit: r17472 - /release/hbase/1.1.7/

[ndimiduk]

Author: ndimiduk
Date: Sun Dec 18 20:12:23 2016
New Revision: 17472

Log:
rm old 1.1.7 release

Removed:
release/hbase/1.1.7/

svn commit: r17471 - /dev/hbase/hbase-1.1.8RC0/ /release/hbase/1.1.8/

[ndimiduk]

Author: ndimiduk
Date: Sun Dec 18 20:10:23 2016
New Revision: 17471

Log:
publish HBase 1.1.8

Added:
release/hbase/1.1.8/
  - copied from r17470, dev/hbase/hbase-1.1.8RC0/
Removed:
dev/hbase/hbase-1.1.8RC0/

hbase git commit: HBASE-17331 Avoid busy waiting in ThrottledInputStream (ChiaPing Tsai)

[tedyu]

Repository: hbase
Updated Branches:
  refs/heads/master f041306cd -> 665fe3eef


HBASE-17331 Avoid busy waiting in ThrottledInputStream (ChiaPing Tsai)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/665fe3ee
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/665fe3ee
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/665fe3ee

Branch: refs/heads/master
Commit: 665fe3eef17a96da037901cba776880d0499a9c1
Parents: f041306
Author: tedyu 
Authored: Sun Dec 18 08:15:15 2016 -0800
Committer: tedyu 
Committed: Sun Dec 18 08:15:15 2016 -0800

--
 .../io/hadoopbackport/ThrottledInputStream.java | 43 ++
 .../TestThrottledInputStream.java   | 48 
 .../org/apache/hadoop/hbase/SplitLogTask.java   |  2 +-
 3 files changed, 82 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/665fe3ee/hbase-common/src/main/java/org/apache/hadoop/hbase/io/hadoopbackport/ThrottledInputStream.java
--
diff --git 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/hadoopbackport/ThrottledInputStream.java
 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/hadoopbackport/ThrottledInputStream.java
index 1bef221..88c1b37 100644
--- 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/hadoopbackport/ThrottledInputStream.java
+++ 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/hadoopbackport/ThrottledInputStream.java
@@ -18,11 +18,15 @@
 
 package org.apache.hadoop.hbase.io.hadoopbackport;
 
+import com.google.common.annotations.VisibleForTesting;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.InterruptedIOException;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.hadoop.fs.PositionedReadable;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 
 /**
  * The ThrottleInputStream provides bandwidth throttling on a specified
@@ -44,8 +48,6 @@ public class ThrottledInputStream extends InputStream {
   private long bytesRead = 0;
   private long totalSleepTime = 0;
 
-  private static final long SLEEP_DURATION_MS = 50;
-
   public ThrottledInputStream(InputStream rawStream) {
 this(rawStream, Long.MAX_VALUE);
   }
@@ -118,14 +120,35 @@ public class ThrottledInputStream extends InputStream {
 return readLen;
   }
 
-  private void throttle() throws IOException {
-while (getBytesPerSec() > maxBytesPerSec) {
-  try {
-Thread.sleep(SLEEP_DURATION_MS);
-totalSleepTime += SLEEP_DURATION_MS;
-  } catch (InterruptedException e) {
-throw new IOException("Thread aborted", e);
-  }
+  private long calSleepTimeMs() {
+return calSleepTimeMs(bytesRead, maxBytesPerSec,
+  EnvironmentEdgeManager.currentTime() - startTime);
+  }
+
+  @VisibleForTesting
+  static long calSleepTimeMs(long bytesRead, long maxBytesPerSec, long 
elapsed) {
+assert elapsed > 0 : "The elapsed time should be greater than zero";
+if (bytesRead <= 0 || maxBytesPerSec <= 0) {
+  return 0;
+}
+// We use this class to load the single source file, so the bytesRead
+// and maxBytesPerSec aren't greater than Double.MAX_VALUE.
+// We can get the precise sleep time by using the double value.
+long rval = (long) double) bytesRead) / ((double) maxBytesPerSec)) * 
1000 - elapsed);
+if (rval <= 0) {
+  return 0;
+} else {
+  return rval;
+}
+  }
+
+  private void throttle() throws InterruptedIOException {
+long sleepTime = calSleepTimeMs();
+totalSleepTime += sleepTime;
+try {
+  TimeUnit.MILLISECONDS.sleep(sleepTime);
+} catch (InterruptedException e) {
+  throw new InterruptedIOException("Thread aborted");
 }
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/665fe3ee/hbase-common/src/test/java/org/apache/hadoop/hbase/io/hadoopbackport/TestThrottledInputStream.java
--
diff --git 
a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/hadoopbackport/TestThrottledInputStream.java
 
b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/hadoopbackport/TestThrottledInputStream.java
new file mode 100644
index 000..15d292d
--- /dev/null
+++ 
b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/hadoopbackport/TestThrottledInputStream.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+