kudu git commit: [iwyu_tool.py] fix on the IWYU tool invocation

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master 4ee3efed5 -> 4935b1b29


[iwyu_tool.py] fix on the IWYU tool invocation

Fixed the way include-what-you-use binary is invoked by the
iwyu_tool.py wrapper script.  Prior to this fix, subprocess.Popen()
had the 'shell' argument set to 'True', so any issue with starting
the binary was silenced because there would be no OSError exception
raised.  Overall, if using 'shell=True' for subprocess.Popen(),
it wouldn't be feasible to detect that sort of error via examining
the exit code of the started process because include-what-you-use
always returns exit code 1 regardless of its findings.

I also updated the iwyu.sh script to be more robust in handling
possible errors, if any.

Prior to this fix, the 'iwyu' CMake target returned success if
include-what-you-use binary was not in place.  This patch fixes that
problem and introduces more robust handling of other possible issues
while working with the include-what-you-use tool.

Change-Id: Ifa4284857583fff6543f65a9c2a71a9e445f39e2
Reviewed-on: http://gerrit.cloudera.org:8080/7989
Reviewed-by: Alexey Serbin 
Tested-by: Alexey Serbin 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/4935b1b2
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/4935b1b2
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/4935b1b2

Branch: refs/heads/master
Commit: 4935b1b298782aabcdac5b8a5c49c3c15d80d8f6
Parents: 4ee3efe
Author: Alexey Serbin 
Authored: Wed Sep 6 18:47:47 2017 -0700
Committer: Alexey Serbin 
Committed: Thu Sep 7 04:21:55 2017 +

--
 build-support/iwyu/iwyu.sh  | 29 +
 build-support/iwyu/iwyu_tool.py | 17 +
 2 files changed, 30 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/4935b1b2/build-support/iwyu/iwyu.sh
--
diff --git a/build-support/iwyu/iwyu.sh b/build-support/iwyu/iwyu.sh
index 6c529fc..52e81eb 100755
--- a/build-support/iwyu/iwyu.sh
+++ b/build-support/iwyu/iwyu.sh
@@ -17,19 +17,26 @@
 # specific language governing permissions and limitations
 # under the License.
 
-ROOT=$(cd $(dirname $BASH_SOURCE)/../..; pwd)
+set -e
+set -o pipefail
 
-IWYU_LOG=$(mktemp -t kudu-iwyu.XX)
-trap "rm -f $IWYU_LOG" EXIT
+ROOT=$(cd $(dirname $BASH_SOURCE)/../..; pwd)
 
 # Build the list of updated files which are of IWYU interest.
+# Since '-e' is set, transform the exit code from the grep accordingly:
+# grep returns 1 if no lines were selected.
 file_list_tmp=$(git diff --name-only \
-$($ROOT/build-support/get-upstream-commit.sh) | grep -E '\.(c|cc|h)$')
+$($ROOT/build-support/get-upstream-commit.sh) | \
+(grep -E '\.(c|cc|h)$' || [ $? -eq 1 ]))
 if [ -z "$file_list_tmp" ]; then
   echo "IWYU verification: no updates on related files, declaring success"
   exit 0
 fi
 
+IWYU_LOG=$(mktemp -t kudu-iwyu.XX)
+UNFILTERED_IWYU_LOG=${IWYU_LOG}.unfiltered
+trap "rm -f $IWYU_LOG $UNFILTERED_IWYU_LOG" EXIT
+
 # Adjust the path for every element in the list. The iwyu_tool.py normalizes
 # paths (via realpath) to match the records from the compilation database.
 IWYU_FILE_LIST=
@@ -47,13 +54,19 @@ IWYU_ARGS="\
 --mapping_file=$IWYU_MAPPINGS_PATH/gtest.imp \
 --mapping_file=$IWYU_MAPPINGS_PATH/libstdcpp.imp"
 
-PATH="$PATH:$PWD/../../thirdparty/clang-toolchain/bin" \
+if ! PATH="$PATH:$PWD/../../thirdparty/clang-toolchain/bin" \
 python $ROOT/build-support/iwyu/iwyu_tool.py -p . $IWYU_FILE_LIST -- \
-$IWYU_ARGS | awk -f $ROOT/build-support/iwyu/iwyu-filter.awk | \
-tee $IWYU_LOG
+$IWYU_ARGS > $UNFILTERED_IWYU_LOG 2>&1; then
+  echo "IWYU verification: failed to run the tool, see below for details"
+  cat $UNFILTERED_IWYU_LOG
+  exit 2
+fi
 
+awk -f $ROOT/build-support/iwyu/iwyu-filter.awk $UNFILTERED_IWYU_LOG | \
+tee $IWYU_LOG
 if [ -s "$IWYU_LOG" ]; then
-  # The output is not empty: the changelist needs correction.
+  # The output is not empty: IWYU finds the set of headers to be inconsistent.
+  echo "IWYU verification: changelist needs correction, see above for details"
   exit 1
 fi
 

http://git-wip-us.apache.org/repos/asf/kudu/blob/4935b1b2/build-support/iwyu/iwyu_tool.py
--
diff --git a/build-support/iwyu/iwyu_tool.py b/build-support/iwyu/iwyu_tool.py
index a3dd4a1..8943226 100755
--- a/build-support/iwyu/iwyu_tool.py
+++ b/build-support/iwyu/iwyu_tool.py
@@ -135,11 +135,10 @@ FORMATTERS = {
 'clang': clang_formatter
 }
 
-def get_outp

kudu git commit: Fix typo in partitioning error message

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master 4935b1b29 -> 9af2046e6


Fix typo in partitioning error message

Change-Id: I5502956e7947a67dba3e68e5add529c7b9e26866
Reviewed-on: http://gerrit.cloudera.org:8080/7997
Tested-by: Kudu Jenkins
Reviewed-by: Alexey Serbin 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/9af2046e
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/9af2046e
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/9af2046e

Branch: refs/heads/master
Commit: 9af2046e610156a9064c87976eec2b5a42e0693b
Parents: 4935b1b
Author: Dan Burkert 
Authored: Thu Sep 7 09:16:06 2017 -0700
Committer: Alexey Serbin 
Committed: Thu Sep 7 20:18:56 2017 +

--
 src/kudu/common/partition.cc   | 2 +-
 src/kudu/master/master-test.cc | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/9af2046e/src/kudu/common/partition.cc
--
diff --git a/src/kudu/common/partition.cc b/src/kudu/common/partition.cc
index a09189b..b1388db 100644
--- a/src/kudu/common/partition.cc
+++ b/src/kudu/common/partition.cc
@@ -285,7 +285,7 @@ Status PartitionSchema::EncodeRangeBounds(const 
vector= upper) {
   return Status::InvalidArgument(
-  "range partition lower bound must be less than or equal to the upper 
bound",
+  "range partition lower bound must be less than the upper bound",
   RangePartitionDebugString(bound.first, bound.second));
 }
 range_partitions->emplace_back(std::move(lower), std::move(upper));

http://git-wip-us.apache.org/repos/asf/kudu/blob/9af2046e/src/kudu/master/master-test.cc
--
diff --git a/src/kudu/master/master-test.cc b/src/kudu/master/master-test.cc
index 9a19400..7d35e00 100644
--- a/src/kudu/master/master-test.cc
+++ b/src/kudu/master/master-test.cc
@@ -602,7 +602,7 @@ TEST_F(MasterTest, TestCreateTableCheckRangeInvariants) {
 ASSERT_TRUE(s.IsInvalidArgument());
 ASSERT_STR_CONTAINS(s.ToString(),
 "Invalid argument: range partition lower bound must be 
"
-"less than or equal to the upper bound");
+"less than the upper bound");
   }
   { // Lower bound equals upper bound.
 KuduPartialRow bound_lower(&kTableSchema);
@@ -614,7 +614,7 @@ TEST_F(MasterTest, TestCreateTableCheckRangeInvariants) {
 ASSERT_TRUE(s.IsInvalidArgument());
 ASSERT_STR_CONTAINS(s.ToString(),
 "Invalid argument: range partition lower bound must be 
"
-"less than or equal to the upper bound");
+"less than the upper bound");
   }
   { // Split equals lower bound
 KuduPartialRow bound_lower(&kTableSchema);

kudu git commit: [iwyu] kudu-specific mappings for std::tr1 entities

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master 9af2046e6 -> 01003ee6b


[iwyu] kudu-specific mappings for std::tr1 entities

Added Kudu-specific IWYU mappings for std::tr1::shared_ptr and friends
from TR1 (those are used in the Kudu C++ client API).

Added --max_line_length=256 IWYU option for better reporting
on the reason for header files inclusion.  If keeping the
default setting of 80 characters, IWYU sometimes omits the information
which symbols require particular header file to be included.

Change-Id: Ia448dedcbb09ee23b30a922c7143df9036490d84
Reviewed-on: http://gerrit.cloudera.org:8080/7998
Reviewed-by: Adar Dembo 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/01003ee6
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/01003ee6
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/01003ee6

Branch: refs/heads/master
Commit: 01003ee6b4ece19c5e803dcac9d777a421d7c520
Parents: 9af2046
Author: Alexey Serbin 
Authored: Wed Sep 6 23:30:56 2017 -0700
Committer: Alexey Serbin 
Committed: Thu Sep 7 22:16:34 2017 +

--
 .gitignore   |  1 +
 build-support/iwyu/iwyu-filter.awk   | 37 ---
 build-support/iwyu/iwyu.sh   |  2 ++
 build-support/iwyu/mappings/kudu.imp | 34 
 4 files changed, 56 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/01003ee6/.gitignore
--
diff --git a/.gitignore b/.gitignore
index d009ccc..9271774 100644
--- a/.gitignore
+++ b/.gitignore
@@ -30,6 +30,7 @@ oprofile_data
 .kdev4/
 .metadata/
 *.iml
+README.html
 
 # VIM/emacs/other editors stuff
 *.swp

http://git-wip-us.apache.org/repos/asf/kudu/blob/01003ee6/build-support/iwyu/iwyu-filter.awk
--
diff --git a/build-support/iwyu/iwyu-filter.awk 
b/build-support/iwyu/iwyu-filter.awk
index 0d7fa79..6576a72 100644
--- a/build-support/iwyu/iwyu-filter.awk
+++ b/build-support/iwyu/iwyu-filter.awk
@@ -26,22 +26,34 @@
 #   
https://github.com/include-what-you-use/include-what-you-use/blob/master/docs/IWYUMappings.md
 #
 # We are using mappings for the boost library (comes with IWYU) and a few
-# custom mappings for gflags, glog, and gtest libraries to address some IWYU
-# quirks (hopefully, those should be resolved as IWYU gets better).
+# custom mappings for gflags, glog, gtest and other libraries to address some
+# IWYU quirks (hopefully, those should be resolved as IWYU gets better).
+# The kudu.imp mappings file is used to provide Kudu-specific mappings.
+#
+# To run the IWYU tool on every C++ source file in the project,
+# use the recipe below.
 #
-# Usage:
 #  1. Run the CMake with -DCMAKE_CXX_INCLUDE_WHAT_YOU_USE=
 #
 # The path to the IWYU binary should be absolute. The path to the binary
 # and the command-line options should be separated by semicolon
 # (that's for feeding it into CMake list variables).
 #
-# E.g., from the build directory (line breaks are just for readability):
+# Below is the set of commands to run from the build directory to configure
+# the build accordingly (line breaks are for better readability):
 #
 # CC=../../thirdparty/clang-toolchain/bin/clang
 # CXX=../../thirdparty/clang-toolchain/bin/clang++
-# IWYU="`pwd`../../thirdparty/clang-toolchain/bin/include-what-you-use;\
-#   -Xiwyu;--mapping_file=`pwd`../../build-support/iwyu/mappings/map.imp"
+# IWYU="`pwd`/../../thirdparty/clang-toolchain/bin/include-what-you-use;\
+#   -Xiwyu;--max_line_length=256;\
+#   
-Xiwyu;--mapping_file=`pwd`/../../build-support/iwyu/mappings/boost-all.imp;\
+#   
-Xiwyu;--mapping_file=`pwd`/../../build-support/iwyu/mappings/boost-all-private.imp;\
+#   
-Xiwyu;--mapping_file=`pwd`/../../build-support/iwyu/mappings/boost-extra.imp;\
+#   
-Xiwyu;--mapping_file=`pwd`/../../build-support/iwyu/mappings/gtest.imp;\
+#   
-Xiwyu;--mapping_file=`pwd`/../../build-support/iwyu/mappings/glog.imp;\
+#   
-Xiwyu;--mapping_file=`pwd`/../../build-support/iwyu/mappings/gflags.imp;\
+#   
-Xiwyu;--mapping_file=`pwd`/../../build-support/iwyu/mappings/kudu.imp;\
+#   
-Xiwyu;--mapping_file=`pwd`/../../build-support/iwyu/mappings/libstdcpp.imp"
 #
 # ../../build-support/enable_devtoolset.sh \
 #   env CC=$CC CXX=$CXX \
@@ -52,18 +64,7 @@
 # NOTE:
 #   Since the Kudu code has some 'ifdef NDEBUG' directives, it's possible
 #   that IWYU would produce different results if run against release, not
-#   debug build. However, we plan to use the tool only with debug builds.
-#
-# NOTE:
-#   As of now, the ap

kudu git commit: [util] fix compilation breakage

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master d45eb2700 -> 0ed75067c


[util] fix compilation breakage

On some Linux systems (old ones), there isn't linux/sysinfo.h
header file.  IWYU might suggest to include linux/sysinfo.h
instead of linux/kernel.h if compiling on a new system, so I added
corresponding IWYU pragmas.

I'll address the IWYU issue in a generic way in a separate change
list (so no pragmas would be necessary).  This patch is a quick
fix for the compilation breakage.

Change-Id: Ie9d13815052d8e4c1360db3f94e15ffd77768af5
Reviewed-on: http://gerrit.cloudera.org:8080/8065
Reviewed-by: Adar Dembo 
Reviewed-by: Andrew Wong 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/0ed75067
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/0ed75067
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/0ed75067

Branch: refs/heads/master
Commit: 0ed75067cf805dda8784cca5ea7a3e7a43616ac6
Parents: d45eb27
Author: Alexey Serbin 
Authored: Wed Sep 13 21:50:18 2017 -0700
Committer: Alexey Serbin 
Committed: Thu Sep 14 05:25:46 2017 +

--
 src/kudu/util/env_posix.cc | 9 +
 1 file changed, 5 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/0ed75067/src/kudu/util/env_posix.cc
--
diff --git a/src/kudu/util/env_posix.cc b/src/kudu/util/env_posix.cc
index 9dd5e2b..3352853 100644
--- a/src/kudu/util/env_posix.cc
+++ b/src/kudu/util/env_posix.cc
@@ -15,9 +15,6 @@
 #include 
 #include 
 #include 
-// IWYU pragma: no_include 
-// IWYU pragma: no_include 
-// IWYU pragma: no_include 
 
 #include 
 #include 
@@ -73,12 +70,16 @@
 #include 
 #include 
 #include 
+#include  // IWYU pragma: keep
 #include 
-#include 
 #include 
 #include 
 #include   // IWYU pragma: keep
 #endif  // defined(__APPLE__)
+// IWYU pragma: no_include 
+// IWYU pragma: no_include 
+// IWYU pragma: no_include 
+// IWYU pragma: no_include 
 
 using base::subtle::Atomic64;
 using base::subtle::Barrier_AtomicIncrement;

kudu git commit: [iwyu] introduce Linux system mappings

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master e2c0e1dbd -> fa8620cc7


[iwyu] introduce Linux system mappings

For starters, added the following mappings to achieve better
compatibility across different Linus versions and distros:

  * use  even if  is available
  * use  even if  is available
  * use  as an umbrella header file for various asm types

Change-Id: I4e50f9135fa076e429b26e3919ef19eb5d430173
Reviewed-on: http://gerrit.cloudera.org:8080/8071
Reviewed-by: Adar Dembo 
Reviewed-by: Andrew Wong 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/fa8620cc
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/fa8620cc
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/fa8620cc

Branch: refs/heads/master
Commit: fa8620cc7b4dd4205a5a58737a7cc3c21957fe05
Parents: e2c0e1d
Author: Alexey Serbin 
Authored: Thu Sep 14 08:37:40 2017 -0700
Committer: Alexey Serbin 
Committed: Thu Sep 14 19:58:43 2017 +

--
 build-support/iwyu/iwyu-filter.awk   |  3 +-
 build-support/iwyu/iwyu.sh   |  3 +-
 build-support/iwyu/mappings/system-linux.imp | 35 +++
 src/kudu/util/env_posix.cc   | 20 ++---
 4 files changed, 48 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/fa8620cc/build-support/iwyu/iwyu-filter.awk
--
diff --git a/build-support/iwyu/iwyu-filter.awk 
b/build-support/iwyu/iwyu-filter.awk
index 6576a72..eddcb8c 100644
--- a/build-support/iwyu/iwyu-filter.awk
+++ b/build-support/iwyu/iwyu-filter.awk
@@ -53,7 +53,8 @@
 #   
-Xiwyu;--mapping_file=`pwd`/../../build-support/iwyu/mappings/glog.imp;\
 #   
-Xiwyu;--mapping_file=`pwd`/../../build-support/iwyu/mappings/gflags.imp;\
 #   
-Xiwyu;--mapping_file=`pwd`/../../build-support/iwyu/mappings/kudu.imp;\
-#   
-Xiwyu;--mapping_file=`pwd`/../../build-support/iwyu/mappings/libstdcpp.imp"
+#   
-Xiwyu;--mapping_file=`pwd`/../../build-support/iwyu/mappings/libstdcpp.imp\
+#   
-Xiwyu;--mapping_file=`pwd`/../../build-support/iwyu/mappings/system-linux.imp"
 #
 # ../../build-support/enable_devtoolset.sh \
 #   env CC=$CC CXX=$CXX \

http://git-wip-us.apache.org/repos/asf/kudu/blob/fa8620cc/build-support/iwyu/iwyu.sh
--
diff --git a/build-support/iwyu/iwyu.sh b/build-support/iwyu/iwyu.sh
index a2c16cf..6ee6287 100755
--- a/build-support/iwyu/iwyu.sh
+++ b/build-support/iwyu/iwyu.sh
@@ -54,7 +54,8 @@ IWYU_ARGS="\
 --mapping_file=$IWYU_MAPPINGS_PATH/glog.imp \
 --mapping_file=$IWYU_MAPPINGS_PATH/gtest.imp \
 --mapping_file=$IWYU_MAPPINGS_PATH/kudu.imp \
---mapping_file=$IWYU_MAPPINGS_PATH/libstdcpp.imp"
+--mapping_file=$IWYU_MAPPINGS_PATH/libstdcpp.imp\
+--mapping_file=$IWYU_MAPPINGS_PATH/system-linux.imp"
 
 if ! PATH="$PATH:$PWD/../../thirdparty/clang-toolchain/bin" \
 python $ROOT/build-support/iwyu/iwyu_tool.py -p . $IWYU_FILE_LIST -- \

http://git-wip-us.apache.org/repos/asf/kudu/blob/fa8620cc/build-support/iwyu/mappings/system-linux.imp
--
diff --git a/build-support/iwyu/mappings/system-linux.imp 
b/build-support/iwyu/mappings/system-linux.imp
new file mode 100644
index 000..22235f5
--- /dev/null
+++ b/build-support/iwyu/mappings/system-linux.imp
@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+#
+# This file contains Linux-specific mappings for IWYU.
+#
+# For instance, some of these mappings take care of variation in system
+# headers between different versions of Linux. From the point of better
+# compatibility, it makes sense to favor the most compatible option
+# among different system versions and flavors.
+#
+
+[
+  { include: ["", private, "", public] 
},
+  { include: ["", private, "", public] },

kudu git commit: standard notation for fixed-size integer type names

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master 052d1def8 -> 5c8a8a2ab


standard notation for fixed-size integer type names

Prefer the standard notation for fixed-size integer types
in the Kudu code, e.g., use uint32_t instead of uint32.  In addition
to unification of type naming, this helps to avoid including the
kudu/gutil/integral_types.h extra header file.

The gutil is left unchanged in that regard.

This patch does not contain any functional changes.

Change-Id: If8588275d0d81e5915c798d4d244bba560b6d25f
Reviewed-on: http://gerrit.cloudera.org:8080/8099
Tested-by: Kudu Jenkins
Reviewed-by: Todd Lipcon 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/5c8a8a2a
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/5c8a8a2a
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/5c8a8a2a

Branch: refs/heads/master
Commit: 5c8a8a2ab9d20ef89e6264cc43b0a3747567434e
Parents: 052d1de
Author: Alexey Serbin 
Authored: Mon Sep 18 16:50:56 2017 -0700
Committer: Alexey Serbin 
Committed: Tue Sep 19 18:26:19 2017 +

--
 src/kudu/cfile/bloomfile-test.cc|  3 +--
 src/kudu/client/client-test.cc  |  2 +-
 src/kudu/client/scan_configuration.h|  3 +--
 src/kudu/common/generic_iterators-test.cc   |  7 +++--
 src/kudu/common/row_changelist-test.cc  |  6 ++---
 src/kudu/common/types-test.cc   |  5 ++--
 src/kudu/consensus/raft_consensus.cc|  9 +++
 src/kudu/fs/block_id.h  |  1 -
 src/kudu/gutil/ref_counted.h|  2 +-
 src/kudu/integration-tests/tablet_copy-itest.cc |  5 ++--
 src/kudu/integration-tests/ts_itest-base.h  |  2 +-
 src/kudu/rpc/retriable_rpc.h|  2 +-
 src/kudu/rpc/rpc-test-base.h|  4 +--
 src/kudu/security/token_signer.cc   |  9 +++
 src/kudu/tablet/tablet_metadata.cc  |  2 +-
 src/kudu/tablet/tablet_metadata.h   |  3 +--
 src/kudu/tools/kudu-admin-test.cc   |  9 +++
 src/kudu/tools/tool_action_tablet.cc|  4 +--
 src/kudu/tserver/scanners.cc|  3 +--
 src/kudu/tserver/tablet_copy_client.h   |  3 +--
 src/kudu/tserver/tablet_copy_source_session.h   |  3 +--
 src/kudu/util/async_logger.cc   |  2 +-
 src/kudu/util/async_logger.h|  3 +--
 src/kudu/util/bloom_filter.h|  5 ++--
 src/kudu/util/condition_variable.cc |  4 +--
 src/kudu/util/debug/trace_event_impl.cc | 28 ++--
 src/kudu/util/debug/trace_event_impl.h  | 17 ++--
 src/kudu/util/file_cache-stress-test.cc |  2 +-
 src/kudu/util/logging-test.cc   |  4 +--
 src/kudu/util/memory/memory.h   |  7 +++--
 src/kudu/util/os-util.cc|  3 +--
 src/kudu/util/pb_util-internal.h|  4 +--
 src/kudu/util/spinlock_profiling.cc |  7 +++--
 src/kudu/util/test_util.cc  |  3 +--
 src/kudu/util/thread.cc |  5 ++--
 35 files changed, 80 insertions(+), 101 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/5c8a8a2a/src/kudu/cfile/bloomfile-test.cc
--
diff --git a/src/kudu/cfile/bloomfile-test.cc b/src/kudu/cfile/bloomfile-test.cc
index 34aead3..afd8171 100644
--- a/src/kudu/cfile/bloomfile-test.cc
+++ b/src/kudu/cfile/bloomfile-test.cc
@@ -32,7 +32,6 @@
 #include "kudu/fs/fs_manager.h"
 #include "kudu/gutil/endian.h"
 #include "kudu/gutil/gscoped_ptr.h"
-#include "kudu/gutil/integral_types.h"
 #include "kudu/util/bloom_filter.h"
 #include "kudu/util/mem_tracker.h"
 #include "kudu/util/slice.h"
@@ -63,7 +62,7 @@ class BloomFileTest : public BloomFileTestBase {
 
 int positive_count = 0;
 // Check that the FP rate for keys we didn't insert is what we expect.
-for (uint64 i = 0; i < FLAGS_n_keys; i++) {
+for (uint64_t i = 0; i < FLAGS_n_keys; i++) {
   uint64_t key = random();
   Slice s(reinterpret_cast(&key), sizeof(key));
 

http://git-wip-us.apache.org/repos/asf/kudu/blob/5c8a8a2a/src/kudu/client/client-test.cc
--
diff --git a/src/kudu/client/client-test.cc b/src/kudu/client/client-test.cc
index cb1ab12..6ed26be 100644
--- a/src/kudu/client/client-test.cc
+++ b/src/kudu/client/client-test.cc
@@ -3024,7 +3024,7 @@ TEST_F(ClientTest, TestAutoFlushBackgroundPreFlush) {
 // For details on this behavior please see the diagram in the body of the
 // KuduSession::Data::ApplyWriteOp() method.
 

[1/2] kudu git commit: [iwyu] update std::move

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master 02ba0ec6c -> 3e6e12a9f


http://git-wip-us.apache.org/repos/asf/kudu/blob/3e6e12a9/src/kudu/tablet/tablet_bootstrap.cc
--
diff --git a/src/kudu/tablet/tablet_bootstrap.cc 
b/src/kudu/tablet/tablet_bootstrap.cc
index 1eb6533..d5c8a3c 100644
--- a/src/kudu/tablet/tablet_bootstrap.cc
+++ b/src/kudu/tablet/tablet_bootstrap.cc
@@ -17,7 +17,6 @@
 
 #include "kudu/tablet/tablet_bootstrap.h"
 
-#include 
 #include 
 #include 
 #include 

http://git-wip-us.apache.org/repos/asf/kudu/blob/3e6e12a9/src/kudu/tablet/tablet_history_gc-test.cc
--
diff --git a/src/kudu/tablet/tablet_history_gc-test.cc 
b/src/kudu/tablet/tablet_history_gc-test.cc
index d12bba7..7aec88a 100644
--- a/src/kudu/tablet/tablet_history_gc-test.cc
+++ b/src/kudu/tablet/tablet_history_gc-test.cc
@@ -32,6 +32,7 @@
 #include "kudu/clock/clock.h"
 #include "kudu/clock/hybrid_clock.h"
 #include "kudu/clock/mock_ntp.h"
+#include "kudu/clock/time_service.h"
 #include "kudu/common/common.pb.h"
 #include "kudu/common/partial_row.h"
 #include "kudu/common/schema.h"
@@ -39,6 +40,7 @@
 #include "kudu/gutil/casts.h"
 #include "kudu/gutil/move.h"
 #include "kudu/gutil/port.h"
+#include "kudu/gutil/ref_counted.h"
 #include "kudu/gutil/strings/substitute.h"
 #include "kudu/gutil/walltime.h"
 #include "kudu/tablet/compaction.h"
@@ -51,7 +53,7 @@
 #include "kudu/tablet/tablet.h"
 #include "kudu/tablet/tablet_metadata.h"
 #include "kudu/tablet/tablet_metrics.h"
-#include "kudu/util/make_shared.h"
+#include "kudu/util/metrics.h"
 #include "kudu/util/monotime.h"
 #include "kudu/util/status.h"
 #include "kudu/util/test_macros.h"
@@ -65,17 +67,6 @@ using std::string;
 using std::vector;
 using strings::Substitute;
 
-// Specify row regex to match on. Empty string means don't match anything.
-#define ASSERT_DEBUG_DUMP_ROWS_MATCH(pattern) do { \
-  const std::string& _pat = (pattern); \
-  vector _rows; \
-  ASSERT_OK(tablet()->DebugDump(&_rows)); \
-  /* Ignore the non-data (formattting) lines in the output. */ \
-  std::string _base_pat = R"(^Dumping|^-|^MRS|^RowSet)"; \
-  if (!_pat.empty()) _base_pat += "|"; \
-  ASSERT_STRINGS_ALL_MATCH(_rows, _base_pat + _pat); \
-} while (0)
-
 namespace kudu {
 namespace tablet {
 
@@ -113,6 +104,8 @@ class TabletHistoryGcTest : public 
TabletTestBase> {
 uint64_t new_time = now + delta.ToMicroseconds();
 SetMockTime(new_time);
   }
+  // Specify row regex to match on. Empty string means don't match anything.
+  void VerifyDebugDumpRowsMatch(const string& pattern) const;
 
   int64_t TotalNumRows() const { return num_rowsets_ * rows_per_rowset_; }
 
@@ -126,7 +119,7 @@ class TabletHistoryGcTest : public 
TabletTestBase> {
 
   const int kStartRow = 0;
   int num_rowsets_ = 3;
-  int rows_per_rowset_ = 300;
+  int64_t rows_per_rowset_ = 300;
 };
 
 void TabletHistoryGcTest::InsertOriginalRows(int64_t num_rowsets, int64_t 
rows_per_rowset) {
@@ -146,6 +139,17 @@ void TabletHistoryGcTest::UpdateOriginalRows(int64_t 
num_rowsets, int64_t rows_p
   ASSERT_EQ(num_rowsets, tablet()->num_rowsets());
 }
 
+void TabletHistoryGcTest::VerifyDebugDumpRowsMatch(const string& pattern) 
const {
+  vector rows;
+  ASSERT_OK(tablet()->DebugDump(&rows)); \
+  // Ignore the non-data (formattting) lines in the output.
+  std::string base_pattern = R"(^Dumping|^-|^MRS|^RowSet)";
+  if (!pattern.empty()) {
+base_pattern += "|";
+  }
+  ASSERT_STRINGS_ALL_MATCH(rows, base_pattern + pattern);
+}
+
 // Test that we do not generate undos for redo operations that are older than
 // the AHM during major delta compaction.
 TEST_F(TabletHistoryGcTest, TestNoGenerateUndoOnMajorDeltaCompaction) {
@@ -191,8 +195,9 @@ TEST_F(TabletHistoryGcTest, 
TestNoGenerateUndoOnMajorDeltaCompaction) {
 
   // Now, we should have base data = 2 with no other historical values.
   // Major delta compaction will not remove UNDOs, so we expect a single UNDO 
DELETE as well.
-  ASSERT_DEBUG_DUMP_ROWS_MATCH(R"(int32 val=2\); Undo Mutations: 
\[@[[:digit:]]+\(DELETE\)\]; )"
-   R"(Redo Mutations: \[\];$)");
+  NO_FATALS(VerifyDebugDumpRowsMatch(
+  R"(int32 val=2\); Undo Mutations: \[@[[:digit:]]+\(DELETE\)\]; )"
+  R"(Redo Mutations: \[\];$)"));
 }
 
 // Test that major delta compaction works when run on a subset of columns:
@@ -233,8 +238,9 @@ TEST_F(TabletHistoryGcTest, 
TestMajorDeltaCompactionOnSubsetOfColumns) {
 
tablet()->GetHistoryGcOpts()));
   }
 
-  ASSERT_DEBUG_DUMP_ROWS_MATCH(R"(int32 val=2\); Undo Mutations: 
\[@[[:digit:]]+\(DELETE\)\]; )"
-   R"(Redo Mutations: \[@[[:digit:]]+\(SET 
key_idx=1\)\];$)");
+  NO_FATALS(VerifyDebugDumpRowsMatch(
+  R"(int32 val=2\); Undo Mutations: \[@[[:digit:]]+\(DELETE\)\]; )"
+  R"(Redo Mutations: \[@[[:digit:]]

[2/2] kudu git commit: [iwyu] update std::move

[alexey]

[iwyu] update std::move

Updated IWYU mappings to use  for std::move() and std::swap()
instead of .  Unsorted micro-cleanups in some tests.

Also, updated the cpplint.py script from the google-styleguide
to reflect std::swap() migration from  to . That
is to keep things consistent between 'iwyu' and 'lint/ilint' targets.
The latest version of the google-styleguide at [1] already contains the
corresponding update along with other C++11-related changes, but I would
like to bump the google-styleguide version in a separate changelist.

References:
  [1] https://github.com/google/styleguide

Change-Id: Ic5972f9a43f4c1db4a0dff5f983f7dba90c8db31
Reviewed-on: http://gerrit.cloudera.org:8080/8088
Reviewed-by: Adar Dembo 
Tested-by: Alexey Serbin 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/3e6e12a9
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/3e6e12a9
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/3e6e12a9

Branch: refs/heads/master
Commit: 3e6e12a9f41abb0f354a81af46698b697c2dcb6f
Parents: 02ba0ec
Author: Alexey Serbin 
Authored: Fri Sep 15 20:25:44 2017 -0700
Committer: Alexey Serbin 
Committed: Wed Sep 20 19:25:59 2017 +

--
 build-support/iwyu/iwyu-filter.awk  | 12 
 build-support/iwyu/mappings/boost-extra.imp |  1 +
 build-support/iwyu/mappings/libstdcpp.imp   |  6 +-
 src/kudu/benchmarks/tpch/tpch_real_world.cc |  6 +-
 src/kudu/cfile/block_cache.h|  2 +-
 src/kudu/cfile/bloomfile-test.cc|  2 +-
 src/kudu/cfile/bloomfile.cc |  2 +-
 src/kudu/cfile/cfile-test.cc|  2 +-
 src/kudu/cfile/cfile_reader.cc  |  1 +
 src/kudu/cfile/cfile_writer.cc  |  3 +-
 src/kudu/client/batcher.cc  |  1 -
 src/kudu/client/client.cc   |  1 -
 src/kudu/client/meta_cache.h|  2 +-
 src/kudu/client/partitioner-internal.h  |  2 +-
 src/kudu/client/replica-internal.cc |  2 +-
 src/kudu/client/scan_configuration.cc   |  2 +-
 src/kudu/client/scan_token-test.cc  |  2 +-
 src/kudu/client/schema.cc   |  2 +-
 src/kudu/client/session-internal.cc |  2 +-
 src/kudu/client/table-internal.cc   |  2 +-
 src/kudu/client/table_alterer-internal.cc   |  2 +-
 src/kudu/client/tablet_server-internal.cc   |  2 +-
 src/kudu/codegen/jit_wrapper.cc |  2 +-
 src/kudu/codegen/module_builder.cc  |  2 +-
 src/kudu/common/partial_row.cc  |  2 +-
 src/kudu/common/partition_pruner-test.cc|  1 -
 src/kudu/common/schema.h|  4 +-
 src/kudu/consensus/consensus_meta-test.cc   | 13 ++--
 src/kudu/consensus/consensus_meta.cc|  2 +-
 .../consensus_meta_manager-stress-test.cc   |  2 +-
 src/kudu/consensus/consensus_meta_manager.cc|  2 +-
 src/kudu/consensus/consensus_peers-test.cc  |  2 +-
 src/kudu/consensus/leader_election-test.cc  |  4 +-
 src/kudu/consensus/log.cc   |  2 +-
 src/kudu/consensus/log_util.cc  |  1 +
 src/kudu/consensus/raft_consensus.h |  2 +-
 .../consensus/raft_consensus_quorum-test.cc | 76 ++--
 src/kudu/fs/block_manager-stress-test.cc|  2 +-
 src/kudu/fs/block_manager-test.cc   |  1 +
 src/kudu/fs/block_manager.h |  6 +-
 src/kudu/fs/block_manager_util-test.cc  |  2 +-
 src/kudu/fs/block_manager_util.cc   |  2 +-
 src/kudu/fs/data_dirs.h |  2 +-
 src/kudu/fs/file_block_manager.cc   |  2 +-
 src/kudu/fs/fs-test-util.h  |  1 +
 src/kudu/fs/fs_manager.cc   |  2 +-
 src/kudu/gutil/atomicops-internals-x86.cc   |  3 +-
 src/kudu/gutil/cpu.cc   |  2 +-
 src/kudu/gutil/hash/city.cc |  2 +-
 src/kudu/gutil/ref_counted_memory.cc|  2 +-
 src/kudu/gutil/strings/split.cc |  1 +
 src/kudu/gutil/strings/split.h  |  3 +-
 src/kudu/gutil/strings/strcat.h |  3 +-
 src/kudu/gutil/strings/stringpiece.cc   | 10 +--
 src/kudu/gutil/strings/strip.cc |  8 ---
 src/kudu/gutil/strings/util.cc  | 11 +--
 .../exactly_once_writes-itest.cc|  2 +-
 .../external_mini_cluster-itest-base.cc |  2 +-
 .../external_mini_cluster-test.cc   |  2 +-
 .../integration-tests/internal_mini_cluster.cc  |  2 +-
 .../integration-tests/master-stress-test.cc |  4 +-
 .../integration-tests/open-readonly-fs-itest.cc |  2 +-
 .../integration-tests/token_signer-itest.cc | 15 ++--
 .../integration-tests/version_migration-test.cc |  4 +-
 src/kudu/m

[1/2] kudu git commit: Fix changes that were left out of KUDU-501 and /table template patches

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master 9c5d6d4df -> 9c1997ad3


Fix changes that were left out of KUDU-501 and /table template patches

Not sure how I did this, but a typo in a comment and some missing
substitute args were fixed but then somehow got left out of the final
changelist.

Change-Id: I97a17f60f23a2ba62f0fa219353b264daeaf7f48
Reviewed-on: http://gerrit.cloudera.org:8080/8125
Reviewed-by: Adar Dembo 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/06a19bd2
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/06a19bd2
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/06a19bd2

Branch: refs/heads/master
Commit: 06a19bd25a964664db08ef89ab0ebe26369519ab
Parents: 9c5d6d4
Author: Will Berkeley 
Authored: Fri Sep 22 10:09:30 2017 -0700
Committer: Will Berkeley 
Committed: Fri Sep 22 22:22:29 2017 +

--
 src/kudu/master/master-path-handlers.cc | 4 ++--
 src/kudu/master/master-path-handlers.h  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/06a19bd2/src/kudu/master/master-path-handlers.cc
--
diff --git a/src/kudu/master/master-path-handlers.cc 
b/src/kudu/master/master-path-handlers.cc
index bf3d12d..e8fa753 100644
--- a/src/kudu/master/master-path-handlers.cc
+++ b/src/kudu/master/master-path-handlers.cc
@@ -271,13 +271,13 @@ void MasterPathHandlers::HandleTablePage(const 
Webserver::WebRequest& req,
 
 s = SchemaFromPB(l.data().pb.schema(), &schema);
 if (!s.ok()) {
-  (*output)["error"] = Substitute("Unable to decode schema: ", 
s.ToString());
+  (*output)["error"] = Substitute("Unable to decode schema: $0", 
s.ToString());
   return;
 }
 s = PartitionSchema::FromPB(l.data().pb.partition_schema(), schema, 
&partition_schema);
 if (!s.ok()) {
   (*output)["error"] =
-  Substitute("Unable to decode partition schema: ", s.ToString());
+  Substitute("Unable to decode partition schema: $0", s.ToString());
   return;
 }
 table->GetAllTablets(&tablets);

http://git-wip-us.apache.org/repos/asf/kudu/blob/06a19bd2/src/kudu/master/master-path-handlers.h
--
diff --git a/src/kudu/master/master-path-handlers.h 
b/src/kudu/master/master-path-handlers.h
index 6195a74..72cff3f 100644
--- a/src/kudu/master/master-path-handlers.h
+++ b/src/kudu/master/master-path-handlers.h
@@ -59,7 +59,7 @@ class MasterPathHandlers {
 
   // Returns a pair (text, target) given a tserver's TSDescriptor and a tablet 
id.
   // - text is the http host and port for the tserver, if available, or the 
tserver's uuid.
-  // - target is a url to to the tablet page for the tablet on the tserver's 
webui,
+  // - target is a url to the tablet page for the tablet on the tserver's 
webui,
   //   or an empty string if no http address is available for the tserver.
   std::pair TSDescToLinkPair(const TSDescriptor& 
desc,
const std::string& 
tablet_id) const;

[2/2] kudu git commit: Revert "[tests] fix flakiness in catalog_manager_tsk-itest"

[alexey]

Revert "[tests] fix flakiness in catalog_manager_tsk-itest"

This reverts commit faa0b14effb6e15f9989d686e5a1f8e1040a1dd6.

Since the fix for KUDU-2149 is committed, the catalog_manager_tsk-itest
is no longer flaky even with original parameters. We want to
use the catalog_manager_tsk-itest with its original settings as a
canary for any further regressions in that area.

Notes on verification:
  With this change, and the fix for KUDU-2149 rolled-back,
  10 out of 1024 failed in when running the catalog_manager_tsk-itest
  test with --stress_cpu_threads=16:
http://dist-test.cloudera.org//job?job_id=aserbin.1506033479.20652

  With this change, no failures were spotted in multiple 1K runs of the
  test with --stress_cpu_threads=16:
http://dist-test.cloudera.org//job?job_id=aserbin.1506109518.28895

Change-Id: Ic7db0eef852a417da5d11480ccdb4f15019ea1da
Reviewed-on: http://gerrit.cloudera.org:8080/8129
Reviewed-by: Adar Dembo 
Tested-by: Kudu Jenkins
Reviewed-by: David Ribeiro Alves 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/9c1997ad
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/9c1997ad
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/9c1997ad

Branch: refs/heads/master
Commit: 9c1997ad3473ab3ce29061c4cd2aa11552967a98
Parents: 06a19bd
Author: Alexey Serbin 
Authored: Fri Sep 22 15:02:23 2017 -0700
Committer: Alexey Serbin 
Committed: Fri Sep 22 22:46:34 2017 +

--
 src/kudu/integration-tests/catalog_manager_tsk-itest.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/9c1997ad/src/kudu/integration-tests/catalog_manager_tsk-itest.cc
--
diff --git a/src/kudu/integration-tests/catalog_manager_tsk-itest.cc 
b/src/kudu/integration-tests/catalog_manager_tsk-itest.cc
index e573097..acdeba9 100644
--- a/src/kudu/integration-tests/catalog_manager_tsk-itest.cc
+++ b/src/kudu/integration-tests/catalog_manager_tsk-itest.cc
@@ -61,7 +61,7 @@ class CatalogManagerTskITest : public KuduTest {
   : num_masters_(3),
 num_tservers_(1),
 #if defined(ADDRESS_SANITIZER) || defined(THREAD_SANITIZER)
-hb_interval_ms_(64),
+hb_interval_ms_(32),
 run_time_seconds_(5)
 #else
 hb_interval_ms_(16),

kudu git commit: [consensus] don't crash in RaftConsensus::ChangeConfig

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master 26a32fc4b -> 98b66574b


[consensus] don't crash in RaftConsensus::ChangeConfig

Don't crash if ReplicateConfigChangeUnlocked() returns an error.
Instead, just return corresponding error to the caller.  This is
to avoid crashing a tablet server if an invalid Raft configuration is
passed to ChangeConfig() RPC with ChangeConfigRequestPB parameter.

Also, use move semantics for all parameters of the
RaftConsensus::ReplicateConfigChangeUnlocked() method.

Change-Id: Ieac3692adae029d098fea254badf2c53c6ac4fd8
Reviewed-on: http://gerrit.cloudera.org:8080/8124
Tested-by: Kudu Jenkins
Reviewed-by: Alexey Serbin 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/98b66574
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/98b66574
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/98b66574

Branch: refs/heads/master
Commit: 98b66574b06838347e323b997778f964acba1abd
Parents: 26a32fc
Author: Alexey Serbin 
Authored: Thu Sep 21 18:34:08 2017 -0700
Committer: Alexey Serbin 
Committed: Mon Sep 25 23:24:44 2017 +

--
 src/kudu/consensus/raft_consensus.cc | 16 +++-
 src/kudu/consensus/raft_consensus.h  |  8 
 2 files changed, 11 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/98b66574/src/kudu/consensus/raft_consensus.cc
--
diff --git a/src/kudu/consensus/raft_consensus.cc 
b/src/kudu/consensus/raft_consensus.cc
index 6b4e968..d70d566 100644
--- a/src/kudu/consensus/raft_consensus.cc
+++ b/src/kudu/consensus/raft_consensus.cc
@@ -568,9 +568,8 @@ Status RaftConsensus::BecomeLeaderUnlocked() {
   round.get(),
   &DoNothingStatusCB,
   std::placeholders::_1));
-  RETURN_NOT_OK(AppendNewRoundToQueueUnlocked(round));
 
-  return Status::OK();
+  return AppendNewRoundToQueueUnlocked(round);
 }
 
 Status RaftConsensus::BecomeReplicaUnlocked() {
@@ -1643,7 +1642,7 @@ Status RaftConsensus::ChangeConfig(const 
ChangeConfigRequestPB& req,
 }
 
 RETURN_NOT_OK(ReplicateConfigChangeUnlocked(
-committed_config, new_config, std::bind(
+std::move(committed_config), std::move(new_config), std::bind(
 &RaftConsensus::MarkDirtyOnSuccess,
 this,
 string("Config change replication complete"),
@@ -2080,16 +2079,16 @@ void RaftConsensus::SetLeaderUuidUnlocked(const string& 
uuid) {
 }
 
 Status RaftConsensus::ReplicateConfigChangeUnlocked(
-const RaftConfigPB& old_config,
-const RaftConfigPB& new_config,
+RaftConfigPB old_config,
+RaftConfigPB new_config,
 StdStatusCallback client_cb) {
   DCHECK(lock_.is_locked());
   auto cc_replicate = new ReplicateMsg();
   cc_replicate->set_op_type(CHANGE_CONFIG_OP);
   ChangeConfigRecordPB* cc_req = cc_replicate->mutable_change_config_record();
   cc_req->set_tablet_id(options_.tablet_id);
-  *cc_req->mutable_old_config() = old_config;
-  *cc_req->mutable_new_config() = new_config;
+  *cc_req->mutable_old_config() = std::move(old_config);
+  *cc_req->mutable_new_config() = std::move(new_config);
   CHECK_OK(time_manager_->AssignTimestamp(cc_replicate));
 
   scoped_refptr round(
@@ -2101,8 +2100,7 @@ Status RaftConsensus::ReplicateConfigChangeUnlocked(
   std::move(client_cb),
   std::placeholders::_1));
 
-  CHECK_OK(AppendNewRoundToQueueUnlocked(round));
-  return Status::OK();
+  return AppendNewRoundToQueueUnlocked(round);
 }
 
 Status RaftConsensus::RefreshConsensusQueueAndPeersUnlocked() {

http://git-wip-us.apache.org/repos/asf/kudu/blob/98b66574/src/kudu/consensus/raft_consensus.h
--
diff --git a/src/kudu/consensus/raft_consensus.h 
b/src/kudu/consensus/raft_consensus.h
index b82afb3..080bc7e 100644
--- a/src/kudu/consensus/raft_consensus.h
+++ b/src/kudu/consensus/raft_consensus.h
@@ -415,12 +415,12 @@ class RaftConsensus : public 
std::enable_shared_from_this,
   // reporting to the master.
   void SetLeaderUuidUnlocked(const std::string& uuid);
 
-  // Replicate (as leader) a pre-validated config change. This includes
-  // updating the peers and setting the new_configuration as pending.
+  // Replicate (as leader) a config change. This includes validating the new
+  // config and updating the peers and setting the new_configuration as 
pending.
   // The old_configuration must be the currently-committed configuration.
   Status ReplicateConfigChangeUnlocked(
-  const RaftConfigPB& old_config,
-  const RaftConfigPB& new_config,
+  RaftConfigPB old_config,
+  RaftConfigPB new_config,
   StdStatusCallback client_cb);
 
   // Update the peers and queue to be consistent with a new active 
configuration.

kudu git commit: [consensus] micro-cleanup on VerifyRaftConfig()

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master 98b66574b -> bab96a2b2


[consensus] micro-cleanup on VerifyRaftConfig()

Removed unused RaftConfigState parameter for VerifyRaftConfig().
In addition, fixed typos in metadata.proto inline documentation.

Change-Id: Ic5b4cb6ceacde7a0c0b2f1c75dca4b7d1681ef33
Reviewed-on: http://gerrit.cloudera.org:8080/8078
Tested-by: Kudu Jenkins
Reviewed-by: Mike Percy 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/bab96a2b
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/bab96a2b
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/bab96a2b

Branch: refs/heads/master
Commit: bab96a2b299de4f1f911e23c80167938ed69394d
Parents: 98b6657
Author: Alexey Serbin 
Authored: Thu Sep 14 19:57:43 2017 -0700
Committer: Alexey Serbin 
Committed: Tue Sep 26 00:56:27 2017 +

--
 src/kudu/consensus/consensus_meta.cc |  2 +-
 src/kudu/consensus/metadata.proto|  8 
 src/kudu/consensus/quorum_util.cc| 10 +-
 src/kudu/consensus/quorum_util.h |  5 +
 src/kudu/consensus/raft_consensus.cc |  8 
 src/kudu/master/sys_catalog.cc   |  5 ++---
 6 files changed, 17 insertions(+), 21 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/bab96a2b/src/kudu/consensus/consensus_meta.cc
--
diff --git a/src/kudu/consensus/consensus_meta.cc 
b/src/kudu/consensus/consensus_meta.cc
index 75de554..9954c8a 100644
--- a/src/kudu/consensus/consensus_meta.cc
+++ b/src/kudu/consensus/consensus_meta.cc
@@ -267,7 +267,7 @@ Status ConsensusMetadata::Flush(FlushMode flush_mode) {
 
   flush_count_for_tests_++;
   // Sanity test to ensure we never write out a bad configuration.
-  RETURN_NOT_OK_PREPEND(VerifyRaftConfig(pb_.committed_config(), 
COMMITTED_CONFIG),
+  RETURN_NOT_OK_PREPEND(VerifyRaftConfig(pb_.committed_config()),
 "Invalid config in ConsensusMetadata, cannot flush to 
disk");
 
   // Create directories if needed.

http://git-wip-us.apache.org/repos/asf/kudu/blob/bab96a2b/src/kudu/consensus/metadata.proto
--
diff --git a/src/kudu/consensus/metadata.proto 
b/src/kudu/consensus/metadata.proto
index cc7223d..718d9eb 100644
--- a/src/kudu/consensus/metadata.proto
+++ b/src/kudu/consensus/metadata.proto
@@ -59,7 +59,7 @@ message RaftPeerPB {
   // Permanent uuid is optional: RaftPeerPB/RaftConfigPB instances may
   // be created before the permanent uuid is known (e.g., when
   // manually specifying a configuration for Master/CatalogManager);
-  // permament uuid can be retrieved at a later time through RPC.
+  // permanent uuid can be retrieved at a later time through RPC.
   optional bytes permanent_uuid = 1;
   optional MemberType member_type = 2;
   optional HostPortPB last_known_addr = 3;
@@ -97,8 +97,8 @@ message ConsensusStatePB {
   // to a term check on the UpdateConsensus() RPC request.
   //
   // Whenever the local peer sees a new term, the leader flag is cleared until
-  // a new leader is acknowledged based on the above critera. Simply casting a
-  // vote for a peer is not sufficient to assume that that peer has won the
+  // a new leader is acknowledged based on the above criteria. Simply casting a
+  // vote for a peer is not sufficient to assume that the peer has won the
   // election, so we do not update this field based on our vote.
   //
   // The leader may be a part of the committed or the pending configuration 
(or both).
@@ -125,7 +125,7 @@ message ConsensusMetadataPB {
   // Whenever a new election is started, the candidate increments this by one
   // and requests votes from peers.
   //
-  // If any RPC or RPC response is received from another node containing a 
term higher
+  // If any RPC request or response is received from another node containing a 
term higher
   // than this one, the server should step down to FOLLOWER and set its 
current_term to
   // match the caller's term.
   //

http://git-wip-us.apache.org/repos/asf/kudu/blob/bab96a2b/src/kudu/consensus/quorum_util.cc
--
diff --git a/src/kudu/consensus/quorum_util.cc 
b/src/kudu/consensus/quorum_util.cc
index 6c791b3..ca074b4 100644
--- a/src/kudu/consensus/quorum_util.cc
+++ b/src/kudu/consensus/quorum_util.cc
@@ -134,9 +134,9 @@ RaftPeerPB::Role GetConsensusRole(const std::string& uuid, 
const ConsensusStateP
   return RaftPeerPB::NON_PARTICIPANT;
 }
 
-Status VerifyRaftConfig(const RaftConfigPB& config, RaftConfigState type) {
+Status VerifyRaftConfig(const RaftConfigPB& config) {
   std::set uuids;
-  if (config.peers_size() == 0) {
+  if (config.peers().empty()) {
  

kudu git commit: [cluster_itest_util] fix typo in format string

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master 36f4259f4 -> 9b0544ae3


[cluster_itest_util] fix typo in format string

Change-Id: Ied30a36d63a331eca7ff96e0ffc3225db5803b22
Reviewed-on: http://gerrit.cloudera.org:8080/8156
Reviewed-by: Adar Dembo 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/9b0544ae
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/9b0544ae
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/9b0544ae

Branch: refs/heads/master
Commit: 9b0544ae3466235cb5bc699ec1a4def3d2aa1f02
Parents: 36f4259
Author: Alexey Serbin 
Authored: Wed Sep 27 15:52:23 2017 -0700
Committer: Alexey Serbin 
Committed: Wed Sep 27 23:18:04 2017 +

--
 src/kudu/integration-tests/cluster_itest_util.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/9b0544ae/src/kudu/integration-tests/cluster_itest_util.cc
--
diff --git a/src/kudu/integration-tests/cluster_itest_util.cc 
b/src/kudu/integration-tests/cluster_itest_util.cc
index 326dd53..fa4c1d4 100644
--- a/src/kudu/integration-tests/cluster_itest_util.cc
+++ b/src/kudu/integration-tests/cluster_itest_util.cc
@@ -451,8 +451,8 @@ Status WaitForReplicasReportedToMaster(
   }
 }
 if (deadline < MonoTime::Now()) {
-  return Status::TimedOut(Substitute("Timed out after waiting "
-  "for tablet $1 expected to report master with $2 replicas, 
has_leader: $3",
+  return Status::TimedOut(Substitute("Timed out while waiting "
+  "for tablet $0 reporting to master with $1 replicas, has_leader: $2",
   tablet_id, num_replicas, *has_leader));
 }
 SleepFor(MonoDelta::FromMilliseconds(20));

kudu git commit: block_compression: fix arg to Substitute

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master 4c7d8e040 -> 77dea331d


block_compression: fix arg to Substitute

Change-Id: If00035e39ba3f837f42e398ca6aa31bbc9afb3a9
Reviewed-on: http://gerrit.cloudera.org:8080/8203
Tested-by: Kudu Jenkins
Reviewed-by: Alexey Serbin 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/77dea331
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/77dea331
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/77dea331

Branch: refs/heads/master
Commit: 77dea331d8d0919d3b9ba2c8cc7f776b5d5f3a6f
Parents: 4c7d8e0
Author: Adar Dembo 
Authored: Tue Oct 3 16:09:26 2017 -0700
Committer: Alexey Serbin 
Committed: Wed Oct 4 01:26:20 2017 +

--
 src/kudu/cfile/block_compression.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/77dea331/src/kudu/cfile/block_compression.cc
--
diff --git a/src/kudu/cfile/block_compression.cc 
b/src/kudu/cfile/block_compression.cc
index e7bb5df..aa342bb 100644
--- a/src/kudu/cfile/block_compression.cc
+++ b/src/kudu/cfile/block_compression.cc
@@ -167,7 +167,7 @@ Status CompressedBlockDecoder::Init() {
   if (uncompressed_size_ > FLAGS_max_cfile_block_size) {
 return Status::Corruption(
   Substitute("uncompressed size $0 overflows the maximum length $1, 
buffer",
- compressed_size, FLAGS_max_cfile_block_size),
+ uncompressed_size_, FLAGS_max_cfile_block_size),
   KUDU_REDACT(data_.ToDebugString(50)));
   }
 

kudu git commit: [tablet copy] comment on TabletCopyClient lifecycle

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master 77dea331d -> 3b0ef599b


[tablet copy] comment on TabletCopyClient lifecycle

Added an explanatory comment on the TabletCopyClient instance lifecycle.
That was the result of an attempt to clean-up corresponding code:
  https://gerrit.cloudera.org/#/c/8133/

Change-Id: I7ae41f12fe75f05c84518c26b6875f34efae0172
Reviewed-on: http://gerrit.cloudera.org:8080/8197
Reviewed-by: Mike Percy 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/3b0ef599
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/3b0ef599
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/3b0ef599

Branch: refs/heads/master
Commit: 3b0ef599b2fc3fc9b895937548e98c57552b021f
Parents: 77dea33
Author: Alexey Serbin 
Authored: Mon Oct 2 18:24:45 2017 -0700
Committer: Alexey Serbin 
Committed: Wed Oct 4 18:30:55 2017 +

--
 src/kudu/tserver/ts_tablet_manager.cc | 33 ++
 1 file changed, 33 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/3b0ef599/src/kudu/tserver/ts_tablet_manager.cc
--
diff --git a/src/kudu/tserver/ts_tablet_manager.cc 
b/src/kudu/tserver/ts_tablet_manager.cc
index 5b6f9e7..2967fe7 100644
--- a/src/kudu/tserver/ts_tablet_manager.cc
+++ b/src/kudu/tserver/ts_tablet_manager.cc
@@ -626,6 +626,39 @@ void TSTabletManager::RunTabletCopy(
   LOG(INFO) << init_msg;
   TRACE(init_msg);
 
+  // The TabletCopyClient instance should be kept alive until the tablet
+  // is successfully copied over and opened/started. This is because we want
+  // to maintain the LogAnchor until the replica starts up. Upon destruction,
+  // the TabletCopyClient instance sends an RPC explicitly ending the tablet
+  // copy session. The source replica then destroys the corresponding
+  // TabletCopySourceSession object, releasing its LogAnchor and allowing
+  // the WAL segments being copied to be GCed.
+  //
+  // See below for more details on why anchoring of WAL segments is necessary.
+  //
+  // * Assume there are WAL segments 0-10 when tablet copy starts. Tablet copy
+  //   will anchor 0 until its destroyed, meaning the source replica wont
+  //   delete it.
+  //
+  // * When tablet copy is done the tablet still needs to bootstrap which will
+  //   take some time.
+  //
+  // * When tablet bootstrap is done, the new replica will need to continue
+  //   catching up to the leader, this time through consensus. It needs segment
+  //   11 to be still available. We need the anchor to still be alive at this
+  //   point, otherwise there is nothing preventing the leader from deleting
+  //   segment 11 and thus making the new replica unable to catch up. Yes, it's
+  //   not optimal: we're anchoring 0 and we might only need to anchor 10/11.
+  //   However, having no anchor at all is likely to cause replicas to start
+  //   fail copying.
+  //
+  // NOTE:
+  //   Ideally, we should wait until the leader starts tracking of the target
+  //   replica's log watermark. As for current implementation, the intent is
+  //   to at least try preventing GC of logs before the tablet replica connects
+  //   to the leader.
+  //
+  // TODO(aserbin): make this robust and more optimal than it is now.
   TabletCopyClient tc_client(tablet_id, fs_manager_, cmeta_manager_,
  server_->messenger(), &tablet_copy_metrics_);
 

[3/3] kudu git commit: tsan: flush TSAN memory every 10 seconds

[alexey]

tsan: flush TSAN memory every 10 seconds

I'm seeing a TSAN precommit test fail in a hard-to-diagnose way because the
Linux OOM killer is killing tservers underneath it. It doesn't appear to
be any leak -- rather, TSAN is just using too much memory due to the
test doing a lot of allocation.

This enables a TSAN option to flush its memory every 10 seconds. This
has the downside of potentially missing races that happen far apart in
time (but those are unlikely to actually trigger in practice) but
reduces RSS significantly.

Change-Id: I721f126eb731c7f35afb732cebe48abd52c3ebd1
Reviewed-on: http://gerrit.cloudera.org:8080/8251
Tested-by: Alexey Serbin 
Reviewed-by: Alexey Serbin 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/24308aa1
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/24308aa1
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/24308aa1

Branch: refs/heads/master
Commit: 24308aa1a5fd05e616dad2e8ee7482a74b9350de
Parents: 83b6931
Author: Todd Lipcon 
Authored: Tue Oct 10 14:08:26 2017 -0700
Committer: Alexey Serbin 
Committed: Sun Oct 15 04:09:19 2017 +

--
 build-support/run-test.sh | 3 +++
 1 file changed, 3 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/24308aa1/build-support/run-test.sh
--
diff --git a/build-support/run-test.sh b/build-support/run-test.sh
index 941912b..6073f19 100755
--- a/build-support/run-test.sh
+++ b/build-support/run-test.sh
@@ -104,6 +104,9 @@ fi
 # Configure TSAN (ignored if this isn't a TSAN build).
 TSAN_OPTIONS="$TSAN_OPTIONS 
suppressions=$SOURCE_ROOT/build-support/tsan-suppressions.txt"
 TSAN_OPTIONS="$TSAN_OPTIONS history_size=7"
+#   Flush TSAN memory every 10 seconds - this prevents RSS blowup in unit tests
+#   which can cause tests to get killed by the OOM killer.
+TSAN_OPTIONS="$TSAN_OPTIONS flush_memory_ms=1"
 TSAN_OPTIONS="$TSAN_OPTIONS external_symbolizer_path=$ASAN_SYMBOLIZER_PATH"
 export TSAN_OPTIONS
 

[1/3] kudu git commit: KUDU-2184. Avoid allocations larger than 1MB

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master b6f727ea2 -> 24308aa1a


KUDU-2184. Avoid allocations larger than 1MB

tcmalloc doesn't perform well for allocations larger than 1MB[1]. We
currently use large allocations for full arenas (in particular the MRS)
which can cause large latency blips when a thread tries to allocate a
new arena component.

This just changes the max buffer size for all arenas to be 1MB and adds
a DCHECK so we avoid this problem in the future.

Additionally I increased the buffer size for the /memz output so that we
can get more stats on the occupancy of different allocator size classes.

[1] https://github.com/gperftools/gperftools/issues/535

Change-Id: I19c8ae9ce268e2623a89624e19db673931a093c0
Reviewed-on: http://gerrit.cloudera.org:8080/8239
Tested-by: Kudu Jenkins
Reviewed-by: David Ribeiro Alves 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/eb5c7e2f
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/eb5c7e2f
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/eb5c7e2f

Branch: refs/heads/master
Commit: eb5c7e2f0c73729a4486bc71512739b0788b1574
Parents: b6f727e
Author: Todd Lipcon 
Authored: Mon Oct 9 14:22:48 2017 -0700
Committer: Todd Lipcon 
Committed: Fri Oct 13 01:58:04 2017 +

--
 src/kudu/cfile/binary_dict_block.cc  | 2 +-
 src/kudu/cfile/cfile-test-base.h | 2 +-
 src/kudu/cfile/cfile-test.cc | 2 +-
 src/kudu/cfile/cfile_util.cc | 2 +-
 src/kudu/server/default-path-handlers.cc | 7 ---
 src/kudu/tablet/concurrent_btree.h   | 5 +
 src/kudu/tablet/deltamemstore.cc | 2 +-
 src/kudu/tablet/memrowset.cc | 2 +-
 src/kudu/tablet/tablet-test-base.h   | 2 +-
 src/kudu/tablet/tablet_random_access-test.cc | 2 +-
 src/kudu/tablet/transactions/transaction.cc  | 2 +-
 src/kudu/util/memory/arena.cc| 9 +
 12 files changed, 19 insertions(+), 20 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/eb5c7e2f/src/kudu/cfile/binary_dict_block.cc
--
diff --git a/src/kudu/cfile/binary_dict_block.cc 
b/src/kudu/cfile/binary_dict_block.cc
index 9e96163..f00d75d 100644
--- a/src/kudu/cfile/binary_dict_block.cc
+++ b/src/kudu/cfile/binary_dict_block.cc
@@ -52,7 +52,7 @@ namespace cfile {
 BinaryDictBlockBuilder::BinaryDictBlockBuilder(const WriterOptions* options)
 : options_(options),
   dict_block_(options_),
-  dictionary_strings_arena_(1024, 32*1024*1024),
+  dictionary_strings_arena_(1024, 1024*1024),
   mode_(kCodeWordMode) {
   data_builder_.reset(new BShufBlockBuilder(options_));
   // We use this invalid StringPiece for the "empty key". It's safe to build 
such

http://git-wip-us.apache.org/repos/asf/kudu/blob/eb5c7e2f/src/kudu/cfile/cfile-test-base.h
--
diff --git a/src/kudu/cfile/cfile-test-base.h b/src/kudu/cfile/cfile-test-base.h
index f4f0c8f..64fd35d 100644
--- a/src/kudu/cfile/cfile-test-base.h
+++ b/src/kudu/cfile/cfile-test-base.h
@@ -467,7 +467,7 @@ void TimeReadFile(FsManager* fs_manager, const BlockId& 
block_id, size_t *count_
   ASSERT_OK(reader->NewIterator(&iter, CFileReader::CACHE_BLOCK));
   ASSERT_OK(iter->SeekToOrdinal(0));
 
-  Arena arena(8192, 8*1024*1024);
+  Arena arena(8192, 1024 * 1024);
   int count = 0;
   switch (reader->type_info()->physical_type()) {
 case UINT8:

http://git-wip-us.apache.org/repos/asf/kudu/blob/eb5c7e2f/src/kudu/cfile/cfile-test.cc
--
diff --git a/src/kudu/cfile/cfile-test.cc b/src/kudu/cfile/cfile-test.cc
index 364fb93..f7ffe85 100644
--- a/src/kudu/cfile/cfile-test.cc
+++ b/src/kudu/cfile/cfile-test.cc
@@ -200,7 +200,7 @@ class TestCFile : public CFileTestBase {
 gscoped_ptr iter;
 ASSERT_OK(reader->NewIterator(&iter, CFileReader::CACHE_BLOCK));
 
-Arena arena(8192, 8*1024*1024);
+Arena arena(8192, 1024 * 1024);
 ScopedColumnBlock cb(10);
 
 SelectionVector sel(10);

http://git-wip-us.apache.org/repos/asf/kudu/blob/eb5c7e2f/src/kudu/cfile/cfile_util.cc
--
diff --git a/src/kudu/cfile/cfile_util.cc b/src/kudu/cfile/cfile_util.cc
index 2995407..31c5af3 100644
--- a/src/kudu/cfile/cfile_util.cc
+++ b/src/kudu/cfile/cfile_util.cc
@@ -46,7 +46,7 @@ Status DumpIterator(const CFileReader& reader,
 int num_rows,
 int indent) {
 
-  Arena arena(8192, 8*1024*1024);
+  Arena arena(8192, 1024 * 1024);
   uint8_t buf[kBufSize];
   const TypeInfo *type = reader.type_info();
   size_t max_rows = kBufSize/type->size();

http://git-wip-u

[2/3] kudu git commit: Reduce log spew from rpcz_store.cc

[alexey]

Reduce log spew from rpcz_store.cc

The sampled RPC call statement has been filing the log. Let's dial down its log 
level.
This particularly impacts Impala as its default log level is 1.

If it turns out that this log statement is useful for Kudu deployment, we can
consider doing this change in Impala only.

Testing done:
  ctest -R rpc-test;
  Built debug and release builds;

Change-Id: I8efe68be0ae7f9ab05937d5c81fb53a709a881f1
Reviewed-on: http://gerrit.cloudera.org:8080/8273
Reviewed-by: Todd Lipcon 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/83b69315
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/83b69315
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/83b69315

Branch: refs/heads/master
Commit: 83b693150d0c5772ae52cf9bc2bad58a46c25a92
Parents: eb5c7e2
Author: Michael Ho 
Authored: Thu Oct 12 16:36:27 2017 -0700
Committer: Todd Lipcon 
Committed: Sat Oct 14 00:46:29 2017 +

--
 src/kudu/rpc/rpcz_store.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/83b69315/src/kudu/rpc/rpcz_store.cc
--
diff --git a/src/kudu/rpc/rpcz_store.cc b/src/kudu/rpc/rpcz_store.cc
index ffbbf2f..8c8c04a 100644
--- a/src/kudu/rpc/rpcz_store.cc
+++ b/src/kudu/rpc/rpcz_store.cc
@@ -159,7 +159,7 @@ void MethodSampler::SampleCall(InboundCall* call) {
   std::swap(bucket->sample, new_sample);
   bucket->last_sample_time.Store(now);
 }
-VLOG(1) << "Sampled call " << call->ToString();
+VLOG(2) << "Sampled call " << call->ToString();
   }
 }
 

kudu git commit: [security] fix typo on rpc_tls_ciphers

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master 24308aa1a -> 78843dd81


[security] fix typo on rpc_tls_ciphers

Fixed typo in the default list of TLS ciphers.  Due to the typo,
the prior revisions didn't use the following ciphers:

ECDHE-RSA-AES128-SHA256 TLSv1.2 Kx=ECDH Au=RSA Enc=AES(128)  Mac=SHA256
AES256-GCM-SHA384   TLSv1.2 Kx=RSA  Au=RSA Enc=AESGCM(256) Mac=AEAD

Change-Id: I8508bf9a9002e77cb38911837b9c3881acc01cf4
Reviewed-on: http://gerrit.cloudera.org:8080/8281
Tested-by: Kudu Jenkins
Reviewed-by: Dan Burkert 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/78843dd8
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/78843dd8
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/78843dd8

Branch: refs/heads/master
Commit: 78843dd81c628239a5fdd962208f949c63273384
Parents: 24308aa
Author: Alexey Serbin 
Authored: Mon Oct 16 09:41:19 2017 -0700
Committer: Alexey Serbin 
Committed: Mon Oct 16 18:21:49 2017 +

--
 src/kudu/security/tls_context.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/78843dd8/src/kudu/security/tls_context.cc
--
diff --git a/src/kudu/security/tls_context.cc b/src/kudu/security/tls_context.cc
index 3ab0131..1cea173 100644
--- a/src/kudu/security/tls_context.cc
+++ b/src/kudu/security/tls_context.cc
@@ -67,7 +67,7 @@ DEFINE_string(rpc_tls_ciphers,
   "ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:"
   "ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:"
   "ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:"
-  "ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256"
+  "ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:"
   "AES256-GCM-SHA384:AES128-GCM-SHA256:"
   "AES256-SHA256:AES128-SHA256:"
   "AES256-SHA:AES128-SHA",

[1/2] kudu git commit: [ts_itest-base] split declaration and definition

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master 92064c023 -> 3a7342a31


http://git-wip-us.apache.org/repos/asf/kudu/blob/3a7342a3/src/kudu/tserver/tablet_server-test-base.h
--
diff --git a/src/kudu/tserver/tablet_server-test-base.h 
b/src/kudu/tserver/tablet_server-test-base.h
index ec015dc..a34c9b5 100644
--- a/src/kudu/tserver/tablet_server-test-base.h
+++ b/src/kudu/tserver/tablet_server-test-base.h
@@ -15,447 +15,109 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#ifndef KUDU_TSERVER_TABLET_SERVER_TEST_BASE_H_
-#define KUDU_TSERVER_TABLET_SERVER_TEST_BASE_H_
+#pragma once
 
-#include 
-#include 
-#include 
-#include 
-#include 
-
-#include 
-#include 
+#include 
 #include 
 #include 
 #include 
 #include 
 
-#include 
-
-#include "kudu/common/wire_protocol-test-util.h"
+#include "kudu/common/row.h"
+#include "kudu/common/schema.h"
 #include "kudu/consensus/consensus.proxy.h"
-#include "kudu/consensus/log_reader.h"
-#include "kudu/gutil/atomicops.h"
 #include "kudu/gutil/gscoped_ptr.h"
-#include "kudu/gutil/stl_util.h"
-#include "kudu/gutil/strings/substitute.h"
-#include "kudu/rpc/messenger.h"
-#include "kudu/rpc/rpc_controller.h"
+#include "kudu/gutil/ref_counted.h"
 #include "kudu/server/server_base.proxy.h"
-#include "kudu/tablet/local_tablet_writer.h"
-#include "kudu/tablet/tablet.h"
 #include "kudu/tablet/tablet_replica.h"
 #include "kudu/tserver/mini_tablet_server.h"
 #include "kudu/tserver/tablet_copy.proxy.h"
-#include "kudu/tserver/scanners.h"
-#include "kudu/tserver/tablet_server.h"
-#include "kudu/tserver/tablet_server_test_util.h"
+#include "kudu/tserver/tserver.pb.h"
 #include "kudu/tserver/tserver_admin.proxy.h"
 #include "kudu/tserver/tserver_service.proxy.h"
-#include "kudu/tserver/ts_tablet_manager.h"
-#include "kudu/util/maintenance_manager.h"
 #include "kudu/util/metrics.h"
-#include "kudu/util/pb_util.h"
-#include "kudu/util/test_graph.h"
-#include "kudu/util/test_macros.h"
+#include "kudu/util/status.h"
 #include "kudu/util/test_util.h"
 
-DEFINE_int32(rpc_timeout, 1000, "Timeout for RPC calls, in seconds");
-DEFINE_int32(num_updater_threads, 1, "Number of updating threads to launch");
-DECLARE_bool(enable_maintenance_manager);
-DECLARE_int32(heartbeat_rpc_timeout_ms);
+namespace kudu {
 
-METRIC_DEFINE_entity(test);
+class KuduPartialRow;
+class TimeSeries;
+
+namespace rpc {
+class Messenger;
+class RpcController;
+} // namespace rpc
 
-namespace kudu {
 namespace tserver {
 
 class TabletServerTestBase : public KuduTest {
  public:
   typedef std::pair KeyValue;
 
-  TabletServerTestBase()
-: schema_(GetSimpleTestSchema()),
-  ts_test_metric_entity_(METRIC_ENTITY_test.Instantiate(
- &ts_test_metric_registry_, "ts_server-test")) 
{
+  TabletServerTestBase();
+
+  // Starts the tablet server, override to start it later.
+  void SetUp() override;
 
-// Disable the maintenance ops manager since we want to trigger our own
-// maintenance operations at predetermined times.
-FLAGS_enable_maintenance_manager = false;
+  virtual void StartTabletServer(int num_data_dirs);
 
-// Decrease heartbeat timeout: we keep re-trying heartbeats when a
-// single master server fails due to a network error. Decreasing
-// the hearbeat timeout to 1 second speeds up unit tests which
-// purposefully specify non-running Master servers.
-FLAGS_heartbeat_rpc_timeout_ms = 1000;
-  }
+  Status WaitForTabletRunning(const char *tablet_id);
 
-  // Starts the tablet server, override to start it later.
-  virtual void SetUp() OVERRIDE {
-KuduTest::SetUp();
-
-key_schema_ = schema_.CreateKeyProjection();
-rb_.reset(new RowBuilder(schema_));
-
-rpc::MessengerBuilder bld("Client");
-ASSERT_OK(bld.Build(&client_messenger_));
-  }
-
-  virtual void StartTabletServer(int num_data_dirs) {
-CHECK(!mini_server_);
-
-// Start server with an invalid master address, so it never successfully
-// heartbeats, even if there happens to be a master running on this 
machine.
-mini_server_.reset(new 
MiniTabletServer(GetTestPath("TabletServerTest-fsroot"),
-HostPort("127.0.0.1", 0), 
num_data_dirs));
-mini_server_->options()->master_addresses.clear();
-mini_server_->options()->master_addresses.emplace_back("255.255.255.255", 
1);
-ASSERT_OK(mini_server_->Start());
-
-// Set up a tablet inside the server.
-ASSERT_OK(mini_server_->AddTestTablet(kTableId, kTabletId, schema_));
-
ASSERT_TRUE(mini_server_->server()->tablet_manager()->LookupTablet(kTabletId,
-   
&tablet_replica_));
-
-// Creating a tablet is async, we wait here instead of having to handle 
errors later.
-ASSERT_OK(WaitForTabletRunning(kTabletId));
-
-// Connect to it.
-ResetClientProxies();
-  }
-
-

[2/2] kudu git commit: [ts_itest-base] split declaration and definition

[alexey]

[ts_itest-base] split declaration and definition

Splitted declaration and definition of methods in
TabletServerIntegrationTestBase and TabletServerTestBase test classes.
In addition to shorter compilation times (no need to compile the same
code multiple times), this is also necessary for upcoming changelist
which splits set of tests in raft_consensus-itest into separate files.

This changelist does not contain any functional changes.

Change-Id: I31085ad2a6df0df795dad70e9e1c90d34cdd3716
Reviewed-on: http://gerrit.cloudera.org:8080/8277
Tested-by: Kudu Jenkins
Reviewed-by: Adar Dembo 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/3a7342a3
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/3a7342a3
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/3a7342a3

Branch: refs/heads/master
Commit: 3a7342a31b2c0e30f5175a2b26b6fdb6acac0dff
Parents: 92064c0
Author: Alexey Serbin 
Authored: Sat Oct 14 12:09:20 2017 -0700
Committer: Alexey Serbin 
Committed: Mon Oct 16 20:42:25 2017 +

--
 src/kudu/integration-tests/CMakeLists.txt   |   1 +
 .../exactly_once_writes-itest.cc|  10 +-
 src/kudu/integration-tests/linked_list-test.cc  |  19 +-
 .../integration-tests/raft_consensus-itest.cc   |  96 ++-
 src/kudu/integration-tests/ts_itest-base.cc | 580 +++
 src/kudu/integration-tests/ts_itest-base.h  | 496 ++--
 src/kudu/tools/kudu-admin-test.cc   |  57 +-
 src/kudu/tserver/CMakeLists.txt |   4 +-
 src/kudu/tserver/tablet_copy-test-base.h|   3 +-
 src/kudu/tserver/tablet_server-stress-test.cc   |   2 +-
 src/kudu/tserver/tablet_server-test-base.cc | 467 +++
 src/kudu/tserver/tablet_server-test-base.h  | 436 ++
 src/kudu/tserver/tablet_server-test.cc  |  68 ++-
 13 files changed, 1271 insertions(+), 968 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/3a7342a3/src/kudu/integration-tests/CMakeLists.txt
--
diff --git a/src/kudu/integration-tests/CMakeLists.txt 
b/src/kudu/integration-tests/CMakeLists.txt
index 968ef19..d124662 100644
--- a/src/kudu/integration-tests/CMakeLists.txt
+++ b/src/kudu/integration-tests/CMakeLists.txt
@@ -28,6 +28,7 @@ set(INTEGRATION_TESTS_SRCS
   internal_mini_cluster-itest-base.cc
   log_verifier.cc
   test_workload.cc
+  ts_itest-base.cc
 )
 
 add_library(itest_util ${INTEGRATION_TESTS_SRCS})

http://git-wip-us.apache.org/repos/asf/kudu/blob/3a7342a3/src/kudu/integration-tests/exactly_once_writes-itest.cc
--
diff --git a/src/kudu/integration-tests/exactly_once_writes-itest.cc 
b/src/kudu/integration-tests/exactly_once_writes-itest.cc
index 1595432..b291bc9 100644
--- a/src/kudu/integration-tests/exactly_once_writes-itest.cc
+++ b/src/kudu/integration-tests/exactly_once_writes-itest.cc
@@ -22,6 +22,7 @@
 #include 
 #include 
 
+#include 
 #include 
 #include 
 
@@ -53,7 +54,12 @@
 #include "kudu/util/test_util.h"
 #include "kudu/util/thread.h"
 
+DECLARE_int32(consensus_rpc_timeout_ms);
+DECLARE_int32(num_replicas);
+DECLARE_int32(num_tablet_servers);
+
 using std::string;
+using std::unique_ptr;
 using std::vector;
 
 namespace kudu {
@@ -112,7 +118,7 @@ void 
ExactlyOnceSemanticsITest::WriteRowsAndCollectResponses(int thread_idx,
   rpc::MessengerBuilder bld("Client");
   ASSERT_OK(bld.Build(&client_messenger));
 
-  std::unique_ptr proxy(new TabletServerServiceProxy(
+  unique_ptr proxy(new TabletServerServiceProxy(
   client_messenger, address, address.host()));
   for (int i = 0; i < num_batches; i++) {
 // Wait for all of the other writer threads to finish their attempts of 
the prior
@@ -146,7 +152,7 @@ void 
ExactlyOnceSemanticsITest::WriteRowsAndCollectResponses(int thread_idx,
   controller.Reset();
   WriteResponsePB response;
 
-  std::unique_ptr request_id(new rpc::RequestIdPB());
+  unique_ptr request_id(new rpc::RequestIdPB());
   request_id->set_client_id("test_client");
   request_id->set_seq_no(i);
   request_id->set_attempt_no(base_attempt_idx * kMaxAttempts + 
num_attempts);

http://git-wip-us.apache.org/repos/asf/kudu/blob/3a7342a3/src/kudu/integration-tests/linked_list-test.cc
--
diff --git a/src/kudu/integration-tests/linked_list-test.cc 
b/src/kudu/integration-tests/linked_list-test.cc
index f7479d7..b12eb14 100644
--- a/src/kudu/integration-tests/linked_list-test.cc
+++ b/src/kudu/integration-tests/linked_list-test.cc
@@ -39,6 +39,7 @@
 
 #include 
 #include 
+#include 
 #include 
 #include 
 
@@ -57,6 +58,9 @@

kudu git commit: [consensus] do not call EnableFailureDetector twice

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master 3a7342a31 -> 441408908


[consensus] do not call EnableFailureDetector twice

This minor clean-up removes second call to EnableFailureDetector()
upon starting the Raft consensus state machine.

Change-Id: I87dac245b20d85dcda991ea02b51d6ee0fe41822
Reviewed-on: http://gerrit.cloudera.org:8080/8271
Reviewed-by: Adar Dembo 
Tested-by: Alexey Serbin 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/44140890
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/44140890
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/44140890

Branch: refs/heads/master
Commit: 4414089085876b6acfc4025391cd2b96bf833944
Parents: 3a7342a
Author: Alexey Serbin 
Authored: Thu Oct 12 23:38:50 2017 -0700
Committer: Alexey Serbin 
Committed: Mon Oct 16 21:50:29 2017 +

--
 src/kudu/consensus/raft_consensus.cc | 15 ++-
 src/kudu/consensus/raft_consensus.h  |  9 +
 2 files changed, 11 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/44140890/src/kudu/consensus/raft_consensus.cc
--
diff --git a/src/kudu/consensus/raft_consensus.cc 
b/src/kudu/consensus/raft_consensus.cc
index 53268db..50b60bf 100644
--- a/src/kudu/consensus/raft_consensus.cc
+++ b/src/kudu/consensus/raft_consensus.cc
@@ -298,7 +298,7 @@ Status RaftConsensus::Start(const ConsensusBootstrapInfo& 
info,
 
 // If this is the first term expire the FD immediately so that we have a
 // fast first election, otherwise we just let the timer expire normally.
-boost::optional initial_delta;
+boost::optional fd_initial_delta;
 if (CurrentTermUnlocked() == 0) {
   // The failure detector is initialized to a low value to trigger an early
   // election (unless someone else requested a vote from us first, which
@@ -311,14 +311,13 @@ Status RaftConsensus::Start(const ConsensusBootstrapInfo& 
info,
   if (PREDICT_TRUE(FLAGS_enable_leader_failure_detection)) {
 LOG_WITH_PREFIX_UNLOCKED(INFO) << "Consensus starting up: Expiring 
failure detector timer "
   "to make a prompt election more 
likely";
-initial_delta = MonoDelta::FromMilliseconds(
+fd_initial_delta = MonoDelta::FromMilliseconds(
 rng_.Uniform(FLAGS_raft_heartbeat_interval_ms));
   }
 }
-EnableFailureDetector(initial_delta);
 
 // Now assume "follower" duties.
-RETURN_NOT_OK(BecomeReplicaUnlocked());
+RETURN_NOT_OK(BecomeReplicaUnlocked(fd_initial_delta));
 
 SetStateUnlocked(kRunning);
   }
@@ -572,16 +571,15 @@ Status RaftConsensus::BecomeLeaderUnlocked() {
   return AppendNewRoundToQueueUnlocked(round);
 }
 
-Status RaftConsensus::BecomeReplicaUnlocked() {
+Status RaftConsensus::BecomeReplicaUnlocked(boost::optional 
fd_delta) {
   DCHECK(lock_.is_locked());
 
   LOG_WITH_PREFIX_UNLOCKED(INFO) << "Becoming Follower/Learner. State: "
  << ToStringUnlocked();
-
   ClearLeaderUnlocked();
 
   // FD should be running while we are a follower.
-  EnableFailureDetector();
+  EnableFailureDetector(std::move(fd_delta));
 
   // Now that we're a replica, we can allow voting for other nodes.
   withhold_votes_until_ = MonoTime::Min();
@@ -1496,8 +1494,7 @@ Status RaftConsensus::RequestVote(const VoteRequestPB* 
request,
   //
   // See also https://ramcloud.stanford.edu/~ongaro/thesis.pdf
   // section 4.2.3.
-  MonoTime now = MonoTime::Now();
-  if (!request->ignore_live_leader() && now < withhold_votes_until_) {
+  if (!request->ignore_live_leader() && MonoTime::Now() < 
withhold_votes_until_) {
 return RequestVoteRespondLeaderIsAlive(request, response);
   }
 

http://git-wip-us.apache.org/repos/asf/kudu/blob/44140890/src/kudu/consensus/raft_consensus.h
--
diff --git a/src/kudu/consensus/raft_consensus.h 
b/src/kudu/consensus/raft_consensus.h
index 080bc7e..5ab93b8 100644
--- a/src/kudu/consensus/raft_consensus.h
+++ b/src/kudu/consensus/raft_consensus.h
@@ -435,9 +435,10 @@ class RaftConsensus : public 
std::enable_shared_from_this,
   Status BecomeLeaderUnlocked();
 
   // Makes the peer become a replica, i.e. a FOLLOWER or a LEARNER.
+  // See EnableFailureDetector() for description of the 'fd_delta' parameter.
   //
   // 'lock_' must be held for configuration change before calling.
-  Status BecomeReplicaUnlocked();
+  Status BecomeReplicaUnlocked(boost::optional fd_delta = 
boost::none);
 
   // Updates the state in a replica by storing the received operations in the 
log
   // and tr

[3/3] kudu git commit: [raft_consensus-itest] separate Raft election tests

[alexey]

[raft_consensus-itest] separate Raft election tests

Separated election-related tests from raft_consensus-itest
into raft_consensus-election-itest.

This changelist does not contain any functional changes.

Change-Id: Ic5289b3de097f9a2ca152034fa2635b67ccddb93
Reviewed-on: http://gerrit.cloudera.org:8080/8278
Reviewed-by: Adar Dembo 
Tested-by: Alexey Serbin 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/ab77ce02
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/ab77ce02
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/ab77ce02

Branch: refs/heads/master
Commit: ab77ce025d9fe68c993d26e15b604c4188e36860
Parents: 4414089
Author: Alexey Serbin 
Authored: Sat Oct 14 23:18:59 2017 -0700
Committer: Alexey Serbin 
Committed: Tue Oct 17 01:09:23 2017 +

--
 build-support/dist_test.py  |2 +-
 src/kudu/integration-tests/CMakeLists.txt   |2 +
 .../raft_consensus-itest-base.cc|  273 
 .../raft_consensus-itest-base.h |   68 +
 .../integration-tests/raft_consensus-itest.cc   | 1507 +-
 .../raft_consensus_election-itest.cc|  532 +++
 src/kudu/tablet/mt-tablet-test.cc   |2 +-
 src/kudu/tserver/tablet_server-test-base.cc |1 -
 8 files changed, 1316 insertions(+), 1071 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/ab77ce02/build-support/dist_test.py
--
diff --git a/build-support/dist_test.py b/build-support/dist_test.py
index a91d7d3..328d2b8 100755
--- a/build-support/dist_test.py
+++ b/build-support/dist_test.py
@@ -89,7 +89,7 @@ NUM_SHARDS_BY_TEST = {
   'delete_table-test': 8,
   'flex_partitioning-itest': 8,
   'mt-tablet-test': 4,
-  'raft_consensus-itest': 8
+  'raft_consensus-itest': 6
 }
 
 

http://git-wip-us.apache.org/repos/asf/kudu/blob/ab77ce02/src/kudu/integration-tests/CMakeLists.txt
--
diff --git a/src/kudu/integration-tests/CMakeLists.txt 
b/src/kudu/integration-tests/CMakeLists.txt
index d124662..ecdda20 100644
--- a/src/kudu/integration-tests/CMakeLists.txt
+++ b/src/kudu/integration-tests/CMakeLists.txt
@@ -27,6 +27,7 @@ set(INTEGRATION_TESTS_SRCS
   external_mini_cluster_fs_inspector.cc
   internal_mini_cluster-itest-base.cc
   log_verifier.cc
+  raft_consensus-itest-base.cc
   test_workload.cc
   ts_itest-base.cc
 )
@@ -86,6 +87,7 @@ ADD_KUDU_TEST(multidir_cluster-itest)
 ADD_KUDU_TEST(open-readonly-fs-itest)
 ADD_KUDU_TEST(raft_config_change-itest)
 ADD_KUDU_TEST(raft_consensus-itest RUN_SERIAL true)
+ADD_KUDU_TEST(raft_consensus_election-itest)
 ADD_KUDU_TEST(registration-test RESOURCE_LOCK "master-web-port")
 ADD_KUDU_TEST(security-faults-itest)
 ADD_KUDU_TEST(security-itest)

http://git-wip-us.apache.org/repos/asf/kudu/blob/ab77ce02/src/kudu/integration-tests/raft_consensus-itest-base.cc
--
diff --git a/src/kudu/integration-tests/raft_consensus-itest-base.cc 
b/src/kudu/integration-tests/raft_consensus-itest-base.cc
new file mode 100644
index 000..afd8c64
--- /dev/null
+++ b/src/kudu/integration-tests/raft_consensus-itest-base.cc
@@ -0,0 +1,273 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "kudu/integration-tests/raft_consensus-itest-base.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include 
+#include 
+#include 
+#include 
+
+#include "kudu/client/client-test-util.h"
+#include "kudu/client/client.h"
+#include "kudu/client/shared_ptr.h"
+#include "kudu/client/write_op.h"
+#include "kudu/common/partial_row.h"
+#include "kudu/common/wire_protocol.h"
+#include "kudu/consensus/consensus.pb.h"
+#include "kudu/consensus/opid.pb.h&

[1/3] kudu git commit: [raft_consensus-itest] separate Raft election tests

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master 441408908 -> ab77ce025


http://git-wip-us.apache.org/repos/asf/kudu/blob/ab77ce02/src/kudu/integration-tests/raft_consensus_election-itest.cc
--
diff --git a/src/kudu/integration-tests/raft_consensus_election-itest.cc 
b/src/kudu/integration-tests/raft_consensus_election-itest.cc
new file mode 100644
index 000..ba4d538
--- /dev/null
+++ b/src/kudu/integration-tests/raft_consensus_election-itest.cc
@@ -0,0 +1,532 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include 
+#include 
+#include 
+#include 
+
+#include "kudu/common/wire_protocol.pb.h"
+#include "kudu/consensus/metadata.pb.h"
+#include "kudu/gutil/gscoped_ptr.h"
+#include "kudu/gutil/map-util.h"
+#include "kudu/gutil/strings/substitute.h"
+#include "kudu/integration-tests/cluster_itest_util.h"
+#include "kudu/integration-tests/cluster_verifier.h"
+#include "kudu/integration-tests/external_mini_cluster_fs_inspector.h"
+#include "kudu/integration-tests/raft_consensus-itest-base.h"
+#include "kudu/integration-tests/test_workload.h"
+#include "kudu/mini-cluster/external_mini_cluster.h"
+#include "kudu/tablet/metadata.pb.h"
+#include "kudu/tserver/tablet_server-test-base.h"
+#include "kudu/tserver/tserver.pb.h"
+#include "kudu/util/metrics.h"
+#include "kudu/util/monotime.h"
+#include "kudu/util/pb_util.h"
+#include "kudu/util/status.h"
+#include "kudu/util/stopwatch.h"
+#include "kudu/util/test_macros.h"
+#include "kudu/util/test_util.h"
+
+DECLARE_int64(client_inserts_per_thread);
+DECLARE_int64(client_num_batches_per_thread);
+DECLARE_int32(consensus_rpc_timeout_ms);
+DECLARE_int32(num_client_threads);
+DECLARE_int32(num_replicas);
+DECLARE_int32(num_tablet_servers);
+
+METRIC_DECLARE_entity(tablet);
+METRIC_DECLARE_counter(transaction_memory_pressure_rejections);
+METRIC_DECLARE_gauge_int64(raft_term);
+
+using kudu::cluster::ExternalTabletServer;
+using kudu::consensus::RaftPeerPB;
+using kudu::itest::AddServer;
+using kudu::itest::GetReplicaStatusAndCheckIfLeader;
+using kudu::itest::LeaderStepDown;
+using kudu::itest::RemoveServer;
+using kudu::itest::StartElection;
+using kudu::itest::TabletServerMap;
+using kudu::itest::TServerDetails;
+using kudu::itest::WaitUntilLeader;
+using kudu::itest::WriteSimpleTestRow;
+using kudu::pb_util::SecureShortDebugString;
+using kudu::tablet::TABLET_DATA_COPYING;
+using std::string;
+using std::unordered_set;
+using std::vector;
+using strings::Substitute;
+
+namespace kudu {
+namespace tserver {
+
+static const int kTestRowKey = 1234;
+static const int kTestRowIntVal = 5678;
+
+class RaftConsensusElectionITest : public RaftConsensusITestBase {
+ protected:
+  void CreateClusterForChurnyElectionsTests(const vector& 
extra_ts_flags);
+  void DoTestChurnyElections(TestWorkload* workload, int max_rows_to_insert);
+};
+
+void RaftConsensusElectionITest::CreateClusterForChurnyElectionsTests(
+const vector& extra_ts_flags) {
+  vector ts_flags;
+
+#ifdef THREAD_SANITIZER
+  // On TSAN builds, we need to be a little bit less churny in order to make
+  // any progress at all.
+  ts_flags.push_back("--raft_heartbeat_interval_ms=5");
+#else
+  ts_flags.emplace_back("--raft_heartbeat_interval_ms=1");
+#endif
+
+  ts_flags.insert(ts_flags.end(), extra_ts_flags.cbegin(), 
extra_ts_flags.cend());
+
+  CreateCluster("raft_consensus-itest-cluster", ts_flags, {});
+}
+
+void RaftConsensusElectionITest::DoTestChurnyElections(TestWorkload* workload,
+   int max_rows_to_insert) 
{
+  workload->set_num_replicas(FLAGS_num_replicas);
+  // Set a really high write timeout so that even in the presence of many 
failures we
+  // can verify an exact number of rows in the end, thanks to exactly once 
semantics.
+  workload->set_write_timeout_millis(60 * 1000 /* 60 seconds */);
+  workload->set_num_write_threads(2);
+  workload->set_write_batch_size(1);
+  workload->Setup();
+  workload->Start();
+
+  // Run for either a prescribed number of writes, or 30 seconds,
+  // whichever comes first. This prevents test timeouts on slower
+  // buil

[2/3] kudu git commit: [raft_consensus-itest] separate Raft election tests

[alexey]

http://git-wip-us.apache.org/repos/asf/kudu/blob/ab77ce02/src/kudu/integration-tests/raft_consensus-itest.cc
--
diff --git a/src/kudu/integration-tests/raft_consensus-itest.cc 
b/src/kudu/integration-tests/raft_consensus-itest.cc
index b3b63f9..4d030c5 100644
--- a/src/kudu/integration-tests/raft_consensus-itest.cc
+++ b/src/kudu/integration-tests/raft_consensus-itest.cc
@@ -15,7 +15,6 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include 
 #include 
 #include 
 #include 
@@ -27,13 +26,11 @@
 #include 
 
 #include 
-#include 
 #include 
 #include 
 #include 
 #include 
 
-#include "kudu/client/client-test-util.h"
 #include "kudu/client/client.h"
 #include "kudu/client/shared_ptr.h"
 #include "kudu/client/write_op.h"
@@ -52,10 +49,8 @@
 #include "kudu/gutil/basictypes.h"
 #include "kudu/gutil/gscoped_ptr.h"
 #include "kudu/gutil/map-util.h"
-#include "kudu/gutil/port.h"
 #include "kudu/gutil/ref_counted.h"
 #include "kudu/gutil/stl_util.h"
-#include "kudu/gutil/stringprintf.h"
 #include "kudu/gutil/strings/strcat.h"
 #include "kudu/gutil/strings/substitute.h"
 #include "kudu/gutil/strings/util.h"
@@ -63,8 +58,8 @@
 #include "kudu/integration-tests/cluster_verifier.h"
 #include "kudu/integration-tests/external_mini_cluster_fs_inspector.h"
 #include "kudu/integration-tests/log_verifier.h"
+#include "kudu/integration-tests/raft_consensus-itest-base.h"
 #include "kudu/integration-tests/test_workload.h"
-#include "kudu/integration-tests/ts_itest-base.h"
 #include "kudu/master/master.pb.h"
 #include "kudu/mini-cluster/external_mini_cluster.h"
 #include "kudu/rpc/messenger.h"
@@ -81,23 +76,18 @@
 #include "kudu/util/monotime.h"
 #include "kudu/util/pb_util.h"
 #include "kudu/util/random.h"
-#include "kudu/util/slice.h"
 #include "kudu/util/status.h"
-#include "kudu/util/stopwatch.h"
 #include "kudu/util/test_macros.h"
 #include "kudu/util/test_util.h"
 #include "kudu/util/thread.h"
 
+DECLARE_int64(client_inserts_per_thread);
+DECLARE_int64(client_num_batches_per_thread);
 DECLARE_int32(consensus_rpc_timeout_ms);
+DECLARE_int32(num_client_threads);
 DECLARE_int32(num_replicas);
 DECLARE_int32(num_tablet_servers);
 DECLARE_int32(rpc_timeout);
-DEFINE_int32(num_client_threads, 8,
- "Number of client threads to launch");
-DEFINE_int64(client_inserts_per_thread, 50,
- "Number of rows inserted by each client thread");
-DEFINE_int64(client_num_batches_per_thread, 5,
- "In how many batches to group the rows, for each client");
 
 METRIC_DECLARE_entity(tablet);
 METRIC_DECLARE_counter(transaction_memory_pressure_rejections);
@@ -148,50 +138,9 @@ static const int kTestRowIntVal = 5678;
 
 // Integration test for the raft consensus implementation.
 // Uses the whole tablet server stack with ExternalMiniCluster.
-class RaftConsensusITest : public TabletServerIntegrationTestBase {
+class RaftConsensusITest : public RaftConsensusITestBase {
  public:
-  RaftConsensusITest()
-  : inserters_(FLAGS_num_client_threads) {
-  }
-
-  virtual void SetUp() OVERRIDE {
-TabletServerIntegrationTestBase::SetUp();
-FLAGS_consensus_rpc_timeout_ms = kConsensusRpcTimeoutForTests;
-  }
-
-  void ScanReplica(TabletServerServiceProxy* replica_proxy,
-   vector* results) {
-
-ScanRequestPB req;
-ScanResponsePB resp;
-RpcController rpc;
-rpc.set_timeout(MonoDelta::FromSeconds(10)); // Squelch warnings.
-
-NewScanRequestPB* scan = req.mutable_new_scan_request();
-scan->set_tablet_id(tablet_id_);
-ASSERT_OK(SchemaToColumnPBs(schema_, scan->mutable_projected_columns()));
-
-// Send the call
-{
-  req.set_batch_size_bytes(0);
-  SCOPED_TRACE(SecureDebugString(req));
-  ASSERT_OK(replica_proxy->Scan(req, &resp, &rpc));
-  SCOPED_TRACE(SecureDebugString(resp));
-  if (resp.has_error()) {
-ASSERT_OK(StatusFromPB(resp.error().status()));
-  }
-}
-
-if (!resp.has_more_results())
-  return;
-
-// Drain all the rows from the scanner.
-NO_FATALS(DrainScannerToStrings(resp.scanner_id(),
-schema_,
-results,
-replica_proxy));
-
-std::sort(results->begin(), results->end());
+  RaftConsensusITest() {
   }
 
   // Scan the given replica in a loop until the number of rows
@@ -199,28 +148,7 @@ class RaftConsensusITest : public 
TabletServerIntegrationTestBase {
   // fails the test.
   void WaitForRowCount(TabletServerServiceProxy* replica_proxy,
int expected_count,
-   vector* results) {
-LOG(INFO) << "Waiting for row count " << expected_count << "...";
-MonoTime start = MonoTime::Now();
-MonoTime deadline = start + MonoDelta::FromSeconds(90);
-while (true) {
-  results->clear();
-  NO_FATALS(ScanReplica(replica_proxy, results));
-

[2/5] kudu git commit: [webui] Add templates for master web ui.

[alexey]

[webui]  Add templates for master web ui.

This adds templates to the rest of the master webui endpoints for which
it makes sense:
- /tablet-servers
- /masters

Change-Id: I4520261dd617a160da51cdd089f42f078a0da728
Reviewed-on: http://gerrit.cloudera.org:8080/8227
Tested-by: Kudu Jenkins
Reviewed-by: Todd Lipcon 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/33c06692
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/33c06692
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/33c06692

Branch: refs/heads/master
Commit: 33c06692e60d0347063fb92a0affddd471e06b3f
Parents: c18815f
Author: Will Berkeley 
Authored: Sat Sep 16 23:35:21 2017 -0700
Committer: Todd Lipcon 
Committed: Tue Oct 17 04:36:24 2017 +

--
 src/kudu/master/master-path-handlers.cc | 140 +++
 src/kudu/master/master-path-handlers.h  |  11 +--
 www/masters.mustache|  45 +
 www/tables.mustache |   2 +-
 www/tablet-servers.mustache |  81 
 5 files changed, 185 insertions(+), 94 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/33c06692/src/kudu/master/master-path-handlers.cc
--
diff --git a/src/kudu/master/master-path-handlers.cc 
b/src/kudu/master/master-path-handlers.cc
index 28b604a..397776b 100644
--- a/src/kudu/master/master-path-handlers.cc
+++ b/src/kudu/master/master-path-handlers.cc
@@ -86,63 +86,53 @@ MasterPathHandlers::~MasterPathHandlers() {
 }
 
 void MasterPathHandlers::HandleTabletServers(const Webserver::WebRequest& 
/*req*/,
- 
Webserver::PrerenderedWebResponse* resp) {
-  ostringstream* output = resp->output;
-  vector > descs;
+ Webserver::WebResponse* resp) {
+  EasyJson* output = resp->output;
+  vector> descs;
   master_->ts_manager()->GetAllDescriptors(&descs);
 
-  *output << "Tablet Servers\n";
-  *output << Substitute("There are $0 registered tablet servers.", 
descs.size());
-
+  (*output)["num_ts"] = std::to_string(descs.size());
+
+  // In mustache, when conditionally rendering a section of the template based
+  // on a key, the value becomes the context. In the subcontext, searches for
+  // keys used to display a value (e.g. {{my_key}}) recurse into the parent
+  // context-- but not when used to define a subsection (e.g. {{#my_list}}.
+  // Thus, we use a negative test {{^has_no_live_ts}} to decide if we
+  // should render the live and dead tablet server tables.
+  bool has_no_live_ts = true;
+  bool has_no_dead_ts = true;
+  output->Set("live_tservers", EasyJson::kArray);
+  output->Set("dead_tservers", EasyJson::kArray);
   map> version_counts;
-  vector live_tserver_rows;
-  vector dead_tserver_rows;
   for (const std::shared_ptr& desc : descs) {
-const string time_since_hb = StringPrintf("%.1fs", 
desc->TimeSinceHeartbeat().ToSeconds());
+string ts_key = desc->PresumedDead() ? "dead_tservers" : "live_tservers";
+EasyJson ts_json = (*output)[ts_key].PushBack(EasyJson::kObject);
+
 ServerRegistrationPB reg;
 desc->GetRegistration(®);
-
-string row = 
Substitute("$0$1$2\n",
-RegistrationToHtml(reg, desc->permanent_uuid()),
-time_since_hb,
-
EscapeForHtmlToString(pb_util::SecureShortDebugString(reg)));
-
-if (desc->PresumedDead()) {
-  version_counts[reg.software_version()][1]++;
-  dead_tserver_rows.push_back(row);
-} else {
-  version_counts[reg.software_version()][0]++;
-  live_tserver_rows.push_back(row);
+ts_json["uuid"] = desc->permanent_uuid();
+if (!reg.http_addresses().empty()) {
+  ts_json["target"] = Substitute("$0://$1:$2/",
+ reg.https_enabled() ? "https" : "http",
+ reg.http_addresses(0).host(),
+ reg.http_addresses(0).port());
 }
-
+ts_json["time_since_hb"] = StringPrintf("%.1fs", 
desc->TimeSinceHeartbeat().ToSeconds());
+ts_json["registration"] = pb_util::SecureShortDebugString(reg);
+version_counts[reg.software_version()][desc->PresumedDead() ? 1 : 0]++;
+has_no_live_ts &= desc->PresumedDead();
+has_no_dead_ts &= !desc->PresumedDead();
   }
+  (*output)["has_no_live_ts"] = has_no_live_ts;
+  (*output)["has_no_dead_ts"] = has_no_dead_ts;
 
-  *output << "Version Summary";
-  *output << "\n";
-  *output << "VersionCount (Live)Count 
(Dead)\n";
-  *output << "\n";
+  output->Set("version_counts", EasyJson::kArray);
   for (const auto& entry : version_counts) {
-*output << Substitute("$0$1$2\n",
-  entry.first, entry.se

[5/5] kudu git commit: [consensus] adding/removing NON_VOTER members

[alexey]

[consensus] adding/removing NON_VOTER members

Added ability to add and remove NON_VOTER member replicas.
Updated the kudu CLI accordingly.  Also, added new integration test:
  * RaftConsensusITest.AddNonVoterReplica
  * RaftConsensusITest.AddThenRemoveNonVoterReplica
  * RaftConsensusITest.NonVoterReplicasDoNotVote

Change-Id: I2662d45ad9bb6a4bf325d4202c2ee619ffad02b7
Reviewed-on: http://gerrit.cloudera.org:8080/8138
Reviewed-by: Todd Lipcon 
Reviewed-by: Mike Percy 
Tested-by: Alexey Serbin 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/fe23710c
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/fe23710c
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/fe23710c

Branch: refs/heads/master
Commit: fe23710c63c90e561b2622f6f67563572d10dbf9
Parents: 8c23c97
Author: Alexey Serbin 
Authored: Mon Oct 16 16:23:57 2017 -0700
Committer: Alexey Serbin 
Committed: Tue Oct 17 15:20:31 2017 +

--
 src/kudu/consensus/leader_election-test.cc  |   5 +-
 src/kudu/consensus/leader_election.cc   |  22 +-
 src/kudu/consensus/leader_election.h|   4 +-
 src/kudu/consensus/quorum_util-test.cc  |  47 +-
 src/kudu/consensus/quorum_util.cc   |  10 +-
 src/kudu/consensus/quorum_util.h|   4 +
 src/kudu/consensus/raft_consensus.cc|  22 +-
 src/kudu/integration-tests/CMakeLists.txt   |   1 +
 .../integration-tests/cluster_itest_util.cc |   4 +-
 .../raft_consensus-itest-base.cc|   6 +
 .../raft_consensus-itest-base.h |  12 +-
 .../integration-tests/raft_consensus-itest.cc   |  82 ++-
 .../raft_consensus_nonvoter-itest.cc| 534 +++
 13 files changed, 671 insertions(+), 82 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/fe23710c/src/kudu/consensus/leader_election-test.cc
--
diff --git a/src/kudu/consensus/leader_election-test.cc 
b/src/kudu/consensus/leader_election-test.cc
index cec04c3..1dc26d9 100644
--- a/src/kudu/consensus/leader_election-test.cc
+++ b/src/kudu/consensus/leader_election-test.cc
@@ -161,7 +161,8 @@ void LeaderElectionTest::ElectionCallback(const 
ElectionResult& result) {
 
 void LeaderElectionTest::InitUUIDs(int num_voters) {
   voter_uuids_ = GenVoterUUIDs(num_voters);
-  candidate_uuid_ = voter_uuids_[num_voters - 1];
+  CHECK(!voter_uuids_.empty());
+  candidate_uuid_ = voter_uuids_.back();
   voter_uuids_.pop_back();
 }
 
@@ -170,6 +171,7 @@ void LeaderElectionTest::InitNoOpPeerProxies() {
   for (const string& uuid : voter_uuids_) {
 RaftPeerPB* peer_pb = config_.add_peers();
 peer_pb->set_permanent_uuid(uuid);
+peer_pb->set_member_type(RaftPeerPB::VOTER);
 PeerProxy* proxy = new NoOpTestPeerProxy(pool_.get(), *peer_pb);
 InsertOrDie(&proxies_, uuid, proxy);
   }
@@ -180,6 +182,7 @@ void LeaderElectionTest::InitDelayableMockedProxies(bool 
enable_delay) {
   for (const string& uuid : voter_uuids_) {
 RaftPeerPB* peer_pb = config_.add_peers();
 peer_pb->set_permanent_uuid(uuid);
+peer_pb->set_member_type(RaftPeerPB::VOTER);
 auto proxy = new DelayablePeerProxy(pool_.get(),
  new 
MockedPeerProxy(pool_.get()));
 if (enable_delay) {

http://git-wip-us.apache.org/repos/asf/kudu/blob/fe23710c/src/kudu/consensus/leader_election.cc
--
diff --git a/src/kudu/consensus/leader_election.cc 
b/src/kudu/consensus/leader_election.cc
index 07b8a01..b8cbddc 100644
--- a/src/kudu/consensus/leader_election.cc
+++ b/src/kudu/consensus/leader_election.cc
@@ -161,8 +161,18 @@ LeaderElection::LeaderElection(const RaftConfigPB& config,
   decision_callback_(std::move(decision_callback)),
   highest_voter_term_(0) {
   for (const RaftPeerPB& peer : config.peers()) {
-if (request.candidate_uuid() == peer.permanent_uuid()) continue;
-follower_uuids_.push_back(peer.permanent_uuid());
+if (request.candidate_uuid() == peer.permanent_uuid()) {
+  DCHECK_EQ(peer.member_type(), RaftPeerPB::VOTER)
+  << Substitute("non-voter member $0 tried to start an election; "
+"Raft config {$1}",
+peer.permanent_uuid(),
+pb_util::SecureShortDebugString(config));
+  continue;
+}
+if (peer.member_type() != RaftPeerPB::VOTER) {
+  continue;
+}
+other_voter_uuids_.push_back(peer.permanent_uuid());
 
 gscoped_ptr state(new VoterState());
 state->proxy_status = proxy_factory->NewProxy(peer, &state->proxy);
@@ -173,10 +183,10 @@ LeaderElecti

[4/5] kudu git commit: KUDU-2044 Tombstoned tablets show up in /metrics

[alexey]

KUDU-2044 Tombstoned tablets show up in /metrics

This patch stops tombstoned tablets from showing up in /metrics. When a
tablet replica is shut down, the tablet's metric entity is marked as
unpublished. All its child metrics will be marked for retirement, then
retired after the retirement interval has passed. Then the entity
itself is unregistered, i.e. removed from the metric registry's map of
entities. If a new replica of the same tablet is added to the server,
it will create a new entity that will be registered with the
metric_registry, either as a new insertion or overwriting the old
replica's entity if the entity had been unpublished but not yet
unregistered.

The tombstoned tablet's metric entity is not destroyed when it's
deregistered, since there are still refs to it and its metrics in the
TabletReplica class, Tablet class, and many other classes associated
with a TabletReplica. The entity will be destroyed when the
TabletReplica is deleted (since it either contains or holds final
references to all the other classes), which happens if the tablet is
replaced or deleted. While it's not ideal to hold the memory for the
entity when it's no longer used, the reason this occurs is because the
TabletReplica instance mostly stays alive. Releasing all the metric
references would require specifically dropping refs to those metrics in
all the surviving subcomponents of a TabletReplica instance that has
been shut down; I think this problem would be better solved by more
completely cleaning up a shut down TabletReplica instance, but that's a
much larger scope than suppressing the metrics.

Change-Id: I21b52262203410ded1e514502b59a79be12f8fb3
Reviewed-on: http://gerrit.cloudera.org:8080/7981
Tested-by: Kudu Jenkins
Reviewed-by: David Ribeiro Alves 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/8c23c97b
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/8c23c97b
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/8c23c97b

Branch: refs/heads/master
Commit: 8c23c97b82708b96e163e75e0002ce96adf8d8fc
Parents: 60ea4a6
Author: Will Berkeley 
Authored: Wed Sep 6 10:57:39 2017 -0700
Committer: Will Berkeley 
Committed: Tue Oct 17 14:54:51 2017 +

--
 src/kudu/integration-tests/tablet_copy-itest.cc | 72 ++--
 src/kudu/tablet/tablet.cc   |  3 +
 src/kudu/tserver/tablet_server-test.cc  | 47 +
 src/kudu/util/metrics.cc| 18 +++--
 src/kudu/util/metrics.h | 17 -
 5 files changed, 146 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/8c23c97b/src/kudu/integration-tests/tablet_copy-itest.cc
--
diff --git a/src/kudu/integration-tests/tablet_copy-itest.cc 
b/src/kudu/integration-tests/tablet_copy-itest.cc
index 88a9350..a36e65e 100644
--- a/src/kudu/integration-tests/tablet_copy-itest.cc
+++ b/src/kudu/integration-tests/tablet_copy-itest.cc
@@ -124,10 +124,12 @@ using std::unordered_map;
 using std::vector;
 using strings::Substitute;
 
+METRIC_DECLARE_entity(tablet);
 
METRIC_DECLARE_histogram(handler_latency_kudu_consensus_ConsensusService_UpdateConsensus);
 METRIC_DECLARE_counter(glog_info_messages);
 METRIC_DECLARE_counter(glog_warning_messages);
 METRIC_DECLARE_counter(glog_error_messages);
+METRIC_DECLARE_counter(rows_inserted);
 METRIC_DECLARE_counter(tablet_copy_bytes_fetched);
 METRIC_DECLARE_counter(tablet_copy_bytes_sent);
 METRIC_DECLARE_gauge_int32(tablet_copy_open_client_sessions);
@@ -1624,6 +1626,17 @@ int64_t CountBootstrappingTablets(ExternalTabletServer 
*ets) {
   &ret));
   return ret;
 }
+
+int64_t CountRowsInserted(ExternalTabletServer *ets) {
+  int64_t ret;
+  CHECK_OK(ets->GetInt64Metric(
+  &METRIC_ENTITY_tablet,
+  nullptr,
+  &METRIC_rows_inserted,
+  "value",
+  &ret));
+  return ret;
+}
 } // anonymous namespace
 
 // Check that state metrics work during tablet copy.
@@ -1676,10 +1689,9 @@ TEST_F(TabletCopyITest, 
TestTabletStateMetricsDuringTabletCopy) {
   });
 
   // Wait for the tablet to be revived via tablet copy.
-  ASSERT_OK(inspect_->WaitForTabletDataStateOnTS(
-  follower_index, tablet_id,
-  { tablet::TABLET_DATA_READY },
-  kTimeout));
+  ASSERT_OK(inspect_->WaitForTabletDataStateOnTS(follower_index, tablet_id,
+ { tablet::TABLET_DATA_READY },
+ kTimeout));
 
   // State: 1 running or bootstrapping tablet.
   ASSERT_EVENTUALLY([&] {
@@ -1688,6 +1700,58 @@ TEST_F(TabletCopyITest, 
TestTabletStateMetricsDuringTabletCopy) {
   });
 }
 
+TEST_F(TabletCopyITest, TestMetricsResetAfterRevival) {
+  MonoDelta kTimeout = MonoDelta::FromSeconds(3

[3/5] kudu git commit: Implement per-tablet active scanner metric

[alexey]

Implement per-tablet active scanner metric

The active_scanners metric would be useful scoped per-tablet.
This patch exposes it as tablet_active_scanners.

It also adds coverage of scanner expiration in tablet_server-test.

Change-Id: I3b2d85f78c655e02654f8581e9521452b149b7d5
Reviewed-on: http://gerrit.cloudera.org:8080/8228
Tested-by: Kudu Jenkins
Reviewed-by: Todd Lipcon 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/60ea4a64
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/60ea4a64
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/60ea4a64

Branch: refs/heads/master
Commit: 60ea4a64366d46a7b2e86aef5c9b38c32e6af15b
Parents: 33c0669
Author: Will Berkeley 
Authored: Mon Oct 2 15:50:33 2017 -0700
Committer: Todd Lipcon 
Committed: Tue Oct 17 04:39:03 2017 +

--
 src/kudu/tablet/tablet_metrics.cc  | 22 ++
 src/kudu/tablet/tablet_metrics.h   |  4 ++
 src/kudu/tserver/scanner_metrics.cc|  4 +-
 src/kudu/tserver/scanners.cc   | 16 ++-
 src/kudu/tserver/tablet_server-test.cc | 67 +++--
 5 files changed, 96 insertions(+), 17 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/60ea4a64/src/kudu/tablet/tablet_metrics.cc
--
diff --git a/src/kudu/tablet/tablet_metrics.cc 
b/src/kudu/tablet/tablet_metrics.cc
index b5469e9..a3b6b41 100644
--- a/src/kudu/tablet/tablet_metrics.cc
+++ b/src/kudu/tablet/tablet_metrics.cc
@@ -42,6 +42,15 @@ METRIC_DEFINE_counter(tablet, rows_deleted, "Rows Deleted",
 kudu::MetricUnit::kRows,
 "Number of row delete operations performed on this tablet since service 
start");
 
+METRIC_DEFINE_counter(tablet, insertions_failed_dup_key, "Duplicate Key 
Inserts",
+  kudu::MetricUnit::kRows,
+  "Number of inserts which failed because the key already 
existed");
+
+METRIC_DEFINE_counter(tablet, upserts_as_updates, "Upserts converted into 
updates",
+  kudu::MetricUnit::kRows,
+  "Number of upserts which were applied as updates because 
the key already "
+  "existed.");
+
 METRIC_DEFINE_counter(tablet, scanner_rows_returned, "Scanner Rows Returned",
   kudu::MetricUnit::kRows,
   "Number of rows returned by scanners to clients. This 
count "
@@ -59,7 +68,6 @@ METRIC_DEFINE_counter(tablet, scanner_bytes_returned, 
"Scanner Bytes Returned",
   "for consumption by clients, and thus is not "
   "a reflection of the amount of work being done by 
scanners.");
 
-
 METRIC_DEFINE_counter(tablet, scanner_rows_scanned, "Scanner Rows Scanned",
   kudu::MetricUnit::kRows,
   "Number of rows processed by scan requests. This is 
measured "
@@ -90,17 +98,12 @@ METRIC_DEFINE_counter(tablet, 
scanner_bytes_scanned_from_disk, "Scanner Bytes Sc
   "and does not include data read from in-memory stores. 
However, it"
   "includes both cache misses and cache hits.");
 
-
-METRIC_DEFINE_counter(tablet, insertions_failed_dup_key, "Duplicate Key 
Inserts",
-  kudu::MetricUnit::kRows,
-  "Number of inserts which failed because the key already 
existed");
-METRIC_DEFINE_counter(tablet, upserts_as_updates, "Upserts converted into 
updates",
-  kudu::MetricUnit::kRows,
-  "Number of upserts which were applied as updates because 
the key already "
-  "existed.");
 METRIC_DEFINE_counter(tablet, scans_started, "Scans Started",
   kudu::MetricUnit::kScanners,
   "Number of scanners which have been started on this 
tablet");
+METRIC_DEFINE_gauge_size(tablet, tablet_active_scanners, "Active Scanners",
+ kudu::MetricUnit::kScanners,
+ "Number of scanners that are currently active on this 
tablet");
 
 METRIC_DEFINE_counter(tablet, bloom_lookups, "Bloom Filter Lookups",
   kudu::MetricUnit::kProbes,
@@ -275,6 +278,7 @@ TabletMetrics::TabletMetrics(const 
scoped_refptr& entity)
 MINIT(scanner_cells_scanned_from_disk),
 MINIT(scanner_bytes_scanned_from_disk),
 MINIT(scans_started),
+GINIT(tablet_active_scanners),
 MINIT(bloom_lookups),
 MINIT(key_file_lookups),
 MINIT(delta_file_lookups),

http://git-wip-us.apache.org/repos/asf/kudu/blob/60ea4a64/src/kudu/tablet/tablet_metrics.h
--
diff --git a/src/kudu/tablet/tablet_metrics.h b/src/kudu/tablet/tablet_metrics.h
index 6fa5a64..9fe87b7 100644
--- a/src/kudu/tablet/tabl

[1/5] kudu git commit: Allow configuration of values passed into kerberos env vars

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master f5e1203a3 -> fe23710c6


Allow configuration of values passed into kerberos env vars

We always used hardcoded constants for the following kerberos
environment variables:

KRB5CCNAME and KRB5RCACHETYPE.

This patch allows for the configuration of these variables by taking
arguments to InitKerberosForServer().

Callsites within Kudu have not been changed as all the parameters have
default values.

The motivation for this patch is that, Impala as a user of the
KuduRPC and Kudu security libraries, needs to have a file based
credential cache since the kinit happens on the C++ side and this cache
needs to be read by the Java side too. Hence, we cannot have it in memory.
Also, Impala still requires replay protection, since some Impala services
use Thrift which lacks the nonce mechanism that KRPC uses for replay
protection.

Change-Id: Iab4ce72c04ec4056dc89fb4c1c540a6fdaca4404
Reviewed-on: http://gerrit.cloudera.org:8080/8247
Reviewed-by: Todd Lipcon 
Tested-by: Todd Lipcon 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/c18815f1
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/c18815f1
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/c18815f1

Branch: refs/heads/master
Commit: c18815f1edd3fc23abc874ffe9fe564c3114fc8e
Parents: f5e1203
Author: Sailesh Mukil 
Authored: Mon Oct 9 23:20:39 2017 -0700
Committer: Todd Lipcon 
Committed: Tue Oct 17 04:21:34 2017 +

--
 src/kudu/security/init.cc | 20 +---
 src/kudu/security/init.h  | 11 ++-
 2 files changed, 19 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/c18815f1/src/kudu/security/init.cc
--
diff --git a/src/kudu/security/init.cc b/src/kudu/security/init.cc
index ded848b..58dd48d 100644
--- a/src/kudu/security/init.cc
+++ b/src/kudu/security/init.cc
@@ -459,21 +459,19 @@ boost::optional GetLoggedInUsernameFromKeytab() {
   return g_kinit_ctx->username_str();
 }
 
-Status InitKerberosForServer() {
+Status InitKerberosForServer(const std::string& krb5ccname, bool 
disable_krb5_replay_cache) {
   if (FLAGS_keytab_file.empty()) return Status::OK();
 
-  // Have the daemons use an in-memory ticket cache, so they don't accidentally
-  // pick up credentials from test cases or any other daemon.
-  // TODO(todd): extract these krb5 env vars into some constants since they're
-  // typo-prone.
-  setenv("KRB5CCNAME", "MEMORY:kudu", 1);
+  setenv("KRB5CCNAME", krb5ccname.c_str(), 1);
   setenv("KRB5_KTNAME", FLAGS_keytab_file.c_str(), 1);
 
-  // KUDU-1897: disable the Kerberos replay cache. The KRPC protocol includes a
-  // per-connection server-generated nonce to protect against replay attacks
-  // when authenticating via Kerberos. The replay cache has many performance 
and
-  // implementation issues.
-  setenv("KRB5RCACHETYPE", "none", 1);
+  if (disable_krb5_replay_cache) {
+// KUDU-1897: disable the Kerberos replay cache. The KRPC protocol 
includes a
+// per-connection server-generated nonce to protect against replay attacks
+// when authenticating via Kerberos. The replay cache has many performance 
and
+// implementation issues.
+setenv("KRB5RCACHETYPE", "none", 1);
+  }
 
   g_kinit_ctx = new KinitContext();
   string principal;

http://git-wip-us.apache.org/repos/asf/kudu/blob/c18815f1/src/kudu/security/init.h
--
diff --git a/src/kudu/security/init.h b/src/kudu/security/init.h
index 913585f..0832256 100644
--- a/src/kudu/security/init.h
+++ b/src/kudu/security/init.h
@@ -30,9 +30,18 @@ class Status;
 
 namespace security {
 
+// The default kerberos credential cache name.
+// Have the daemons use an in-memory ticket cache, so they don't accidentally
+// pick up credentials from test cases or any other daemon.
+static const std::string kKrb5CCName = "MEMORY:kudu";
+
 // Initializes Kerberos for a server. In particular, this processes
 // the '--keytab_file' command line flag.
-Status InitKerberosForServer();
+// 'krb5ccname' is passed into the KRB5CCNAME env var.
+// 'disable_krb5_replay_cache' if set to true, disables the kerberos replay 
cache by setting
+// the KRB5RCACHETYPE env var to "none".
+Status InitKerberosForServer(const std::string& krb5ccname = kKrb5CCName,
+ bool disable_krb5_replay_cache = true);
 
 // Returns the process lock 'kerberos_reinit_lock'
 // This lock is taken in write mode while the ticket is being reacquired, and

kudu git commit: consensus: Rename CHANGE_ROLE op => CHANGE_REPLICA_TYPE

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master 827c35bc3 -> abbfa49de


consensus: Rename CHANGE_ROLE op => CHANGE_REPLICA_TYPE

"Role" is currently a pretty ambiguous term in Kudu because role is
currently defined as LEADER, FOLLOWER, LEARNER, etc. "member_type" is
the standard type name but "replica" is more descriptive. Let's rename
this operation CHANGE_REPLICA_TYPE.

This is a wire-compatible change.

Change-Id: I29c8bed1f6b5c84b86be3997b76ac4f825fa2f0e
Reviewed-on: http://gerrit.cloudera.org:8080/8295
Tested-by: Kudu Jenkins
Reviewed-by: Alexey Serbin 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/abbfa49d
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/abbfa49d
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/abbfa49d

Branch: refs/heads/master
Commit: abbfa49dee52d49dc4658c1ef57ed8467e43bb5d
Parents: 827c35b
Author: Mike Percy 
Authored: Tue Oct 10 12:56:13 2017 -0700
Committer: Alexey Serbin 
Committed: Tue Oct 17 21:50:28 2017 +

--
 src/kudu/consensus/consensus.proto   | 2 +-
 src/kudu/consensus/raft_consensus.cc | 4 ++--
 src/kudu/tools/tool_action_tablet.cc | 7 ---
 3 files changed, 7 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/abbfa49d/src/kudu/consensus/consensus.proto
--
diff --git a/src/kudu/consensus/consensus.proto 
b/src/kudu/consensus/consensus.proto
index c2acf7b..c8abd5d 100644
--- a/src/kudu/consensus/consensus.proto
+++ b/src/kudu/consensus/consensus.proto
@@ -120,7 +120,7 @@ enum ChangeConfigType {
   UNKNOWN_CHANGE = 0;
   ADD_SERVER = 1;
   REMOVE_SERVER = 2;
-  CHANGE_ROLE = 3;
+  CHANGE_REPLICA_TYPE = 3;
 }
 
 // A configuration change request for the tablet with 'tablet_id'.

http://git-wip-us.apache.org/repos/asf/kudu/blob/abbfa49d/src/kudu/consensus/raft_consensus.cc
--
diff --git a/src/kudu/consensus/raft_consensus.cc 
b/src/kudu/consensus/raft_consensus.cc
index 66a3c81..581bace 100644
--- a/src/kudu/consensus/raft_consensus.cc
+++ b/src/kudu/consensus/raft_consensus.cc
@@ -1646,8 +1646,8 @@ Status RaftConsensus::ChangeConfig(const 
ChangeConfigRequestPB& req,
 }
 break;
 
-  // TODO: Support role change.
-  case CHANGE_ROLE:
+  // TODO(mpercy): Support changing between VOTER and NON_VOTER.
+  case CHANGE_REPLICA_TYPE:
   default:
 return Status::NotSupported("Role change is not yet implemented.");
 }

http://git-wip-us.apache.org/repos/asf/kudu/blob/abbfa49d/src/kudu/tools/tool_action_tablet.cc
--
diff --git a/src/kudu/tools/tool_action_tablet.cc 
b/src/kudu/tools/tool_action_tablet.cc
index 3677cf9..8faa214 100644
--- a/src/kudu/tools/tool_action_tablet.cc
+++ b/src/kudu/tools/tool_action_tablet.cc
@@ -166,7 +166,8 @@ Status DoChangeConfig(const vector& 
master_addresses,
   if (cc_type == consensus::REMOVE_SERVER && member_type) {
 return Status::InvalidArgument("cannot supply Raft member type when 
removing a server");
   }
-  if ((cc_type == consensus::ADD_SERVER || cc_type == consensus::CHANGE_ROLE) 
&& !member_type) {
+  if ((cc_type == consensus::ADD_SERVER || cc_type == 
consensus::CHANGE_REPLICA_TYPE) &&
+  !member_type) {
 return Status::InvalidArgument(
 "must specify member type when adding a server or changing member 
type");
   }
@@ -219,7 +220,7 @@ Status ChangeConfig(const RunnerContext& context, 
ChangeConfigType cc_type) {
   const string& tablet_id = FindOrDie(context.required_args, kTabletIdArg);
   const string& replica_uuid = FindOrDie(context.required_args, kTsUuidArg);
   boost::optional member_type;
-  if (cc_type == consensus::ADD_SERVER || cc_type == consensus::CHANGE_ROLE) {
+  if (cc_type == consensus::ADD_SERVER || cc_type == 
consensus::CHANGE_REPLICA_TYPE) {
 const string& replica_type = FindOrDie(context.required_args, 
kReplicaTypeArg);
 string uppercase_peer_type;
 ToUpperCase(replica_type, &uppercase_peer_type);
@@ -238,7 +239,7 @@ Status AddReplica(const RunnerContext& context) {
 }
 
 Status ChangeReplicaType(const RunnerContext& context) {
-  return ChangeConfig(context, consensus::CHANGE_ROLE);
+  return ChangeConfig(context, consensus::CHANGE_REPLICA_TYPE);
 }
 
 Status RemoveReplica(const RunnerContext& context) {

kudu git commit: consensus: Make it easier to swap in a role change

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master 57d5a3151 -> aa7d6a97b


consensus: Make it easier to swap in a role change

This patch simply updates some helper functions to make it easier in a
follow-on patch to modify the role of a member in a copied Raft config.

Change-Id: Icca40d5ee0baeb542360b7047cb88102ff0432bd
Reviewed-on: http://gerrit.cloudera.org:8080/8296
Tested-by: Kudu Jenkins
Reviewed-by: Alexey Serbin 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/aa7d6a97
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/aa7d6a97
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/aa7d6a97

Branch: refs/heads/master
Commit: aa7d6a97bc8561ae4bc96a5c923736cb4676ecfa
Parents: 57d5a31
Author: Mike Percy 
Authored: Tue Oct 10 13:39:55 2017 -0700
Committer: Alexey Serbin 
Committed: Wed Oct 18 02:12:48 2017 +

--
 src/kudu/consensus/quorum_util-test.cc   | 14 +++---
 src/kudu/consensus/quorum_util.cc| 14 +++---
 src/kudu/consensus/quorum_util.h |  6 +++---
 src/kudu/consensus/raft_consensus_quorum-test.cc |  4 ++--
 src/kudu/tserver/tablet_copy_client-test.cc  |  6 +-
 5 files changed, 24 insertions(+), 20 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/aa7d6a97/src/kudu/consensus/quorum_util-test.cc
--
diff --git a/src/kudu/consensus/quorum_util-test.cc 
b/src/kudu/consensus/quorum_util-test.cc
index 9982c74..4b2ad0c 100644
--- a/src/kudu/consensus/quorum_util-test.cc
+++ b/src/kudu/consensus/quorum_util-test.cc
@@ -47,20 +47,20 @@ TEST(QuorumUtilTest, TestMemberExtraction) {
   AddPeer(&config, "C", RaftPeerPB::VOTER);
 
   // Basic test for GetRaftConfigMember().
-  RaftPeerPB peer_pb;
-  Status s = GetRaftConfigMember(config, "invalid", &peer_pb);
+  RaftPeerPB* peer_pb;
+  Status s = GetRaftConfigMember(&config, "invalid", &peer_pb);
   ASSERT_TRUE(s.IsNotFound()) << s.ToString();
-  ASSERT_OK(GetRaftConfigMember(config, "A", &peer_pb));
-  ASSERT_EQ("A", peer_pb.permanent_uuid());
+  ASSERT_OK(GetRaftConfigMember(&config, "A", &peer_pb));
+  ASSERT_EQ("A", peer_pb->permanent_uuid());
 
   // Basic test for GetRaftConfigLeader().
   ConsensusStatePB cstate;
   *cstate.mutable_committed_config() = config;
-  s = GetRaftConfigLeader(cstate, &peer_pb);
+  s = GetRaftConfigLeader(&cstate, &peer_pb);
   ASSERT_TRUE(s.IsNotFound()) << s.ToString();
   cstate.set_leader_uuid("B");
-  ASSERT_OK(GetRaftConfigLeader(cstate, &peer_pb));
-  ASSERT_EQ("B", peer_pb.permanent_uuid());
+  ASSERT_OK(GetRaftConfigLeader(&cstate, &peer_pb));
+  ASSERT_EQ("B", peer_pb->permanent_uuid());
 }
 
 TEST(QuorumUtilTest, TestDiffConsensusStates) {

http://git-wip-us.apache.org/repos/asf/kudu/blob/aa7d6a97/src/kudu/consensus/quorum_util.cc
--
diff --git a/src/kudu/consensus/quorum_util.cc 
b/src/kudu/consensus/quorum_util.cc
index 55c5bb9..f81f2a7 100644
--- a/src/kudu/consensus/quorum_util.cc
+++ b/src/kudu/consensus/quorum_util.cc
@@ -64,23 +64,23 @@ bool IsVoterRole(RaftPeerPB::Role role) {
   return role == RaftPeerPB::LEADER || role == RaftPeerPB::FOLLOWER;
 }
 
-Status GetRaftConfigMember(const RaftConfigPB& config,
+Status GetRaftConfigMember(RaftConfigPB* config,
const std::string& uuid,
-   RaftPeerPB* peer_pb) {
-  for (const RaftPeerPB& peer : config.peers()) {
+   RaftPeerPB** peer_pb) {
+  for (RaftPeerPB& peer : *config->mutable_peers()) {
 if (peer.permanent_uuid() == uuid) {
-  *peer_pb = peer;
+  *peer_pb = &peer;
   return Status::OK();
 }
   }
   return Status::NotFound(Substitute("Peer with uuid $0 not found in consensus 
config", uuid));
 }
 
-Status GetRaftConfigLeader(const ConsensusStatePB& cstate, RaftPeerPB* 
peer_pb) {
-  if (cstate.leader_uuid().empty()) {
+Status GetRaftConfigLeader(ConsensusStatePB* cstate, RaftPeerPB** peer_pb) {
+  if (cstate->leader_uuid().empty()) {
 return Status::NotFound("Consensus config has no leader");
   }
-  return GetRaftConfigMember(cstate.committed_config(), cstate.leader_uuid(), 
peer_pb);
+  return GetRaftConfigMember(cstate->mutable_committed_config(), 
cstate->leader_uuid(), peer_pb);
 }
 
 bool RemoveFromRaftConfig(RaftConfigPB* config, const string& uuid) {

http://git-wip-us.apache.org/repos/asf/kudu/blob/aa7d6a97/src/kudu/consensus/quorum_util.h
---

kudu git commit: [itests] fix flake in TombstonedVoteAfterFailedTabletCopy

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master 20118c7e4 -> bba425e8d


[itests] fix flake in TombstonedVoteAfterFailedTabletCopy

Fixed flakiness in the TombstonedVoteAfterFailedTabletCopy scenario
of the raft_consensus_election-itest.  A race between adding a new
replica and DeleteTablet was possible if adding a new server before
the catalog manager acknowledges eviction of the former replica and
sends DeleteTablet request.  In other words, prior to this fix,
a delayed DeleteTablet request might be processed _after_ a new
server has been added.

The race was easily reproducible by running the test scenario with
high number of stress-cpu-threads:

  ./bin/raft_consensus_election-itest \
--stress_cpu_threads=128 \
--gtest_filter=\
  'RaftConsensusElectionITest.TombstonedVoteAfterFailedTabletCopy'

Prior to this change, the test failed almost 100% if running as
described above. After the fix, no failures observed in about 100 runs.

Change-Id: Ifaba16d1acfee6ff01e475525482f21ff64133f4
Reviewed-on: http://gerrit.cloudera.org:8080/8298
Reviewed-by: Todd Lipcon 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/bba425e8
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/bba425e8
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/bba425e8

Branch: refs/heads/master
Commit: bba425e8de0f6fcbd78013960faa327fa0be7b85
Parents: 20118c7
Author: Alexey Serbin 
Authored: Tue Oct 17 08:21:52 2017 -0700
Committer: Alexey Serbin 
Committed: Wed Oct 18 22:16:22 2017 +

--
 .../integration-tests/raft_consensus-itest.cc   |  3 ++-
 .../raft_consensus_election-itest.cc| 21 ++--
 2 files changed, 17 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/bba425e8/src/kudu/integration-tests/raft_consensus-itest.cc
--
diff --git a/src/kudu/integration-tests/raft_consensus-itest.cc 
b/src/kudu/integration-tests/raft_consensus-itest.cc
index d012867..4c45849 100644
--- a/src/kudu/integration-tests/raft_consensus-itest.cc
+++ b/src/kudu/integration-tests/raft_consensus-itest.cc
@@ -113,6 +113,7 @@ using kudu::itest::TServerDetails;
 using kudu::itest::TabletServerMap;
 using kudu::itest::WAIT_FOR_LEADER;
 using kudu::itest::WaitForReplicasReportedToMaster;
+using kudu::itest::WaitUntilCommittedOpIdIndexIs;
 using kudu::itest::WaitUntilLeader;
 using kudu::itest::WriteSimpleTestRow;
 using kudu::master::TabletLocationsPB;
@@ -1708,7 +1709,7 @@ TEST_F(RaftConsensusITest, 
TestMasterNotifiedOnConfigChange) {
   // Wait for initial NO_OP to be committed by the leader.
   TServerDetails* leader_ts;
   ASSERT_OK(FindTabletLeader(tablet_servers_, tablet_id, timeout, &leader_ts));
-  ASSERT_OK(itest::WaitUntilCommittedOpIdIndexIs(1, leader_ts, tablet_id, 
timeout));
+  ASSERT_OK(WaitUntilCommittedOpIdIndexIs(1, leader_ts, tablet_id, timeout));
 
   // Change the config.
   TServerDetails* tserver_to_add = tablet_servers_[uuid_to_add];

http://git-wip-us.apache.org/repos/asf/kudu/blob/bba425e8/src/kudu/integration-tests/raft_consensus_election-itest.cc
--
diff --git a/src/kudu/integration-tests/raft_consensus_election-itest.cc 
b/src/kudu/integration-tests/raft_consensus_election-itest.cc
index ba4d538..23f5010 100644
--- a/src/kudu/integration-tests/raft_consensus_election-itest.cc
+++ b/src/kudu/integration-tests/raft_consensus_election-itest.cc
@@ -72,6 +72,7 @@ using kudu::itest::WaitUntilLeader;
 using kudu::itest::WriteSimpleTestRow;
 using kudu::pb_util::SecureShortDebugString;
 using kudu::tablet::TABLET_DATA_COPYING;
+using kudu::tablet::TABLET_DATA_TOMBSTONED;
 using std::string;
 using std::unordered_set;
 using std::vector;
@@ -491,11 +492,19 @@ TEST_F(RaftConsensusElectionITest, 
TombstonedVoteAfterFailedTabletCopy) {
   ASSERT_OK(WaitUntilCommittedConfigNumVotersIs(2, 
tablet_servers_[leader_uuid],
 tablet_id_, kTimeout));
   ASSERT_EQ(1, active_tablet_servers.erase(follower_uuid));
-  ASSERT_OK(WaitForServersToAgree(kTimeout, active_tablet_servers, tablet_id_, 
2));
 
-  // Now the follower is evicted and will be tombstoned.
-  // We'll add it back to the config and then crash the follower during the
-  // resulting tablet copy.
+  // Wait for the deleted tablet to be tombstoned, meaning the catalog manager
+  // has already sent DeleteTablet request and it has been processed. A race
+  // between adding a new server and processing DeleteTablet request is
+  // possible: the DeleteTablet request might be sent/processed _after_
+  // a new server has been added.
+  const int follower_idx =
+  cluster_->ta

[2/2] kudu git commit: tool: speed up test runtime a bit

[alexey]

tool: speed up test runtime a bit

We're good about setting --never_fsync in ExternalMiniClusters and in all
gtests, but test-based invocations of the Kudu CLI don't get it. This patch
changes that by centralizing all test-based CLI invocations and ensuring
they pass --never_fsync.

Change-Id: I27c67368b736f864f7cfa3482fdae72acafce851
Reviewed-on: http://gerrit.cloudera.org:8080/8337
Tested-by: Kudu Jenkins
Reviewed-by: Alexey Serbin 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/a6f2322b
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/a6f2322b
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/a6f2322b

Branch: refs/heads/master
Commit: a6f2322b241189d0a32d1d07165dee8268e4dcaa
Parents: 4d84eee
Author: Adar Dembo 
Authored: Thu Oct 19 11:55:01 2017 -0700
Committer: Alexey Serbin 
Committed: Thu Oct 19 21:47:22 2017 +

--
 src/kudu/tools/kudu-admin-test.cc  | 80 -
 src/kudu/tools/kudu-tool-test.cc   | 40 +++--
 src/kudu/tools/kudu-ts-cli-test.cc | 16 +++
 src/kudu/tools/tool_test_util.cc   | 16 ++-
 src/kudu/tools/tool_test_util.h| 13 +-
 5 files changed, 89 insertions(+), 76 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/a6f2322b/src/kudu/tools/kudu-admin-test.cc
--
diff --git a/src/kudu/tools/kudu-admin-test.cc 
b/src/kudu/tools/kudu-admin-test.cc
index f49e9bd..696ba89 100644
--- a/src/kudu/tools/kudu-admin-test.cc
+++ b/src/kudu/tools/kudu-admin-test.cc
@@ -58,7 +58,6 @@
 #include "kudu/util/monotime.h"
 #include "kudu/util/net/sockaddr.h"
 #include "kudu/util/status.h"
-#include "kudu/util/subprocess.h"
 #include "kudu/util/test_macros.h"
 #include "kudu/util/test_util.h"
 
@@ -85,6 +84,8 @@ using kudu::itest::WaitUntilCommittedOpIdIndexIs;
 using kudu::itest::WaitUntilTabletInState;
 using kudu::itest::WaitUntilTabletRunning;
 using kudu::pb_util::SecureDebugString;
+using std::back_inserter;
+using std::copy;
 using std::deque;
 using std::string;
 using std::vector;
@@ -152,8 +153,7 @@ TEST_F(AdminCliTest, TestChangeConfig) {
   ASSERT_EQ(leader->uuid(), master_observed_leader->uuid());
 
   LOG(INFO) << "Adding tserver with uuid " << new_node->uuid() << " as 
VOTER...";
-  ASSERT_OK(Subprocess::Call({
-GetKuduCtlAbsolutePath(),
+  ASSERT_OK(RunKuduTool({
 "tablet",
 "change_config",
 "add_replica",
@@ -190,8 +190,7 @@ TEST_F(AdminCliTest, TestChangeConfig) {
 
   // Now remove the server once again.
   LOG(INFO) << "Removing tserver with uuid " << new_node->uuid() << " from the 
config...";
-  ASSERT_OK(Subprocess::Call({
-GetKuduCtlAbsolutePath(),
+  ASSERT_OK(RunKuduTool({
 "tablet",
 "change_config",
 "remove_replica",
@@ -250,16 +249,15 @@ TEST_F(AdminCliTest, TestMoveTablet) {
   for (int i = 0; i < num_moves; i++) {
 const string remove = active_tservers.front();
 const string add = inactive_tservers.front();
-ASSERT_OK(Subprocess::Call({
-   GetKuduCtlAbsolutePath(),
-   "tablet",
-   "change_config",
-   "move_replica",
-   
cluster_->master()->bound_rpc_addr().ToString(),
-   tablet_id_,
-   remove,
-   add
-   }));
+ASSERT_OK(RunKuduTool({
+  "tablet",
+  "change_config",
+  "move_replica",
+  cluster_->master()->bound_rpc_addr().ToString(),
+  tablet_id_,
+  remove,
+  add
+}));
 active_tservers.pop_front();
 active_tservers.push_back(add);
 inactive_tservers.pop_front();
@@ -283,12 +281,14 @@ TEST_F(AdminCliTest, TestMoveTablet) {
 Status RunUnsafeChangeConfig(const string& tablet_id,
  const string& dst_host,
  vector peer_uuid_list) {
-  vector command_args = { GetKuduCtlAbsolutePath(), "remote_replica",
-  "unsafe_change_config", dst_host, tablet_id };
-  for (const auto& peer_uuid : peer_uuid_list) {
-command_args.push_back(peer_uuid);
-  }
-  return Subprocess::Call(command_args);
+  vector command_args = {
+  "remote_replica",
+  "unsafe_change_config",
+  dst_host,
+  tablet_id
+  };
+  copy(peer_uuid_list.begin(), peer_uuid_lis

[1/2] kudu git commit: fs: adjust directory sync behavior

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master 741bdfa41 -> a6f2322b2


fs: adjust directory sync behavior

This patch centralizes the synchronization of fs roots and data directories
in a new env_util helper method, which uses the set of all created files and
directories as input to synchronization.

There are a few related fixes:
- DataDirManager::Create wasn't considering enable_data_block_fsync when
  synchronizing. It does now.
- It was also synchronizing the directories themselves instead of their
  parent directories; the latter is what's needed for durability.

Change-Id: I27e7bf6a5a31720cf80a002c3553536ee0975758
Reviewed-on: http://gerrit.cloudera.org:8080/8290
Reviewed-by: Andrew Wong 
Tested-by: Kudu Jenkins
Reviewed-by: Todd Lipcon 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/4d84eeea
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/4d84eeea
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/4d84eeea

Branch: refs/heads/master
Commit: 4d84eeea1695734f2723af0e200f134b9fe72005
Parents: 741bdfa
Author: Adar Dembo 
Authored: Fri Oct 13 17:38:57 2017 -0700
Committer: Todd Lipcon 
Committed: Thu Oct 19 18:49:33 2017 +

--
 src/kudu/fs/data_dirs.cc  | 11 ---
 src/kudu/fs/fs_manager.cc | 13 -
 src/kudu/util/env_util.cc | 22 +-
 src/kudu/util/env_util.h  |  7 +++
 4 files changed, 36 insertions(+), 17 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/4d84eeea/src/kudu/fs/data_dirs.cc
--
diff --git a/src/kudu/fs/data_dirs.cc b/src/kudu/fs/data_dirs.cc
index 9ece90d..635a4a8 100644
--- a/src/kudu/fs/data_dirs.cc
+++ b/src/kudu/fs/data_dirs.cc
@@ -28,7 +28,6 @@
 #include 
 #include 
 #include 
-#include 
 #include 
 #include 
 
@@ -104,6 +103,7 @@ METRIC_DEFINE_gauge_uint64(server, data_dirs_full,
kudu::MetricUnit::kDataDirectories,
"Number of data directories whose disks are 
currently full");
 
+DECLARE_bool(enable_data_block_fsync);
 DECLARE_string(block_manager);
 
 namespace kudu {
@@ -117,7 +117,6 @@ using std::set;
 using std::shuffle;
 using std::string;
 using std::unique_ptr;
-using std::unordered_set;
 using std::vector;
 using strings::Substitute;
 
@@ -393,7 +392,6 @@ Status DataDirManager::Create() {
   int idx = 0;
 
   // Ensure the data dirs exist and create the instance files.
-  unordered_set to_sync;
   for (const auto& root : canonicalized_data_fs_roots_) {
 RETURN_NOT_OK_PREPEND(root.status, "Could not create directory manager 
with disks failed");
 string data_dir = JoinPathSegments(root.path, kDataDirName);
@@ -402,7 +400,6 @@ Status DataDirManager::Create() {
 Substitute("Could not create directory $0", data_dir));
 if (created) {
   dirs_to_delete.emplace_back(data_dir);
-  to_sync.insert(root.path);
 }
 
 if (block_manager_type_ == "log") {
@@ -419,9 +416,9 @@ Status DataDirManager::Create() {
   }
 
   // Ensure newly created directories are synchronized to disk.
-  for (const string& dir : to_sync) {
-RETURN_NOT_OK_PREPEND(env_->SyncDir(dir),
-  Substitute("Unable to synchronize directory $0", 
dir));
+  if (FLAGS_enable_data_block_fsync) {
+RETURN_NOT_OK_PREPEND(env_util::SyncAllParentDirs(
+env_, dirs_to_delete, files_to_delete), "could not sync data 
directories");
   }
 
   // Success: don't delete any files.

http://git-wip-us.apache.org/repos/asf/kudu/blob/4d84eeea/src/kudu/fs/fs_manager.cc
--
diff --git a/src/kudu/fs/fs_manager.cc b/src/kudu/fs/fs_manager.cc
index bf88c63..5f62858 100644
--- a/src/kudu/fs/fs_manager.cc
+++ b/src/kudu/fs/fs_manager.cc
@@ -22,7 +22,6 @@
 #include 
 #include 
 #include 
-#include 
 #include 
 
 #include 
@@ -107,7 +106,6 @@ using std::ostream;
 using std::set;
 using std::string;
 using std::unique_ptr;
-using std::unordered_set;
 using std::vector;
 using strings::Substitute;
 
@@ -400,7 +398,6 @@ Status 
FsManager::CreateInitialFileSystemLayout(boost::optional uuid) {
 
   InstanceMetadataPB metadata;
   RETURN_NOT_OK(CreateInstanceMetadata(std::move(uuid), &metadata));
-  unordered_set to_sync;
   for (const auto& root : canonicalized_all_fs_roots_) {
 if (!root.status.ok()) {
   continue;
@@ -411,7 +408,6 @@ Status 
FsManager::CreateInitialFileSystemLayout(boost::optional uuid) {
   "Unable to create FSManager root");
 if (created) {
   dirs_to_delete.emplace_back(root_name);
-  to_sync.insert(DirName(root_name));
 }
 RETURN_NOT_OK_PREPEND(WriteInstanceMetadata(metadata, root_name),
   "Unable to write instance

kudu git commit: [tests] clean-up on cluster_itest_util

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master ac18aca75 -> 407f55327


[tests] clean-up on cluster_itest_util

Updated signature of the AddServer(), RemoveServer(), and DeleteTablet()
methods to accommodate the majority of their use cases.

Also, changed the signature of the TSTabletManager::DeleteTablet() to be
more lightweight.

This changelist does not contain any functional modifications.

Change-Id: Icc5c85fb58750dec286a8ae546db955e3bd4073c
Reviewed-on: http://gerrit.cloudera.org:8080/8318
Tested-by: Kudu Jenkins
Reviewed-by: Dan Burkert 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/407f5532
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/407f5532
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/407f5532

Branch: refs/heads/master
Commit: 407f55327e34008929edabbceda3a16a349e9045
Parents: ac18aca
Author: Alexey Serbin 
Authored: Wed Oct 18 12:29:44 2017 -0700
Committer: Alexey Serbin 
Committed: Mon Oct 23 17:49:43 2017 +

--
 .../integration-tests/client_failover-itest.cc  |  11 +-
 .../integration-tests/cluster_itest_util.cc |  68 
 src/kudu/integration-tests/cluster_itest_util.h |  19 ++-
 .../integration-tests/delete_table-itest.cc |  59 +++
 .../integration-tests/delete_tablet-itest.cc|   8 +-
 .../raft_config_change-itest.cc |   6 +-
 .../integration-tests/raft_consensus-itest.cc   |  41 ++---
 .../raft_consensus_election-itest.cc|   7 +-
 .../raft_consensus_nonvoter-itest.cc|   6 +-
 src/kudu/integration-tests/tablet_copy-itest.cc | 165 +--
 .../tablet_copy_client_session-itest.cc |   9 +-
 .../tablet_replacement-itest.cc |  21 ++-
 .../tombstoned_voting-imc-itest.cc  |  18 +-
 .../tombstoned_voting-stress-test.cc|   4 +-
 src/kudu/tools/kudu-admin-test.cc   |   9 +-
 src/kudu/tserver/tablet_service.cc  |   2 +-
 src/kudu/tserver/ts_tablet_manager.cc   |  24 ++-
 src/kudu/tserver/ts_tablet_manager.h|  10 +-
 18 files changed, 227 insertions(+), 260 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/407f5532/src/kudu/integration-tests/client_failover-itest.cc
--
diff --git a/src/kudu/integration-tests/client_failover-itest.cc 
b/src/kudu/integration-tests/client_failover-itest.cc
index afbc786..47cedf4 100644
--- a/src/kudu/integration-tests/client_failover-itest.cc
+++ b/src/kudu/integration-tests/client_failover-itest.cc
@@ -15,7 +15,6 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include 
 #include 
 #include 
 #include 
@@ -23,7 +22,6 @@
 #include 
 #include 
 
-#include 
 #include 
 #include 
 
@@ -58,6 +56,7 @@ using kudu::client::sp::shared_ptr;
 using kudu::cluster::ExternalTabletServer;
 using kudu::cluster::ScopedResumeExternalDaemon;
 using kudu::itest::TServerDetails;
+using kudu::itest::DeleteTablet;
 using kudu::tablet::TABLET_DATA_TOMBSTONED;
 using std::set;
 using std::string;
@@ -167,8 +166,7 @@ TEST_P(ClientFailoverParamITest, 
TestDeleteLeaderWhileScanning) {
 
   // Delete the leader replica. This will cause the next scan to the same
   // leader to get a TABLET_NOT_FOUND error.
-  ASSERT_OK(itest::DeleteTablet(leader, tablet_id, TABLET_DATA_TOMBSTONED,
-boost::none, kTimeout));
+  ASSERT_OK(DeleteTablet(leader, tablet_id, TABLET_DATA_TOMBSTONED, kTimeout));
 
   int old_leader_index = leader_index;
   TServerDetails* old_leader = leader;
@@ -185,14 +183,13 @@ TEST_P(ClientFailoverParamITest, 
TestDeleteLeaderWhileScanning) {
 
   // Do a config change to remove the old replica and add a new one.
   // Cause the new replica to become leader, then do the scan again.
-  ASSERT_OK(RemoveServer(leader, tablet_id, old_leader, boost::none, 
kTimeout));
+  ASSERT_OK(RemoveServer(leader, tablet_id, old_leader, kTimeout));
   // Wait until the config is committed, otherwise AddServer() will fail.
   ASSERT_OK(WaitUntilCommittedConfigOpIdIndexIs(workload.batches_completed() + 
4, leader, tablet_id,
 kTimeout));
 
   TServerDetails* to_add = 
ts_map_[cluster_->tablet_server(missing_replica_index)->uuid()];
-  ASSERT_OK(AddServer(leader, tablet_id, to_add, consensus::RaftPeerPB::VOTER,
-  boost::none, kTimeout));
+  ASSERT_OK(AddServer(leader, tablet_id, to_add, consensus::RaftPeerPB::VOTER, 
kTimeout));
   HostPort hp;
   ASSERT_OK(HostPortFromPB(leader->registration.rpc_addresses(0), &hp));
   ASSERT_OK(StartTabletCopy(to_add, tablet_id, leader->uuid(), hp, 1, 
kTimeout));

http://git-wip-us.apache.org/repos/asf/kudu/blob/407f5532/src/kud

kudu git commit: consensus: support changing between VOTER and NON_VOTER

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master e938aaa21 -> 28a671365


consensus: support changing between VOTER and NON_VOTER

This patch implements changing from VOTER to NON_VOTER and vice-versa.

Added an integration test scenario for NON_VOTER --> VOTER,
VOTER --> NON_VOTER changes and other use cases.

Change-Id: I978cbed683b0e95a903f2bd2b57496aee71cb33a
Reviewed-on: http://gerrit.cloudera.org:8080/8297
Tested-by: Kudu Jenkins
Reviewed-by: Todd Lipcon 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/28a67136
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/28a67136
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/28a67136

Branch: refs/heads/master
Commit: 28a671365d6d38da966481daf937b3776e3d4852
Parents: e938aaa
Author: Mike Percy 
Authored: Wed Oct 18 01:12:26 2017 +0900
Committer: Todd Lipcon 
Committed: Tue Oct 24 18:16:36 2017 +

--
 src/kudu/consensus/quorum_util-test.cc  |  10 +
 src/kudu/consensus/quorum_util.cc   |   6 +
 src/kudu/consensus/quorum_util.h|   4 +
 src/kudu/consensus/raft_consensus.cc|  61 -
 src/kudu/consensus/raft_consensus.h |   8 +
 .../integration-tests/cluster_itest_util.cc |  35 ++-
 src/kudu/integration-tests/cluster_itest_util.h |  10 +
 .../raft_consensus_nonvoter-itest.cc| 225 ++-
 8 files changed, 345 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/28a67136/src/kudu/consensus/quorum_util-test.cc
--
diff --git a/src/kudu/consensus/quorum_util-test.cc 
b/src/kudu/consensus/quorum_util-test.cc
index 4b2ad0c..8864977 100644
--- a/src/kudu/consensus/quorum_util-test.cc
+++ b/src/kudu/consensus/quorum_util-test.cc
@@ -179,6 +179,16 @@ TEST(QuorumUtilTest, TestIsRaftConfigVoter) {
   ASSERT_FALSE(IsRaftConfigVoter("B", config));
   ASSERT_FALSE(IsRaftConfigVoter("C", config));
   ASSERT_FALSE(IsRaftConfigVoter(no_member_type_peer_uuid, config));
+
+  RaftPeerPB* peer_a;
+  ASSERT_OK(GetRaftConfigMember(&config, "A", &peer_a));
+  RaftPeerPB* peer_b;
+  ASSERT_OK(GetRaftConfigMember(&config, "B", &peer_b));
+  ASSERT_FALSE(ReplicaTypesEqual(*peer_a, *peer_b));
+  ASSERT_TRUE(ReplicaTypesEqual(*peer_b, *peer_b));
+  RaftPeerPB* peer_c;
+  ASSERT_OK(GetRaftConfigMember(&config, "C", &peer_c));
+  ASSERT_FALSE(ReplicaTypesEqual(*peer_b, *peer_c));
 }
 
 } // namespace consensus

http://git-wip-us.apache.org/repos/asf/kudu/blob/28a67136/src/kudu/consensus/quorum_util.cc
--
diff --git a/src/kudu/consensus/quorum_util.cc 
b/src/kudu/consensus/quorum_util.cc
index f81f2a7..90d9671 100644
--- a/src/kudu/consensus/quorum_util.cc
+++ b/src/kudu/consensus/quorum_util.cc
@@ -98,6 +98,12 @@ bool RemoveFromRaftConfig(RaftConfigPB* config, const 
string& uuid) {
   return true;
 }
 
+bool ReplicaTypesEqual(const RaftPeerPB& peer1, const RaftPeerPB& peer2) {
+  // TODO(mpercy): Include comparison of replica intentions once they are
+  // implemented.
+  return peer1.member_type() == peer2.member_type();
+}
+
 int CountVoters(const RaftConfigPB& config) {
   int voters = 0;
   for (const RaftPeerPB& peer : config.peers()) {

http://git-wip-us.apache.org/repos/asf/kudu/blob/28a67136/src/kudu/consensus/quorum_util.h
--
diff --git a/src/kudu/consensus/quorum_util.h b/src/kudu/consensus/quorum_util.h
index 26b6e78..3bd0afc 100644
--- a/src/kudu/consensus/quorum_util.h
+++ b/src/kudu/consensus/quorum_util.h
@@ -56,6 +56,10 @@ Status GetRaftConfigLeader(ConsensusStatePB* cstate, 
RaftPeerPB** peer_pb);
 // Returns true on success.
 bool RemoveFromRaftConfig(RaftConfigPB* config, const std::string& uuid);
 
+// Returns true iff the two peers have equivalent replica types and associated
+// options.
+bool ReplicaTypesEqual(const RaftPeerPB& peer1, const RaftPeerPB& peer2);
+
 // Counts the number of voters in the configuration.
 int CountVoters(const RaftConfigPB& config);
 

http://git-wip-us.apache.org/repos/asf/kudu/blob/28a67136/src/kudu/consensus/raft_consensus.cc
--
diff --git a/src/kudu/consensus/raft_consensus.cc 
b/src/kudu/consensus/raft_consensus.cc
index 581bace..5f7f392 100644
--- a/src/kudu/consensus/raft_consensus.cc
+++ b/src/kudu/consensus/raft_consensus.cc
@@ -316,7 +316,7 @@ Status RaftConsensus::Start(const ConsensusBootstrapInfo& 
info,
   }
 }
 
-// Now assume "follower" duties.
+// Now assume non-leader replica duties.
 RETURN_NOT_OK(BecomeReplicaUnlocked(fd_initial_delta));
 
 SetStateUnlocked(kRunning);
@@ -585,15 +585,9 @@ S

[1/2] kudu git commit: thirdparty: treat cc the same as gcc when building boost

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master c8603ab77 -> 6097e2d1a


thirdparty: treat cc the same as gcc when building boost

Change-Id: I6c26723b2bcbcba524aad441e39b2d3fbe580079
Reviewed-on: http://gerrit.cloudera.org:8080/8391
Tested-by: Adar Dembo 
Reviewed-by: Todd Lipcon 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/49a7d971
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/49a7d971
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/49a7d971

Branch: refs/heads/master
Commit: 49a7d9712bf367b89946e0b81f5db26a85ef00aa
Parents: c8603ab
Author: Adar Dembo 
Authored: Wed Oct 25 16:21:43 2017 -0700
Committer: Todd Lipcon 
Committed: Thu Oct 26 20:39:40 2017 +

--
 thirdparty/build-definitions.sh | 6 ++
 1 file changed, 6 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/49a7d971/thirdparty/build-definitions.sh
--
diff --git a/thirdparty/build-definitions.sh b/thirdparty/build-definitions.sh
index acfdf30..e5b8045 100644
--- a/thirdparty/build-definitions.sh
+++ b/thirdparty/build-definitions.sh
@@ -686,6 +686,12 @@ build_boost() {
 # and clang, even when they're called through ccache.
 local COMPILER=$($CC --version | awk 'NR==1 {print $1;}')
 
+# If the compiler binary used was 'cc' and not 'gcc', it will also report
+# itself as 'cc'. Coerce it to gcc.
+if [ "$COMPILER" = "cc" ]; then
+  COMPILER=gcc
+fi
+
 TOOLSET="toolset=${COMPILER}"
 echo "Using $TOOLSET"
 echo "using ${COMPILER} : : $CXX ;" > $USER_JAMFILE

[2/2] kudu git commit: [tools] limit per-session error buffer space

[alexey]

[tools] limit per-session error buffer space

Introduced --error-buffer-size-bytes flag to control the size of the
per-session error buffer.  By default, the limit is set to 16MB per
session (i.e. per worker thread).

Prior to this patch, the per-session error buffer had no size limit
and could grow very large because of huge number of errors during the
run (e.g., because of duplicate value errors).

I also updated the in-line documentation to add an example of using
the tool for subsequent runs against the same table using the
--seq-start flag.

Change-Id: I2b7663c52e57b2d3a1e11b22301b804da784fa74
Reviewed-on: http://gerrit.cloudera.org:8080/8396
Reviewed-by: Adar Dembo 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/6097e2d1
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/6097e2d1
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/6097e2d1

Branch: refs/heads/master
Commit: 6097e2d1a4287563ef22a4f6e32e6f4ee2604031
Parents: 49a7d97
Author: Alexey Serbin 
Authored: Wed Oct 25 17:47:29 2017 -0700
Committer: Alexey Serbin 
Committed: Thu Oct 26 20:43:51 2017 +

--
 src/kudu/tools/tool_action_perf.cc | 52 ++---
 1 file changed, 41 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/6097e2d1/src/kudu/tools/tool_action_perf.cc
--
diff --git a/src/kudu/tools/tool_action_perf.cc 
b/src/kudu/tools/tool_action_perf.cc
index 568b0a0..1e4e30c 100644
--- a/src/kudu/tools/tool_action_perf.cc
+++ b/src/kudu/tools/tool_action_perf.cc
@@ -30,7 +30,7 @@
 // of length 64 for binary and string fields
 // with Kudu master server listening on the default port at localhost:
 //
-//   kudu test loadgen \
+//   kudu perf loadgen \
 // --num_threads=1 \
 // --num_rows_per_thread=800 \
 // --string_len=64 \
@@ -46,7 +46,7 @@
 // using the specified pre-set string for binary and string fields
 // with Kudu master server listening on the default port at localhost:
 //
-//   kudu test loadgen \
+//   kudu perf loadgen \
 // --num_threads=2 \
 // --num_rows_per_thread=400 \
 // --string_fixed=012345678901234567890123456789012 \
@@ -62,7 +62,7 @@
 // using the specified pre-set string for binary and string fields
 // with Kudu master server listening at 127.0.0.1:8765
 //
-//   kudu test loadgen \
+//   kudu perf loadgen \
 // --num_threads=4 \
 // --num_rows_per_thread=200 \
 // --string_fixed=0123456789 \
@@ -79,10 +79,31 @@
 // plus run post-insertion row scan to verify
 // that the count of the inserted rows matches the expected number:
 //
-//   kudu test loadgen \
+//   kudu perf loadgen \
 // --run_scan=true \
 // 127.0.0.1
 //
+//
+// If running the tool against already existing table multiple times,
+// use the '--seq_start' flag to avoid errors on duplicate values in subsequent
+// runs. For example: an already existing table 't3' has 5 columns.
+// The sequence below contains 3 runs which insert 6000 rows in total
+// (3 runs * 1000 rows per thread * 2 threads)
+// with no duplicate values across all columns:
+//
+//   kudu perf loadgen 127.0.0.1 --table_name=t3 --num_threads=2 \
+// --num_rows_per_thread=1000 --seq_start=0
+//
+//   kudu perf loadgen 127.0.0.1 --table_name=t3 --num_threads=2 \
+// --num_rows_per_thread=1000 --seq_start=1
+//
+//   perf perf loadgen 127.0.0.1 --table_name=t3 --num_threads=2 \
+// --num_rows_per_thread=1000 --seq_start=2
+//
+// The single sequence number is used to generate values for all table columns,
+// so for the example above each run increments the sequence number by 1:
+// 1000 rows per thread * 2 threads * 5 columns
+//
 
 #include "kudu/tools/tool_action.h"
 
@@ -103,7 +124,6 @@
 #include 
 
 #include "kudu/client/client.h"
-#include "kudu/client/row_result.h"
 #include "kudu/client/scan_batch.h"
 #include "kudu/client/schema.h"
 #include "kudu/client/shared_ptr.h"
@@ -131,11 +151,10 @@ using kudu::client::KuduClientBuilder;
 using kudu::client::KuduColumnSchema;
 using kudu::client::KuduError;
 using kudu::client::KuduInsert;
-using kudu::client::KuduRowResult;
-using kudu::client::KuduSchema;
-using kudu::client::KuduSchemaBuilder;
 using kudu::client::KuduScanBatch;
 using kudu::client::KuduScanner;
+using kudu::client::KuduSchema;
+using kudu::client::KuduSchemaBuilder;
 using kudu::client::KuduSession;
 using kudu::client::KuduTable;
 using kudu::client::KuduTableCreator;
@@ -150,8 +169,8 @@ using std::numeric_limits;
 using std::ostringstream;
 using std::string;
 using std::thread;
-using std::vector;
 using std::unique_ptr;
+using std::vector;
 usi

kudu git commit: [build] fix OPENSSL_ROOT_DIR override on RH/CentOS

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master c08f27fc5 -> 9f85f04a9


[build] fix OPENSSL_ROOT_DIR override on RH/CentOS

Due to an extra level of indirection in the condition for one if()
expression, the OpenSSL location was not overridden on RedHat/CentOS
machines where the OpenSSL workaround build existed in
$KUDU_ROOT/thirdparty/installed/openssl-el6-workaround,
even if the OPENSSL_ROOT_DIR was overridden for cmake invocation
(i.e. -DOPENSSL_ROOT_DIR=/path/to/alt-openssl-location was specified).

Removing the level of indirection for the OPENSSL_ROOT_DIR variable
in the expression fixed the issue, which seems to be in sync with
  https://cmake.org/cmake/help/v3.9/command/if.html

At least, with this patch the issue is gone if running cmake 3.9.0.

Change-Id: Ibe69b5d62ef4054e60138a154a85569d92166aff
Reviewed-on: http://gerrit.cloudera.org:8080/8407
Tested-by: Alexey Serbin 
Reviewed-by: Todd Lipcon 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/9f85f04a
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/9f85f04a
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/9f85f04a

Branch: refs/heads/master
Commit: 9f85f04a91c38229649ccb4aa4773a45fc742dc3
Parents: c08f27f
Author: Alexey Serbin 
Authored: Fri Oct 27 14:58:55 2017 -0700
Committer: Alexey Serbin 
Committed: Mon Oct 30 19:33:07 2017 +

--
 CMakeLists.txt | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/9f85f04a/CMakeLists.txt
--
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 7e1350c..59542cf 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -912,8 +912,7 @@ ADD_THIRDPARTY_LIB(mustache
 ## binary in thirdparty (deposited there by 
thirdparty/install-openssl-el6-workaround.sh)
 ## then we'll use that one. See that script for more information.
 set(CENTOS_6_4_OPENSSL_DIR 
"${THIRDPARTY_INSTALL_DIR}/openssl-el6-workaround/usr/")
-if (NOT "${OPENSSL_ROOT_DIR}" AND
-EXISTS "${CENTOS_6_4_OPENSSL_DIR}")
+if (NOT OPENSSL_ROOT_DIR AND EXISTS "${CENTOS_6_4_OPENSSL_DIR}")
   set(OPENSSL_ROOT_DIR "${CENTOS_6_4_OPENSSL_DIR}")
 endif()
 find_package(OpenSSL 1.0.0 REQUIRED)

kudu git commit: Update auth_token_validity_seconds description

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master 563f41b5c -> 354f4c445


Update auth_token_validity_seconds description

The caveats are no longer relevant since KUDU-2013 landed.

Change-Id: I050e39d0377049cdaac0afb5085a8a9a19d620a5
Reviewed-on: http://gerrit.cloudera.org:8080/8413
Tested-by: Kudu Jenkins
Reviewed-by: Alexey Serbin 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/354f4c44
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/354f4c44
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/354f4c44

Branch: refs/heads/master
Commit: 354f4c44562de03e20064a5349c306f3dd05fa19
Parents: 563f41b
Author: Dan Burkert 
Authored: Mon Oct 30 13:48:08 2017 -0400
Committer: Alexey Serbin 
Committed: Tue Oct 31 16:02:53 2017 +

--
 src/kudu/master/master.cc | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/354f4c44/src/kudu/master/master.cc
--
diff --git a/src/kudu/master/master.cc b/src/kudu/master/master.cc
index 0b2d7d9..0bb45cf 100644
--- a/src/kudu/master/master.cc
+++ b/src/kudu/master/master.cc
@@ -75,9 +75,8 @@ TAG_FLAG(tsk_rotation_seconds, experimental);
 
 DEFINE_int64(authn_token_validity_seconds, 60 * 60 * 24 * 7,
  "Period of time for which an issued authentication token is 
valid. "
- "It's not possible to renew a token, hence the token validity "
- "interval defines the longest possible lifetime of an external "
- "job which uses a token for authentication.");
+ "Clients will automatically attempt to reacquire a token after 
the "
+ "validity period expires.");
 TAG_FLAG(authn_token_validity_seconds, experimental);
 
 using std::min;

kudu git commit: KUDU-2209. HybridClock doesn't handle changes in STA_NANO flag

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master b418e88b6 -> 10f6164b1


KUDU-2209. HybridClock doesn't handle changes in STA_NANO flag

Users have occasionally reported spurious crashes due to Kudu thinking
that another node has a time stamp from the future. After some debugging
I realized that the issue is that we currently capture the flag
'STA_NANO' from the kernel only at startup. This flag indicates whether
the kernel's sub-second timestamp is in nanoseconds or microseconds. We
initially assumed this was a static property of the kernel. However it
turns out that this flag can get toggled at runtime by ntp in certain
circumstances. Given this, it was possible for us to interpret a number
of nanoseconds as if it were microseconds, resulting in a timestamp up
to 1000 seconds in the future.

This patch changes the SystemNtp time source to always use the 'adjtime'
call to fetch the clock, and looks at the STA_NANO flag on every such
call rather than only at startup.

I checked the source for the ntp_gettime call that we used to use, and
it turns out it was implemented in terms of the same adjtime() call, so
there should be no performance penalty in this change.

This patch doesn't include tests since it's based on some kernel
functionality. However, I was able to test it as follows:

- wrote a simple program to print the time once a second
- stopped ntp, ran chrony, stopped chrony, and started ntpd
-- this has the side effect of clearing STA_NANO
- waited 10 minutes or so, and eventually ntp reset back to STA_NANO
-- this caused my test program to start printing incorrect timestamps
   as it was interpreting nanosecond values from the kernel as if they
   were microseconds

Change-Id: Iaa9241dea1304076ed5cfaec78779d15b11293ff
Reviewed-on: http://gerrit.cloudera.org:8080/8450
Reviewed-by: Alexey Serbin 
Tested-by: Alexey Serbin 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/10f6164b
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/10f6164b
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/10f6164b

Branch: refs/heads/master
Commit: 10f6164b1217e0299bcfedc061d2c57581c389bd
Parents: b418e88
Author: Todd Lipcon 
Authored: Wed Nov 1 18:41:07 2017 -0700
Committer: Alexey Serbin 
Committed: Fri Nov 3 05:12:00 2017 +

--
 src/kudu/clock/system_ntp.cc | 67 ---
 src/kudu/clock/system_ntp.h  |  7 +---
 2 files changed, 21 insertions(+), 53 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/10f6164b/src/kudu/clock/system_ntp.cc
--
diff --git a/src/kudu/clock/system_ntp.cc b/src/kudu/clock/system_ntp.cc
index 9cdac05..d1e8d56 100644
--- a/src/kudu/clock/system_ntp.cc
+++ b/src/kudu/clock/system_ntp.cc
@@ -22,12 +22,9 @@
 
 #include 
 #include 
-#include 
 
 #include 
 
-#include "kudu/gutil/port.h"
-#include "kudu/gutil/strings/substitute.h"
 #include "kudu/util/errno.h"
 #include "kudu/util/logging.h"
 #include "kudu/util/status.h"
@@ -35,37 +32,21 @@
 namespace kudu {
 namespace clock {
 
-using strings::Substitute;
-
 const double SystemNtp::kAdjtimexScalingFactor = 65536;
-const uint64_t SystemNtp::kNanosPerSec = 100;
+const uint64_t SystemNtp::kMicrosPerSec = 100;
 
 namespace {
 
-// Returns the clock modes and checks if the clock is synchronized.
-Status GetClockModes(timex* timex) {
-  // this makes ntp_adjtime a read-only call
-  timex->modes = 0;
-  int rc = ntp_adjtime(timex);
-  if (PREDICT_FALSE(rc == TIME_ERROR)) {
-return Status::ServiceUnavailable(
-Substitute("Error reading clock. Clock considered unsynchronized. 
Return code: $0", rc));
-  }
-  // TODO what to do about leap seconds? see KUDU-146
-  if (PREDICT_FALSE(rc != TIME_OK)) {
-LOG(ERROR) << Substitute("TODO Server undergoing leap second. Return code: 
$0", rc);
-  }
-  return Status::OK();
-}
-
 // Returns the current time/max error and checks if the clock is synchronized.
-Status GetClockTime(ntptimeval* timeval) {
-  int rc = ntp_gettime(timeval);
+Status CallAdjTime(timex* tx) {
+  // Set mode to 0 to query the current time.
+  tx->modes = 0;
+  int rc = ntp_adjtime(tx);
   switch (rc) {
 case TIME_OK:
   return Status::OK();
 case -1: // generic error
-  return Status::ServiceUnavailable("Error reading clock. ntp_gettime() 
failed",
+  return Status::ServiceUnavailable("Error reading clock. ntp_adjtime() 
failed",
 ErrnoToString(errno));
 case TIME_ERROR:
   return Status::ServiceUnavailable("Error reading clock. Clock considered 
unsynchronized");
@@

[1/4] kudu git commit: rpc: fix service lookup to use the proper lock type

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master b167b98e4 -> ed45d7207


rpc: fix service lookup to use the proper lock type

Looking up the RPC service in the service map was using an exclusive
lock rather than a per-CPU read-lock.

Fixing this yielded a ~5% improvement on the average throughput
of rpc-bench. I ran:

rpc-bench --server-reactors 96 --async-call-concurrency 100 \
  -worker-threads 40 -client-threads 48 -run-seconds 1 \
  --gtest_filter=\*Async --gtest_repeat=100 2>&1 | tee /tmp/log

before and after the change. Running 100 short benchmarks is preferable
to a long one since this benchmark has quite a bit of variability
depending on reactor thread hashing, etc. I grepped out the reqs/sec
number and then ran a t-test using R:

data:  d.before and d.after
t = -4.538, df = 165.27, p-value = 1.088e-05
alternative hypothesis: true difference in means is not equal to 0
95 percent confidence interval:
 -59048.87 -23244.07
sample estimates:
mean of x mean of y
 785314.9  826461.4

Special thanks to Sergei Politov from Yugabyte for finding this.

Change-Id: I871765dffc5208dbdfc6b7f9f7fa2160611631b9
Reviewed-on: http://gerrit.cloudera.org:8080/8478
Tested-by: Kudu Jenkins
Reviewed-by: Alexey Serbin 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/10ed76cc
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/10ed76cc
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/10ed76cc

Branch: refs/heads/master
Commit: 10ed76cc1e31e0d226ee5411e738d6675072956a
Parents: b167b98
Author: Todd Lipcon 
Authored: Sun Nov 5 20:22:20 2017 -0800
Committer: Todd Lipcon 
Committed: Mon Nov 6 05:26:15 2017 +

--
 src/kudu/rpc/messenger.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/10ed76cc/src/kudu/rpc/messenger.cc
--
diff --git a/src/kudu/rpc/messenger.cc b/src/kudu/rpc/messenger.cc
index 46b144f..1c8b5b2 100644
--- a/src/kudu/rpc/messenger.cc
+++ b/src/kudu/rpc/messenger.cc
@@ -567,7 +567,7 @@ void Messenger::ScheduleOnReactor(const 
boost::function& fu
 const scoped_refptr Messenger::rpc_service(const string& 
service_name) const {
   scoped_refptr service;
   {
-std::lock_guard guard(lock_);
+shared_lock guard(lock_.get_lock());
 if (!FindCopy(rpc_services_, service_name, &service)) {
   return scoped_refptr(nullptr);
 }

[4/4] kudu git commit: [tools] simplify code in DumpRowSet

[alexey]

[tools] simplify code in DumpRowSet

This changelist does not contain any functional modifications.

Change-Id: Id5347732023ea13e7275927be8ef346cd03e45f1
Reviewed-on: http://gerrit.cloudera.org:8080/8475
Tested-by: Alexey Serbin 
Reviewed-by: Todd Lipcon 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/ed45d720
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/ed45d720
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/ed45d720

Branch: refs/heads/master
Commit: ed45d72070bb4c3f70357d4c67077023c6d186fa
Parents: f7b54d8
Author: Alexey Serbin 
Authored: Sat Nov 4 22:44:29 2017 -0700
Committer: Alexey Serbin 
Committed: Mon Nov 6 05:45:30 2017 +

--
 src/kudu/tools/tool_action_local_replica.cc | 26 ++--
 1 file changed, 11 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/ed45d720/src/kudu/tools/tool_action_local_replica.cc
--
diff --git a/src/kudu/tools/tool_action_local_replica.cc 
b/src/kudu/tools/tool_action_local_replica.cc
index 94b3ae7..784def5 100644
--- a/src/kudu/tools/tool_action_local_replica.cc
+++ b/src/kudu/tools/tool_action_local_replica.cc
@@ -849,27 +849,23 @@ Status DumpRowSet(const RunnerContext& context) {
 
   // If rowset index is provided, only dump that rowset.
   if (FLAGS_rowset_index != -1) {
-bool found = false;
 for (const shared_ptr& rs_meta : meta->rowsets())  {
   if (rs_meta->id() == FLAGS_rowset_index) {
-found = true;
 return DumpRowSetInternal(fs_manager.get(), meta->schema(),
   rs_meta, 0);
   }
 }
-if (!found) {
-  return Status::InvalidArgument(
-  Substitute("Could not find rowset $0 in tablet id $1",
- FLAGS_rowset_index, tablet_id));
-}
-  } else {
-// Rowset index not provided, dump all rowsets
-size_t idx = 0;
-for (const shared_ptr& rs_meta : meta->rowsets())  {
-  cout << endl << "Dumping rowset " << idx++ << endl << kSeparatorLine;
-  RETURN_NOT_OK(DumpRowSetInternal(fs_manager.get(), meta->schema(),
-   rs_meta, 2));
-}
+return Status::InvalidArgument(
+Substitute("Could not find rowset $0 in tablet id $1",
+   FLAGS_rowset_index, tablet_id));
+  }
+
+  // Rowset index not provided, dump all rowsets
+  size_t idx = 0;
+  for (const shared_ptr& rs_meta : meta->rowsets())  {
+cout << endl << "Dumping rowset " << idx++ << endl << kSeparatorLine;
+RETURN_NOT_OK(DumpRowSetInternal(fs_manager.get(), meta->schema(),
+ rs_meta, 2));
   }
   return Status::OK();
 }

[3/4] kudu git commit: [TabletServerTestBase] fix typo in DeleteTestRowsRemote

[alexey]

[TabletServerTestBase] fix typo in DeleteTestRowsRemote

Change-Id: I80638320b312060b303fdea83b5f83c93a1b9fc4
Reviewed-on: http://gerrit.cloudera.org:8080/8474
Tested-by: Alexey Serbin 
Reviewed-by: Todd Lipcon 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/f7b54d85
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/f7b54d85
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/f7b54d85

Branch: refs/heads/master
Commit: f7b54d85b28e59a065fb6d644a9364a69ea578b4
Parents: 7f51045
Author: Alexey Serbin 
Authored: Sat Nov 4 22:39:26 2017 -0700
Committer: Todd Lipcon 
Committed: Mon Nov 6 05:27:32 2017 +

--
 src/kudu/tserver/tablet_server-test-base.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/f7b54d85/src/kudu/tserver/tablet_server-test-base.cc
--
diff --git a/src/kudu/tserver/tablet_server-test-base.cc 
b/src/kudu/tserver/tablet_server-test-base.cc
index 0ab114e..5107934 100644
--- a/src/kudu/tserver/tablet_server-test-base.cc
+++ b/src/kudu/tserver/tablet_server-test-base.cc
@@ -278,7 +278,7 @@ void TabletServerTestBase::DeleteTestRowsRemote(int64_t 
first_row,
   }
 
   SCOPED_TRACE(SecureDebugString(req));
-  ASSERT_OK(proxy_->Write(req, &resp, &controller));
+  ASSERT_OK(DCHECK_NOTNULL(proxy)->Write(req, &resp, &controller));
   SCOPED_TRACE(SecureDebugString(resp));
   ASSERT_FALSE(resp.has_error()) << SecureShortDebugString(resp);
 }

[2/4] kudu git commit: KUDU-1578. Hybrid clock should ride over brief interruptions in NTP service

[alexey]

KUDU-1578. Hybrid clock should ride over brief interruptions in NTP service

This changes HybridClock so that, if the clock loses synchronization for
a brief period of time, it will continue to use the most recent clock
reading, adjusting it forward based on the local monotonic clock and a
conservative estimate for max error.

In the case that the clock is out of sync for a significantly long time,
the max error will grow large enough to eclipse the 10-second default,
at which point it will still crash as before. But, if NTP is properly
restored within a few minutes, the server should remain operational.

A simple test is included which injects a fake error into the NTP
source. I also ran a kudu-master while briefly disabling NTP and ensured
that the log messages showed up as expected and then went away when I
resumed NTP synchronization.

Change-Id: I0d03a77033070e6bcdfdefc4341981f28a1477a0
Reviewed-on: http://gerrit.cloudera.org:8080/8451
Reviewed-by: Alexey Serbin 
Tested-by: Todd Lipcon 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/7f510458
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/7f510458
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/7f510458

Branch: refs/heads/master
Commit: 7f5104586fa381347c582260df539b8cbb02f08b
Parents: 10ed76c
Author: Todd Lipcon 
Authored: Wed Nov 1 19:13:05 2017 -0700
Committer: Todd Lipcon 
Committed: Mon Nov 6 05:26:38 2017 +

--
 src/kudu/clock/hybrid_clock-test.cc | 38 +
 src/kudu/clock/hybrid_clock.cc  | 71 
 src/kudu/clock/hybrid_clock.h   | 12 --
 src/kudu/clock/system_ntp.cc| 10 +
 4 files changed, 119 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/7f510458/src/kudu/clock/hybrid_clock-test.cc
--
diff --git a/src/kudu/clock/hybrid_clock-test.cc 
b/src/kudu/clock/hybrid_clock-test.cc
index e3d2afd..0524a84 100644
--- a/src/kudu/clock/hybrid_clock-test.cc
+++ b/src/kudu/clock/hybrid_clock-test.cc
@@ -41,6 +41,7 @@
 #include "kudu/util/test_util.h"
 #include "kudu/util/thread.h"
 
+DECLARE_bool(inject_adjtimex_errors);
 DECLARE_string(time_source);
 
 namespace kudu {
@@ -309,5 +310,42 @@ TEST_F(HybridClockTest, 
TestGetPhysicalComponentDifference) {
   ASSERT_EQ(-100, negative_delta.ToMicroseconds());
 }
 
+
+TEST_F(HybridClockTest, TestRideOverNtpInterruption) {
+  Timestamp timestamps[3];
+  uint64_t max_error_usec[3];
+
+  // Get the clock once, with a working NTP.
+  clock_->NowWithError(×tamps[0], &max_error_usec[0]);
+
+  // Try to read the clock again a second later, but with an error
+  // injected. It should extrapolate from the first read.
+  SleepFor(MonoDelta::FromSeconds(1));
+  FLAGS_inject_adjtimex_errors = true;
+  clock_->NowWithError(×tamps[1], &max_error_usec[1]);
+
+  // The new clock reading should be a second or longer from the
+  // first one, since SleepFor guarantees sleeping at least as long
+  // as specified.
+  MonoDelta phys_diff = clock_->GetPhysicalComponentDifference(
+  timestamps[1], timestamps[0]);
+  ASSERT_GE(phys_diff.ToSeconds(), 1);
+
+  // The new clock reading should have higher error than the first.
+  // The error should have increased based on the clock skew.
+  int64_t error_diff = max_error_usec[1] - max_error_usec[0];
+  ASSERT_NEAR(error_diff, clock_->time_service()->skew_ppm() * 
phys_diff.ToSeconds(),
+  10);
+
+  // Now restore the ability to read the system clock, and
+  // read it again.
+  FLAGS_inject_adjtimex_errors = false;
+  clock_->NowWithError(×tamps[2], &max_error_usec[2]);
+
+  ASSERT_LT(timestamps[0].ToUint64(), timestamps[1].ToUint64());
+  ASSERT_LT(timestamps[1].ToUint64(), timestamps[2].ToUint64());
+}
+
+
 }  // namespace clock
 }  // namespace kudu

http://git-wip-us.apache.org/repos/asf/kudu/blob/7f510458/src/kudu/clock/hybrid_clock.cc
--
diff --git a/src/kudu/clock/hybrid_clock.cc b/src/kudu/clock/hybrid_clock.cc
index 6ee7129..0518afe 100644
--- a/src/kudu/clock/hybrid_clock.cc
+++ b/src/kudu/clock/hybrid_clock.cc
@@ -34,6 +34,7 @@
 #include "kudu/gutil/strings/substitute.h"
 #include "kudu/util/debug/trace_event.h"
 #include "kudu/util/flag_tags.h"
+#include "kudu/util/logging.h"
 #include "kudu/util/metrics.h"
 #include "kudu/util/monotime.h"
 #include "kudu/util/status.h"
@@ -170,11 +171,8 @@ void HybridClock::NowWithError(Timestamp* timestamp, 
uint64_t* max_error_usec) {
 
   uint64_t now_usec;
   uint64_t error_usec;
-  Status s = WalltimeWithError(&now_usec, &e

kudu git commit: [stopwatch] update Stopwatch::GetTimes() on OS X

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master 26b06c5c5 -> 1520b3958


[stopwatch] update Stopwatch::GetTimes() on OS X

Updated implementation of Stopwatch::GetTimes() for OS X to output
relevant numbers corresponding to getrusage(..., RUSAGE_THREAD).
The new implementation is modeled after OS X-specific code from
  https://codereview.chromium.org/16818003

Example of output prior to the fix:
  Time spent creating directory manager: \
real 0.009s user 18446744046.974s   sys 0.005s

Example of output after the fix:
  Time spent creating directory manager: \
real 0.008s user 0.002s sys 0.006s

Change-Id: I13584e12342e3548c50a38c890b3f2cb18fd3f07
Reviewed-on: http://gerrit.cloudera.org:8080/8473
Tested-by: Alexey Serbin 
Reviewed-by: Dan Burkert 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/1520b395
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/1520b395
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/1520b395

Branch: refs/heads/master
Commit: 1520b39585e0bdc4a9b64bf1b900f3ecb05e0276
Parents: 26b06c5
Author: Alexey Serbin 
Authored: Sat Nov 4 20:40:38 2017 -0700
Committer: Alexey Serbin 
Committed: Mon Nov 6 18:18:11 2017 +

--
 src/kudu/gutil/walltime.h |  2 +-
 src/kudu/util/stopwatch.h | 20 +++-
 2 files changed, 12 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/1520b395/src/kudu/gutil/walltime.h
--
diff --git a/src/kudu/gutil/walltime.h b/src/kudu/gutil/walltime.h
index 40af48e..02338a8 100644
--- a/src/kudu/gutil/walltime.h
+++ b/src/kudu/gutil/walltime.h
@@ -74,7 +74,7 @@ extern void InitializeTimebaseInfo();
 inline void GetCurrentTime(mach_timespec_t* ts) {
   clock_serv_t cclock;
   host_get_clock_service(mach_host_self(), CALENDAR_CLOCK, &cclock);
-  clock_get_time(cclock, ts);
+  CHECK_EQ(KERN_SUCCESS, clock_get_time(cclock, ts));
   mach_port_deallocate(mach_task_self(), cclock);
 }
 

http://git-wip-us.apache.org/repos/asf/kudu/blob/1520b395/src/kudu/util/stopwatch.h
--
diff --git a/src/kudu/util/stopwatch.h b/src/kudu/util/stopwatch.h
index 66d54d1..617ee8b 100644
--- a/src/kudu/util/stopwatch.h
+++ b/src/kudu/util/stopwatch.h
@@ -25,6 +25,7 @@
 #if defined(__APPLE__)
 #include 
 #include 
+#include 
 #endif  // defined(__APPLE__)
 
 #include "kudu/gutil/macros.h"
@@ -167,8 +168,8 @@ class Stopwatch {
 
   // Construct a new stopwatch. The stopwatch is initially stopped.
   explicit Stopwatch(Mode mode = THIS_THREAD)
-: stopped_(true),
-  mode_(mode) {
+  : mode_(mode),
+stopped_(true) {
 times_.clear();
   }
 
@@ -240,15 +241,17 @@ class Stopwatch {
 
 #if defined(__APPLE__)
 if (mode_ == THIS_THREAD) {
-  //Adapted from http://blog.kuriositaet.de/?p=257.
-  struct task_basic_info t_info;
-  mach_msg_type_number_t t_info_count = TASK_BASIC_INFO_COUNT;
-  CHECK_EQ(KERN_SUCCESS, task_info(mach_task_self(), 
TASK_THREAD_TIMES_INFO,
-   (task_info_t)&t_info, &t_info_count));
+  // Adapted from https://codereview.chromium.org/16818003
+  thread_basic_info_data_t t_info;
+  mach_msg_type_number_t count = THREAD_BASIC_INFO_COUNT;
+  CHECK_EQ(KERN_SUCCESS, thread_info(mach_thread_self(), THREAD_BASIC_INFO,
+ (thread_info_t)&t_info, &count));
   usage.ru_utime.tv_sec = t_info.user_time.seconds;
   usage.ru_utime.tv_usec = t_info.user_time.microseconds;
   usage.ru_stime.tv_sec = t_info.system_time.seconds;
   usage.ru_stime.tv_usec = t_info.system_time.microseconds;
+  usage.ru_nivcsw = t_info.suspend_count;
+  usage.ru_nvcsw = 0;
 } else {
   CHECK_EQ(0, getrusage(RUSAGE_SELF, &usage));
 }
@@ -267,10 +270,9 @@ class Stopwatch {
 times->context_switches = usage.ru_nvcsw + usage.ru_nivcsw;
   }
 
+  const Mode mode_;
   bool stopped_;
-
   CpuTimes times_;
-  Mode mode_;
 };
 
 

kudu git commit: [security] test and fixes for TLS socket EINTR issues

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master 1520b3958 -> 18e024cf8


[security] test and fixes for TLS socket EINTR issues

SSL_{read,write}() can return SSL_ERROR_WANT_{READ,WRITE}
correspondingly when signal interrupts recv()/send() calls even if
SSL_MODE_AUTO_RETRY is set in the TLS context.  To handle that
properly in Socket::Blocking{Recv,Write}() methods, return
NetworkError() with appropriate POSIX error code from
TlsSocket::{Recv,Write}().

As a by-product, this changelist fixes flakiness in TestUniqueClientIds
scenario of the ClientStressTest test and other flaky tests which failed
with errors like below:

  Bad status: IO error: Could not connect to the cluster: \
Client connection negotiation failed: client connection to \
IP:port: Read zero bytes on a blocking Recv() call: \
Transferred 0 of 4 bytes

Prior to this fix, the test failure ratio observed with dist-test
for TSAN builds was about 6% in multiple 1K runs.  After the fix,
no failures observed.

Change-Id: Ibec9049186f79f1c43295e4735538ed7ba4e516e
Reviewed-on: http://gerrit.cloudera.org:8080/8462
Tested-by: Kudu Jenkins
Reviewed-by: Dan Burkert 
Reviewed-by: Alexey Serbin 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/18e024cf
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/18e024cf
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/18e024cf

Branch: refs/heads/master
Commit: 18e024cf8bcaea192efb63780802cc4c799bbb9c
Parents: 1520b39
Author: Todd Lipcon 
Authored: Fri Nov 3 12:39:25 2017 -0700
Committer: Alexey Serbin 
Committed: Mon Nov 6 19:20:09 2017 +

--
 src/kudu/security/CMakeLists.txt |   1 +
 src/kudu/security/tls_handshake.cc   |   1 -
 src/kudu/security/tls_socket-test.cc | 196 ++
 src/kudu/security/tls_socket.cc  |   9 ++
 src/kudu/util/net/socket.cc  |  27 +++-
 5 files changed, 227 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/18e024cf/src/kudu/security/CMakeLists.txt
--
diff --git a/src/kudu/security/CMakeLists.txt b/src/kudu/security/CMakeLists.txt
index c669e25..b6ae528 100644
--- a/src/kudu/security/CMakeLists.txt
+++ b/src/kudu/security/CMakeLists.txt
@@ -126,5 +126,6 @@ if (NOT NO_TESTS)
   ADD_KUDU_TEST(crypto-test)
   ADD_KUDU_TEST(test/mini_kdc-test)
   ADD_KUDU_TEST(tls_handshake-test)
+  ADD_KUDU_TEST(tls_socket-test)
   ADD_KUDU_TEST(token-test)
 endif()

http://git-wip-us.apache.org/repos/asf/kudu/blob/18e024cf/src/kudu/security/tls_handshake.cc
--
diff --git a/src/kudu/security/tls_handshake.cc 
b/src/kudu/security/tls_handshake.cc
index 57f9c7c..6daea6b 100644
--- a/src/kudu/security/tls_handshake.cc
+++ b/src/kudu/security/tls_handshake.cc
@@ -116,7 +116,6 @@ Status TlsHandshake::Continue(const string& recv, string* 
send) {
 DCHECK_GE(send->size(), 0);
 return Status::OK();
   }
-  DCHECK_GT(send->size(), 0);
   return Status::Incomplete("TLS Handshake incomplete");
 }
 

http://git-wip-us.apache.org/repos/asf/kudu/blob/18e024cf/src/kudu/security/tls_socket-test.cc
--
diff --git a/src/kudu/security/tls_socket-test.cc 
b/src/kudu/security/tls_socket-test.cc
new file mode 100644
index 000..a978e68
--- /dev/null
+++ b/src/kudu/security/tls_socket-test.cc
@@ -0,0 +1,196 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "kudu/security/tls_handshake.h"
+
+#include 
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include 
+#include 
+
+#include "kudu/gutil/macros.h"
+#include "kudu/security/tls_context.h"
+#include "kudu/util/countdown_latch.h"
+#include "kudu/util/monotime.h"
+#include "kudu/util/net/sockaddr.h"
+#include "kudu/util/net/socket

kudu git commit: [hybrid_clock-test] adapt the test for OS X

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master 9748fd2c8 -> 28fdec405


[hybrid_clock-test] adapt the test for OS X

Fixed linkage breakage on OS X: adapted the hybrid_clock-test for OS X.

Change-Id: I50a4b139eb6b0b5c79bce48eedb24bc7c445970c
Reviewed-on: http://gerrit.cloudera.org:8080/8488
Tested-by: Alexey Serbin 
Reviewed-by: Todd Lipcon 
Reviewed-by: Dan Burkert 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/28fdec40
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/28fdec40
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/28fdec40

Branch: refs/heads/master
Commit: 28fdec4053c86f08f410d0ef460e4c82a78f9e1c
Parents: 9748fd2
Author: Alexey Serbin 
Authored: Mon Nov 6 19:08:57 2017 -0800
Committer: Alexey Serbin 
Committed: Wed Nov 8 18:34:40 2017 +

--
 src/kudu/clock/hybrid_clock-test.cc  |  8 +++-
 src/kudu/clock/system_ntp.cc |  9 +++--
 src/kudu/clock/system_unsync_time.cc | 14 ++
 3 files changed, 20 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/28fdec40/src/kudu/clock/hybrid_clock-test.cc
--
diff --git a/src/kudu/clock/hybrid_clock-test.cc 
b/src/kudu/clock/hybrid_clock-test.cc
index 0524a84..8857b32 100644
--- a/src/kudu/clock/hybrid_clock-test.cc
+++ b/src/kudu/clock/hybrid_clock-test.cc
@@ -41,7 +41,7 @@
 #include "kudu/util/test_util.h"
 #include "kudu/util/thread.h"
 
-DECLARE_bool(inject_adjtimex_errors);
+DECLARE_bool(inject_unsync_time_errors);
 DECLARE_string(time_source);
 
 namespace kudu {
@@ -310,7 +310,6 @@ TEST_F(HybridClockTest, TestGetPhysicalComponentDifference) 
{
   ASSERT_EQ(-100, negative_delta.ToMicroseconds());
 }
 
-
 TEST_F(HybridClockTest, TestRideOverNtpInterruption) {
   Timestamp timestamps[3];
   uint64_t max_error_usec[3];
@@ -321,7 +320,7 @@ TEST_F(HybridClockTest, TestRideOverNtpInterruption) {
   // Try to read the clock again a second later, but with an error
   // injected. It should extrapolate from the first read.
   SleepFor(MonoDelta::FromSeconds(1));
-  FLAGS_inject_adjtimex_errors = true;
+  FLAGS_inject_unsync_time_errors = true;
   clock_->NowWithError(×tamps[1], &max_error_usec[1]);
 
   // The new clock reading should be a second or longer from the
@@ -339,13 +338,12 @@ TEST_F(HybridClockTest, TestRideOverNtpInterruption) {
 
   // Now restore the ability to read the system clock, and
   // read it again.
-  FLAGS_inject_adjtimex_errors = false;
+  FLAGS_inject_unsync_time_errors = false;
   clock_->NowWithError(×tamps[2], &max_error_usec[2]);
 
   ASSERT_LT(timestamps[0].ToUint64(), timestamps[1].ToUint64());
   ASSERT_LT(timestamps[1].ToUint64(), timestamps[2].ToUint64());
 }
 
-
 }  // namespace clock
 }  // namespace kudu

http://git-wip-us.apache.org/repos/asf/kudu/blob/28fdec40/src/kudu/clock/system_ntp.cc
--
diff --git a/src/kudu/clock/system_ntp.cc b/src/kudu/clock/system_ntp.cc
index ff2c2a6..770a3fa 100644
--- a/src/kudu/clock/system_ntp.cc
+++ b/src/kudu/clock/system_ntp.cc
@@ -23,18 +23,15 @@
 #include 
 #include 
 
-#include 
+#include 
 #include 
 
 #include "kudu/gutil/port.h"
 #include "kudu/util/errno.h"
-#include "kudu/util/flag_tags.h"
 #include "kudu/util/logging.h"
 #include "kudu/util/status.h"
 
-DEFINE_bool(inject_adjtimex_errors, false,
-"If true, will return a fake 'unsynchronized' status from NTP.");
-TAG_FLAG(inject_adjtimex_errors, unsafe);
+DECLARE_bool(inject_unsync_time_errors);
 
 namespace kudu {
 namespace clock {
@@ -49,7 +46,7 @@ Status CallAdjTime(timex* tx) {
   // Set mode to 0 to query the current time.
   tx->modes = 0;
   int rc = ntp_adjtime(tx);
-  if (PREDICT_FALSE(FLAGS_inject_adjtimex_errors)) {
+  if (PREDICT_FALSE(FLAGS_inject_unsync_time_errors)) {
 rc = TIME_ERROR;
   }
   switch (rc) {

http://git-wip-us.apache.org/repos/asf/kudu/blob/28fdec40/src/kudu/clock/system_unsync_time.cc
--
diff --git a/src/kudu/clock/system_unsync_time.cc 
b/src/kudu/clock/system_unsync_time.cc
index 453a2b2..12da925 100644
--- a/src/kudu/clock/system_unsync_time.cc
+++ b/src/kudu/clock/system_unsync_time.cc
@@ -19,9 +19,17 @@
 
 #include 
 
+#include 
 #include 
 
+#include "kudu/gutil/port.h"
 #include "kudu/gutil/walltime.h"
+#include "kudu/util/flag_tags.h"
+
+DEFINE_bool(inject_unsync_time_errors, false,
+"If true, both Init() and WalltimeWithError() return "
+"'clock considered unsynchronized' error.");
+TAG_FLAG(in

kudu git commit: [tools] Fix doc'd endianness for kudu test mini_cluster

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master 04c23bc84 -> b8e7dd423


[tools] Fix doc'd endianness for kudu test mini_cluster

See ControlShellProtocol::ReceiveMessage in tool_action_common.cc.

Change-Id: Ida22188ed5b477888e2ae88d6bab9e935db72066
Reviewed-on: http://gerrit.cloudera.org:8080/8504
Tested-by: Kudu Jenkins
Reviewed-by: Alexey Serbin 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/b8e7dd42
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/b8e7dd42
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/b8e7dd42

Branch: refs/heads/master
Commit: b8e7dd423a66aa19958470a51be74768ef4103a1
Parents: 04c23bc
Author: Will Berkeley 
Authored: Wed Nov 8 12:55:48 2017 -0800
Committer: Alexey Serbin 
Committed: Thu Nov 9 17:18:21 2017 +

--
 src/kudu/tools/tool.proto   | 2 +-
 src/kudu/tools/tool_action_common.h | 2 +-
 src/kudu/tools/tool_action_test.cc  | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/b8e7dd42/src/kudu/tools/tool.proto
--
diff --git a/src/kudu/tools/tool.proto b/src/kudu/tools/tool.proto
index 8a30bf1..6107c26 100644
--- a/src/kudu/tools/tool.proto
+++ b/src/kudu/tools/tool.proto
@@ -76,7 +76,7 @@ enum DaemonType {
 
 // Identifier for a cluster daemon, unique to the cluster.
 message DaemonIdentifierPB {
-  // Whether the daemon is amaster, tserver, or whatever.
+  // Whether the daemon is a master, tserver, or whatever.
   optional DaemonType type = 1;
 
   // Index of the daemon in the cluster, if the cluster has multiple daemons

http://git-wip-us.apache.org/repos/asf/kudu/blob/b8e7dd42/src/kudu/tools/tool_action_common.h
--
diff --git a/src/kudu/tools/tool_action_common.h 
b/src/kudu/tools/tool_action_common.h
index ea08643..9edf079 100644
--- a/src/kudu/tools/tool_action_common.h
+++ b/src/kudu/tools/tool_action_common.h
@@ -177,7 +177,7 @@ class LeaderMasterProxy {
 class ControlShellProtocol {
  public:
   enum class SerializationMode {
-// Each message is serialized as a four byte little-endian size followed by
+// Each message is serialized as a four byte big-endian size followed by
 // the protobuf-encoded message itself.
 PB,
 

http://git-wip-us.apache.org/repos/asf/kudu/blob/b8e7dd42/src/kudu/tools/tool_action_test.cc
--
diff --git a/src/kudu/tools/tool_action_test.cc 
b/src/kudu/tools/tool_action_test.cc
index fd8a7cf..504b4e1 100644
--- a/src/kudu/tools/tool_action_test.cc
+++ b/src/kudu/tools/tool_action_test.cc
@@ -48,7 +48,7 @@
 DEFINE_string(serialization, "json", "Serialization method to be used by the "
   "control shell. Valid values are 'json' (protobuf serialized "
   "into JSON and terminated with a newline character) or 'pb' "
-  "(four byte protobuf message length in little endian followed by 
"
+  "(four byte protobuf message length in big endian followed by "
   "the protobuf message itself).");
 DEFINE_validator(serialization, [](const char* /*n*/, const std::string& v) {
   return boost::iequals(v, "pb") ||

kudu git commit: [tests] fix flake in RaftConsensusNonVoterITest

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master b8e7dd423 -> bf1fcb0df


[tests] fix flake in RaftConsensusNonVoterITest

Fixed flakiness in the RaftConsensusNonVoterITest::PromoteAndDemote
scenario.  The problematic scenario was when the tablet leader had been
paused before the tablet copy completed.  That resulted in the new
leader being unable to commit its own NO_OP because the tablet copying
server could not commit any new messages before completing its (hung)
tablet copy.  Meanwhile, the log was flooded with messages like below:

  Couldn't send request to peer 95f63392a59643d3b15bfbb97662d24e for \
tablet bb1c77bc65544659bfa24d852e2fc728. \
Error code: TABLET_NOT_RUNNING (12). \
Status: Illegal state: Tablet not RUNNING: INITIALIZED. \
Retrying in the next heartbeat period. Already tried 1210 times.

Other issues were related to tablet leader re-elections and also cases
where a newly promoted replica has almost instantly became a new leader.

To deal with those issues and preserve the coverage, I separated part
of the former PromoteAndDemote scenario into a new scenario called
PromotedReplicaCanVote.

Change-Id: I177e45cdfc883b6cec074877fea034374b00c677
Reviewed-on: http://gerrit.cloudera.org:8080/8477
Reviewed-by: Mike Percy 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/bf1fcb0d
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/bf1fcb0d
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/bf1fcb0d

Branch: refs/heads/master
Commit: bf1fcb0df5f2fbb6d1557b6b932bc67045fff531
Parents: b8e7dd4
Author: Alexey Serbin 
Authored: Sun Nov 5 18:24:32 2017 -0800
Committer: Alexey Serbin 
Committed: Fri Nov 10 06:12:45 2017 +

--
 src/kudu/consensus/raft_consensus.cc|   5 +-
 .../integration-tests/cluster_itest_util.cc |   1 -
 .../raft_consensus_nonvoter-itest.cc| 355 +--
 src/kudu/integration-tests/ts_itest-base.cc |  52 ++-
 src/kudu/integration-tests/ts_itest-base.h  |  10 +
 5 files changed, 294 insertions(+), 129 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/bf1fcb0d/src/kudu/consensus/raft_consensus.cc
--
diff --git a/src/kudu/consensus/raft_consensus.cc 
b/src/kudu/consensus/raft_consensus.cc
index 5f7f392..99ecefe 100644
--- a/src/kudu/consensus/raft_consensus.cc
+++ b/src/kudu/consensus/raft_consensus.cc
@@ -385,7 +385,7 @@ string ReasonString(RaftConsensus::ElectionReason reason, 
StringPiece leader_uui
 } // anonymous namespace
 
 Status RaftConsensus::StartElection(ElectionMode mode, ElectionReason reason) {
-  const char* mode_str = ModeString(mode);
+  const char* const mode_str = ModeString(mode);
 
   TRACE_EVENT2("consensus", "RaftConsensus::StartElection",
"peer", LogPrefixThreadSafe(),
@@ -398,7 +398,8 @@ Status RaftConsensus::StartElection(ElectionMode mode, 
ElectionReason reason) {
 
 RaftPeerPB::Role active_role = cmeta_->active_role();
 if (active_role == RaftPeerPB::LEADER) {
-  LOG_WITH_PREFIX_UNLOCKED(INFO) << "Not starting " << mode << " -- 
already leader";
+  LOG_WITH_PREFIX_UNLOCKED(INFO) << Substitute(
+  "Not starting $0 -- already a leader", mode_str);
   return Status::OK();
 }
 if (PREDICT_FALSE(!consensus::IsVoterRole(active_role))) {

http://git-wip-us.apache.org/repos/asf/kudu/blob/bf1fcb0d/src/kudu/integration-tests/cluster_itest_util.cc
--
diff --git a/src/kudu/integration-tests/cluster_itest_util.cc 
b/src/kudu/integration-tests/cluster_itest_util.cc
index cac682d..114c239 100644
--- a/src/kudu/integration-tests/cluster_itest_util.cc
+++ b/src/kudu/integration-tests/cluster_itest_util.cc
@@ -1071,7 +1071,6 @@ Status BeginTabletCopySession(const TServerDetails* ts,
   return Status::OK();
 }
 
-
 Status GetInt64Metric(const HostPort& http_hp,
   const MetricEntityPrototype* entity_proto,
   const char* entity_id,

http://git-wip-us.apache.org/repos/asf/kudu/blob/bf1fcb0d/src/kudu/integration-tests/raft_consensus_nonvoter-itest.cc
--
diff --git a/src/kudu/integration-tests/raft_consensus_nonvoter-itest.cc 
b/src/kudu/integration-tests/raft_consensus_nonvoter-itest.cc
index 4722185..a190ebe 100644
--- a/src/kudu/integration-tests/raft_consensus_nonvoter-itest.cc
+++ b/src/kudu/integration-tests/raft_consensus_nonvoter-itest.cc
@@ -26,10 +26,13 @@
 #include 
 #include 
 
-#include "kudu/consensus/consensus.pb.h"
+#include "kudu/cl

kudu git commit: Fix compiler warnings in tablet_replacement-itest

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master 338cf61cb -> 471feccd0


Fix compiler warnings in tablet_replacement-itest

This patch removes the following compiler warnings:

  [879/1023] Building CXX object 
src/kudu/inte...ment-itest.dir/tablet_replacement-itest.cc.o
  ../../src/kudu/integration-tests/tablet_replacement-itest.cc:370:7: warning: 
ignoring return value of function declared with 'warn_unused_result' attribute 
[-Wunused-result]
cluster_->tablet_server(kFollower1Index)->Pause();
^~~
  ../../src/kudu/integration-tests/tablet_replacement-itest.cc:371:7: warning: 
ignoring return value of function declared with 'warn_unused_result' attribute 
[-Wunused-result]
cluster_->tablet_server(kFollower2Index)->Pause();
^~~

Change-Id: I5f9f65a73d2fe0e4b6d75010628fd36a57f98def
Reviewed-on: http://gerrit.cloudera.org:8080/8584
Reviewed-by: Alexey Serbin 
Tested-by: Alexey Serbin 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/471feccd
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/471feccd
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/471feccd

Branch: refs/heads/master
Commit: 471feccd073e90ac767ce6d6339b4fda71f72ff0
Parents: 338cf61
Author: Mike Percy 
Authored: Thu Nov 16 18:22:08 2017 -0800
Committer: Alexey Serbin 
Committed: Fri Nov 17 07:40:36 2017 +

--
 src/kudu/integration-tests/tablet_replacement-itest.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/471feccd/src/kudu/integration-tests/tablet_replacement-itest.cc
--
diff --git a/src/kudu/integration-tests/tablet_replacement-itest.cc 
b/src/kudu/integration-tests/tablet_replacement-itest.cc
index c012f02..eb724f4 100644
--- a/src/kudu/integration-tests/tablet_replacement-itest.cc
+++ b/src/kudu/integration-tests/tablet_replacement-itest.cc
@@ -367,8 +367,8 @@ void 
TabletReplacementITest::TestDontEvictIfRemainingConfigIsUnstable(Instabilit
   cluster_->tablet_server(kFollower2Index)->Shutdown();
   break;
 case NODE_STOPPED:
-  cluster_->tablet_server(kFollower1Index)->Pause();
-  cluster_->tablet_server(kFollower2Index)->Pause();
+  ASSERT_OK(cluster_->tablet_server(kFollower1Index)->Pause());
+  ASSERT_OK(cluster_->tablet_server(kFollower2Index)->Pause());
   break;
   }
 

kudu git commit: KUDU-2218. tls_socket: properly handle temporary socket errors in Writev

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master 471feccd0 -> 64eb9f37b


KUDU-2218. tls_socket: properly handle temporary socket errors in Writev

This fixes a bug which caused RaftConsensusITest.TestLargeBatches to
fail when run under stress, as in the following command line:

taskset -c 0-4 \
 build/latest/bin/raft_consensus-itest \
   --gtest_filter=\*LargeBat\* \
   --stress-cpu-threads=8

This would produce an error like:
Network error: failed to write to TLS socket: error:1409F07F:SSL 
routines:SSL3_WRITE_PENDING:bad write retry:s3_pkt.c:878

This means that we were retrying a write after getting EAGAIN, but with
a different buffer than the first time.

I tracked this down to mishandling of temporary socket errors in
TlsSocket::Writev(). In the case that we successfully write part of the
io vector but hit such an error trying to write a later element in the
vector, we were still propagating the error back up to the caller. The
caller didn't realize that part of the write was successful, and thus it
would retry the write from the beginning.

The fix is to fix the above, but also to enable partial writes in
TlsContext. The new test fails if either of the above two changes are
backed out.

Change-Id: If797f220f42bfb2e6f452b66f15e7a758e883472
Reviewed-on: http://gerrit.cloudera.org:8080/8570
Tested-by: Kudu Jenkins
Reviewed-by: Alexey Serbin 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/64eb9f37
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/64eb9f37
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/64eb9f37

Branch: refs/heads/master
Commit: 64eb9f37b171419ed12a3795efe28faf2fd33b3d
Parents: 471fecc
Author: Todd Lipcon 
Authored: Wed Nov 15 22:55:44 2017 -0800
Committer: Alexey Serbin 
Committed: Fri Nov 17 17:42:14 2017 +

--
 src/kudu/security/tls_context.cc |   2 +-
 src/kudu/security/tls_socket-test.cc | 277 ++
 src/kudu/security/tls_socket.cc  |  10 +-
 3 files changed, 219 insertions(+), 70 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/64eb9f37/src/kudu/security/tls_context.cc
--
diff --git a/src/kudu/security/tls_context.cc b/src/kudu/security/tls_context.cc
index 1cea173..28e180b 100644
--- a/src/kudu/security/tls_context.cc
+++ b/src/kudu/security/tls_context.cc
@@ -115,7 +115,7 @@ Status TlsContext::Init() {
   if (!ctx_) {
 return Status::RuntimeError("failed to create TLS context", 
GetOpenSSLErrors());
   }
-  SSL_CTX_set_mode(ctx_.get(), SSL_MODE_AUTO_RETRY);
+  SSL_CTX_set_mode(ctx_.get(), SSL_MODE_AUTO_RETRY | 
SSL_MODE_ENABLE_PARTIAL_WRITE);
 
   // Disable SSLv2 and SSLv3 which are vulnerable to various issues such as 
POODLE.
   // We support versions back to TLSv1.0 since OpenSSL on RHEL 6.4 and earlier 
does not

http://git-wip-us.apache.org/repos/asf/kudu/blob/64eb9f37/src/kudu/security/tls_socket-test.cc
--
diff --git a/src/kudu/security/tls_socket-test.cc 
b/src/kudu/security/tls_socket-test.cc
index a978e68..214d2bf 100644
--- a/src/kudu/security/tls_socket-test.cc
+++ b/src/kudu/security/tls_socket-test.cc
@@ -17,7 +17,11 @@
 
 #include "kudu/security/tls_handshake.h"
 
+#include 
 #include 
+#include 
+#include 
+#include 
 
 #include 
 #include 
@@ -28,6 +32,7 @@
 #include 
 #include 
 #include 
+#include 
 
 #include 
 #include 
@@ -38,6 +43,8 @@
 #include "kudu/util/monotime.h"
 #include "kudu/util/net/sockaddr.h"
 #include "kudu/util/net/socket.h"
+#include "kudu/util/random.h"
+#include "kudu/util/random_util.h"
 #include "kudu/util/scoped_cleanup.h"
 #include "kudu/util/status.h"
 #include "kudu/util/test_macros.h"
@@ -46,25 +53,27 @@
 using std::string;
 using std::thread;
 using std::unique_ptr;
-
+using std::vector;
 
 namespace kudu {
 namespace security {
 
+const MonoDelta kTimeout = MonoDelta::FromSeconds(10);
+
+// Size is big enough to not fit into output socket buffer of default size
+// (controlled by setsockopt() with SO_SNDBUF).
+constexpr size_t kEchoChunkSize = 32 * 1024 * 1024;
 
 class TlsSocketTest : public KuduTest {
  public:
   void SetUp() override {
 KuduTest::SetUp();
-
 ASSERT_OK(client_tls_.Init());
-ASSERT_OK(server_tls_.Init());
-ASSERT_OK(server_tls_.GenerateSelfSignedCertAndKey());
   }
 
  protected:
+  void ConnectClient(const Sockaddr& addr, unique_ptr* sock);
   TlsContext client_tls_;
-  TlsContext server_tls_;
 };
 
 Status DoNegotiationSide(Socket* sock, TlsHandshake* tls, const char* side) {
@@ -101,19 +110,112 @@ Status DoNegotiationSide(Socket* sock, TlsHandshake* 
tls, const char*

[2/2] kudu git commit: catalog_manager_tsk-itest: ensure that test eventually makes progress

[alexey]

catalog_manager_tsk-itest: ensure that test eventually makes progress

This test previously tried to introduce a lot of master leader elections
by setting a very low heartbeat and failure interval. This worked, but
sometimes worked so well that the test never made progress and couldn't
obtain a stable leader long enough to create a table.

This patch changes the test to instead use a separate thread which
triggers elections manually on all the leaders. The elections start off
very frequent and then back off as the test progresses to ensure that by
the end, the leaders do actually make progress.

I verified that this still covers the case of a failed write when
writing TSKs by changing the RETURN_NOT_OK to a CHECK_OK when storing
the TSK. With the CHECK_OK, the test failed nearly immediately.

Change-Id: I3ecda0c269225e7674bc384fee652576b110ae7b
Reviewed-on: http://gerrit.cloudera.org:8080/8567
Tested-by: Kudu Jenkins
Reviewed-by: Alexey Serbin 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/8e6bfa9f
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/8e6bfa9f
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/8e6bfa9f

Branch: refs/heads/master
Commit: 8e6bfa9fb8aa7292e75a9fd3dcf9c50c8ea5191e
Parents: c4006ae
Author: Todd Lipcon 
Authored: Wed Nov 15 21:49:17 2017 -0800
Committer: Alexey Serbin 
Committed: Fri Nov 17 23:43:13 2017 +

--
 .../catalog_manager_tsk-itest.cc| 66 ++--
 1 file changed, 46 insertions(+), 20 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/8e6bfa9f/src/kudu/integration-tests/catalog_manager_tsk-itest.cc
--
diff --git a/src/kudu/integration-tests/catalog_manager_tsk-itest.cc 
b/src/kudu/integration-tests/catalog_manager_tsk-itest.cc
index 0a0a0e4..9812102 100644
--- a/src/kudu/integration-tests/catalog_manager_tsk-itest.cc
+++ b/src/kudu/integration-tests/catalog_manager_tsk-itest.cc
@@ -16,12 +16,17 @@
 // under the License.
 
 #include 
+#include 
+#include 
 #include 
 #include 
 #include 
+#include 
 #include 
+#include 
 #include 
 
+#include 
 #include 
 
 #include "kudu/client/client-test-util.h"
@@ -30,11 +35,18 @@
 #include "kudu/client/shared_ptr.h"
 #include "kudu/client/write_op.h"
 #include "kudu/common/partial_row.h"
+#include "kudu/consensus/consensus.pb.h"
+#include "kudu/consensus/consensus.proxy.h"
 #include "kudu/gutil/gscoped_ptr.h"
 #include "kudu/gutil/strings/substitute.h"
+#include "kudu/master/sys_catalog.h"
 #include "kudu/mini-cluster/external_mini_cluster.h"
+#include "kudu/rpc/rpc_controller.h"
 #include "kudu/tablet/key_value_test_schema.h"
 #include "kudu/util/monotime.h"
+#include "kudu/util/net/sockaddr.h"
+#include "kudu/util/scoped_cleanup.h"
+#include "kudu/util/status.h"
 #include "kudu/util/test_macros.h"
 #include "kudu/util/test_util.h"
 
@@ -47,6 +59,7 @@ using kudu::client::KuduTable;
 using kudu::client::KuduTableCreator;
 using kudu::cluster::ExternalMiniCluster;
 using kudu::cluster::ExternalMiniClusterOptions;
+using std::atomic;
 using std::back_inserter;
 using std::copy;
 using std::string;
@@ -74,22 +87,10 @@ class CatalogManagerTskITest : public KuduTest {
 cluster_opts_.master_rpc_ports = { 11030, 11031, 11032 };
 cluster_opts_.num_tablet_servers = num_tservers_;
 
-// Add common flags for both masters and tservers.
-const vector common_flags = {
-  Substitute("--raft_heartbeat_interval_ms=$0", hb_interval_ms_),
-};
-copy(common_flags.begin(), common_flags.end(),
-back_inserter(cluster_opts_.extra_master_flags));
-copy(common_flags.begin(), common_flags.end(),
-back_inserter(cluster_opts_.extra_tserver_flags));
-
 // Add master-only flags.
 const vector master_flags = {
   "--catalog_manager_inject_latency_prior_tsk_write_ms=1000",
   "--raft_enable_pre_election=false",
-  Substitute("--leader_failure_exp_backoff_max_delta_ms=$0",
-  hb_interval_ms_ * 4),
-  "--leader_failure_max_missed_heartbeat_periods=1.0",
   "--master_non_leader_masters_propagate_tsk",
   "--tsk_rotation_seconds=2",
 };
@@ -113,7 +114,7 @@ class CatalogManagerTskITest : public KuduTest {
 using ::kudu::client::sp::shared_ptr;
 static const char* kTableName = "test-table";
 // Using the setting for both RPC and admin operation timeout.
-const MonoDelta timeout = MonoDelta::FromSeconds(600);
+const MonoDelta timeout = MonoDelta::FromSeconds(120);

[1/2] kudu git commit: dist_test: enable sharding of a few more tests

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master 64eb9f37b -> 8e6bfa9fb


dist_test: enable sharding of a few more tests

Adds sharding for rowset_tree-test, tablet_copy-itest, and
delete_tablet-itest which are the longest-running non-sharded tests in
recent builds such as [1]

[1] http://dist-test.cloudera.org/job?job_id=jenkins-slave.1510801703.18407

Change-Id: Iff1e0ac39f1834e8ac22283e1dc0c336d328ae35
Reviewed-on: http://gerrit.cloudera.org:8080/8565
Reviewed-by: Dan Burkert 
Tested-by: Todd Lipcon 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/c4006ae0
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/c4006ae0
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/c4006ae0

Branch: refs/heads/master
Commit: c4006ae028b44e9b7f8f006d64ed7a3a3f34fe9a
Parents: 64eb9f3
Author: Todd Lipcon 
Authored: Wed Nov 15 20:58:02 2017 -0800
Committer: Todd Lipcon 
Committed: Fri Nov 17 21:26:12 2017 +

--
 build-support/dist_test.py | 5 -
 1 file changed, 4 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/c4006ae0/build-support/dist_test.py
--
diff --git a/build-support/dist_test.py b/build-support/dist_test.py
index a6f97d4..78fe5cb 100755
--- a/build-support/dist_test.py
+++ b/build-support/dist_test.py
@@ -90,10 +90,13 @@ DEPS_FOR_ALL = \
 NUM_SHARDS_BY_TEST = {
   'cfile-test': 4,
   'client-test': 8,
+  'delete_table-itest': 4,
   'delete_table-test': 8,
   'flex_partitioning-itest': 8,
   'mt-tablet-test': 4,
-  'raft_consensus-itest': 6
+  'raft_consensus-itest': 6,
+  'rowset_tree-test': 6,
+  'tablet_copy-itest': 6
 }
 
 

kudu git commit: tablet_copy_service: fix Status message with misplaced $0

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master f9d85f53b -> a4642a41e


tablet_copy_service: fix Status message with misplaced $0

Change-Id: I0443d58f32b306a0d59dd76fa901ad384ceac6e0
Reviewed-on: http://gerrit.cloudera.org:8080/8598
Reviewed-by: Alexey Serbin 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/a4642a41
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/a4642a41
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/a4642a41

Branch: refs/heads/master
Commit: a4642a41e5db7a7a887b39c1cb661a6de1dca47e
Parents: f9d85f5
Author: Todd Lipcon 
Authored: Mon Nov 20 10:50:54 2017 -0800
Committer: Alexey Serbin 
Committed: Tue Nov 21 01:07:35 2017 +

--
 src/kudu/tserver/tablet_copy_service.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/a4642a41/src/kudu/tserver/tablet_copy_service.cc
--
diff --git a/src/kudu/tserver/tablet_copy_service.cc 
b/src/kudu/tserver/tablet_copy_service.cc
index 6cfb9e7..2d10eb7 100644
--- a/src/kudu/tserver/tablet_copy_service.cc
+++ b/src/kudu/tserver/tablet_copy_service.cc
@@ -170,8 +170,8 @@ void TabletCopyServiceImpl::BeginTabletCopySession(
 
   if (!new_session && !session->IsInitialized()) {
 RPC_RETURN_NOT_OK(
-Status::ServiceUnavailable("tablet copy session for tablet $0 is 
initializing",
-   tablet_id),
+Status::ServiceUnavailable(
+Substitute("tablet copy session for tablet $0 is initializing", 
tablet_id)),
 TabletCopyErrorPB::UNKNOWN_ERROR,
 "try again later",
 context);

kudu git commit: consensus: Add gflag to enable improved re-replication

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master 8a81f4ff1 -> 6c02bea97


consensus: Add gflag to enable improved re-replication

This gflag will be used to control whether improved re-replication is
enabled on a cluster.

Change-Id: I247842af7d0172d6e6b167a29613f6b261990a01
Reviewed-on: http://gerrit.cloudera.org:8080/8626
Tested-by: Kudu Jenkins
Reviewed-by: Alexey Serbin 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/6c02bea9
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/6c02bea9
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/6c02bea9

Branch: refs/heads/master
Commit: 6c02bea97db29ef178b2f907c6d7fb561d9ac82c
Parents: 8a81f4f
Author: Mike Percy 
Authored: Tue Nov 21 14:55:36 2017 -0800
Committer: Alexey Serbin 
Committed: Wed Nov 22 19:35:30 2017 +

--
 src/kudu/consensus/raft_consensus.cc | 7 +++
 1 file changed, 7 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/6c02bea9/src/kudu/consensus/raft_consensus.cc
--
diff --git a/src/kudu/consensus/raft_consensus.cc 
b/src/kudu/consensus/raft_consensus.cc
index ce4ba54..58cc087 100644
--- a/src/kudu/consensus/raft_consensus.cc
+++ b/src/kudu/consensus/raft_consensus.cc
@@ -120,6 +120,13 @@ DEFINE_bool(raft_enable_tombstoned_voting, true,
 TAG_FLAG(raft_enable_tombstoned_voting, experimental);
 TAG_FLAG(raft_enable_tombstoned_voting, runtime);
 
+// Enable improved re-replication (KUDU-1097).
+DEFINE_bool(raft_prepare_replacement_before_eviction, false,
+"When enabled, failed replicas will only be evicted after a "
+"replacement has been prepared for them.");
+TAG_FLAG(raft_prepare_replacement_before_eviction, advanced);
+TAG_FLAG(raft_prepare_replacement_before_eviction, experimental);
+
 DECLARE_int32(memory_limit_warn_threshold_percentage);
 
 // Metrics

kudu git commit: KUDU-1097: 3-4-3 behavior for catalog manager

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master 88e39bad1 -> 87dcaf34d


KUDU-1097: 3-4-3 behavior for catalog manager

Updated the catalog_manager to behave as specified in the 3-4-3
re-replication improvement v1 design implementation plan.  This
update includes changes referred as 'patch 2' and 'patch 4'.

Change-Id: I6f0469ac641bf7a03dbef01eaa3f1b58a5bf5d27
Reviewed-on: http://gerrit.cloudera.org:8080/8619
Tested-by: Kudu Jenkins
Reviewed-by: Mike Percy 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/87dcaf34
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/87dcaf34
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/87dcaf34

Branch: refs/heads/master
Commit: 87dcaf34d3495f87164b80e03b757efc12be2fe0
Parents: 88e39ba
Author: Alexey Serbin 
Authored: Mon Nov 20 12:04:40 2017 -0800
Committer: Alexey Serbin 
Committed: Mon Nov 27 06:17:45 2017 +

--
 src/kudu/consensus/quorum_util-test.cc  | 442 ++-
 src/kudu/consensus/quorum_util.cc   | 187 
 src/kudu/consensus/quorum_util.h|  12 +
 .../raft_consensus_nonvoter-itest.cc| 153 ++-
 src/kudu/master/catalog_manager.cc  | 343 +-
 src/kudu/master/catalog_manager.h   |   2 +-
 6 files changed, 1015 insertions(+), 124 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/87dcaf34/src/kudu/consensus/quorum_util-test.cc
--
diff --git a/src/kudu/consensus/quorum_util-test.cc 
b/src/kudu/consensus/quorum_util-test.cc
index 8864977..cede1db 100644
--- a/src/kudu/consensus/quorum_util-test.cc
+++ b/src/kudu/consensus/quorum_util-test.cc
@@ -15,8 +15,12 @@
 // specific language governing permissions and limitations
 // under the License.
 
+#include 
 #include 
+#include 
+#include 
 
+#include 
 #include 
 
 #include "kudu/common/common.pb.h"
@@ -25,26 +29,68 @@
 #include "kudu/util/status.h"
 #include "kudu/util/test_macros.h"
 
+using std::string;
+using std::unique_ptr;
+using std::vector;
+
 namespace kudu {
 namespace consensus {
 
-using std::string;
+// Handy notation of membership types used by AddPeer(), etc.
+constexpr auto N = RaftPeerPB::NON_VOTER;
+constexpr auto U = RaftPeerPB::UNKNOWN_MEMBER_TYPE;
+constexpr auto V = RaftPeerPB::VOTER;
+
+typedef std::pair Attr;
 
 // Add a consensus peer into the specified configuration.
 static void AddPeer(RaftConfigPB* config,
 const string& uuid,
-RaftPeerPB::MemberType type) {
+RaftPeerPB::MemberType type,
+boost::optional overall_health = boost::none,
+vector attrs = {}) {
   RaftPeerPB* peer = config->add_peers();
   peer->set_permanent_uuid(uuid);
   peer->mutable_last_known_addr()->set_host(uuid + ".example.com");
   peer->set_member_type(type);
+  if (overall_health) {
+unique_ptr health_report(new HealthReportPB);
+switch (*overall_health) {
+  case '+':
+health_report->set_overall_health(HealthReportPB::HEALTHY);
+break;
+  case '-':
+health_report->set_overall_health(HealthReportPB::FAILED);
+break;
+  case '?':
+health_report->set_overall_health(HealthReportPB::UNKNOWN);
+break;
+  default:
+FAIL() << *overall_health << ": unexpected replica health status";
+break;
+}
+peer->set_allocated_health_report(health_report.release());
+  }
+  if (!attrs.empty()) {
+unique_ptr attrs_pb(new RaftPeerAttrsPB);
+for (const auto& attr : attrs) {
+  if (attr.first == "PROMOTE") {
+attrs_pb->set_promote(attr.second);
+  } else if (attr.first == "REPLACE") {
+attrs_pb->set_replace(attr.second);
+  } else {
+FAIL() << attr.first << ": unexpected attribute to set";
+  }
+}
+peer->set_allocated_attrs(attrs_pb.release());
+  }
 }
 
 TEST(QuorumUtilTest, TestMemberExtraction) {
   RaftConfigPB config;
-  AddPeer(&config, "A", RaftPeerPB::VOTER);
-  AddPeer(&config, "B", RaftPeerPB::VOTER);
-  AddPeer(&config, "C", RaftPeerPB::VOTER);
+  AddPeer(&config, "A", V);
+  AddPeer(&config, "B", V);
+  AddPeer(&config, "C", V);
 
   // Basic test for GetRaftConfigMember().
   RaftPeerPB* peer_pb;
@@ -65,9 +111,9 @@ TEST(QuorumUtilTest, TestMemberExtraction) {
 
 TEST(QuorumUtilTest, TestDiffConsensusStates) {
   ConsensusStatePB old_cs;
-  AddPeer(old_cs.mutable_committed_config(), &q

kudu git commit: KUDU-1097 (patch1): test for replica health reporting

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master 87dcaf34d -> 37b89924e


KUDU-1097 (patch1): test for replica health reporting

Added a test to verify that the leader tablet replica reports on the
replica health changes. The tests verifies that the health reports
are present in the Raft consensus state reported by the leader replica.
The test also verifies that the incremental tablet reports contain
appropriate information once the replica health status changes.

Change-Id: Ie62b49efebad9a123eec51dd302e375e46e0682d
Reviewed-on: http://gerrit.cloudera.org:8080/8642
Tested-by: Kudu Jenkins
Reviewed-by: Mike Percy 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/37b89924
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/37b89924
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/37b89924

Branch: refs/heads/master
Commit: 37b89924ee807f39ae30343b84e837daa7a979b3
Parents: 87dcaf3
Author: Alexey Serbin 
Authored: Wed Nov 22 22:03:59 2017 -0800
Committer: Alexey Serbin 
Committed: Mon Nov 27 06:33:39 2017 +

--
 .../ts_tablet_manager-itest.cc  | 340 ---
 1 file changed, 291 insertions(+), 49 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/37b89924/src/kudu/integration-tests/ts_tablet_manager-itest.cc
--
diff --git a/src/kudu/integration-tests/ts_tablet_manager-itest.cc 
b/src/kudu/integration-tests/ts_tablet_manager-itest.cc
index 98b0e26..ee5a957 100644
--- a/src/kudu/integration-tests/ts_tablet_manager-itest.cc
+++ b/src/kudu/integration-tests/ts_tablet_manager-itest.cc
@@ -16,10 +16,12 @@
 // under the License.
 
 #include 
+#include 
 #include 
 #include 
 #include 
 #include 
+#include 
 #include 
 
 #include 
@@ -56,37 +58,43 @@
 #include "kudu/util/test_macros.h"
 #include "kudu/util/test_util.h"
 
-DECLARE_bool(enable_leader_failure_detection);
-DECLARE_bool(catalog_manager_wait_for_new_tablets_to_elect_leader);
 DECLARE_bool(allow_unsafe_replication_factor);
+DECLARE_bool(catalog_manager_wait_for_new_tablets_to_elect_leader);
+DECLARE_bool(enable_leader_failure_detection);
+DECLARE_bool(raft_prepare_replacement_before_eviction);
 DEFINE_int32(num_election_test_loops, 3,
  "Number of random EmulateElection() loops to execute in "
  "TestReportNewLeaderOnLeaderChange");
 
-namespace kudu {
-namespace tserver {
-
-using client::KuduClient;
-using client::KuduSchema;
-using client::KuduTable;
-using client::KuduTableCreator;
-using cluster::InternalMiniCluster;
-using cluster::InternalMiniClusterOptions;
-using consensus::GetConsensusRole;
-using consensus::RaftPeerPB;
-using itest::SimpleIntKeyKuduSchema;
-using master::MasterServiceProxy;
-using master::ReportedTabletPB;
-using master::TabletReportPB;
-using rpc::Messenger;
-using rpc::MessengerBuilder;
+using kudu::client::KuduClient;
+using kudu::client::KuduSchema;
+using kudu::client::KuduTable;
+using kudu::client::KuduTableCreator;
+using kudu::cluster::InternalMiniCluster;
+using kudu::cluster::InternalMiniClusterOptions;
+using kudu::consensus::ConsensusStatePB;
+using kudu::consensus::GetConsensusRole;
+using kudu::consensus::HealthReportPB;
+using kudu::consensus::RaftConfigPB;
+using kudu::consensus::RaftConsensus;
+using kudu::consensus::RaftPeerPB;
+using kudu::itest::SimpleIntKeyKuduSchema;
+using kudu::master::MasterServiceProxy;
+using kudu::master::ReportedTabletPB;
+using kudu::master::TabletReportPB;
+using kudu::rpc::Messenger;
+using kudu::rpc::MessengerBuilder;
+using kudu::tablet::TabletReplica;
+using kudu::tserver::MiniTabletServer;
+using std::map;
 using std::shared_ptr;
 using std::string;
-using std::vector;
 using std::unique_ptr;
+using std::vector;
 using strings::Substitute;
-using tablet::TabletReplica;
-using tserver::MiniTabletServer;
+
+namespace kudu {
+namespace tserver {
 
 static const char* const kTableName = "test-table";
 
@@ -95,7 +103,7 @@ class TsTabletManagerITest : public KuduTest {
   TsTabletManagerITest()
   : schema_(SimpleIntKeyKuduSchema()) {
   }
-  virtual void SetUp() override {
+  void SetUp() override {
 KuduTest::SetUp();
 
 MessengerBuilder bld("client");
@@ -111,6 +119,21 @@ class TsTabletManagerITest : public KuduTest {
   }
 
  protected:
+  void DisableHeartbeatingToMaster();
+
+  // Populate the 'replicas' container with corresponding objects representing
+  // tablet replicas running at tablet servers in the test cluster. It's 
assumed
+  // there is at least one tablet replica per tablet server. Also, this utility
+  // method awaits up to the specified timeout for the consensus to be running
+  // before adding an elemen

kudu git commit: consensus: Rename ChangeConfigType enum constants

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master 79a255bbf -> fc143c01d


consensus: Rename ChangeConfigType enum constants

Since we now have attributes in the RaftConfigPeerPB message, it would
be more useful to allow a peer to be modified such that only its
attributes change, but not its member type. In order to provide
meaningful semantics, that means we should change the constant name for
the "modify peer" operation as well. This patch changes
CHANGE_REPLICA_TYPE to MODIFY_PEER.

While we're in there, let's also rename the other constants to more
accurately reflect the terminology we tend to use in this part of the code.

There are no functional or wire-protocol changes in this patch. The
functional changes to go along with this patch will come in a follow-up
patch.

Change-Id: I77b838f1b7ddcf176d1a5146e04fa1fddc1042fa
Reviewed-on: http://gerrit.cloudera.org:8080/8643
Tested-by: Kudu Jenkins
Reviewed-by: Mike Percy 
Reviewed-by: Alexey Serbin 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/fc143c01
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/fc143c01
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/fc143c01

Branch: refs/heads/master
Commit: fc143c01dd9091ef212f9b572752875f11524567
Parents: 79a255b
Author: Mike Percy 
Authored: Sun Nov 26 21:17:29 2017 -0800
Committer: Alexey Serbin 
Committed: Mon Nov 27 19:41:44 2017 +

--
 src/kudu/consensus/consensus.proto|  8 
 src/kudu/consensus/raft_consensus.cc  | 14 +++---
 src/kudu/integration-tests/cluster_itest_util.cc  |  6 +++---
 src/kudu/integration-tests/cluster_itest_util.h   |  4 ++--
 .../raft_consensus_nonvoter-itest.cc  |  4 ++--
 src/kudu/master/catalog_manager.cc|  8 
 src/kudu/tools/tool_action_tablet.cc  | 18 +-
 7 files changed, 31 insertions(+), 31 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/fc143c01/src/kudu/consensus/consensus.proto
--
diff --git a/src/kudu/consensus/consensus.proto 
b/src/kudu/consensus/consensus.proto
index c8abd5d..6ff4186 100644
--- a/src/kudu/consensus/consensus.proto
+++ b/src/kudu/consensus/consensus.proto
@@ -118,9 +118,9 @@ message ChangeConfigRecordPB {
 
 enum ChangeConfigType {
   UNKNOWN_CHANGE = 0;
-  ADD_SERVER = 1;
-  REMOVE_SERVER = 2;
-  CHANGE_REPLICA_TYPE = 3;
+  ADD_PEER = 1;
+  REMOVE_PEER = 2;
+  MODIFY_PEER = 3;
 }
 
 // A configuration change request for the tablet with 'tablet_id'.
@@ -139,7 +139,7 @@ message ChangeConfigRequestPB {
   optional ChangeConfigType type = 2;
 
   // The peer to add or remove.
-  // When 'type' == ADD_SERVER, both the permanent_uuid and last_known_addr
+  // When 'type' == ADD_PEER, both the permanent_uuid and last_known_addr
   // fields must be set. Otherwise, only the permanent_uuid field is required.
   optional RaftPeerPB server = 3;
 

http://git-wip-us.apache.org/repos/asf/kudu/blob/fc143c01/src/kudu/consensus/raft_consensus.cc
--
diff --git a/src/kudu/consensus/raft_consensus.cc 
b/src/kudu/consensus/raft_consensus.cc
index f1e0650..cdfb0fb 100644
--- a/src/kudu/consensus/raft_consensus.cc
+++ b/src/kudu/consensus/raft_consensus.cc
@@ -802,7 +802,7 @@ void RaftConsensus::TryRemoveFollowerTask(const string& 
uuid,
   ChangeConfigRequestPB req;
   req.set_tablet_id(options_.tablet_id);
   req.mutable_server()->set_permanent_uuid(uuid);
-  req.set_type(REMOVE_SERVER);
+  req.set_type(REMOVE_PEER);
   req.set_cas_config_opid_index(committed_config.opid_index());
   LOG(INFO) << LogPrefixThreadSafe() << "Attempting to remove follower "
 << uuid << " from the Raft config. Reason: " << reason;
@@ -849,7 +849,7 @@ void RaftConsensus::TryPromoteNonVoterTask(const 
std::string& peer_uuid,
 
   ChangeConfigRequestPB req;
   req.set_tablet_id(options_.tablet_id);
-  req.set_type(CHANGE_REPLICA_TYPE);
+  req.set_type(MODIFY_PEER);
   req.mutable_server()->set_permanent_uuid(peer_uuid);
   req.mutable_server()->set_member_type(RaftPeerPB::VOTER);
   req.mutable_server()->mutable_attrs()->set_promote(false);
@@ -1682,7 +1682,7 @@ Status RaftConsensus::ChangeConfig(const 
ChangeConfigRequestPB& req,
 new_config.clear_opid_index();
 const string& server_uuid = server.permanent_uuid();
 switch (type) {
-  case ADD_SERVER:
+  case ADD_PEER:
 // Ensure the server we are adding is not already a member of the 
configuration.
 if (IsRaftConfigMember(server_uuid, committed_config)) {

kudu git commit: [tablet_metadata] protect pre_flush_callback_ by flush_lock_

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master b29c93734 -> fa65d647a


[tablet_metadata] protect pre_flush_callback_ by flush_lock_

This patch makes the pre_flush_callback_ protected by the flush_lock_
to address the case of concurrent calls to
TabletMetadata::DeleteTabletData() and Tablet::Shutdown().  The call to
Tablet::Shutdown() was originated from TabletReplica::OnDiskSize() when
the local shared_ptr variable ended up keeping the last reference to
the object corresponding to the tablet being deleted.

The issue contributed to the flakiness at least of the following tests:
   * CreateTableStressTest.CreateAndDeleteBigTable
   * DeleteTableWhileScanInProgressParamTest

Prior to this patch, TSAN reported about read/write race for the callback
with the traces like the following:
  Read of size 8 at 0x7b5703d0 by thread T56 (mutexes: write M2161):
#0 kudu::Callback::Run() const 
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/gutil/callback.h:394:45
 (libtablet.so+0x1caa81)
#1 kudu::tablet::TabletMetadata::Flush() 
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tablet/tablet_metadata.cc:549:23
 (libtablet.so+0x1c30b8)
#2 
kudu::tablet::TabletMetadata::DeleteTabletData(kudu::tablet::TabletDataState, 
boost::optional const&) 
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tablet/tablet_metadata.cc:232:3
 (libtablet.so+0x1c411a)
#3 
kudu::tserver::TSTabletManager::DeleteTabletData(scoped_refptr
 const&, scoped_refptr const&, 
kudu::tablet::TabletDataState, boost::optional) 
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tserver/ts_tablet_manager.cc:1281:3
 (libtserver.so+0xfad26)
#4 
kudu::tserver::TSTabletManager::DeleteTablet(std::__1::basic_string, std::__1::allocator > const&, 
kudu::tablet::TabletDataState, boost::optional const&, 
kudu::tserver::TabletServerErrorPB_Code*) 
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tserver/ts_tablet_manager.cc:826:14
 (libtserver.so+0xfc365)
#5 
kudu::tserver::TabletServiceAdminImpl::DeleteTablet(kudu::tserver::DeleteTabletRequestPB
 const*, kudu::tserver::DeleteTabletResponsePB*, kudu::rpc::RpcContext*) 
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tserver/tablet_service.cc:804:41
 (libtserver.so+0xd5dfd)
...

  Previous write of size 8 at 0x7b5703d0 by thread T12 (mutexes: write 
M90770251150680224, write M623321216325058272):
#0 kudu::internal::CallbackBase::operator=(kudu::internal::CallbackBase 
const&) 
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/gutil/callback_internal.h:34:7
 (libtserver.so+0xb0596)
#1 kudu::Callback::operator=(kudu::Callback&&) 
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/gutil/callback.h:358:7
 (libtablet.so+0x115cb0)
#2 
kudu::tablet::TabletMetadata::SetPreFlushCallback(kudu::Callback) 
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tablet/tablet_metadata.h:229:74
 (libtablet.so+0x10e437)
#3 kudu::tablet::Tablet::Shutdown() 
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tablet/tablet.cc:352:14
 (libtablet.so+0xf7fdc)
#4 kudu::tablet::Tablet::~Tablet() 
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tablet/tablet.cc:248:3
 (libtablet.so+0xf7c9e)
...
#10 kudu::tablet::TabletReplica::OnDiskSize() const 
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tablet/tablet_replica.cc:742:1
 (libtablet.so+0x143af7)
...

Change-Id: I21d3195183584d1a51aeec64b049ac49994f69be
Reviewed-on: http://gerrit.cloudera.org:8080/8649
Reviewed-by: Mike Percy 
Reviewed-by: Andrew Wong 
Tested-by: Alexey Serbin 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/fa65d647
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/fa65d647
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/fa65d647

Branch: refs/heads/master
Commit: fa65d647a450c6895bac279dd5e5ad5fb6f8d228
Parents: b29c937
Author: Alexey Serbin 
Authored: Mon Nov 27 12:16:49 2017 -0800
Committer: Alexey Serbin 
Committed: Tue Nov 28 02:00:39 2017 +

--
 src/kudu/tablet/tablet_metadata.cc | 5 +
 src/kudu/tablet/tablet_metadata.h  | 5 ++---
 2 files changed, 7 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/fa65d647/src/kudu/tablet/tablet_metadata.cc
--
diff --git a/src/kudu/tablet/tablet_metadata.cc 
b/src/kudu/tablet/tablet_metadata.cc
index d848c1f..92eb8f8 100644
--- a/src/kudu/tablet/tablet_metadata.cc
+++ b/src/kudu/tablet/tablet_metadata.cc
@@ -617,6 +617,11 @@ Status TabletMetadata::ReplaceSuperBlockUnlocked(const 
TabletSuperBlockPB &pb) {
   return Status::OK();
 }
 
+void TabletMetadata::SetPreFlushCallb

kudu git commit: KUDU-1097: scenario for fallen-behind-the-WAL replica

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master 5c87c22be -> aafed6d1d


KUDU-1097: scenario for fallen-behind-the-WAL replica

Added end-to-end test scenario for the case when an existing follower
replica falls behind the WAL GC threshold.  First, the system attempts
to replace it with a non-voter where tablet copying fails.  As soon as
that detected, a new non-voter replica is added and the replacement
succeeds, so the tablet configuration eventually gets rid of failed
replicas and contains exactly 3 healthy replicas.

Change-Id: Ifcaff45c9fa3beb51d99dc53d88138da7dc120af
Reviewed-on: http://gerrit.cloudera.org:8080/8675
Reviewed-by: Mike Percy 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/aafed6d1
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/aafed6d1
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/aafed6d1

Branch: refs/heads/master
Commit: aafed6d1d749978899a78dae84dd25c028a1763a
Parents: 5c87c22
Author: Alexey Serbin 
Authored: Tue Nov 28 15:56:31 2017 -0800
Committer: Alexey Serbin 
Committed: Wed Nov 29 10:28:17 2017 +

--
 src/kudu/consensus/quorum_util-test.cc  |  18 +++
 src/kudu/consensus/quorum_util.cc   |  13 +-
 .../raft_consensus-itest-base.cc|  12 +-
 .../raft_consensus-itest-base.h |   4 +-
 .../integration-tests/raft_consensus-itest.cc   |   9 +-
 .../raft_consensus_election-itest.cc|   3 +-
 .../raft_consensus_nonvoter-itest.cc| 147 +++
 src/kudu/master/catalog_manager.cc  |   4 +
 src/kudu/tserver/tablet_copy_service.cc |   7 +-
 9 files changed, 202 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/aafed6d1/src/kudu/consensus/quorum_util-test.cc
--
diff --git a/src/kudu/consensus/quorum_util-test.cc 
b/src/kudu/consensus/quorum_util-test.cc
index cede1db..461ae7a 100644
--- a/src/kudu/consensus/quorum_util-test.cc
+++ b/src/kudu/consensus/quorum_util-test.cc
@@ -382,6 +382,24 @@ TEST(QuorumUtilTest, IsUnderReplicated) {
 EXPECT_FALSE(IsUnderReplicated(config, 2));
 EXPECT_TRUE(IsUnderReplicated(config, 3));
   }
+  {
+RaftConfigPB config;
+AddPeer(&config, "A", V, '-');
+AddPeer(&config, "B", V, '+');
+AddPeer(&config, "C", V, '+');
+AddPeer(&config, "D", N, '-', {{"PROMOTE", true}});
+AddPeer(&config, "E", N, '+', {{"PROMOTE", true}});
+EXPECT_FALSE(IsUnderReplicated(config, 3));
+  }
+  {
+RaftConfigPB config;
+AddPeer(&config, "A", V, '-');
+AddPeer(&config, "B", V, '+');
+AddPeer(&config, "C", V, '+');
+AddPeer(&config, "D", N, '-', {{"PROMOTE", true}});
+AddPeer(&config, "E", N, '+', {{"PROMOTE", false}});
+EXPECT_TRUE(IsUnderReplicated(config, 3));
+  }
 }
 
 // Verify logic of the kudu::consensus::CanEvictReplica(), anticipating

http://git-wip-us.apache.org/repos/asf/kudu/blob/aafed6d1/src/kudu/consensus/quorum_util.cc
--
diff --git a/src/kudu/consensus/quorum_util.cc 
b/src/kudu/consensus/quorum_util.cc
index e002070..8632e3c 100644
--- a/src/kudu/consensus/quorum_util.cc
+++ b/src/kudu/consensus/quorum_util.cc
@@ -34,6 +34,7 @@
 
 using google::protobuf::RepeatedPtrField;
 using kudu::pb_util::SecureShortDebugString;
+using kudu::pb_util::SecureDebugString;
 using std::map;
 using std::pair;
 using std::set;
@@ -408,6 +409,7 @@ bool IsUnderReplicated(const RaftConfigPB& config, int 
replication_factor) {
   // While working with the optional fields related to per-replica health 
status
   // and attributes, has_a_field()-like methods are not called because of
   // the appropriate default values of those fields.
+  VLOG(2) << "config to evaluate: " << SecureDebugString(config);
   for (const RaftPeerPB& peer : config.peers()) {
 switch (peer.member_type()) {
   case RaftPeerPB::VOTER:
@@ -428,8 +430,11 @@ bool IsUnderReplicated(const RaftConfigPB& config, int 
replication_factor) {
 break;
 }
   }
-  return replication_factor > (num_voters_total - num_voters_need_replacement) 
+
-  num_non_voters_to_promote;
+  const bool is_under_replicated = replication_factor >
+  (num_voters_total - num_voters_need_replacement) + 
num_non_voters_to_promote;
+  VLOG(2) << "decision: the config is" << (is_under_replicated

kudu git commit: KUDU-2230: the leader is always a viable voter

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master 3f4253753 -> dc497fec2


KUDU-2230: the leader is always a viable voter

While iterating over peers in PeerMessageQueue::SafeToEvictUnlocked(),
assume that the leader is always a viable voter.  Prior to this patch,
the leader itself was not counted as a viable voter since the leader
does not update its own last_communication_time.

Both test scenarios TestDontEvictIfRemainingConfigIsUnstable_NodesDown
and TestDontEvictIfRemainingConfigIsUnstable_NodesStopped of the
TabletReplacementITest test have been modified to cover the fixed issue.
I verified that commenting out the fix in consensus_queue.cc
makes them fail.

Change-Id: Id200ec8e562e2b21c8eef09e7b38a8d85d23239c
Reviewed-on: http://gerrit.cloudera.org:8080/8709
Reviewed-by: Mike Percy 
Reviewed-by: Andrew Wong 
Tested-by: Alexey Serbin 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/dc497fec
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/dc497fec
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/dc497fec

Branch: refs/heads/master
Commit: dc497fec2cb1cabec17eb250ca7d8a7e3d6a9e8f
Parents: 3f42537
Author: Alexey Serbin 
Authored: Thu Nov 30 19:44:22 2017 -0800
Committer: Alexey Serbin 
Committed: Fri Dec 1 22:47:21 2017 +

--
 src/kudu/consensus/consensus_queue.cc   | 37 +
 .../tablet_replacement-itest.cc | 82 +++-
 2 files changed, 84 insertions(+), 35 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/dc497fec/src/kudu/consensus/consensus_queue.cc
--
diff --git a/src/kudu/consensus/consensus_queue.cc 
b/src/kudu/consensus/consensus_queue.cc
index 008b488..6d09b33 100644
--- a/src/kudu/consensus/consensus_queue.cc
+++ b/src/kudu/consensus/consensus_queue.cc
@@ -401,6 +401,7 @@ OpId PeerMessageQueue::GetNextOpId() const {
 
 bool PeerMessageQueue::SafeToEvictUnlocked(const string& evict_uuid) const {
   DCHECK(queue_lock_.is_locked());
+  DCHECK_EQ(LEADER, queue_state_.mode);
   auto now = MonoTime::Now();
 
   int remaining_voters = 0;
@@ -418,24 +419,26 @@ bool PeerMessageQueue::SafeToEvictUnlocked(const string& 
evict_uuid) const {
 remaining_voters++;
 
 bool viable = true;
+// Being alive, the local peer itself (the leader) is always a viable
+// voter: the criteria below apply only to non-local peers.
+if (uuid != local_peer_pb_.permanent_uuid()) {
+  // Only consider a peer to be a viable voter if...
+  // ...its last exchange was successful
+  viable &= peer->last_exchange_status == PeerStatus::OK;
+
+  // ...the peer is up to date with the latest majority.
+  //
+  //This indicates that it's actively participating in majorities and 
likely to
+  //replicate a config change immediately when we propose it.
+  viable &= peer->last_received.index() >= 
queue_state_.majority_replicated_index;
 
-// Only consider a peer to be a viable voter if...
-// ...its last exchange was successful
-viable &= peer->last_exchange_status == PeerStatus::OK;
-
-// ...the peer is up to date with the latest majority.
-//
-//This indicates that it's actively participating in majorities and 
likely to
-//replicate a config change immediately when we propose it.
-viable &= peer->last_received.index() >= 
queue_state_.majority_replicated_index;
-
-// ...we have communicated successfully with it recently.
-//
-//This handles the case where the tablet has had no recent writes and 
therefore
-//even a replica that is down would have participated in the latest 
majority.
-auto unreachable_time = now - peer->last_communication_time;
-viable &= unreachable_time.ToMilliseconds() < 
FLAGS_consensus_rpc_timeout_ms;
-
+  // ...we have communicated successfully with it recently.
+  //
+  //This handles the case where the tablet has had no recent writes 
and therefore
+  //even a replica that is down would have participated in the latest 
majority.
+  auto unreachable_time = now - peer->last_communication_time;
+  viable &= unreachable_time.ToMilliseconds() < 
FLAGS_consensus_rpc_timeout_ms;
+}
 if (viable) {
   remaining_viable_voters++;
 }

http://git-wip-us.apache.org/repos/asf/kudu/blob/dc497fec/src/kudu/integration-tests/tablet_replacement-itest.cc
--
diff --git a/src/kudu/integration-tests/tablet_replacement-itest.cc 
b/src/kudu/integration-tests/tablet_replacement-itest.cc
index eb724f4..c0665b4 100644
--- a/src/kudu/integration-tests/tablet_replacement-ite

kudu git commit: KUDU-2229. consensus: Leader should not start FD

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master dc497fec2 -> 1c9e424c2


KUDU-2229. consensus: Leader should not start FD

This patch fixes a log spam issue caused by commit
28a671365d6d38da966481daf937b3776e3d4852 that erroneously enables the
failure detector on the leader when a configuration takes place. That
bug had no consensus safety implications, but it did generate a large
volume of confusing log messages in a neverending loop.

This patch also includes some minor updates to InternalMiniCluster for
convenience purposes.

Change-Id: Ie2ec9c5499e8e4c1659333bd53dd2d7f6dae81f5
Reviewed-on: http://gerrit.cloudera.org:8080/8711
Reviewed-by: Alexey Serbin 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/1c9e424c
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/1c9e424c
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/1c9e424c

Branch: refs/heads/master
Commit: 1c9e424c251d6b47fa8a161a94358634fbd13549
Parents: dc497fe
Author: Mike Percy 
Authored: Thu Nov 30 19:40:23 2017 -0800
Committer: Alexey Serbin 
Committed: Fri Dec 1 23:15:12 2017 +

--
 src/kudu/consensus/raft_consensus.cc|  21 ++-
 src/kudu/consensus/raft_consensus.h |  19 ++-
 src/kudu/integration-tests/CMakeLists.txt   |   3 +-
 ...raft_consensus_failure_detector-imc-itest.cc | 144 +++
 src/kudu/mini-cluster/external_mini_cluster.h   |   4 +-
 src/kudu/mini-cluster/internal_mini_cluster.cc  |  18 +++
 src/kudu/mini-cluster/internal_mini_cluster.h   |   5 +
 src/kudu/mini-cluster/mini_cluster.h|   4 +
 src/kudu/rpc/periodic.cc|   5 +
 src/kudu/rpc/periodic.h |   5 +-
 10 files changed, 206 insertions(+), 22 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/1c9e424c/src/kudu/consensus/raft_consensus.cc
--
diff --git a/src/kudu/consensus/raft_consensus.cc 
b/src/kudu/consensus/raft_consensus.cc
index 291c146..cb3118e 100644
--- a/src/kudu/consensus/raft_consensus.cc
+++ b/src/kudu/consensus/raft_consensus.cc
@@ -599,7 +599,7 @@ Status 
RaftConsensus::BecomeReplicaUnlocked(boost::optional fd_delta)
 
   // Enable/disable leader failure detection if becoming VOTER/NON_VOTER 
replica
   // correspondingly.
-  ToggleFailureDetector(std::move(fd_delta));
+  UpdateFailureDetectorState(std::move(fd_delta));
 
   // Now that we're a replica, we can allow voting for other nodes.
   withhold_votes_until_ = MonoTime::Min();
@@ -2610,7 +2610,7 @@ void 
RaftConsensus::CompleteConfigChangeRoundUnlocked(ConsensusRound* round, con
 
   // Disable leader failure detection if transitioning from VOTER to
   // NON_VOTER and vice versa.
-  ToggleFailureDetector();
+  UpdateFailureDetectorState();
 } else {
   LOG_WITH_PREFIX_UNLOCKED(INFO)
   << "Skipping abort of non-pending config change with OpId "
@@ -2670,14 +2670,15 @@ void RaftConsensus::DisableFailureDetector() {
   }
 }
 
-void RaftConsensus::ToggleFailureDetector(boost::optional delta) {
-  if (IsRaftConfigVoter(peer_uuid(), cmeta_->ActiveConfig())) {
-// A non-leader voter replica should run failure detector.
+void RaftConsensus::UpdateFailureDetectorState(boost::optional 
delta) {
+  const auto& uuid = peer_uuid();
+  if (uuid != cmeta_->leader_uuid() &&
+  cmeta_->IsVoterInConfig(uuid, ACTIVE_CONFIG)) {
+// A voter that is not the leader should run the failure detector.
 EnableFailureDetector(std::move(delta));
   } else {
-// A non-voter should not start leader elections. The leader failure
-// detector should be re-enabled once the non-voter replica is promoted
-// to voter replica.
+// Otherwise, the local peer should not start leader elections
+// (e.g. if it is the leader, a non-voter, a non-participant, etc).
 DisableFailureDetector();
   }
 }
@@ -2805,9 +2806,7 @@ Status RaftConsensus::SetPendingConfigUnlocked(const 
RaftConfigPB& new_config) {
   }
   cmeta_->set_pending_config(new_config);
 
-  // Disable leader failure detection if transitioning from VOTER to NON_VOTER
-  // and vice versa.
-  ToggleFailureDetector();
+  UpdateFailureDetectorState();
 
   return Status::OK();
 }

http://git-wip-us.apache.org/repos/asf/kudu/blob/1c9e424c/src/kudu/consensus/raft_consensus.h
--
diff --git a/src/kudu/consensus/raft_consensus.h 
b/src/kudu/consensus/raft_consensus.h
index 2c438d1..5ace791 100644
--- a/src/kudu/consensus/raft_consensus.h
+++ b/src/kudu/consensus/raft_consensus.h
@@ -158,6 +158,11 @@ class RaftConsensus : public 
std::enable_shared_from_this,
   // Retur

kudu git commit: [client] expose non-voter replicas for kudu CLI tool

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master 1c9e424c2 -> d9754746b


[client] expose non-voter replicas for kudu CLI tool

Added private interface to expose non-voter replica in the kudu CLI
tools.  Also, use that functionality to list table's replicas.

Updated output of ksck to include information on non-voters replicas:
  * renamed column 'Voters' into 'Replicas'
  * a non-voter is displayed with '~' extra suffix, e.g. in the example
below two non-voter replicas, B and C, are in the config along with
leader replica A:

 Config source |   Replicas   | Current term | Config index | Committed?
---+--+--+--+---
 master| A*  B~  C~   |  |  | Yes
 A | A*  B~  C~   | 4| 5| Yes
 B | A*  B~  C~   | 4| 5| Yes
 C | A*  B~  C~   | 4| 5| Yes

I also updated the format of output to make it more readable.  An
example of new output for 'kudu table list --list_tablets':

loadgen_auto_1e5f57a73dd34fa7bde7465219bd1866
  T 69e22ffecb5b459db8a7fbf2f751a76f
L 5acf108a9d364178b1aa672e52214ab8 127.0.0.1:9872
N f2e726948d3143e184097cd953dd2230 127.0.0.1:9878
N 3ae7fdf107cf424dbf9c5e33c33eafd0 127.0.0.1:9876

  T 662bae40e4ec46588e90670bec16b2b5
L 5acf108a9d364178b1aa672e52214ab8 127.0.0.1:9872

  T 5ba95da9b2f9455db9d690bdd35ae5cb
L df8296cf97df42eaacd78e794c028c79 127.0.0.1:9874

  T 528ae88cce1e4415a0d075dead5c983c
L 20e66417713446039c22987fec13b800 127.0.0.1:9870

  T 157ea108357e469e8fee001aec521d7e
L 5acf108a9d364178b1aa672e52214ab8 127.0.0.1:9872

  T 9c6a4d493fff4bda9bfe8f158cb08e69
L 5acf108a9d364178b1aa672e52214ab8 127.0.0.1:9872

  T 36d03adab6ad494ea4e769ef92ef5577
L df8296cf97df42eaacd78e794c028c79 127.0.0.1:9874

  T 7ce6d42984734614bd627d872e99c833
L 20e66417713446039c22987fec13b800 127.0.0.1:9870

Change-Id: I19317fdf5a2d5c8bb5f37b27bb83067a4df4ea20
Reviewed-on: http://gerrit.cloudera.org:8080/8586
Reviewed-by: Mike Percy 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/d9754746
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/d9754746
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/d9754746

Branch: refs/heads/master
Commit: d9754746b64098fdd4fe0e35d9344ba62d8c7a25
Parents: 1c9e424
Author: Alexey Serbin 
Authored: Thu Nov 16 18:03:51 2017 -0800
Committer: Alexey Serbin 
Committed: Fri Dec 1 23:28:22 2017 +

--
 src/kudu/client/CMakeLists.txt  |   1 +
 src/kudu/client/client.cc   |  10 +-
 src/kudu/client/client.h|   4 +
 src/kudu/client/client_builder-internal.cc  |   6 +-
 src/kudu/client/client_builder-internal.h   |   2 +
 src/kudu/client/meta_cache.cc   |  41 --
 src/kudu/client/meta_cache.h|   6 +-
 src/kudu/client/replica-internal.cc |   3 +-
 src/kudu/client/replica-internal.h  |   3 +-
 src/kudu/client/replica_controller-internal.cc  |  40 ++
 src/kudu/client/replica_controller-internal.h   |  57 
 src/kudu/client/scan_token-internal.cc  |   3 +-
 .../raft_consensus_nonvoter-itest.cc|  86 
 src/kudu/tools/ksck-test.cc |  30 ++--
 src/kudu/tools/ksck.cc  | 140 ---
 src/kudu/tools/ksck.h   |  51 ---
 src/kudu/tools/ksck_remote.cc   |   9 +-
 src/kudu/tools/kudu-admin-test.cc   |   2 +-
 src/kudu/tools/tool_action_table.cc |  83 ++-
 19 files changed, 428 insertions(+), 149 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/d9754746/src/kudu/client/CMakeLists.txt
--
diff --git a/src/kudu/client/CMakeLists.txt b/src/kudu/client/CMakeLists.txt
index a78d300..92a1c7e 100644
--- a/src/kudu/client/CMakeLists.txt
+++ b/src/kudu/client/CMakeLists.txt
@@ -44,6 +44,7 @@ set(CLIENT_SRCS
   scan_token-internal.cc
   scanner-internal.cc
   replica-internal.cc
+  replica_controller-internal.cc
   resource_metrics.cc
   schema.cc
   session-internal.cc

http://git-wip-us.apache.org/repos/asf/kudu/blob/d9754746/src/kudu/client/client.cc
--
diff --git a/src/kudu/client/client.cc b/src/kudu/client/client.cc
index aed362d..93431b7 100644
--- a/src/kudu/client/client.cc
+++ b/src/kudu/client/client.cc
@@ -324,7 +324,7 @@ Status KuduClientBuilder::Build(shared_ptr* 
client) {
 RETURN_NOT_OK(ImportAuthnCred

kudu git commit: KUDU-1097: more robust criteria for replica eviction

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master ff535b521 -> 2a6de8ea4


KUDU-1097: more robust criteria for replica eviction

Updated the replica eviction criteria to address scenarios where newly
added replicas fail (before or after promotion) one after another
prior to eviction of the original failed voter replica.  With this
patch, the replacement process should not end up with placing failed
replicas at every available tablet server, deadlocking the replacement
process.  Added corresponding unit tests to cover the updated behavior
of the consensus::CanEvictReplica() utility function.

Updated the replica eviction criteria for more robust handling of the
'replace' attribute.  In this context, the case when the leader replica
itself is marked with the 'replace' attribute is now handled properly.
Added unit tests to cover appropriate cases.

Also, enabled the AdminCliTest.TestMoveTablet_KUDU_1097 test scenario
since now it should not be flaky anymore.

As an additional touch, updated the FailedTabletCopy scenario of the
RaftConsensusNonVoterITest to provide more comprehensive coverage.

Change-Id: I2062c2963fface43062b3154bf5ffa61b4fa684f
Reviewed-on: http://gerrit.cloudera.org:8080/8679
Tested-by: Alexey Serbin 
Reviewed-by: Mike Percy 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/2a6de8ea
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/2a6de8ea
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/2a6de8ea

Branch: refs/heads/master
Commit: 2a6de8ea46ae699bffb0435119fc62a15200eab7
Parents: ff535b5
Author: Alexey Serbin 
Authored: Wed Nov 29 02:15:06 2017 -0800
Committer: Alexey Serbin 
Committed: Tue Dec 12 19:20:25 2017 +

--
 src/kudu/consensus/quorum_util-test.cc  | 682 +--
 src/kudu/consensus/quorum_util.cc   | 183 -
 .../raft_consensus_nonvoter-itest.cc|  66 +-
 src/kudu/master/catalog_manager.cc  |  10 +-
 src/kudu/tools/kudu-admin-test.cc   |   2 +-
 5 files changed, 847 insertions(+), 96 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/2a6de8ea/src/kudu/consensus/quorum_util-test.cc
--
diff --git a/src/kudu/consensus/quorum_util-test.cc 
b/src/kudu/consensus/quorum_util-test.cc
index cd038fc..3f525c0 100644
--- a/src/kudu/consensus/quorum_util-test.cc
+++ b/src/kudu/consensus/quorum_util-test.cc
@@ -26,23 +26,46 @@
 #include "kudu/common/common.pb.h"
 #include "kudu/consensus/metadata.pb.h"
 #include "kudu/consensus/quorum_util.h"
+#include "kudu/gutil/strings/substitute.h"
 #include "kudu/util/status.h"
 #include "kudu/util/test_macros.h"
 
 using std::string;
 using std::unique_ptr;
 using std::vector;
+using strings::Substitute;
 
 namespace kudu {
 namespace consensus {
 
 // Handy notation of membership types used by AddPeer(), etc.
-constexpr auto N = RaftPeerPB::NON_VOTER;
-constexpr auto U = RaftPeerPB::UNKNOWN_MEMBER_TYPE;
-constexpr auto V = RaftPeerPB::VOTER;
+constexpr auto N = RaftPeerPB::NON_VOTER;   // 
NOLINT(readability-identifier-naming)
+constexpr auto U = RaftPeerPB::UNKNOWN_MEMBER_TYPE; // 
NOLINT(readability-identifier-naming)
+constexpr auto V = RaftPeerPB::VOTER;   // 
NOLINT(readability-identifier-naming)
+
+// The various possible health statuses.
+constexpr auto kHealthStatuses = { '?', '-', '+' };
 
 typedef std::pair Attr;
 
+static void SetOverallHealth(HealthReportPB* health_report,
+ char overall_health) {
+  switch (overall_health) {
+case '+':
+  health_report->set_overall_health(HealthReportPB::HEALTHY);
+  break;
+case '-':
+  health_report->set_overall_health(HealthReportPB::FAILED);
+  break;
+case '?':
+  health_report->set_overall_health(HealthReportPB::UNKNOWN);
+  break;
+default:
+  FAIL() << overall_health << ": unexpected replica health status";
+  break;
+  }
+}
+
 // Add a consensus peer into the specified configuration.
 static void AddPeer(RaftConfigPB* config,
 const string& uuid,
@@ -55,20 +78,7 @@ static void AddPeer(RaftConfigPB* config,
   peer->set_member_type(type);
   if (overall_health) {
 unique_ptr health_report(new HealthReportPB);
-switch (*overall_health) {
-  case '+':
-health_report->set_overall_health(HealthReportPB::HEALTHY);
-break;
-  case '-':
-health_report->set_overall_health(HealthReportPB::FAILED);
-break;
-  case '?':
-health_report->set_overall_he

kudu git commit: [itests] fix build

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master bb2620299 -> dbc84a2d7


[itests] fix build

This is a follow-up for 7c8fc625e72ec2bf6ef614df2ec25346e1a0f7b7, which
became a bit outdated after bb2620299c45b3db90590236298a497a619c0bda was
merged.

This patch does not contain any functional changes.

Change-Id: I466d941f8634e37b68dd85f29d39f1eb02b26173
Reviewed-on: http://gerrit.cloudera.org:8080/8844
Reviewed-by: Mike Percy 
Tested-by: Alexey Serbin 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/dbc84a2d
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/dbc84a2d
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/dbc84a2d

Branch: refs/heads/master
Commit: dbc84a2d7469e2de97fe9e35f5b5443e94f2a790
Parents: bb26202
Author: Alexey Serbin 
Authored: Thu Dec 14 10:55:14 2017 -0800
Committer: Alexey Serbin 
Committed: Thu Dec 14 19:01:37 2017 +

--
 .../integration-tests/raft_consensus_nonvoter-itest.cc   | 11 +--
 1 file changed, 5 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/dbc84a2d/src/kudu/integration-tests/raft_consensus_nonvoter-itest.cc
--
diff --git a/src/kudu/integration-tests/raft_consensus_nonvoter-itest.cc 
b/src/kudu/integration-tests/raft_consensus_nonvoter-itest.cc
index 11fbbbe..022fa4f 100644
--- a/src/kudu/integration-tests/raft_consensus_nonvoter-itest.cc
+++ b/src/kudu/integration-tests/raft_consensus_nonvoter-itest.cc
@@ -1585,14 +1585,13 @@ TEST_F(RaftConsensusNonVoterITest, 
RestartClusterWithNonVoter) {
   }
   workload.StopAndJoin();
 
-  vector servers_without_replica;
-  NO_FATALS(GetServersWithoutReplica(&servers_without_replica));
-  ASSERT_EQ(4, servers_without_replica.size());
-
-  ExternalTabletServer* ts_with_replica = GetServerWithReplica();
+  const auto server_uuids_with_replica = GetServersWithReplica(tablet_id_);
+  ASSERT_FALSE(server_uuids_with_replica.empty());
+  const string& failed_replica_uuid = server_uuids_with_replica.front();
+  ExternalTabletServer* ts_with_replica =
+  cluster_->tablet_server_by_uuid(failed_replica_uuid);
   ASSERT_NE(nullptr, ts_with_replica);
   ts_with_replica->Shutdown();
-  const string& failed_replica_uuid = ts_with_replica->uuid();
 
   bool has_leader = false;
   TabletLocationsPB tablet_locations;

kudu git commit: KUDU-2236 pt 2: fix flake in TestKuduClient

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master 5356334ef -> a89c8f395


KUDU-2236 pt 2: fix flake in TestKuduClient

The original patch d78b2727d1246069b2006ee652c3ff9a2005601c was too
loose in what exceptions it would allow. A more elegant way to uphold
the original intention of the test (i.e. to avoid log spew caused by
an unexpected disconnection) is to verify that there are no messages
complaining about unexpectedly-disconnected connections.

Change-Id: I7aafdc0eca00e743048ecc099dcb3241ce7ac8ad
Reviewed-on: http://gerrit.cloudera.org:8080/8908
Tested-by: Kudu Jenkins
Reviewed-by: Alexey Serbin 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/a89c8f39
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/a89c8f39
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/a89c8f39

Branch: refs/heads/master
Commit: a89c8f39544337c6ff36cdd28edec4fadac8427c
Parents: 5356334
Author: Andrew Wong 
Authored: Thu Dec 21 16:55:52 2017 -0800
Committer: Alexey Serbin 
Committed: Sat Dec 23 02:00:02 2017 +

--
 .../src/test/java/org/apache/kudu/client/TestKuduClient.java | 8 +++-
 1 file changed, 3 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/a89c8f39/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java
--
diff --git 
a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java 
b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java
index 2e0ff7a..1081cc6 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java
@@ -836,12 +836,10 @@ public class TestKuduClient extends BaseKuduTest {
 localClient.exportAuthenticationCredentials();
   }
 }
-// Allow "connection disconnected" exceptions, which can come from threads
-// that don't get synchronously joined by client.close().
+// Ensure there is no log spew due to an unexpected lost connection.
 String exception_text = cla.getAppendedText();
-assertTrue("Unexpected exception:\n" + exception_text,
-   !exception_text.contains("Exception") ||
-   exception_text.contains("connection disconnected"));
+assertFalse("Unexpected exception:\n" + exception_text,
+   exception_text.contains("lost connection to peer"));
   }
 
   @Test(timeout = 10)

kudu git commit: [rpc] micro clean-up on RpcContext

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master ae62b8697 -> 07e4cbd4d


[rpc] micro clean-up on RpcContext

Reuse the code from RpcContext::RespondRpcFailure() in
RpcContext::RespondFailure() implementation.

This changelist does not contain any functional changes.

Change-Id: I0ef93200a8491e9cdecd5ebf3f146d1f986977ef
Reviewed-on: http://gerrit.cloudera.org:8080/8919
Tested-by: Kudu Jenkins
Reviewed-by: Will Berkeley 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/07e4cbd4
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/07e4cbd4
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/07e4cbd4

Branch: refs/heads/master
Commit: 07e4cbd4d1edb0d1ce9bd0e9c2dac7b8ce321539
Parents: ae62b86
Author: Alexey Serbin 
Authored: Fri Dec 22 17:55:29 2017 -0800
Committer: Alexey Serbin 
Committed: Thu Dec 28 00:03:21 2017 +

--
 src/kudu/rpc/rpc_context.cc | 13 +
 1 file changed, 1 insertion(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/07e4cbd4/src/kudu/rpc/rpc_context.cc
--
diff --git a/src/kudu/rpc/rpc_context.cc b/src/kudu/rpc/rpc_context.cc
index 9c3d154..7af981b 100644
--- a/src/kudu/rpc/rpc_context.cc
+++ b/src/kudu/rpc/rpc_context.cc
@@ -96,18 +96,7 @@ void RpcContext::RespondNoCache() {
 }
 
 void RpcContext::RespondFailure(const Status &status) {
-  if (AreResultsTracked()) {
-result_tracker_->FailAndRespond(call_->header().request_id(),
-ErrorStatusPB::ERROR_APPLICATION, status);
-  } else {
-VLOG(4) << call_->remote_method().service_name() << ": Sending RPC failure 
response for "
-<< call_->ToString() << ": " << status.ToString();
-TRACE_EVENT_ASYNC_END2("rpc_call", "RPC", this,
-   "status", status.ToString(),
-   "trace", trace()->DumpToString());
-call_->RespondFailure(ErrorStatusPB::ERROR_APPLICATION, status);
-delete this;
-  }
+  return RespondRpcFailure(ErrorStatusPB::ERROR_APPLICATION, status);
 }
 
 void RpcContext::RespondRpcFailure(ErrorStatusPB_RpcErrorCodePB err, const 
Status& status) {

kudu git commit: [thirdparty] fix lib link in PREFIX_DIR

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master cc236542e -> 9bdeddf33


[thirdparty] fix lib link in PREFIX_DIR

Avoid creating the unintended 'lib' soft link in already soft-linked
directory.

Change-Id: I9ff2d57414eb048a97b5494bebab5826a9d83fdf
Reviewed-on: http://gerrit.cloudera.org:8080/8943
Reviewed-by: Todd Lipcon 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/9bdeddf3
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/9bdeddf3
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/9bdeddf3

Branch: refs/heads/master
Commit: 9bdeddf3373369e1406972861682f2a61889040d
Parents: cc23654
Author: Alexey Serbin 
Authored: Thu Jan 4 11:50:43 2018 -0800
Committer: Alexey Serbin 
Committed: Thu Jan 4 20:33:20 2018 +

--
 thirdparty/build-thirdparty.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/9bdeddf3/thirdparty/build-thirdparty.sh
--
diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh
index e239984..c570f9e 100755
--- a/thirdparty/build-thirdparty.sh
+++ b/thirdparty/build-thirdparty.sh
@@ -125,7 +125,7 @@ for PREFIX_DIR in $PREFIX_COMMON $PREFIX_DEPS 
$PREFIX_DEPS_TSAN; do
 # On some systems, autotools installs libraries to lib64 rather than lib.  
Fix
 # this by setting up lib64 as a symlink to lib.  We have to do this step 
first
 # to handle cases where one third-party library depends on another.
-ln -sf "$PREFIX_DIR/lib" "$PREFIX_DIR/lib64"
+ln -nsf "$PREFIX_DIR/lib" "$PREFIX_DIR/lib64"
   fi
 done
 

kudu git commit: [cmake_modules] one more OpenJDK location

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master 9bdeddf33 -> bddbb34e2


[cmake_modules] one more OpenJDK location

Added the location of java-7-openjdk at Ubuntu 14.04 LTS (amd64).

Change-Id: Ia595e6ce03708066421b59ea074ac9f3acd276c4
Reviewed-on: http://gerrit.cloudera.org:8080/8952
Reviewed-by: Dan Burkert 
Tested-by: Alexey Serbin 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/bddbb34e
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/bddbb34e
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/bddbb34e

Branch: refs/heads/master
Commit: bddbb34e2e28fd4affae3d4d37d6509922ffb4fb
Parents: 9bdeddf
Author: Alexey Serbin 
Authored: Fri Jan 5 12:32:23 2018 -0800
Committer: Alexey Serbin 
Committed: Fri Jan 5 21:34:23 2018 +

--
 cmake_modules/FindJavaHome.cmake | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/bddbb34e/cmake_modules/FindJavaHome.cmake
--
diff --git a/cmake_modules/FindJavaHome.cmake b/cmake_modules/FindJavaHome.cmake
index 4cce3bf..0463cd8 100644
--- a/cmake_modules/FindJavaHome.cmake
+++ b/cmake_modules/FindJavaHome.cmake
@@ -49,6 +49,7 @@ set(JAVA_HOME_CANDIDATES
 
 # OpenJDK 7 Candidates
 /usr/lib/jvm/java-1.7.0-openjdk
+/usr/lib/jvm/java-7-openjdk-amd64
 /usr/lib/jvm/java-7-openjdk
 
 # Misc. Candidates

kudu git commit: [tests] update for 3-4-3 replica management scheme (3/3)

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master 36995e260 -> 391e3255d


[tests] update for 3-4-3 replica management scheme (3/3)

Updated scenarios of the following tests to run with 3-4-3 replica
management scheme:
  * DiskFailureITest
  * TabletCopyFailureITest
  * TabletCopyITest
  * TsTabletManagerITest

This patch is a part of the work done in the context of KUDU-1097.

Change-Id: Id68e29fcee4e6dbd166ada9f18dac2d8fe351a3b
Reviewed-on: http://gerrit.cloudera.org:8080/8858
Reviewed-by: Mike Percy 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/391e3255
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/391e3255
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/391e3255

Branch: refs/heads/master
Commit: 391e3255d6f0ae59f375e5532cea985384f636f8
Parents: 36995e2
Author: Alexey Serbin 
Authored: Fri Dec 15 15:09:24 2017 -0800
Committer: Alexey Serbin 
Committed: Sat Jan 6 03:22:24 2018 +

--
 .../integration-tests/disk_failure-itest.cc | 61 +
 src/kudu/integration-tests/tablet_copy-itest.cc | 18 +++-
 .../ts_tablet_manager-itest.cc  | 90 +---
 3 files changed, 120 insertions(+), 49 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/391e3255/src/kudu/integration-tests/disk_failure-itest.cc
--
diff --git a/src/kudu/integration-tests/disk_failure-itest.cc 
b/src/kudu/integration-tests/disk_failure-itest.cc
index 3657846..8d79884 100644
--- a/src/kudu/integration-tests/disk_failure-itest.cc
+++ b/src/kudu/integration-tests/disk_failure-itest.cc
@@ -18,6 +18,7 @@
 #include 
 #include 
 #include 
+#include 
 #include 
 
 #include 
@@ -29,6 +30,7 @@
 #include "kudu/integration-tests/external_mini_cluster-itest-base.h"
 #include "kudu/integration-tests/test_workload.h"
 #include "kudu/mini-cluster/external_mini_cluster.h"
+#include "kudu/tserver/tserver.pb.h"
 #include "kudu/util/metrics.h"
 #include "kudu/util/monotime.h"
 #include "kudu/util/path_util.h"
@@ -38,19 +40,19 @@
 
 METRIC_DECLARE_gauge_uint64(data_dirs_failed);
 
-namespace kudu {
-
-using cluster::ExternalMiniClusterOptions;
-using cluster::ExternalTabletServer;
-using fs::BlockManager;
+using kudu::cluster::ExternalMiniClusterOptions;
+using kudu::cluster::ExternalTabletServer;
+using kudu::fs::BlockManager;
 using std::string;
 using std::vector;
 using strings::Substitute;
 
+namespace kudu {
+
 const MonoDelta kAgreementTimeout = MonoDelta::FromSeconds(30);
 
 class DiskFailureITest : public ExternalMiniClusterITestBase,
- public ::testing::WithParamInterface {
+ public 
::testing::WithParamInterface> {
  public:
 
   // Waits for 'ext_tserver' to experience 'target_failed_disks' disk failures.
@@ -70,14 +72,24 @@ class DiskFailureITest : public 
ExternalMiniClusterITestBase,
 // shut down and restarted. Errors are injected to one of the directories while
 // it is shut down.
 TEST_P(DiskFailureITest, TestFailDuringServerStartup) {
-  // Set up a cluster with three servers with five disks each.
+  const string block_manager_type = std::get<0>(GetParam());
+  const bool is_3_4_3_mode = std::get<1>(GetParam());
+  // Set up a cluster with five disks at each tablet server. In case of 3-4-3
+  // replication scheme one more tablet server is needed to put the replacement
+  // non-voter replica there.
+  const auto kNumTabletServers = is_3_4_3_mode ? 4 : 3;
+  const auto kNumTablets = 5;
+  const auto kNumRows = 100;
+
   ExternalMiniClusterOptions opts;
-  opts.num_tablet_servers = 3;
+  opts.num_tablet_servers = kNumTabletServers;
   opts.num_data_dirs = 5;
-  opts.block_manager_type = GetParam();
+  opts.block_manager_type = block_manager_type;
+  opts.extra_master_flags.push_back(
+  Substitute("--raft_prepare_replacement_before_eviction=$0", 
is_3_4_3_mode));
+  opts.extra_tserver_flags.push_back(
+  Substitute("--raft_prepare_replacement_before_eviction=$0", 
is_3_4_3_mode));
   NO_FATALS(StartClusterWithOpts(opts));
-  const int kNumTablets = 5;
-  const int kNumRows = 100;
 
   // Write some data to a tablet. This will spread blocks across all
   // directories.
@@ -90,14 +102,25 @@ TEST_P(DiskFailureITest, TestFailDuringServerStartup) {
   });
   write_workload.StopAndJoin();
 
-  // Ensure the tablets get to a running state.
-  ExternalTabletServer* ts = cluster_->tablet_server(0);
-  ASSERT_OK(cluster_->WaitForTabletsRunning(ts, kNumTablets, 
kAgreementTimeout));
+  // Arbitrarily select one tablet server which hosts a replica of the tablet.
+  ExternalTabletServer* ts = nu

kudu git commit: KUDU-1097: turn on 3-4-3 replica management scheme

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master 72c83c571 -> 1ac10d536


KUDU-1097: turn on 3-4-3 replica management scheme

This patch enables 3-4-3 replica management scheme for tablets: now
the 3-4-3 scheme is used by default.  To switch back to 3-2-3 scheme,
set --raft_prepare_replacement_before_eviction=false for both masters
and tablet servers.

Change-Id: I781d02fe455140ed3a4693438eece74f07700946
Reviewed-on: http://gerrit.cloudera.org:8080/8862
Tested-by: Kudu Jenkins
Reviewed-by: Mike Percy 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/1ac10d53
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/1ac10d53
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/1ac10d53

Branch: refs/heads/master
Commit: 1ac10d5364db507e5052f5dc13c19d32f62dc4f6
Parents: 72c83c5
Author: Alexey Serbin 
Authored: Sun Dec 17 22:06:16 2017 -0800
Committer: Alexey Serbin 
Committed: Tue Jan 9 01:13:07 2018 +

--
 src/kudu/consensus/raft_consensus.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/1ac10d53/src/kudu/consensus/raft_consensus.cc
--
diff --git a/src/kudu/consensus/raft_consensus.cc 
b/src/kudu/consensus/raft_consensus.cc
index 0892f9f..09f66d9 100644
--- a/src/kudu/consensus/raft_consensus.cc
+++ b/src/kudu/consensus/raft_consensus.cc
@@ -123,7 +123,7 @@ TAG_FLAG(raft_enable_tombstoned_voting, experimental);
 TAG_FLAG(raft_enable_tombstoned_voting, runtime);
 
 // Enable improved re-replication (KUDU-1097).
-DEFINE_bool(raft_prepare_replacement_before_eviction, false,
+DEFINE_bool(raft_prepare_replacement_before_eviction, true,
 "When enabled, failed replicas will only be evicted after a "
 "replacement has been prepared for them.");
 TAG_FLAG(raft_prepare_replacement_before_eviction, advanced);

[2/3] kudu git commit: [mvcc] Fix watermark advancement in the absence of committed transactions

[alexey]

[mvcc] Fix watermark advancement in the absence of committed transactions

MvccSnapshot's 'none_committed_at_or_after_' (ncaoa) watermark
indicates the point after which no transactions are committed.
This is used to cull redo deltas that cannot contain any
committed transactions under a snapshot, thus reducing work when
applying deltas.

By definition, this watermark is never supposed to be lower than
the 'all_committed_before_' watermark. If this invariant is broken
delta selection might be wrong by skipping whole delta files.

Normally the ncaoa watermark is advanced when transactions are
marked as committed, in which case this is done correctly. However
there is problem when the 'all_committed_before_' watermark
is advanced and there are no committed or in flight transactions.
In this case the ncaoa watermark might be left behind, manifesting
as incorrect delta application/skipping or even crashes.

This patch includes a fix and a very directed test that makes sure
it is correct. A follow up patch includes a new test in fuzz-itest
that would trigger a crash without the fix.

Change-Id: Icf94ea289204adf2eed797c3623cdea1b11b09db
Reviewed-on: http://gerrit.cloudera.org:8080/8964
Tested-by: David Ribeiro Alves 
Reviewed-by: Todd Lipcon 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/ad3b6a63
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/ad3b6a63
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/ad3b6a63

Branch: refs/heads/master
Commit: ad3b6a63a41c5b3d2f9907654f505bf76dba14ab
Parents: 8b873a1
Author: David Alves 
Authored: Mon Jan 8 12:36:35 2018 -0800
Committer: Todd Lipcon 
Committed: Tue Jan 9 20:31:02 2018 +

--
 src/kudu/tablet/mvcc-test.cc | 42 +++
 src/kudu/tablet/mvcc.cc  |  7 +++
 src/kudu/tablet/mvcc.h   |  2 +-
 3 files changed, 50 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/ad3b6a63/src/kudu/tablet/mvcc-test.cc
--
diff --git a/src/kudu/tablet/mvcc-test.cc b/src/kudu/tablet/mvcc-test.cc
index 9bcea5e..8458922 100644
--- a/src/kudu/tablet/mvcc-test.cc
+++ b/src/kudu/tablet/mvcc-test.cc
@@ -740,5 +740,47 @@ TEST_F(MvccTest, TestWaitUntilCleanDeadline) {
   ASSERT_TRUE(s.IsTimedOut()) << s.ToString();
 }
 
+// Test for a bug related to the initialization of the MvccManager without
+// any pending transactions, i.e. when there are only calls to 
AdvanceSafeTime().
+// Prior to the fix we would advance safe/clean time but not the
+// 'none_committed_at_or_after_' watermark, meaning the latter would become 
lower
+// than safe/clean time. This had the effect on compaction of culling delta 
files
+// even though they shouldn't be culled.
+// This test makes sure that watermarks are advanced correctly and that delta
+// files are culled correctly.
+TEST_F(MvccTest, TestCorrectInitWithNoTxns) {
+  MvccManager mgr;
+
+  MvccSnapshot snap;
+  mgr.TakeSnapshot(&snap);
+  EXPECT_EQ(snap.all_committed_before_, Timestamp::kInitialTimestamp);
+  EXPECT_EQ(snap.none_committed_at_or_after_, Timestamp::kInitialTimestamp);
+  EXPECT_EQ(snap.committed_timestamps_.size(), 0);
+
+  // Read the clock a few times to advance the timestamp
+  for (int i = 0; i < 10; i++) clock_->Now();
+
+  // Advance the safe timestamp.
+  Timestamp new_safe_time = clock_->Now();
+  mgr.AdjustSafeTime(new_safe_time);
+
+  // Test that the snapshot reports that a timestamp lower than the safe time
+  // may have be committed transaction before that timestamp.
+  // Conversely, test that the snapshot reports that a timestamp greater
+  // than the safe time (and thus greater than none_committed_at_or_after_'
+  // as there are no other transactions committed) cannot have any committed
+  // transactions after that timestamp.
+  MvccSnapshot snap2;
+  mgr.TakeSnapshot(&snap2);
+  Timestamp before_safe_time(new_safe_time.value() - 1);
+  Timestamp after_safe_time(new_safe_time.value() + 1);
+  EXPECT_TRUE(snap2.MayHaveCommittedTransactionsAtOrAfter(before_safe_time));
+  EXPECT_FALSE(snap2.MayHaveCommittedTransactionsAtOrAfter(after_safe_time));
+
+  EXPECT_EQ(snap2.all_committed_before_, new_safe_time);
+  EXPECT_EQ(snap2.none_committed_at_or_after_, new_safe_time);
+  EXPECT_EQ(snap2.committed_timestamps_.size(), 0);
+}
+
 } // namespace tablet
 } // namespace kudu

http://git-wip-us.apache.org/repos/asf/kudu/blob/ad3b6a63/src/kudu/tablet/mvcc.cc
--
diff --git a/src/kudu/tablet/mvcc.cc b/src/kudu/tablet/mvcc.cc
index 87786b9..62d3d43 100644
--- a/src/kudu/tablet/mvcc.cc
+++ b/src/kudu/tablet/mvcc.cc
@@ -238,6 +238,13 @@ void MvccManager::AdjustCleanTime() {
   // Filter out any committed timestamp

[3/3] kudu git commit: [tests] update tests for replication scheme consistency

[alexey]

[tests] update tests for replication scheme consistency

Fixed and updated some scenarios once the re-replication scheme
consistency is enforced:

 * MasterTest.TestRegisterAndHeartbeat scenario
 * MasterReplicationTest.TestHeartbeatAcceptedByAnyMaster
 * MasterCertAuthorityTest.RefuseToSignInvalidCSR
 * MasterCertAuthorityTest.MasterLeaderSignsCSR

This is a follow-up for 1769eed53ee2c21a88a766cb72bf8c9ae622099d.

Change-Id: I8f5bbb581abeb3a781b522dff90711b2ffc988df
Reviewed-on: http://gerrit.cloudera.org:8080/8979
Reviewed-by: Adar Dembo 
Tested-by: Alexey Serbin 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/746d0a5c
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/746d0a5c
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/746d0a5c

Branch: refs/heads/master
Commit: 746d0a5cd53c8252af0c0b6ad294cdb6b39a0319
Parents: ad3b6a6
Author: Alexey Serbin 
Authored: Tue Jan 9 11:11:43 2018 -0800
Committer: Alexey Serbin 
Committed: Tue Jan 9 21:17:26 2018 +

--
 .../master_cert_authority-itest.cc  | 32 +++
 .../master_replication-itest.cc | 33 +++
 src/kudu/master/master-test.cc  | 59 +++-
 3 files changed, 101 insertions(+), 23 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/746d0a5c/src/kudu/integration-tests/master_cert_authority-itest.cc
--
diff --git a/src/kudu/integration-tests/master_cert_authority-itest.cc 
b/src/kudu/integration-tests/master_cert_authority-itest.cc
index 109f3f0..806b870 100644
--- a/src/kudu/integration-tests/master_cert_authority-itest.cc
+++ b/src/kudu/integration-tests/master_cert_authority-itest.cc
@@ -20,12 +20,13 @@
 #include 
 
 #include 
+#include 
 #include 
 
 #include "kudu/common/common.pb.h"
 #include "kudu/common/wire_protocol.pb.h"
+#include "kudu/consensus/replica_management.pb.h"
 #include "kudu/gutil/gscoped_ptr.h"
-#include "kudu/gutil/port.h"
 #include "kudu/master/master.h"
 #include "kudu/master/master.pb.h"
 #include "kudu/master/master.proxy.h"
@@ -45,19 +46,21 @@
 #include "kudu/util/test_util.h"
 #include "kudu/util/user.h"
 
+DECLARE_bool(raft_prepare_replacement_before_eviction);
+
+using kudu::cluster::InternalMiniCluster;
+using kudu::cluster::InternalMiniClusterOptions;
+using kudu::consensus::ReplicaManagementInfoPB;
+using kudu::security::ca::CertRequestGenerator;
+using kudu::security::Cert;
+using kudu::security::CertSignRequest;
+using kudu::security::DataFormat;
+using kudu::security::PrivateKey;
 using std::string;
 using std::shared_ptr;
 
-namespace kudu {
-
-using cluster::InternalMiniCluster;
-using cluster::InternalMiniClusterOptions;
-using security::ca::CertRequestGenerator;
-using security::Cert;
-using security::CertSignRequest;
-using security::DataFormat;
-using security::PrivateKey;
 
+namespace kudu {
 namespace master {
 
 class MasterCertAuthorityTest : public KuduTest {
@@ -71,7 +74,7 @@ class MasterCertAuthorityTest : public KuduTest {
 opts_.num_masters = num_masters_ = opts_.master_rpc_ports.size();
   }
 
-  virtual void SetUp() OVERRIDE {
+  void SetUp() override {
 KuduTest::SetUp();
 cluster_.reset(new InternalMiniCluster(env_, opts_));
 ASSERT_OK(cluster_->Start());
@@ -117,6 +120,12 @@ class MasterCertAuthorityTest : public KuduTest {
 pb->set_host("localhost");
 pb->set_port(2000);
 
+// Information on the replica management scheme.
+ReplicaManagementInfoPB rmi;
+rmi.set_replacement_scheme(FLAGS_raft_prepare_replacement_before_eviction
+? ReplicaManagementInfoPB::PREPARE_REPLACEMENT_BEFORE_EVICTION
+: ReplicaManagementInfoPB::EVICT_FIRST);
+
 for (int i = 0; i < cluster_->num_masters(); ++i) {
   TSHeartbeatRequestPB req;
   TSHeartbeatResponsePB resp;
@@ -124,6 +133,7 @@ class MasterCertAuthorityTest : public KuduTest {
 
   req.mutable_common()->CopyFrom(common);
   req.mutable_registration()->CopyFrom(fake_reg);
+  req.mutable_replica_management_info()->CopyFrom(rmi);
 
   MiniMaster* m = cluster_->mini_master(i);
   if (!m->is_started()) {

http://git-wip-us.apache.org/repos/asf/kudu/blob/746d0a5c/src/kudu/integration-tests/master_replication-itest.cc
--
diff --git a/src/kudu/integration-tests/master_replication-itest.cc 
b/src/kudu/integration-tests/master_replication-itest.cc
index f75bca5..a521ee0 100644
--- a/src/kudu/integration-tests/master_replication-itest.cc
+++ b/src/kudu/integration-tests/master_replication-itest.cc
@@ -21,6 +21,7 @@
 #includ

[1/3] kudu git commit: Add some DVLOG statements to help in debugging compaction issues

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master f932547b2 -> 746d0a5cd


Add some DVLOG statements to help in debugging compaction issues

This adds some DVLOG statemenst that were crucial in debugging
KUDU-2233, particularly aroung merge compactions, clock advancement
and log gc.

Change-Id: I958045a75feceee19c09961de106bbacd343c739
Reviewed-on: http://gerrit.cloudera.org:8080/8886
Tested-by: Kudu Jenkins
Reviewed-by: Todd Lipcon 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/8b873a12
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/8b873a12
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/8b873a12

Branch: refs/heads/master
Commit: 8b873a1257d11510cdbf80fe4c39ae26398e364a
Parents: f932547
Author: David Alves 
Authored: Tue Dec 19 13:30:35 2017 -0800
Committer: Todd Lipcon 
Committed: Tue Jan 9 20:30:55 2018 +

--
 src/kudu/tablet/compaction.cc | 12 
 src/kudu/tablet/mvcc.cc   |  4 +++-
 src/kudu/tablet/tablet_replica.cc |  7 ++-
 3 files changed, 21 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/8b873a12/src/kudu/tablet/compaction.cc
--
diff --git a/src/kudu/tablet/compaction.cc b/src/kudu/tablet/compaction.cc
index 89c2a3f..62b6302 100644
--- a/src/kudu/tablet/compaction.cc
+++ b/src/kudu/tablet/compaction.cc
@@ -477,17 +477,23 @@ class MergeCompactionInput : public CompactionInput {
 if (smallest_idx < 0) {
   smallest_idx = i;
   smallest = state->next();
+  DVLOG(4) << "Set (initial) smallest from state: " << i << " 
smallest: "
+   << CompactionInputRowToString(*smallest);
   continue;
 }
 int row_comp = schema_->Compare(state->next()->row, smallest->row);
 if (row_comp < 0) {
   smallest_idx = i;
   smallest = state->next();
+  DVLOG(4) << "Set (by comp) smallest from state: " << i << " 
smallest: "
+   << CompactionInputRowToString(*smallest);
   continue;
 }
 // If we found two rows with the same key, we want to make the newer 
one point to the older
 // one, which must be a ghost.
 if (PREDICT_FALSE(row_comp == 0)) {
+  DVLOG(4) << "Duplicate row.\nLeft: " << 
CompactionInputRowToString(*state->next())
+   << "\nRight: " << CompactionInputRowToString(*smallest);
   int mutation_comp = CompareDuplicatedRows(*state->next(), *smallest);
   CHECK_NE(mutation_comp, 0);
   if (mutation_comp > 0) {
@@ -498,11 +504,15 @@ class MergeCompactionInput : public CompactionInput {
 states_[smallest_idx]->pop_front();
 smallest_idx = i;
 smallest = state->next();
+DVLOG(4) << "Set smallest to right duplicate: "
+ << CompactionInputRowToString(*smallest);
 continue;
   }
   // .. otherwise copy and pop the other one.
   RETURN_NOT_OK(SetPreviousGhost(smallest, state->next(), true /* 
clone */,
  smallest->row.row_block()->arena()));
+  DVLOG(4) << "Updated left duplicate smallest: "
+   << CompactionInputRowToString(*smallest);
   states_[i]->pop_front();
   continue;
 }
@@ -510,6 +520,7 @@ class MergeCompactionInput : public CompactionInput {
   DCHECK_GE(smallest_idx, 0);
 
   states_[smallest_idx]->pop_front();
+  DVLOG(4) << "Pushing smallest to block: " << 
CompactionInputRowToString(*smallest);
   block->push_back(*smallest);
 }
 
@@ -1227,6 +1238,7 @@ Status ReupdateMissedDeltas(const string &tablet_name,
   // But was this row GCed at flush time?
   if (decoder.is_delete() &&
   history_gc_opts.IsAncientHistory(mut->timestamp())) {
+DVLOG(3) << "Marking for garbage collection row: " << 
schema->DebugRow(row.row);
 is_garbage_collected = true;
   }
   continue;

http://git-wip-us.apache.org/repos/asf/kudu/blob/8b873a12/src/kudu/tablet/mvcc.cc
--
diff --git a/src/kudu/tablet/mvcc.cc b/src/kudu/tablet/mvcc.cc
index eb2fcf5..87786b9 100644
--- a/src/kudu/tablet/mvcc.cc
+++ b/src/kudu/tablet/mvcc.cc
@@ -170,10 +170,10 @@ void MvccManager::AdvanceEarliestInFlightTimestamp() {
 
 void MvccManager::AdjustSafeTime(Timestamp safe_time) {
   std::lock_guard l(lock_);
-
   // No more transactions will start with a ts that is lower than or equal
   // to 'safe_time', so we adjust the snapshot accordingly.
   if (PREDICT_TRUE(safe_time_ <= safe_time)) {
+DVLOG(4) << "Adjusting safe time to: " << safe_time;

kudu git commit: [raft_consensus-itest] fix for Test_KUDU_1735

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master 746d0a5cd -> 8bcc80eec


[raft_consensus-itest] fix for Test_KUDU_1735

Quick fir for Test_KUDU_1735 after enabling 3-4-3 replication mode
by default.  For some reason, this test slipped under radar and
I discovered that it fails only now (the other relevant tests were
already updated).

I tried to make this test work for 3-4-3, but it was not successful
yet.  So, to fix the build, let's push this easy fix first.

This is a follow-up for 1ac10d5364db507e5052f5dc13c19d32f62dc4f6.

Change-Id: I8d96b66d528f90f7611ef32726401e3208b0e47d
Reviewed-on: http://gerrit.cloudera.org:8080/8987
Reviewed-by: Adar Dembo 
Tested-by: Alexey Serbin 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/8bcc80ee
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/8bcc80ee
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/8bcc80ee

Branch: refs/heads/master
Commit: 8bcc80eec075321faef26b2a0ccac12ac9d7
Parents: 746d0a5
Author: Alexey Serbin 
Authored: Tue Jan 9 14:55:18 2018 -0800
Committer: Alexey Serbin 
Committed: Tue Jan 9 23:19:05 2018 +

--
 src/kudu/integration-tests/raft_consensus-itest.cc | 6 ++
 1 file changed, 6 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/8bcc80ee/src/kudu/integration-tests/raft_consensus-itest.cc
--
diff --git a/src/kudu/integration-tests/raft_consensus-itest.cc 
b/src/kudu/integration-tests/raft_consensus-itest.cc
index fdda2da..281ef57 100644
--- a/src/kudu/integration-tests/raft_consensus-itest.cc
+++ b/src/kudu/integration-tests/raft_consensus-itest.cc
@@ -2402,10 +2402,16 @@ TEST_F(RaftConsensusITest, 
TestChangeConfigRejectedUnlessNoopReplicated) {
 TEST_F(RaftConsensusITest, Test_KUDU_1735) {
   const MonoDelta kTimeout = MonoDelta::FromSeconds(10);
   const vector kTsFlags = {
+// This scenario uses 'manual election' mode and assumes no leader change.
 "--enable_leader_failure_detection=false",
+// This test is specific for the 3-2-3 replica management scheme.
+"--raft_prepare_replacement_before_eviction=false",
   };
   const vector kMasterFlags = {
+// This scenario uses 'manual election' mode and assumes no leader change.
 "--catalog_manager_wait_for_new_tablets_to_elect_leader=false",
+// This test is specific for the 3-2-3 replica management scheme.
+"--raft_prepare_replacement_before_eviction=false",
   };
 
   NO_FATALS(BuildAndStart(kTsFlags, kMasterFlags));

kudu git commit: [tls_socket-test] fix test to pass on Ubuntu 16.04

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master 35fc7b7e5 -> 26fd153b0


[tls_socket-test] fix test to pass on Ubuntu 16.04

Prior to this fix, the TlsSocketTest.TestNonBlockingWritev test failed
on Ubuntu 16.04 (kernel 4.4.0).  After some investigation, it turned
out that setting socket send buffer size for something lower than the
MTU size (which is 64kB for localhost) triggered the delayed
acknowledgement logic.  Since the test passes big chunks of data
(32MB on every iteration), with 40ms delay for almost every 16KB sent,
the test eventually timed out.  I verified that disabling the delayed
TCP ack for TLS sockets fixed the issue.

I think that the proper fix is to remove the custom setting for the
socket send buffer: a 32MB chunk of data is big enough to not fit into
the socket buffer of the default size.

Also, it seems this issue is not going to bite us in real life since
we don't set the size of the socket send buffer anywhere.

NOTE: however, some mystery is left since the failing test would pass
  if running under strace.

Change-Id: I94170d5bdbe17a5952a82faf74d8b868b42460aa
Reviewed-on: http://gerrit.cloudera.org:8080/8996
Tested-by: Kudu Jenkins
Reviewed-by: Todd Lipcon 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/26fd153b
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/26fd153b
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/26fd153b

Branch: refs/heads/master
Commit: 26fd153b055b10bee90275d4f4c9e5f7088d4e83
Parents: 35fc7b7
Author: Alexey Serbin 
Authored: Wed Jan 10 13:39:05 2018 -0800
Committer: Todd Lipcon 
Committed: Fri Jan 12 05:28:47 2018 +

--
 src/kudu/security/tls_socket-test.cc | 4 
 1 file changed, 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/26fd153b/src/kudu/security/tls_socket-test.cc
--
diff --git a/src/kudu/security/tls_socket-test.cc 
b/src/kudu/security/tls_socket-test.cc
index 214d2bf..4f332a5 100644
--- a/src/kudu/security/tls_socket-test.cc
+++ b/src/kudu/security/tls_socket-test.cc
@@ -20,7 +20,6 @@
 #include 
 #include 
 #include 
-#include 
 #include 
 
 #include 
@@ -282,9 +281,6 @@ TEST_F(TlsSocketTest, TestNonBlockingWritev) {
   unique_ptr client_sock;
   NO_FATALS(ConnectClient(server.listen_addr(), &client_sock));
 
-  int sndbuf = 16 * 1024;
-  CHECK_ERR(setsockopt(client_sock->GetFd(), SOL_SOCKET, SO_SNDBUF, &sndbuf, 
sizeof(sndbuf)));
-
   unique_ptr buf(new uint8_t[kEchoChunkSize]);
   unique_ptr rbuf(new uint8_t[kEchoChunkSize]);
   RandomString(buf.get(), kEchoChunkSize, &rng);

[2/2] kudu git commit: [catalog_manager] more info if unable to replace a replica

[alexey]

[catalog_manager] more info if unable to replace a replica

Output actionable warning message when the catalog manager is unable
to find a spot for a replacement replica.  Since the 3-4-3 replication
scheme is now enabled by default, this might be useful in case if
running a cluster with just 3 tablet servers when tables have
replication factor of 3.

Change-Id: Id5f562c6d1ff526daa785ea535e440598c03cd37
Reviewed-on: http://gerrit.cloudera.org:8080/9040
Reviewed-by: Mike Percy 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/c40e0587
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/c40e0587
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/c40e0587

Branch: refs/heads/master
Commit: c40e0587bf6a6aa55e5bd72dd2dd9356b1507f2e
Parents: 5806b80
Author: Alexey Serbin 
Authored: Tue Jan 16 15:01:34 2018 -0800
Committer: Alexey Serbin 
Committed: Wed Jan 17 03:18:11 2018 +

--
 src/kudu/master/catalog_manager.cc | 38 ++---
 src/kudu/master/ts_manager.cc  |  4 ++--
 src/kudu/master/ts_manager.h   |  8 +++
 3 files changed, 41 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/c40e0587/src/kudu/master/catalog_manager.cc
--
diff --git a/src/kudu/master/catalog_manager.cc 
b/src/kudu/master/catalog_manager.cc
index d6b369b..1248144 100644
--- a/src/kudu/master/catalog_manager.cc
+++ b/src/kudu/master/catalog_manager.cc
@@ -3133,11 +3133,43 @@ bool AsyncAddReplicaTask::SendRequest(int attempt) {
   InsertOrDie(&excluded, ts_desc);
 }
   }
+
   auto replacement_replica = SelectReplica(ts_descs, excluded, rng_);
   if (PREDICT_FALSE(!replacement_replica)) {
-KLOG_EVERY_N(WARNING, 100) << LogPrefix()
-   << "No candidate replacement replica found for 
tablet "
-   << tablet_->ToString();
+auto msg = Substitute("no candidate replacement replica found for tablet 
$0",
+  tablet_->ToString());
+// Check whether it's a situation when a replacement replica cannot be 
found
+// due to an inconsistency in cluster configuration. If the tablet has the
+// replication factor of N, and the cluster is configured to use 
N->(N+1)->N
+// replication scheme (see --raft_prepare_replacement_before_eviction 
flag),
+// at least N+1 tablet servers should be registered to find a place
+// for a replacement replica.
+TSDescriptorVector all_descriptors;
+master_->ts_manager()->GetAllDescriptors(&all_descriptors);
+const auto num_tservers_registered = all_descriptors.size();
+
+auto replication_factor = 0;
+{
+  TableMetadataLock l(tablet_->table().get(), LockMode::READ);
+  replication_factor = 
tablet_->table()->metadata().state().pb.num_replicas();
+}
+DCHECK_GE(replication_factor, 0);
+const auto num_tservers_needed =
+FLAGS_raft_prepare_replacement_before_eviction ? replication_factor + 1
+   : replication_factor;
+if (num_tservers_registered < num_tservers_needed) {
+  msg += Substitute(
+  "; the total number of registered tablet servers ($0) does not allow 
"
+  "for replacement of the failed replica: at least $1 tablet servers "
+  "are required", num_tservers_registered, num_tservers_needed);
+  if (FLAGS_raft_prepare_replacement_before_eviction &&
+  num_tservers_registered == replication_factor) {
+msg +=
+  "; consider either adding an additional tablet server or running "
+  "the cluster with --raft_prepare_replacement_before_eviction=false";
+  }
+}
+KLOG_EVERY_N_SECS(WARNING, 60) << LogPrefix() << msg;
 return false;
   }
 

http://git-wip-us.apache.org/repos/asf/kudu/blob/c40e0587/src/kudu/master/ts_manager.cc
--
diff --git a/src/kudu/master/ts_manager.cc b/src/kudu/master/ts_manager.cc
index 65faad8..8b1aa1c 100644
--- a/src/kudu/master/ts_manager.cc
+++ b/src/kudu/master/ts_manager.cc
@@ -92,13 +92,13 @@ Status TSManager::RegisterTS(const NodeInstancePB& instance,
   return Status::OK();
 }
 
-void TSManager::GetAllDescriptors(vector > *descs) 
const {
+void TSManager::GetAllDescriptors(vector> *descs) 
const {
   descs->clear();
   shared_lock l(lock_);
   AppendValuesFromMap(servers_by_id_, descs);
 }
 
-void TSManager::GetAllLiveDescriptors(vector > 
*descs) const {
+void TSManager::GetAllLiveDes

[1/2] kudu git commit: mini-cluster: rename data_root to cluster_root

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master b0d27d3f5 -> c40e0587b


mini-cluster: rename data_root to cluster_root

I found that the name data_root for the root directory in which to place
the cluster's data and logs was a confusing name because:
  1. we use "data" here loosely to describe any old bytes, encompassing
 not only data blocks, but WALs and logs as well, and
  2. the concept of a "data root" already exists to mean a
 user-specified location to place data blocks (i.e. an entry in
 fs_data_dirs).

As such, this patch renames External- and InternalMiniClusterOptions'
"data_root" members to "cluster_root".

Change-Id: Id8ccc9e232f47a684d6b9226fd84639c4c94d0c3
Reviewed-on: http://gerrit.cloudera.org:8080/9033
Reviewed-by: Adar Dembo 
Tested-by: Andrew Wong 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/5806b807
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/5806b807
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/5806b807

Branch: refs/heads/master
Commit: 5806b807a712116e79e8a67dc4dcc55d51ea6b2a
Parents: b0d27d3
Author: Andrew Wong 
Authored: Tue Jan 16 08:44:24 2018 -0800
Committer: Andrew Wong 
Committed: Wed Jan 17 01:06:54 2018 +

--
 src/kudu/benchmarks/tpch/tpch1.cc  |  2 +-
 src/kudu/benchmarks/tpch/tpch_real_world.cc|  2 +-
 src/kudu/integration-tests/ts_itest-base.cc|  4 ++--
 src/kudu/mini-cluster/external_mini_cluster.cc | 22 ++---
 src/kudu/mini-cluster/external_mini_cluster.h  | 10 +-
 src/kudu/mini-cluster/internal_mini_cluster.cc | 14 ++---
 src/kudu/mini-cluster/internal_mini_cluster.h  |  4 ++--
 src/kudu/tools/kudu-tool-test.cc   |  4 ++--
 src/kudu/tools/tool.proto  |  2 +-
 src/kudu/tools/tool_action_test.cc | 20 +--
 10 files changed, 42 insertions(+), 42 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/5806b807/src/kudu/benchmarks/tpch/tpch1.cc
--
diff --git a/src/kudu/benchmarks/tpch/tpch1.cc 
b/src/kudu/benchmarks/tpch/tpch1.cc
index d8d4a11..2feb823 100644
--- a/src/kudu/benchmarks/tpch/tpch1.cc
+++ b/src/kudu/benchmarks/tpch/tpch1.cc
@@ -261,7 +261,7 @@ int main(int argc, char **argv) {
 kudu::Status s = env->CreateDir(FLAGS_mini_cluster_base_dir);
 CHECK(s.IsAlreadyPresent() || s.ok()) << s.ToString();
 kudu::cluster::InternalMiniClusterOptions options;
-options.data_root = FLAGS_mini_cluster_base_dir;
+options.cluster_root = FLAGS_mini_cluster_base_dir;
 cluster.reset(new kudu::cluster::InternalMiniCluster(env, options));
 CHECK_OK(cluster->StartSync());
 master_address = cluster->mini_master()->bound_rpc_addr_str();

http://git-wip-us.apache.org/repos/asf/kudu/blob/5806b807/src/kudu/benchmarks/tpch/tpch_real_world.cc
--
diff --git a/src/kudu/benchmarks/tpch/tpch_real_world.cc 
b/src/kudu/benchmarks/tpch/tpch_real_world.cc
index ce65501..03a25f1 100644
--- a/src/kudu/benchmarks/tpch/tpch_real_world.cc
+++ b/src/kudu/benchmarks/tpch/tpch_real_world.cc
@@ -194,7 +194,7 @@ Status TpchRealWorld::Init() {
 
 ExternalMiniClusterOptions opts;
 opts.num_tablet_servers = 1;
-opts.data_root = FLAGS_tpch_mini_cluster_base_dir;
+opts.cluster_root = FLAGS_tpch_mini_cluster_base_dir;
 if (!FLAGS_tpch_path_to_ts_flags_file.empty()) {
   opts.extra_tserver_flags.push_back("--flagfile=" + 
FLAGS_tpch_path_to_ts_flags_file);
 }

http://git-wip-us.apache.org/repos/asf/kudu/blob/5806b807/src/kudu/integration-tests/ts_itest-base.cc
--
diff --git a/src/kudu/integration-tests/ts_itest-base.cc 
b/src/kudu/integration-tests/ts_itest-base.cc
index f8344f0..26f4010 100644
--- a/src/kudu/integration-tests/ts_itest-base.cc
+++ b/src/kudu/integration-tests/ts_itest-base.cc
@@ -105,7 +105,7 @@ void TabletServerIntegrationTestBase::AddExtraFlags(
 }
 
 void TabletServerIntegrationTestBase::CreateCluster(
-const string& data_root_path,
+const string& cluster_root_path,
 const vector& non_default_ts_flags,
 const vector& non_default_master_flags,
 uint32_t num_data_dirs) {
@@ -118,7 +118,7 @@ void TabletServerIntegrationTestBase::CreateCluster(
 
   cluster::ExternalMiniClusterOptions opts;
   opts.num_tablet_servers = FLAGS_num_tablet_servers;
-  opts.data_root = GetTestPath(data_root_path);
+  opts.cluster_root = GetTestPath(cluster_root_path);
   opts.num_data_dirs = num_data_dirs;
 
   // Enable exactly once semantics for tests.

http://git-wip-us.apache.org/repos/asf/kudu/blob/5806b807/src/kudu/mini-cluster/external_mini_cluster.cc
--

kudu git commit: [rpc] don't issue authn tokens over non-confidential connections

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master d3165ed2b -> 1f346a1be


[rpc] don't issue authn tokens over non-confidential connections

With this patch, master will not issue authn tokens to clients over a
non-confidential connection.  The 'confidentiality' property is based
on the features of the negotiated connection and stored into the
RpcContext object, accessible via the RpcContext::is_confidential()
method.

Change-Id: Ie31aa492bcc460dbd43975bccfe571354f3bf885
Reviewed-on: http://gerrit.cloudera.org:8080/9025
Tested-by: Kudu Jenkins
Reviewed-by: Alexey Serbin 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/1f346a1b
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/1f346a1b
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/1f346a1b

Branch: refs/heads/master
Commit: 1f346a1bec3db99877a4ace0d3682e5341bec9c6
Parents: d3165ed
Author: Alexey Serbin 
Authored: Thu Jan 11 14:55:20 2018 -0800
Committer: Alexey Serbin 
Committed: Thu Jan 18 05:31:15 2018 +

--
 src/kudu/integration-tests/security-itest.cc | 105 ++
 src/kudu/master/master_service.cc|   7 +-
 src/kudu/rpc/connection.cc   |   5 ++
 src/kudu/rpc/connection.h|  12 +++
 src/kudu/rpc/negotiation.cc  |   4 +
 src/kudu/rpc/rpc_context.cc  |   5 ++
 src/kudu/rpc/rpc_context.h   |   7 ++
 7 files changed, 143 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/1f346a1b/src/kudu/integration-tests/security-itest.cc
--
diff --git a/src/kudu/integration-tests/security-itest.cc 
b/src/kudu/integration-tests/security-itest.cc
index 16c0175..6fb76bf 100644
--- a/src/kudu/integration-tests/security-itest.cc
+++ b/src/kudu/integration-tests/security-itest.cc
@@ -22,11 +22,13 @@
 #include 
 #include 
 
+#include 
 #include 
 #include 
 
 #include "kudu/client/client-test-util.h"
 #include "kudu/client/client.h"
+#include "kudu/client/client.pb.h"
 #include "kudu/client/schema.h"
 #include "kudu/client/shared_ptr.h"
 #include "kudu/client/write_op.h"
@@ -37,9 +39,11 @@
 #include "kudu/master/master.pb.h"
 #include "kudu/master/master.proxy.h"
 #include "kudu/mini-cluster/external_mini_cluster.h"
+#include "kudu/mini-cluster/mini_cluster.h"
 #include "kudu/rpc/messenger.h"
 #include "kudu/rpc/rpc_controller.h"
 #include "kudu/security/test/mini_kdc.h"
+#include "kudu/security/token.pb.h"
 #include "kudu/server/server_base.pb.h"
 #include "kudu/server/server_base.proxy.h"
 #include "kudu/tablet/key_value_test_schema.h"
@@ -52,6 +56,8 @@
 #include "kudu/util/test_macros.h"
 #include "kudu/util/test_util.h"
 
+DECLARE_string(local_ip_for_outbound_sockets);
+
 using kudu::client::KuduClient;
 using kudu::client::KuduInsert;
 using kudu::client::KuduSchema;
@@ -304,4 +310,103 @@ TEST_F(SecurityITest, TestWorldReadablePrivateKey) {
   credentials_name));
 }
 
+struct AuthTokenIssuingTestParams {
+  const ExternalMiniCluster::BindMode bind_mode;
+  const string rpc_authentication;
+  const string rpc_encryption;
+  const bool rpc_encrypt_loopback_connections;
+  const bool authn_token_present;
+};
+class AuthTokenIssuingTest :
+public SecurityITest,
+public ::testing::WithParamInterface {
+};
+INSTANTIATE_TEST_CASE_P(, AuthTokenIssuingTest, ::testing::ValuesIn(
+vector{
+  { ExternalMiniCluster::BindMode::LOOPBACK, "required", "required", true, 
 true,  },
+  { ExternalMiniCluster::BindMode::LOOPBACK, "required", "required", 
false, true,  },
+  //ExternalMiniCluster::BindMode::LOOPBACK, "required", "disabled": 
non-acceptable
+  //ExternalMiniCluster::BindMode::LOOPBACK, "required", "disabled": 
non-acceptable
+  { ExternalMiniCluster::BindMode::LOOPBACK, "disabled", "required", true, 
 true,  },
+  { ExternalMiniCluster::BindMode::LOOPBACK, "disabled", "required", 
false, true,  },
+  { ExternalMiniCluster::BindMode::LOOPBACK, "disabled", "disabled", true, 
 false, },
+  { ExternalMiniCluster::BindMode::LOOPBACK, "disabled", "disabled", 
false, true,  },
+#if defined(__linux__)
+  { ExternalMiniCluster::BindMode::UNIQUE_LOOPBACK, "required", 
"required", true,  true,  },
+  { ExternalMiniCluster::BindMode::UNIQUE_LOOPBACK, "required", 
"required&qu

[1/3] kudu git commit: consensus: avoid extra thread wakeups for Peer::SignalRequest

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master 3b51bf337 -> c497c69bb


consensus: avoid extra thread wakeups for Peer::SignalRequest

Looking at a profile on a YCSB workload I see a significant amount of
CPU in submitting tasks to the raft threadpool for requests. My guess is
that, in a workload like this, we often call SignalRequest while the
request was already pending.

This patch avoids submitting the threadpool task if there is already a
task pending.

Change-Id: I62ec4ea91e976590ce2de4fa34db61eaf9aab92a
Reviewed-on: http://gerrit.cloudera.org:8080/9092
Tested-by: Kudu Jenkins
Reviewed-by: David Ribeiro Alves 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/9961902d
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/9961902d
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/9961902d

Branch: refs/heads/master
Commit: 9961902d8b4f18a5e5d68aeb624e27167651b837
Parents: 3b51bf3
Author: Todd Lipcon 
Authored: Mon Jan 22 08:13:15 2018 -0800
Committer: Todd Lipcon 
Committed: Wed Jan 24 19:07:17 2018 +

--
 src/kudu/consensus/consensus_peers.cc | 6 ++
 1 file changed, 6 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/9961902d/src/kudu/consensus/consensus_peers.cc
--
diff --git a/src/kudu/consensus/consensus_peers.cc 
b/src/kudu/consensus/consensus_peers.cc
index 6d68528..187ed6f 100644
--- a/src/kudu/consensus/consensus_peers.cc
+++ b/src/kudu/consensus/consensus_peers.cc
@@ -167,6 +167,12 @@ Status Peer::SignalRequest(bool even_if_queue_empty) {
 return Status::IllegalState("Peer was closed.");
   }
 
+  // Only allow one request at a time. No sense waking up the
+  // raft thread pool if the task will just abort anyway.
+  if (request_pending_) {
+return Status::OK();
+  }
+
   // Capture a weak_ptr reference into the submitted functor so that we can
   // safely handle the functor outliving its peer.
   weak_ptr w_this = shared_from_this();

[3/3] kudu git commit: KUDU-2265 CA-signed server certs for non-leader masters

[alexey]

KUDU-2265 CA-signed server certs for non-leader masters

This changelist addresses KUDU-2265.  Prior to this fix, a non-leader
master had a self-signed server TLS certificates if it hasn't ever
become a leader.

An integration test to verify the new behavior is added as well.

Change-Id: Ia3539d58d10ed319ad1d8686c1259c92822fb710
Reviewed-on: http://gerrit.cloudera.org:8080/9076
Tested-by: Kudu Jenkins
Reviewed-by: Todd Lipcon 
Reviewed-by: Dan Burkert 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/c497c69b
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/c497c69b
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/c497c69b

Branch: refs/heads/master
Commit: c497c69bb2128a8a62738b9b6bb97a5b83da0323
Parents: e0a743d
Author: Alexey Serbin 
Authored: Thu Jan 18 23:52:03 2018 -0800
Committer: Alexey Serbin 
Committed: Wed Jan 24 19:24:32 2018 +

--
 src/kudu/integration-tests/CMakeLists.txt   |   1 +
 .../security-master-certificates-itest.cc   | 100 +++
 src/kudu/master/catalog_manager.cc  |  46 +++--
 src/kudu/master/catalog_manager.h   |  30 +-
 src/kudu/security/tls_context.cc|  10 +-
 src/kudu/util/rw_mutex.cc   |   6 ++
 src/kudu/util/rw_mutex.h|   2 +
 7 files changed, 177 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/c497c69b/src/kudu/integration-tests/CMakeLists.txt
--
diff --git a/src/kudu/integration-tests/CMakeLists.txt 
b/src/kudu/integration-tests/CMakeLists.txt
index 1a10a7d..a6a6938 100644
--- a/src/kudu/integration-tests/CMakeLists.txt
+++ b/src/kudu/integration-tests/CMakeLists.txt
@@ -94,6 +94,7 @@ ADD_KUDU_TEST(raft_consensus-itest RUN_SERIAL true)
 ADD_KUDU_TEST(registration-test RESOURCE_LOCK "master-web-port")
 ADD_KUDU_TEST(security-faults-itest)
 ADD_KUDU_TEST(security-itest)
+ADD_KUDU_TEST(security-master-certificates-itest RESOURCE_LOCK 
"master-rpc-ports")
 ADD_KUDU_TEST(security-unknown-tsk-itest)
 ADD_KUDU_TEST(stop_tablet-itest)
 ADD_KUDU_TEST(table_locations-itest)

http://git-wip-us.apache.org/repos/asf/kudu/blob/c497c69b/src/kudu/integration-tests/security-master-certificates-itest.cc
--
diff --git a/src/kudu/integration-tests/security-master-certificates-itest.cc 
b/src/kudu/integration-tests/security-master-certificates-itest.cc
new file mode 100644
index 000..b521d65
--- /dev/null
+++ b/src/kudu/integration-tests/security-master-certificates-itest.cc
@@ -0,0 +1,100 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include 
+#include 
+
+#include 
+#include 
+
+#include "kudu/consensus/raft_consensus.h"
+#include "kudu/gutil/ref_counted.h"
+#include "kudu/master/catalog_manager.h"
+#include "kudu/master/master.h"
+#include "kudu/master/mini_master.h"
+#include "kudu/master/sys_catalog.h"
+#include "kudu/mini-cluster/internal_mini_cluster.h"
+#include "kudu/security/tls_context.h"
+#include "kudu/tablet/tablet_replica.h"
+#include "kudu/util/status.h"
+#include "kudu/util/test_macros.h"
+#include "kudu/util/test_util.h"
+
+DECLARE_bool(enable_leader_failure_detection);
+
+using kudu::cluster::InternalMiniCluster;
+using kudu::cluster::InternalMiniClusterOptions;
+using kudu::consensus::RaftConsensus;
+
+using std::unique_ptr;
+
+
+namespace kudu {
+
+class SecurityMasterCertsTest : public KuduTest {
+ public:
+  void SetUp() override {
+KuduTest::SetUp();
+
+// This test requires manual system catalog leader election.
+FLAGS_enable_leader_failure_detection = false;
+
+InternalMiniClusterOptions opts;
+opts.master_rpc_ports = { 11010, 11011, 11012, 11013, 11014, };
+opts.num_masters = opts.master_rpc_ports.size

[2/3] kudu git commit: Upgrade to tcmalloc 2.6.3 and enabled sized-deallocation

[alexey]

Upgrade to tcmalloc 2.6.3 and enabled sized-deallocation

This bumps to a new gperftools/tcmalloc version which ought to be a bit
faster. It also enables sized-deallocation support on compilers that
support it.

The upgrade is relatively straigthforward but for a few items:
- for whatever reason, we developed a conflict between our own
  base::SpinLock from gutil and tcmalloc's. This conflict has always
  existed, but apparently tcmalloc's implementation changed sizes
  slightly, meaning that now the linker complains about the duplicate
  symbol definition. I created a simple find-replace patch to change
  tcmalloc to use a 'tcmalloc::' namespace instead of 'base::'.
- our previous patches are all now included in tcmalloc upstream and no
  longer necessary.
- for unknown reasons, gperftools now fails to build with TSAN
  instrumentation: it shows duplicate symbols between the tcmalloc
  'operator new' and clang_rt's 'operator new'. We never actually needed
  to use gperftools in TSAN builds anyway, so this just removes the
  instrumented build and properly ifdefs out a few of the places where
  we'd included gperftools headers unconditionally.

Sized deallocation is a C++14 feature (which can be selectively enabled
on C++11) which adds a new 'operator delete(void*, size_t)'. In the
common case where we delete a Foo*, the compiler already knows
sizeof(Foo) and thus can pass this second argument to 'operator delete'.
This goes through an optimized code path inside tcmalloc -- namely, it
can skip the expensive lookup which otherwise would be required to map
the pointer back to its size class.

Unfortuantely, we can't always rely on the system allocator to provide
sized deallocation, and thus we can't use this in the exported variants
of our code (i.e. the client library). So, this patch only enables it
when building the Kudu binaries and not the client library.

I ran a quick performance check using full_stack-insert-scan-test twice
each before and after:

Before:
 Performance counter stats for 'build/latest/bin/full_stack-insert-scan-test 
--gtest_filter=*MRSOnlyStressTest* --inserts_per_client=20 
--concurrent_inserts=10 --rows_per_batch=1 --skip_scans':

 322623.135418  task-clock (msec) #4.964 CPUs utilized
11,751,640  context-switches  #0.036 M/sec
 3,619,010  cpu-migrations#0.011 M/sec
   117,706  page-faults   #0.365 K/sec
 1,057,656,296,542  cycles#3.278 GHz
   695,668,728,662  instructions  #0.66  insn per cycle
   127,565,245,327  branches  #  395.400 M/sec
 1,480,987,998  branch-misses #1.16% of all branches

  64.996506196 seconds time elapsed

 330206.351604  task-clock (msec) #5.330 CPUs utilized
12,518,370  context-switches  #0.038 M/sec
 3,695,790  cpu-migrations#0.011 M/sec
   113,580  page-faults   #0.344 K/sec
 1,059,061,436,907  cycles#3.207 GHz
   697,782,051,928  instructions  #0.66  insn per cycle
   127,949,774,596  branches  #  387.484 M/sec
 1,371,967,917  branch-misses #1.07% of all branches

  61.952217532 seconds time elapsed

After:

 310788.334202  task-clock (msec) #5.315 CPUs utilized
12,486,276  context-switches  #0.040 M/sec
 3,690,660  cpu-migrations#0.012 M/sec
   113,988  page-faults   #0.367 K/sec
 1,027,033,932,375  cycles#3.305 GHz
   663,508,168,985  instructions  #0.65  insn per cycle
   125,864,966,052  branches  #  404.986 M/sec
 1,442,683,495  branch-misses #1.15% of all branches

  58.479033228 seconds time elapsed

 Performance counter stats for 
'build/latest/bin/full_stack-insert-scan-test.263 
--gtest_filter=*MRSOnlyStressTest* --inserts_per_client=20 
--concurrent_inserts=10 --rows_per_batch=1 --skip_scans':

 317505.548908  task-clock (msec) #5.323 CPUs utilized
12,461,003  context-switches  #0.039 M/sec
 3,688,491  cpu-migrations#0.012 M/sec
   113,952  page-faults   #0.359 K/sec
 1,029,595,155,391  cycles#3.243 GHz
   663,514,269,656  instructions  #0.64  insn per cycle
   125,865,138,975  branches  #  396.419 M/sec
 1,419,717,877  branch-misses #1.13% of all branches

  59.642651558 seconds time elapsed

Seems like a small improvement in cycle count, wall clock, etc.

Change-Id: I3b76c9d71b29d58f0b6dade3fc33b32d0c3de23b
Reviewed-on: http://gerrit.cloudera.org:8080/9108
Tested-b

kudu git commit: [gradle] fetch gradle-wrapper.jar from raw.githubusercontent.com

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master 37a81b82c -> 4ee1c4fd7


[gradle] fetch gradle-wrapper.jar from raw.githubusercontent.com

This patch switches the URL to fetch gradle-wrapper.jar from
github.com to raw.githubusercontent.com. That's to be able to build
the Java part of the project at older Linux distros (RH < 7.4).

The github.com servers dropped TLSv1/TLSv1.1 support some time ago:
see [1] for details.

[1] https://githubengineering.com/crypto-removal-notice/

Change-Id: Ia745ef4d69726852fbe60be0d1778705d5580f14
Reviewed-on: http://gerrit.cloudera.org:8080/11262
Reviewed-by: Grant Henke 
Tested-by: Alexey Serbin 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/4ee1c4fd
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/4ee1c4fd
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/4ee1c4fd

Branch: refs/heads/master
Commit: 4ee1c4fd7a716182abbe4bab7a8a20089158a894
Parents: 37a81b8
Author: Alexey Serbin 
Authored: Fri Aug 17 13:21:14 2018 -0700
Committer: Alexey Serbin 
Committed: Sat Aug 18 00:17:05 2018 +

--
 java/gradle/wrapper.gradle | 8 ++--
 java/gradlew   | 2 +-
 2 files changed, 7 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/4ee1c4fd/java/gradle/wrapper.gradle
--
diff --git a/java/gradle/wrapper.gradle b/java/gradle/wrapper.gradle
index 56071a5..10171d5 100644
--- a/java/gradle/wrapper.gradle
+++ b/java/gradle/wrapper.gradle
@@ -51,8 +51,12 @@ task bootstrapWrapper() {
 
 // Add a trailing zero to the version if needed.
 def fullVersion = versions.gradle.count(".") == 1 ? "${versions.gradle}.0" 
: versions.gradle
-// Leverages the wrapper jar checked into the gradle project on github 
because the jar isn't available elsewhere.
-def wrapperBaseUrl = 
"https://github.com/gradle/gradle/raw/v$fullVersion/gradle/wrapper";
+// Leverages the wrapper jar checked into the gradle project on github 
because the jar isn't
+// available elsewhere. Using raw.githubusercontent.com instead of 
github.com because
+// github.com servers deprecated TLSv1/TLSv1.1 support some time ago, so 
older versions
+// of curl (built against OpenSSL library that doesn't support TLSv1.2) 
would fail to
+// fetch the jar.
+def wrapperBaseUrl = 
"https://raw.githubusercontent.com/gradle/gradle/v$fullVersion/gradle/wrapper";
 def wrapperJarUrl = wrapperBaseUrl + "/gradle-wrapper.jar"
 
 def bootstrapString = """

http://git-wip-us.apache.org/repos/asf/kudu/blob/4ee1c4fd/java/gradlew
--
diff --git a/java/gradlew b/java/gradlew
index 3807ff4..f40bb25 100755
--- a/java/gradlew
+++ b/java/gradlew
@@ -82,7 +82,7 @@ esac
 # Loop in case we encounter an error.
 for attempt in 1 2 3; do
   if [ ! -e $APP_HOME/gradle/wrapper/gradle-wrapper.jar ]; then
-if ! curl -s -S --retry 3 -L -o 
"$APP_HOME/gradle/wrapper/gradle-wrapper.jar" 
"https://github.com/gradle/gradle/raw/v4.9.0/gradle/wrapper/gradle-wrapper.jar";;
 then
+if ! curl -s -S --retry 3 -L -o 
"$APP_HOME/gradle/wrapper/gradle-wrapper.jar" 
"https://raw.githubusercontent.com/gradle/gradle/v4.9.0/gradle/wrapper/gradle-wrapper.jar";;
 then
   rm -f "$APP_HOME/gradle/wrapper/gradle-wrapper.jar"
   # Pause for a bit before looping in case the server throttled us.
   sleep 5

kudu git commit: [gutil] add libunwind along with libglog to fix build

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master be360bf01 -> 0dc19bfb3


[gutil] add libunwind along with libglog to fix build

Since libglog depends on libunwind (at least on older Linux systems),
it's necessary to add libunwind into the list of libraries when
linking in libglog.  Otherwise, in case of a static linkage, there are
errors like below:

./../../lib/../src/kudu/util/CMakeFiles/kudu_util.dir/debug-util.cc.o: In 
function `PrimeLibunwind':
src/kudu/src/kudu/util/debug-util.cc:349: undefined reference to 
`_Ux86_64_getcontext'
src/kudu/src/kudu/util/debug-util.cc:350: undefined reference to 
`_ULx86_64_init_local'
../../../lib/../src/kudu/util/CMakeFiles/kudu_util.dir/debug-util.cc.o: In 
function `kudu::StackTrace::Collect(int)':
src/kudu/src/kudu/util/debug-util.cc:591: undefined reference to 
`_Ux86_64_getcontext'
src/kudu/src/kudu/util/debug-util.cc:592: undefined reference to 
`_ULx86_64_init_local'
src/kudu/src/kudu/util/debug-util.cc:607: undefined reference to 
`_ULx86_64_step'
src/kudu/src/kudu/util/debug-util.cc:598: undefined reference to 
`_ULx86_64_get_reg'
src/kudu/src/kudu/util/debug-util.cc:607: undefined reference to 
`_ULx86_64_step'
../../../../../thirdparty/installed/uninstrumented/lib/libglog.a(libglog_la-utilities.o):
 In function `GetStackTrace':
src/kudu/thirdparty/src/glog-0.3.5/src/stacktrace_libunwind-inl.h:65: undefined 
reference to `_Ux86_64_getcontext'
src/kudu/thirdparty/src/glog-0.3.5/src/stacktrace_libunwind-inl.h:66: undefined 
reference to `_ULx86_64_init_local'
src/kudu/thirdparty/src/glog-0.3.5/src/stacktrace_libunwind-inl.h:78: undefined 
reference to `_ULx86_64_step'
src/kudu/thirdparty/src/glog-0.3.5/src/stacktrace_libunwind-inl.h:70: undefined 
reference to `_ULx86_64_get_reg'
src/kudu/thirdparty/src/glog-0.3.5/src/stacktrace_libunwind-inl.h:78: undefined 
reference to `_ULx86_64_step'
collect2: error: ld returned 1 exit status

This is a follow-up for a8d088e11b8b901a91e4820dcfdca9537a1e801a: in
that changelist KUDU_BASE_LIBS were removed from the linkage
of libraries/binaries where libgutil was linked in (KUDU_BASE_LIBS
contains both libglog and libunwind).

Change-Id: Ia91824b257fbc41a442b484bbc5a5f926db708ec
Reviewed-on: http://gerrit.cloudera.org:8080/11327
Reviewed-by: Andrew Wong 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/0dc19bfb
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/0dc19bfb
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/0dc19bfb

Branch: refs/heads/master
Commit: 0dc19bfb37cf9a20b9e1d7122084c199720d5ca4
Parents: be360bf
Author: Alexey Serbin 
Authored: Sat Aug 25 00:05:52 2018 -0700
Committer: Alexey Serbin 
Committed: Sat Aug 25 08:06:19 2018 +

--
 src/kudu/gutil/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/0dc19bfb/src/kudu/gutil/CMakeLists.txt
--
diff --git a/src/kudu/gutil/CMakeLists.txt b/src/kudu/gutil/CMakeLists.txt
index 3d487eb..093cc1b 100644
--- a/src/kudu/gutil/CMakeLists.txt
+++ b/src/kudu/gutil/CMakeLists.txt
@@ -54,6 +54,7 @@ set(GUTIL_SRCS
 
 set(GUTIL_LIBS
   glog
+  unwind
   protobuf)
 
 if (NOT APPLE)

kudu git commit: [gutil] change libunwind to Linux specific to fix build

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master f5ffcb4d7 -> bb78520a7


[gutil] change libunwind to Linux specific to fix build

This is a follow-up for 0dc19bfb37cf9a20b9e1d7122084c199720d5ca4:
that change breaks the build on macOS.

Change-Id: I879564cc4e6f233bc234890d401d7cb068c3eaf4
Reviewed-on: http://gerrit.cloudera.org:8080/11338
Tested-by: Kudu Jenkins
Reviewed-by: Alexey Serbin 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/bb78520a
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/bb78520a
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/bb78520a

Branch: refs/heads/master
Commit: bb78520a7a636926bba3c68fc85bdca157524ef0
Parents: f5ffcb4
Author: Kiyoshi Mizumaru 
Authored: Tue Aug 28 04:32:41 2018 +0900
Committer: Alexey Serbin 
Committed: Mon Aug 27 20:06:41 2018 +

--
 src/kudu/gutil/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/bb78520a/src/kudu/gutil/CMakeLists.txt
--
diff --git a/src/kudu/gutil/CMakeLists.txt b/src/kudu/gutil/CMakeLists.txt
index 093cc1b..787e6db 100644
--- a/src/kudu/gutil/CMakeLists.txt
+++ b/src/kudu/gutil/CMakeLists.txt
@@ -54,12 +54,12 @@ set(GUTIL_SRCS
 
 set(GUTIL_LIBS
   glog
-  unwind
   protobuf)
 
 if (NOT APPLE)
   set(GUTIL_LIBS
 ${GUTIL_LIBS}
+unwind
 rt) # clock_gettime() requires -lrt
 endif()
 

kudu git commit: KUDU-2560: exported thirdparty library variants should use exported deps

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master b3eb06a01 -> 1d9ce5625


KUDU-2560: exported thirdparty library variants should use exported deps

This addresses a longstanding bug in ADD_THIRDPARTY_LIB that caused
"exported" thirdparty library targets to express their own thirdparty
dependencies in terms of regular (i.e. non-exported) targets.

The first time this mattered was in commit d55df3c6e, where libunwind was
added as a dependency of glog. This caused the exported client library, by
virtue of its glog dependency, to depend on 'unwind' (shared object) instead
of on 'unwind_exported' (static archive). In most systems the linker elided
that dependency because it was unnecessary, but it is preserved by the
linker in devtoolset3. That causes client_examples-test to fail if the
system doesn't have libunwind installed.

While I was there I removed some usages of DEPS that were not actually used.

Change-Id: I28dd499b152d4eadd9b5cf6575650a1a3db163f1
Reviewed-on: http://gerrit.cloudera.org:8080/11354
Reviewed-by: Dan Burkert 
Tested-by: Kudu Jenkins
Reviewed-by: Andrew Wong 
Reviewed-by: Alexey Serbin 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/1d9ce562
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/1d9ce562
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/1d9ce562

Branch: refs/heads/master
Commit: 1d9ce56251bb41e21d58cb5b8904ba0aa35110be
Parents: b3eb06a
Author: Adar Dembo 
Authored: Wed Aug 29 15:14:35 2018 -0700
Committer: Alexey Serbin 
Committed: Thu Aug 30 01:16:30 2018 +

--
 CMakeLists.txt   | 16 
 cmake_modules/FindPmem.cmake |  1 -
 2 files changed, 8 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/1d9ce562/CMakeLists.txt
--
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 0cd6bc1..d7821e8 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -942,7 +942,7 @@ function(ADD_THIRDPARTY_LIB LIB_NAME)
   # A shared object exported variant will force any "leaf" library that
   # transitively depends on it to also depend on it at runtime; this is
   # desirable for some libraries (e.g. cyrus_sasl).
-  set(LIB_NAME_EXPORTED ${LIB_NAME}_exported)
+  set(LIB_NAME_EXPORTED "${LIB_NAME}_exported")
   if(ARG_STATIC_LIB)
 add_library(${LIB_NAME_EXPORTED} STATIC IMPORTED)
 set_target_properties(${LIB_NAME_EXPORTED}
@@ -953,8 +953,11 @@ function(ADD_THIRDPARTY_LIB LIB_NAME)
   PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}")
   endif()
   if(ARG_DEPS)
+foreach(DEP ${ARG_DEPS})
+  list(APPEND EXPORTED_DEPS "${DEP}_exported")
+endforeach()
 set_target_properties(${LIB_NAME_EXPORTED}
-  PROPERTIES IMPORTED_LINK_INTERFACE_LIBRARIES "${ARG_DEPS}")
+  PROPERTIES IMPORTED_LINK_INTERFACE_LIBRARIES "${EXPORTED_DEPS}")
   endif()
 endfunction()
 
@@ -974,15 +977,13 @@ set(CMAKE_PREFIX_PATH
 find_package(CyrusSASL REQUIRED)
 include_directories(SYSTEM ${CYRUS_SASL_INCLUDE_DIR})
 ADD_THIRDPARTY_LIB(cyrus_sasl
-  SHARED_LIB "${CYRUS_SASL_SHARED_LIB}"
-  DEPS ${CYRUS_SASL_LIB_DEPS})
+  SHARED_LIB "${CYRUS_SASL_SHARED_LIB}")
 
 ## GSSAPI
 find_package(GSSAPI REQUIRED)
 include_directories(SYSTEM ${GSSAPI_INCLUDE_DIR})
 ADD_THIRDPARTY_LIB(gssapi_krb5
-  SHARED_LIB "${GSSAPI_SHARED_LIB}"
-  DEPS ${GSSAPI_LIB_DEPS})
+  SHARED_LIB "${GSSAPI_SHARED_LIB}")
 
 ## GLog (depends on libunwind)
 find_package(GLog REQUIRED)
@@ -1157,8 +1158,7 @@ ADD_THIRDPARTY_LIB(pmem
   SHARED_LIB "${PMEM_SHARED_LIB}")
 ADD_THIRDPARTY_LIB(pmemobj
   STATIC_LIB "${PMEMOBJ_STATIC_LIB}"
-  SHARED_LIB "${PMEMOBJ_SHARED_LIB}"
-  DEPS ${PMEMOBJ_DEPS})
+  SHARED_LIB "${PMEMOBJ_SHARED_LIB}")
 endif()
 
 if (EXISTS ${VMEM_SHARED_LIB})

http://git-wip-us.apache.org/repos/asf/kudu/blob/1d9ce562/cmake_modules/FindPmem.cmake
--
diff --git a/cmake_modules/FindPmem.cmake b/cmake_modules/FindPmem.cmake
index 2270e65..a628e2f 100644
--- a/cmake_modules/FindPmem.cmake
+++ b/cmake_modules/FindPmem.cmake
@@ -21,7 +21,6 @@
 #  XXX_STATIC_LIBS, path to *.a
 #  XXX_SHARED_LIBS, path to *.so shared library
 #  PMEM_FOUND, whether PMEM libraries have been found
-#  PMEMOBJ_DEPS, dependencies required for using libpmemobj
 
 find_path(PMEM_INCLUDE_DIR libpmem.h
   NO_CMAKE_SYSTEM_PATH

kudu git commit: [tests] make master-stress-test more stable

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master 1d9ce5625 -> eb15beabd


[tests] make master-stress-test more stable

The master-stress-test has been flaky for some time.  After looking
at those failure closely, I found at least five different issues.

This patch addresses the most prominent one: failures of the test
scenario because of timeout errors in case of TSAN builds.  About 9 out
of 10 failures were due to the issue fixed by this patch.  The timeout
errors were triggered by RPC queue overflow and the timing of master
restarts wrt the retry/back-off pattern used by KuduClient and other
test utility code.

The rest of issues behind the flakiness will be addressed separately.

This patch also introduces rpc_negotiation_timeout as a member for
ExternalMiniClusterOptions: that's to customize connection negotiation
timeout for the cluster's utility messenger.

Some statistics about the flakiness:

before the fix:
  37 out of 256 failed in TSAN build, where almost all failures were
  due to the issues fixed by this patch:
http://dist-test.cloudera.org//job?job_id=aserbin.1535666928.86597

after the fix:
  2 out of 256 failed in TSAN build, where the failure was due to [2],
  which will be addressed separately:
http://dist-test.cloudera.org/job?job_id=aserbin.1535665784.64065

A few of other issues due to which the test is still a bit flaky:
  [1] https://issues.apache.org/jira/browse/KUDU-2561
  [2] https://issues.apache.org/jira/browse/KUDU-2564
  [3] https://issues.apache.org/jira/browse/HIVE-19874

By my understanding, Dan found [3] to be the reason behind one type
of HMS-related failures; and there two more to evaluate.

Change-Id: I6b30d8afd4a24acdbd96481cadeaf8f6a9475adf
Reviewed-on: http://gerrit.cloudera.org:8080/11364
Tested-by: Kudu Jenkins
Reviewed-by: Adar Dembo 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/eb15beab
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/eb15beab
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/eb15beab

Branch: refs/heads/master
Commit: eb15beabd2d4d86649a35dcd027e1749cc695866
Parents: 1d9ce56
Author: Alexey Serbin 
Authored: Wed Aug 29 16:52:09 2018 -0700
Committer: Alexey Serbin 
Committed: Fri Aug 31 18:16:44 2018 +

--
 .../integration-tests/master-stress-test.cc | 117 +--
 src/kudu/mini-cluster/external_mini_cluster.cc  |   5 +-
 src/kudu/mini-cluster/external_mini_cluster.h   |   6 +
 3 files changed, 93 insertions(+), 35 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/eb15beab/src/kudu/integration-tests/master-stress-test.cc
--
diff --git a/src/kudu/integration-tests/master-stress-test.cc 
b/src/kudu/integration-tests/master-stress-test.cc
index 38aa71f..86d37cf 100644
--- a/src/kudu/integration-tests/master-stress-test.cc
+++ b/src/kudu/integration-tests/master-stress-test.cc
@@ -26,6 +26,7 @@
 #include 
 
 #include 
+#include 
 #include 
 #include 
 
@@ -58,6 +59,7 @@
 #include "kudu/util/test_macros.h"
 #include "kudu/util/test_util.h"
 
+DECLARE_int64(timeout_ms);
 DEFINE_int32(num_create_table_threads, 4,
 "Number of threads that should create tables");
 DEFINE_int32(num_alter_table_threads, 2,
@@ -66,36 +68,40 @@ DEFINE_int32(num_delete_table_threads, 2,
 "Number of threads that should delete tables");
 DEFINE_int32(num_replace_tablet_threads, 2,
 "Number of threads that should replace tablets");
-DEFINE_int32(num_seconds_to_run, 5,
+DEFINE_int32(num_seconds_to_run, 10,
  "Number of seconds that the test should run");
 
-namespace kudu {
-
-using client::KuduClient;
-using client::KuduClientBuilder;
-using client::KuduColumnSchema;
-using client::KuduSchema;
-using client::KuduSchemaBuilder;
-using client::KuduTableAlterer;
-using client::KuduTableCreator;
-using cluster::ExternalMaster;
-using cluster::ExternalMiniCluster;
-using cluster::ExternalMiniClusterOptions;
-using master::MasterServiceProxy;
-using master::ListTablesRequestPB;
-using master::ListTablesResponsePB;
-using rpc::Messenger;
-using rpc::MessengerBuilder;
-using rpc::RpcController;
+using kudu::client::KuduClient;
+using kudu::client::KuduClientBuilder;
+using kudu::client::KuduColumnSchema;
+using kudu::client::KuduSchema;
+using kudu::client::KuduSchemaBuilder;
+using kudu::client::KuduTableAlterer;
+using kudu::client::KuduTableCreator;
+using kudu::cluster::ExternalMaster;
+using kudu::cluster::ExternalMiniCluster;
+using kudu::cluster::ExternalMiniClusterOptions;
+using kudu::itest::ListTablets;
+using kudu::master::ListTablesRequestPB;
+using kudu::master::ListTablesResponsePB;
+using kudu::master::ReplaceTabletReq

[2/2] kudu git commit: test: quell warning in disk_failure-itest

[alexey]

test: quell warning in disk_failure-itest

disk_failure-itest will yield the following warning when built:

../../src/kudu/integration-tests/disk_failure-itest.cc:246:3: warning: control 
may reach end of non-void function [-Wreturn-type]

Change-Id: If9cd6beb9855b0d22cf5fbb74c4d253ff84f4bca
Reviewed-on: http://gerrit.cloudera.org:8080/11385
Tested-by: Kudu Jenkins
Reviewed-by: Alexey Serbin 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/a58867c5
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/a58867c5
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/a58867c5

Branch: refs/heads/master
Commit: a58867c59c0ecbdd2c74d8ac6744b7b3cd304a36
Parents: 060d7b0
Author: Andrew Wong 
Authored: Tue Sep 4 12:27:04 2018 -0700
Committer: Alexey Serbin 
Committed: Tue Sep 4 20:06:18 2018 +

--
 src/kudu/integration-tests/disk_failure-itest.cc | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/a58867c5/src/kudu/integration-tests/disk_failure-itest.cc
--
diff --git a/src/kudu/integration-tests/disk_failure-itest.cc 
b/src/kudu/integration-tests/disk_failure-itest.cc
index 111ccf4..7123820 100644
--- a/src/kudu/integration-tests/disk_failure-itest.cc
+++ b/src/kudu/integration-tests/disk_failure-itest.cc
@@ -243,6 +243,7 @@ class DiskErrorITest : public ExternalMiniClusterITestBase,
   };
   return SetFlags(cluster_->tablet_server(i), recovery_flags);
 }
+return Status::OK();
   }
 
   // Waits for the number of failed tablets on the tablet server to reach

[1/2] kudu git commit: [tablet] Add tablet metric for number of diskrowsets

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master cf6927cb1 -> a58867c59


[tablet] Add tablet metric for number of diskrowsets

Change-Id: Ifb1ca202c281da07007eff1f555af2eef9b2393e
Reviewed-on: http://gerrit.cloudera.org:8080/11378
Reviewed-by: Adar Dembo 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/060d7b0f
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/060d7b0f
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/060d7b0f

Branch: refs/heads/master
Commit: 060d7b0f59b287c18c0690a7795ae225a1f534ee
Parents: cf6927c
Author: Will Berkeley 
Authored: Fri Aug 31 17:06:54 2018 -0700
Committer: Will Berkeley 
Committed: Tue Sep 4 18:23:11 2018 +

--
 src/kudu/tablet/tablet.cc  |  8 +++-
 src/kudu/tserver/tablet_server-test.cc | 19 +++
 2 files changed, 26 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/060d7b0f/src/kudu/tablet/tablet.cc
--
diff --git a/src/kudu/tablet/tablet.cc b/src/kudu/tablet/tablet.cc
index f3a5143..7a44b91 100644
--- a/src/kudu/tablet/tablet.cc
+++ b/src/kudu/tablet/tablet.cc
@@ -170,6 +170,9 @@ METRIC_DEFINE_gauge_size(tablet, memrowset_size, "MemRowSet 
Memory Usage",
 METRIC_DEFINE_gauge_size(tablet, on_disk_data_size, "Tablet Data Size On Disk",
  kudu::MetricUnit::kBytes,
  "Space used by this tablet's data blocks.");
+METRIC_DEFINE_gauge_size(tablet, num_rowsets_on_disk, "Tablet Number of 
Rowsets on Disk",
+ kudu::MetricUnit::kUnits,
+ "Number of diskrowsets in this tablet");
 
 using kudu::MaintenanceManager;
 using kudu::clock::HybridClock;
@@ -236,6 +239,9 @@ Tablet::Tablet(scoped_refptr metadata,
 METRIC_on_disk_data_size.InstantiateFunctionGauge(
   metric_entity_, Bind(&Tablet::OnDiskDataSize, Unretained(this)))
   ->AutoDetach(&metric_detacher_);
+METRIC_num_rowsets_on_disk.InstantiateFunctionGauge(
+  metric_entity_, Bind(&Tablet::num_rowsets, Unretained(this)))
+  ->AutoDetach(&metric_detacher_);
   }
 
   if (FLAGS_tablet_throttler_rpc_per_sec > 0 || 
FLAGS_tablet_throttler_bytes_per_sec > 0) {
@@ -2250,7 +2256,7 @@ int64_t Tablet::CountRedoDeltasForTests() const {
 
 size_t Tablet::num_rowsets() const {
   shared_lock l(component_lock_);
-  return components_->rowsets->all_rowsets().size();
+  return components_ ? components_->rowsets->all_rowsets().size() : 0;
 }
 
 void Tablet::PrintRSLayout(ostream* o) {

http://git-wip-us.apache.org/repos/asf/kudu/blob/060d7b0f/src/kudu/tserver/tablet_server-test.cc
--
diff --git a/src/kudu/tserver/tablet_server-test.cc 
b/src/kudu/tserver/tablet_server-test.cc
index 671949f..084da03 100644
--- a/src/kudu/tserver/tablet_server-test.cc
+++ b/src/kudu/tserver/tablet_server-test.cc
@@ -169,6 +169,7 @@ METRIC_DECLARE_gauge_uint64(log_block_manager_containers);
 METRIC_DECLARE_counter(log_block_manager_holes_punched);
 METRIC_DECLARE_gauge_size(active_scanners);
 METRIC_DECLARE_gauge_size(tablet_active_scanners);
+METRIC_DECLARE_gauge_size(num_rowsets_on_disk);
 
 namespace kudu {
 
@@ -3323,6 +3324,24 @@ TEST_F(TabletServerTest, 
TestNoMetricsForTombstonedTablet) {
   }
 }
 
+TEST_F(TabletServerTest, TestTabletNumberOfDiskRowSetsMetric) {
+  scoped_refptr tablet;
+  
ASSERT_TRUE(mini_server_->server()->tablet_manager()->LookupTablet(kTabletId, 
&tablet));
+  ASSERT_TRUE(tablet->tablet()->GetMetricEntity());
+
+  // We don't care what the function is, since the metric is already 
instantiated.
+  auto num_diskrowsets = METRIC_num_rowsets_on_disk.InstantiateFunctionGauge(
+  tablet->tablet()->GetMetricEntity(), Callback());
+
+  // No data, no diskrowsets.
+  ASSERT_EQ(0, num_diskrowsets->value());
+
+  // Insert a row and flush. There should be 1 diskrowset.
+  ASSERT_NO_FATAL_FAILURE(InsertTestRowsRemote(0, 1, 1));
+  ASSERT_OK(tablet->tablet()->Flush());
+  ASSERT_EQ(1, num_diskrowsets->value());
+}
+
 // Test ensuring that when rowset min/max keys are stored with and read from
 // the rowset metadata, the tablet server doesn't read any blocks when
 // bootstrapping.

[1/2] kudu git commit: [tools] Fix help text for replica_type

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master 257dd585a -> 885d8bdaf


[tools] Fix help text for replica_type

'kudu tablet change_config [add_replica|change_replica_type]' take an
argument for 'replica type', which the help text said had to be "VOTER"
or "NON-VOTER", but actually it has to be "VOTER" or "NON_VOTER",
because it is parsed into an enum by a generated PB function, and the PB
enum names are "VOTER" and "NON_VOTER". This fixes the help text.

Change-Id: Icb43e0eaffbfa17967103c7aebe15ea0a06c5598
Reviewed-on: http://gerrit.cloudera.org:8080/11400
Tested-by: Kudu Jenkins
Reviewed-by: Alexey Serbin 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/0653dcde
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/0653dcde
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/0653dcde

Branch: refs/heads/master
Commit: 0653dcdebb8a5dbe0463341c8f12bbc5ae14fec8
Parents: 257dd58
Author: Will Berkeley 
Authored: Fri Sep 7 11:34:44 2018 -0700
Committer: Alexey Serbin 
Committed: Fri Sep 7 20:29:31 2018 +

--
 src/kudu/tools/tool_action_tablet.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/0653dcde/src/kudu/tools/tool_action_tablet.cc
--
diff --git a/src/kudu/tools/tool_action_tablet.cc 
b/src/kudu/tools/tool_action_tablet.cc
index bda4637..da0209b 100644
--- a/src/kudu/tools/tool_action_tablet.cc
+++ b/src/kudu/tools/tool_action_tablet.cc
@@ -242,7 +242,7 @@ unique_ptr BuildTabletMode() {
   .AddRequiredParameter({ kTsUuidArg,
   "UUID of the tablet server that should host the 
new replica" })
   .AddRequiredParameter(
-  { kReplicaTypeArg, "New replica's type. Must be VOTER or NON-VOTER."
+  { kReplicaTypeArg, "New replica's type. Must be VOTER or NON_VOTER."
   })
   .Build();
 
@@ -255,7 +255,7 @@ unique_ptr BuildTabletMode() {
   .AddRequiredParameter({ kTsUuidArg,
   "UUID of the tablet server hosting the existing 
replica" })
   .AddRequiredParameter(
-  { kReplicaTypeArg, "Existing replica's new type. Must be VOTER or 
NON-VOTER."
+  { kReplicaTypeArg, "Existing replica's new type. Must be VOTER or 
NON_VOTER."
   })
   .Build();
 

[2/2] kudu git commit: bitmap: add equality method

[alexey]

bitmap: add equality method

This is useful for tests.

Change-Id: If53e41250435501e158bdbc076dc8e89ea153256
Reviewed-on: http://gerrit.cloudera.org:8080/11266
Tested-by: Kudu Jenkins
Reviewed-by: Alexey Serbin 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/885d8bda
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/885d8bda
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/885d8bda

Branch: refs/heads/master
Commit: 885d8bdafd46b7a4d48cabb8ae929431b78b7cab
Parents: 0653dcd
Author: Adar Dembo 
Authored: Fri Aug 17 17:03:22 2018 -0700
Committer: Alexey Serbin 
Committed: Fri Sep 7 21:07:38 2018 +

--
 src/kudu/util/bitmap-test.cc | 49 +--
 src/kudu/util/bitmap.h   | 21 +
 2 files changed, 68 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/885d8bda/src/kudu/util/bitmap-test.cc
--
diff --git a/src/kudu/util/bitmap-test.cc b/src/kudu/util/bitmap-test.cc
index 089ed3b..18ccb27 100644
--- a/src/kudu/util/bitmap-test.cc
+++ b/src/kudu/util/bitmap-test.cc
@@ -15,6 +15,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
+#include "kudu/util/bitmap.h"
+
 #include 
 #include 
 #include 
@@ -22,7 +24,6 @@
 #include 
 
 #include "kudu/gutil/strings/join.h"
-#include "kudu/util/bitmap.h"
 
 namespace kudu {
 
@@ -73,7 +74,7 @@ TEST(TestBitMap, TestIteration2) {
   ASSERT_EQ("1", JoinElements(read_back, ","));
 }
 
-TEST(TestBitmap, TestSetAndTestBits) {
+TEST(TestBitMap, TestSetAndTestBits) {
   uint8_t bm[1];
   memset(bm, 0, sizeof(bm));
 
@@ -227,4 +228,48 @@ TEST(TestBitMap, TestBitmapIteration) {
   ASSERT_EQ(expected_sizes[i], size);
 }
 
+TEST(TestBitMap, TestEquals) {
+  uint8_t bm1[8] = { 0 };
+  uint8_t bm2[8] = { 0 };
+  size_t num_bits = sizeof(bm1) * 8;
+  ASSERT_TRUE(BitmapEquals(bm1, bm2, num_bits));
+
+  // Loop over each bit starting from the end and going to the beginning. In
+  // each iteration, set the bit in one bitmap and verify that although the two
+  // bitmaps aren't equal, if we were to ignore the changed bits, they are 
still equal.
+  for (int i = num_bits - 1; i >= 0; i--) {
+SCOPED_TRACE(i);
+BitmapChange(bm1, i, true);
+ASSERT_FALSE(BitmapEquals(bm1, bm2, num_bits));
+ASSERT_TRUE(BitmapEquals(bm1, bm2, i));
+  }
+
+  // Now loop in the other direction, setting the second bitmap bit by bit.
+  // As before, if we consider the bitmaps in their entirety, they're not 
equal,
+  // but if we consider just the sequences where both are set, they are equal.
+  for (int i = 0; i < num_bits - 1; i++) {
+SCOPED_TRACE(i);
+BitmapChange(bm2, i, true);
+ASSERT_FALSE(BitmapEquals(bm1, bm2, num_bits));
+ASSERT_TRUE(BitmapEquals(bm1, bm2, i + 1));
+  }
+
+  // If we set the very last bit, both bitmaps are now equal in their entirety.
+  BitmapChange(bm2, num_bits - 1, true);
+  ASSERT_TRUE(BitmapEquals(bm1, bm2, num_bits));
+
+  // Test equality on overlapped bitmaps (i.e. a single underlying bitmap, two
+  // subsequences of which are considered to be two separate bitmaps).
+
+  // Set every third bit; the rest are unset.
+  uint8_t bm3[8] = { 0 };
+  for (int i = 0; i < num_bits; i += 3) {
+BitmapChange(bm3, i, true);
+  }
+
+  ASSERT_TRUE(BitmapEquals(bm3, bm3, num_bits)); // fully overlapped
+  ASSERT_FALSE(BitmapEquals(bm3, bm3 + 1, num_bits - 8)); // off by one byte
+  ASSERT_TRUE(BitmapEquals(bm3, bm3 + 3, num_bits - 24)); // off by three bytes
+}
+
 } // namespace kudu

http://git-wip-us.apache.org/repos/asf/kudu/blob/885d8bda/src/kudu/util/bitmap.h
--
diff --git a/src/kudu/util/bitmap.h b/src/kudu/util/bitmap.h
index d9f5260..947a6c6 100644
--- a/src/kudu/util/bitmap.h
+++ b/src/kudu/util/bitmap.h
@@ -28,6 +28,7 @@
 
 #include "kudu/gutil/bits.h"
 #include "kudu/gutil/port.h"
+#include "kudu/gutil/strings/fastmem.h"
 
 namespace kudu {
 
@@ -98,6 +99,26 @@ inline bool BitmapIsAllZero(const uint8_t *bitmap, size_t 
offset, size_t bitmap_
   return !BitmapFindFirstSet(bitmap, offset, bitmap_size, &idx);
 }
 
+// Returns true if the two bitmaps are equal.
+//
+// It is assumed that both bitmaps have 'bitmap_size' number of bits.
+inline bool BitmapEquals(const uint8_t* bm1, const uint8_t* bm2, size_t 
bitmap_size) {
+  // Use memeq() to check all of the full bytes.
+  size_t num_full_bytes = bitmap_size >> 3;
+  if (!strings::memeq(bm1, bm2, num_full_bytes)) {
+return false;
+  }
+
+  // Check any remaining bits in one extra operation.
+  size_t num_rema

[2/2] kudu git commit: [catalog_manager] updated warning message

[alexey]

[catalog_manager] updated warning message

Updated the warning message logged upon a failure to allocate
an extra replica for a tablet: don't call the new replica a replacement
because AsyncAddReplicaTask is used not only in re-replication scenarios,
but in replica movement scenarios as well.

Also, do not mention unsetting --raft_prepare_replacement_before_eviction
since the 3-2-3 replica management scheme is deprecated at this point
and is no better in the case of a whole tablet server failure.

This is a follow-up to 1fcce42200d22597e7e69baa7232b4de93d5e2a3.

There are no functional changes in this patch.

Change-Id: Ifb6905dc1870acd34553187594008ba34781ce6d
Reviewed-on: http://gerrit.cloudera.org:8080/11402
Tested-by: Kudu Jenkins
Reviewed-by: Adar Dembo 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/953a09b8
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/953a09b8
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/953a09b8

Branch: refs/heads/master
Commit: 953a09b826b4a0ccd379480f7aac441186f8bacb
Parents: e357051
Author: Alexey Serbin 
Authored: Fri Sep 7 16:35:33 2018 -0700
Committer: Alexey Serbin 
Committed: Mon Sep 10 20:35:04 2018 +

--
 src/kudu/master/catalog_manager.cc | 27 +++
 1 file changed, 11 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/953a09b8/src/kudu/master/catalog_manager.cc
--
diff --git a/src/kudu/master/catalog_manager.cc 
b/src/kudu/master/catalog_manager.cc
index 0dd8003..27e9722 100644
--- a/src/kudu/master/catalog_manager.cc
+++ b/src/kudu/master/catalog_manager.cc
@@ -3475,14 +3475,14 @@ bool AsyncAddReplicaTask::SendRequest(int attempt) {
 
   auto replacement_replica = SelectReplica(ts_descs, excluded, rng_);
   if (PREDICT_FALSE(!replacement_replica)) {
-auto msg = Substitute("no candidate replacement replica found for tablet 
$0",
+auto msg = Substitute("no extra replica candidate found for tablet $0",
   tablet_->ToString());
-// Check whether it's a situation when a replacement replica cannot be 
found
+// Check whether it's a situation when an extra replica cannot be found
 // due to an inconsistency in cluster configuration. If the tablet has the
-// replication factor of N, and the cluster is configured to use 
N->(N+1)->N
-// replication scheme (see --raft_prepare_replacement_before_eviction 
flag),
-// at least N+1 tablet servers should be registered to find a place
-// for a replacement replica.
+// replication factor of N, and the cluster is using the N->(N+1)->N
+// replica management scheme (see 
--raft_prepare_replacement_before_eviction
+// flag), at least N+1 tablet servers should be registered to find a place
+// for an extra replica.
 TSDescriptorVector all_descriptors;
 master_->ts_manager()->GetAllDescriptors(&all_descriptors);
 const auto num_tservers_registered = all_descriptors.size();
@@ -3492,21 +3492,16 @@ bool AsyncAddReplicaTask::SendRequest(int attempt) {
   TableMetadataLock l(tablet_->table().get(), LockMode::READ);
   replication_factor = 
tablet_->table()->metadata().state().pb.num_replicas();
 }
-DCHECK_GE(replication_factor, 0);
+DCHECK_GE(replication_factor, 1);
 const auto num_tservers_needed =
 FLAGS_raft_prepare_replacement_before_eviction ? replication_factor + 1
: replication_factor;
 if (num_tservers_registered < num_tservers_needed) {
   msg += Substitute(
-  "; the total number of registered tablet servers ($0) does not allow 
"
-  "for replacement of the failed replica: at least $1 tablet servers "
-  "are required", num_tservers_registered, num_tservers_needed);
-  if (FLAGS_raft_prepare_replacement_before_eviction &&
-  num_tservers_registered == replication_factor) {
-msg +=
-  "; consider either adding an additional tablet server or running "
-  "the cluster with --raft_prepare_replacement_before_eviction=false";
-  }
+  ": the total number of registered tablet servers ($0) does not allow 
"
+  "for adding an extra replica; consider bringing up more "
+  "to have at least $1 tablet servers up and running",
+  num_tservers_registered, num_tservers_needed);
 }
 KLOG_EVERY_N_SECS(WARNING, 60) << LogPrefix() << msg;
 return false;

[1/2] kudu git commit: [docs] Add "one client only" best practice for kudu-spark

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master b552d9118 -> 953a09b82


[docs] Add "one client only" best practice for kudu-spark

Change-Id: Ibaf369315b8627674ba64e6418d153568ded6fe8
Reviewed-on: http://gerrit.cloudera.org:8080/11409
Tested-by: Will Berkeley 
Reviewed-by: Alexey Serbin 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/e3570519
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/e3570519
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/e3570519

Branch: refs/heads/master
Commit: e3570519b200a0ffbd713798bc8aabd6f36ed3b7
Parents: b552d91
Author: Will Berkeley 
Authored: Mon Sep 10 10:45:30 2018 -0700
Committer: Will Berkeley 
Committed: Mon Sep 10 18:43:43 2018 +

--
 docs/developing.adoc | 17 +
 1 file changed, 17 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/e3570519/docs/developing.adoc
--
diff --git a/docs/developing.adoc b/docs/developing.adoc
index 98db2ba..49d8c7e 100644
--- a/docs/developing.adoc
+++ b/docs/developing.adoc
@@ -217,6 +217,23 @@ mode, the submitting user must have an active Kerberos 
ticket granted through
 name and keytab location must be provided through the `--principal` and
 `--keytab` arguments to `spark2-submit`.
 
+=== Spark Integration Best Practices
+
+ Avoid multiple Kudu clients per cluster.
+
+One common Kudu-Spark coding error is instantiating extra `KuduClient` objects.
+In kudu-spark, a `KuduClient` is owned by the `KuduContext`. Spark application 
code
+should not create another `KuduClient` connecting to the same cluster. Instead,
+application code should use the `KuduContext` to access a `KuduClient` using
+`KuduContext#syncClient`.
+
+To diagnose multiple `KuduClient` instances in a Spark job, look for signs in
+the logs of the master being overloaded by many `GetTableLocations` or
+`GetTabletLocations` requests coming from different clients, usually around the
+same time. This symptom is especially likely in Spark Streaming code,
+where creating a `KuduClient` per task will result in periodic waves of master
+requests from new clients.
+
 === Spark Integration Known Issues and Limitations
 
 - Spark 2.2+ requires Java 8 at runtime even though Kudu Spark 2.x integration

[2/2] kudu git commit: [jepsen] set default admin operation timeout to 2 minutes

[alexey]

[jepsen] set default admin operation timeout to 2 minutes

Increased the default admin operation timeout for the Kudu client
used in Jepsen tests up to 2 minutes.  That's to work around slow DNS
resolutions (sometimes up to 30 seconds) that happen intermittently
while running the tests at VMs provisioned in GCE.

Change-Id: Iaa9ade5cf38b4f3247749fa33bd40ada9616f21f
Reviewed-on: http://gerrit.cloudera.org:8080/11405
Reviewed-by: Hao Hao 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/edde4beb
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/edde4beb
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/edde4beb

Branch: refs/heads/master
Commit: edde4bebea8e309a43e3316f78c45df6e6c8c9fe
Parents: 38a4847
Author: Alexey Serbin 
Authored: Fri Sep 7 18:33:30 2018 -0700
Committer: Alexey Serbin 
Committed: Mon Sep 10 22:18:21 2018 +

--
 java/kudu-jepsen/src/main/clojure/jepsen/kudu/client.clj | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/edde4beb/java/kudu-jepsen/src/main/clojure/jepsen/kudu/client.clj
--
diff --git a/java/kudu-jepsen/src/main/clojure/jepsen/kudu/client.clj 
b/java/kudu-jepsen/src/main/clojure/jepsen/kudu/client.clj
index 65b6e5a..c50201c 100644
--- a/java/kudu-jepsen/src/main/clojure/jepsen/kudu/client.clj
+++ b/java/kudu-jepsen/src/main/clojure/jepsen/kudu/client.clj
@@ -41,9 +41,9 @@
 (defn sync-client
   "Builds and returns a new synchronous Kudu client."
   [master-addresses]
-  (let [builder (new KuduClient$KuduClientBuilder master-addresses)
-client (. builder build)]
-client))
+  (let [builder (new KuduClient$KuduClientBuilder master-addresses)]
+(.defaultAdminOperationTimeoutMs builder 12)
+(.build builder)))
 
 (defn close-client
   [sync-client]

[1/2] kudu git commit: [gutil] add LookupOrEmplace()

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master 953a09b82 -> edde4bebe


[gutil] add LookupOrEmplace()

Added LookupOrEmplace() which is similar to LookupOrInsert() but
uses the r-value mechanics.  Also, added unit tests for new
functionality.

Change-Id: I3a177f410c1d4ed33e6f44cb7d6c33d6141f84fc
Reviewed-on: http://gerrit.cloudera.org:8080/11401
Reviewed-by: Adar Dembo 
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/38a48476
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/38a48476
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/38a48476

Branch: refs/heads/master
Commit: 38a48476ee98e180d09e71cd0547119ff43423e4
Parents: 953a09b
Author: Alexey Serbin 
Authored: Fri Sep 7 13:50:29 2018 -0700
Committer: Alexey Serbin 
Committed: Mon Sep 10 21:18:25 2018 +

--
 src/kudu/gutil/map-util.h  | 15 ++
 src/kudu/util/map-util-test.cc | 59 +
 2 files changed, 74 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/38a48476/src/kudu/gutil/map-util.h
--
diff --git a/src/kudu/gutil/map-util.h b/src/kudu/gutil/map-util.h
index 82d266f..ef6baae 100644
--- a/src/kudu/gutil/map-util.h
+++ b/src/kudu/gutil/map-util.h
@@ -495,6 +495,21 @@ LookupOrInsert(Collection* const collection,
   collection, typename Collection::value_type(key, value));
 }
 
+// It's similar to LookupOrInsert() but uses the emplace and r-value mechanics
+// to achieve the desired results. The constructor of the new element is called
+// with exactly the same arguments as supplied to emplace, forwarded via
+// std::forward(args). The element may be constructed even if there
+// already is an element with the same key in the container, in which case the
+// newly constructed element will be destroyed immediately.
+// For details, see
+//   https://en.cppreference.com/w/cpp/container/map/emplace
+//   https://en.cppreference.com/w/cpp/container/unordered_map/emplace
+template 
+typename Collection::mapped_type&
+LookupOrEmplace(Collection* const collection, Args&&... args) {
+  return collection->emplace(std::forward(args)...).first->second;
+}
+
 // Counts the number of equivalent elements in the given "sequence", and stores
 // the results in "count_map" with element as the key and count as the value.
 //

http://git-wip-us.apache.org/repos/asf/kudu/blob/38a48476/src/kudu/util/map-util-test.cc
--
diff --git a/src/kudu/util/map-util-test.cc b/src/kudu/util/map-util-test.cc
index b074007..78a3484 100644
--- a/src/kudu/util/map-util-test.cc
+++ b/src/kudu/util/map-util-test.cc
@@ -123,4 +123,63 @@ TEST(EmplaceTest, TestEmplace) {
   ASSERT_EQ("foobar", *FindOrDie(my_map, key2));
 }
 
+TEST(LookupOrEmplaceTest, IntMap) {
+  const string key = "mega";
+  map int_map;
+
+  {
+const auto& val = LookupOrEmplace(&int_map, key, 0);
+ASSERT_EQ(0, val);
+auto* val_ptr = FindOrNull(int_map, key);
+ASSERT_NE(nullptr, val_ptr);
+ASSERT_EQ(0, *val_ptr);
+  }
+
+  {
+auto& val = LookupOrEmplace(&int_map, key, 10);
+ASSERT_EQ(0, val);
+++val;
+auto* val_ptr = FindOrNull(int_map, key);
+ASSERT_NE(nullptr, val_ptr);
+ASSERT_EQ(1, *val_ptr);
+  }
+
+  {
+LookupOrEmplace(&int_map, key, 100) += 1000;
+auto* val_ptr = FindOrNull(int_map, key);
+ASSERT_NE(nullptr, val_ptr);
+ASSERT_EQ(1001, *val_ptr);
+  }
+}
+
+TEST(LookupOrEmplaceTest, UniquePtrMap) {
+  constexpr int key = 0;
+  const string ref_str = "turbo";
+  map> uptr_map;
+
+  {
+unique_ptr val(new string(ref_str));
+const auto& lookup_val = LookupOrEmplace(&uptr_map, key, std::move(val));
+ASSERT_EQ(nullptr, val.get());
+ASSERT_NE(nullptr, lookup_val.get());
+ASSERT_EQ(ref_str, *lookup_val);
+  }
+
+  {
+unique_ptr val(new string("giga"));
+auto& lookup_val = LookupOrEmplace(&uptr_map, key, std::move(val));
+ASSERT_NE(nullptr, lookup_val.get());
+ASSERT_EQ(ref_str, *lookup_val);
+// Update the stored value.
+*lookup_val = "giga";
+  }
+
+  {
+unique_ptr val(new string(ref_str));
+const auto& lookup_val = LookupOrEmplace(&uptr_map, key, std::move(val));
+ASSERT_NE(nullptr, lookup_val.get());
+ASSERT_EQ("giga", *lookup_val);
+  }
+}
+
 } // namespace kudu

kudu git commit: [jepsen] a bunch of minor updates on README.adoc

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master edde4bebe -> e709db1f3


[jepsen] a bunch of minor updates on README.adoc

Added a link to the reference script for building Kudu and running
Jepsen tests.  Added information on how to quickly locate corresponding
sections in the Jepsen output in case if an error happened during Jepsen
tests run.  Updated the command to build the kudu-jepnsen package.

This patch does not contain any functional changes.

Change-Id: Ibd4c7a522829e774f786a85d4bdfa01d0be553f5
Reviewed-on: http://gerrit.cloudera.org:8080/11414
Tested-by: Kudu Jenkins
Reviewed-by: Hao Hao 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/e709db1f
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/e709db1f
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/e709db1f

Branch: refs/heads/master
Commit: e709db1f379e2375841a2c6357cd3b6a6ce4618b
Parents: edde4be
Author: Alexey Serbin 
Authored: Mon Sep 10 16:27:19 2018 -0700
Committer: Alexey Serbin 
Committed: Tue Sep 11 05:34:53 2018 +

--
 java/kudu-jepsen/README.adoc | 24 +++-
 1 file changed, 19 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/e709db1f/java/kudu-jepsen/README.adoc
--
diff --git a/java/kudu-jepsen/README.adoc b/java/kudu-jepsen/README.adoc
index 7dfaaee..fc7663b 100644
--- a/java/kudu-jepsen/README.adoc
+++ b/java/kudu-jepsen/README.adoc
@@ -47,10 +47,11 @@ To build the library the following components are required:
 * JDK 8
 * Apache Maven version 3.3.6 or higher
 
-To build the project, run in the parent directory (i.e. `$KUDU_HOME/java`)
+To build the project and install necessary Java packages into the local mvn
+repository, run in the parent directory (i.e. `$KUDU_HOME/java`)
 [listing]
 
-$ mvn clean compile test-compile -Pjepsen
+$ mvn -Pjepsen -DskipTests clean install
 
 
 === Running
@@ -101,6 +102,10 @@ In the Jepsen terminology, Kudu master and tserver nodes 
are playing
 *Jepsen DB node* roles. The machine where the above mentioned maven command
 is run plays *Jepsen control node* role.
 
+=== A reference script to build Kudu and run Jepsen tests
+The following link:../../src/kudu/scripts/jepsen.sh[Bourne-again shell script]
+can be used as a reference to build Kudu from source and run Jepsen tests.
+
 === Troubleshooting
 When Jepsen's analysis doesn't find inconsistencies in the history of 
operations
 it outputs the following in the end of a test:
@@ -132,8 +137,16 @@ summary message below reports on 10 errors in 10 tests ran:
 21:41:42 0 failures, 10 errors.
 
 To get more details, take a closer look at the output of `mvn clojure:run`
-or at particular `jepsen.log` files under
-$KUDU_HOME/java/kudu-jepsen/store/rw-register/ directories.
+or at particular `jepsen.log` files in
+`$KUDU_HOME/java/kudu-jepsen/store/rw-register/` directory. A
+quick way to locate the corresponding section in the error log is to search for
+`^ERROR in \(` regex pattern. An example of error message from Jepsen's output:
+[listing]
+
+ERROR in (register-test-tserver-random-halves) (KuduException.java:110)
+expected: (:valid? (:results (jepsen/run! (tcasefun opts
+  actual: org.apache.kudu.client.NonRecoverableException: can not complete 
before timeout: KuduRpc(method=IsCreateTableDone, tablet=null, attempt=28, 
DeadlineTracker(timeout=3, elapsed=28571), ...
+
 
 The latter class represents more serious issue: a manifestation of
 non-linearizable history of operations. This is reported as _failure_ in the
@@ -154,7 +167,8 @@ Analysis invalid! (ﾉಥ益ಥ）ﾉ ┻━┻
 To troubleshoot, first it's necessary to find where the failed test stores
 the results: it should be one of the timestamp-named sub-directories
 (e.g. `20170109T071938.000-0800`) under
-`$KUDU_HOME/java/kudu-jepsen/store/rw-register`. One of the possible ways
+`$KUDU_HOME/java/kudu-jepsen/store/rw-register` in case of a linearizability
+failure in one of the `rw-register` test scenarios. One of the possible ways
 to find the directory:
 [listing]
 

[2/2] kudu git commit: Remove unnecessary friendship

[alexey]

Remove unnecessary friendship

This commit removes unnecessary friendship and declaration in client.h.

Change-Id: Ia7c5f35a8b6a6ec2de1d57bbb4f68b53aae4e700
Reviewed-on: http://gerrit.cloudera.org:8080/11416
Tested-by: Kudu Jenkins
Reviewed-by: Adar Dembo 
Reviewed-by: Alexey Serbin 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/f0716cc5
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/f0716cc5
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/f0716cc5

Branch: refs/heads/master
Commit: f0716cc521554bc6c4b785d339ecf17b8ca983e6
Parents: f02a0c0
Author: Hao Hao 
Authored: Mon Sep 10 17:52:29 2018 -0700
Committer: Alexey Serbin 
Committed: Tue Sep 11 18:15:39 2018 +

--
 src/kudu/client/client.h | 5 -
 1 file changed, 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/f0716cc5/src/kudu/client/client.h
--
diff --git a/src/kudu/client/client.h b/src/kudu/client/client.h
index 640afce..bfc2c2b 100644
--- a/src/kudu/client/client.h
+++ b/src/kudu/client/client.h
@@ -51,8 +51,6 @@ namespace kudu {
 
 class ClientStressTest_TestUniqueClientIds_Test;
 class KuduPartialRow;
-class MasterHmsTest_TestAlterTable_Test;
-class MasterHmsUpgradeTest_TestRenameExistingTables_Test;
 class MonoDelta;
 class PartitionSchema;
 class SecurityUnknownTskTest;
@@ -1229,9 +1227,6 @@ class KUDU_EXPORT KuduTableAlterer {
 
   friend class KuduClient;
 
-  FRIEND_TEST(kudu::MasterHmsTest, TestAlterTable);
-  FRIEND_TEST(kudu::MasterHmsUpgradeTest, TestRenameExistingTables);
-
   KuduTableAlterer(KuduClient* client,
const std::string& name);
 

[1/2] kudu git commit: KUDU-2566: Enhance rowset tree pruning and stop copying strings

[alexey]

Repository: kudu
Updated Branches:
  refs/heads/master e709db1f3 -> f0716cc52


KUDU-2566: Enhance rowset tree pruning and stop copying strings

Three improvements:
1) Support open-ended intervals:
   Previously, the rowset tree could only compute overlap with a
   closed and bounded interval. This changeset adds the ability to
   find overlap for unbounded intervals as well. As a result, tablet
   iterators with a single primary key bound are more efficient
   because they do not iterate over rowsets that don't overlap with
   the key bound.
2) End up fetching one more rowset for the upper bound which is
   exclusive:
   This changeset also adds the ability to query the rowset tree
   using an exclusive upper bound, whereas previously only inclusive
   intervals were supported. This makes scans more efficient since
   primary key upper bounds are passed as exclusive bounds, so now
   rowsets that overlap only in the upper bound are excluded.
3) Perf improvement:
   Using raw slices instead of copying to strings while querying.
   The copying from slices to string is discarded, which could help
   to increase the query performance.

Change-Id: I0e34b4169f93f3519f694c9bf86ca5295c318e79
Reviewed-on: http://gerrit.cloudera.org:8080/11381
Tested-by: Kudu Jenkins
Reviewed-by: Will Berkeley 


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/f02a0c0f
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/f02a0c0f
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/f02a0c0f

Branch: refs/heads/master
Commit: f02a0c0f29b6ef5468137869764119b1a8f8b3ae
Parents: e709db1
Author: helifu 
Authored: Mon Sep 3 19:12:16 2018 +0800
Committer: Will Berkeley 
Committed: Tue Sep 11 16:25:48 2018 +

--
 src/kudu/tablet/rowset_tree-test.cc | 174 +--
 src/kudu/tablet/rowset_tree.cc  |  32 --
 src/kudu/tablet/rowset_tree.h   |  15 ++-
 src/kudu/tablet/tablet.cc   |  22 ++--
 src/kudu/util/interval_tree-inl.h   |  44 
 src/kudu/util/interval_tree-test.cc | 132 +--
 src/kudu/util/interval_tree.h   |  12 ++-
 7 files changed, 349 insertions(+), 82 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/kudu/blob/f02a0c0f/src/kudu/tablet/rowset_tree-test.cc
--
diff --git a/src/kudu/tablet/rowset_tree-test.cc 
b/src/kudu/tablet/rowset_tree-test.cc
index 9cfb7f7..a248a5f 100644
--- a/src/kudu/tablet/rowset_tree-test.cc
+++ b/src/kudu/tablet/rowset_tree-test.cc
@@ -18,15 +18,19 @@
 #include 
 #include 
 #include 
-#include 
 #include 
+#include 
+#include 
+#include 
 #include 
 
+#include 
 #include 
 #include 
 
 #include "kudu/gutil/map-util.h"
 #include "kudu/gutil/stringprintf.h"
+#include "kudu/gutil/strings/substitute.h"
 #include "kudu/tablet/mock-rowsets.h"
 #include "kudu/tablet/rowset.h"
 #include "kudu/tablet/rowset_tree.h"
@@ -39,6 +43,7 @@ using std::shared_ptr;
 using std::string;
 using std::unordered_set;
 using std::vector;
+using strings::Substitute;
 
 namespace kudu { namespace tablet {
 
@@ -86,30 +91,185 @@ TEST_F(TestRowSetTree, TestTree) {
   ASSERT_EQ(vec[3].get(), out[0]); // MemRowSet
   ASSERT_EQ(vec[0].get(), out[1]);
   ASSERT_EQ(vec[1].get(), out[2]);
+
+  // interval [3,4) overlaps 0-5, 3-5 and the MemRowSet
+  out.clear();
+  tree.FindRowSetsIntersectingInterval(Slice("3"), Slice("4"), &out);
+  ASSERT_EQ(3, out.size());
+  ASSERT_EQ(vec[3].get(), out[0]);
+  ASSERT_EQ(vec[0].get(), out[1]);
+  ASSERT_EQ(vec[1].get(), out[2]);
+
+  // interval [0,2) overlaps 0-5 and the MemRowSet
+  out.clear();
+  tree.FindRowSetsIntersectingInterval(Slice("0"), Slice("2"), &out);
+  ASSERT_EQ(2, out.size());
+  ASSERT_EQ(vec[3].get(), out[0]);
+  ASSERT_EQ(vec[0].get(), out[1]);
+
+  // interval [5,7) overlaps 0-5, 3-5, 5-9 and the MemRowSet
+  out.clear();
+  tree.FindRowSetsIntersectingInterval(Slice("5"), Slice("7"), &out);
+  ASSERT_EQ(4, out.size());
+  ASSERT_EQ(vec[3].get(), out[0]);
+  ASSERT_EQ(vec[0].get(), out[1]);
+  ASSERT_EQ(vec[1].get(), out[2]);
+  ASSERT_EQ(vec[2].get(), out[3]);
+
+  // "3" overlaps 0-5, 3-5, and the MemRowSet.
+  out.clear();
+  tree.FindRowSetsWithKeyInRange("3", &out);
+  ASSERT_EQ(3, out.size());
+  ASSERT_EQ(vec[3].get(), out[0]); // MemRowSet
+  ASSERT_EQ(vec[0].get(), out[1]);
+  ASSERT_EQ(vec[1].get(), out[2]);
 
-  // interval (2,4) overlaps 0-5, 3-5 and the MemRowSet
+  // "5" overlaps 0-5, 3-5, 5-9, and the MemRowSet
+  out.clear();
+  tree.FindRowSetsWithKeyInRange("5", &out);
+  ASSERT_EQ(4, out.size());
+  ASSERT_EQ(vec[3].get(), out[0]); // MemRowSet
+  ASSERT_EQ(vec[0].get(), out[1]);
+  ASSERT_EQ(vec[1].get(), out[2]);
+  ASSERT_EQ(vec[2].get(), out[3]);
+
+  // interval [0,5) overlaps 0-5, 3-5, and the MemRowSet
+