kudu git commit: [iwyu_tool.py] fix on the IWYU tool invocation
Repository: kudu Updated Branches: refs/heads/master 4ee3efed5 -> 4935b1b29 [iwyu_tool.py] fix on the IWYU tool invocation Fixed the way include-what-you-use binary is invoked by the iwyu_tool.py wrapper script. Prior to this fix, subprocess.Popen() had the 'shell' argument set to 'True', so any issue with starting the binary was silenced because there would be no OSError exception raised. Overall, if using 'shell=True' for subprocess.Popen(), it wouldn't be feasible to detect that sort of error via examining the exit code of the started process because include-what-you-use always returns exit code 1 regardless of its findings. I also updated the iwyu.sh script to be more robust in handling possible errors, if any. Prior to this fix, the 'iwyu' CMake target returned success if include-what-you-use binary was not in place. This patch fixes that problem and introduces more robust handling of other possible issues while working with the include-what-you-use tool. Change-Id: Ifa4284857583fff6543f65a9c2a71a9e445f39e2 Reviewed-on: http://gerrit.cloudera.org:8080/7989 Reviewed-by: Alexey Serbin Tested-by: Alexey Serbin Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/4935b1b2 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/4935b1b2 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/4935b1b2 Branch: refs/heads/master Commit: 4935b1b298782aabcdac5b8a5c49c3c15d80d8f6 Parents: 4ee3efe Author: Alexey Serbin Authored: Wed Sep 6 18:47:47 2017 -0700 Committer: Alexey Serbin Committed: Thu Sep 7 04:21:55 2017 + -- build-support/iwyu/iwyu.sh | 29 + build-support/iwyu/iwyu_tool.py | 17 + 2 files changed, 30 insertions(+), 16 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/4935b1b2/build-support/iwyu/iwyu.sh -- diff --git a/build-support/iwyu/iwyu.sh b/build-support/iwyu/iwyu.sh index 6c529fc..52e81eb 100755 --- a/build-support/iwyu/iwyu.sh +++ b/build-support/iwyu/iwyu.sh @@ -17,19 +17,26 @@ # specific language governing permissions and limitations # under the License. -ROOT=$(cd $(dirname $BASH_SOURCE)/../..; pwd) +set -e +set -o pipefail -IWYU_LOG=$(mktemp -t kudu-iwyu.XX) -trap "rm -f $IWYU_LOG" EXIT +ROOT=$(cd $(dirname $BASH_SOURCE)/../..; pwd) # Build the list of updated files which are of IWYU interest. +# Since '-e' is set, transform the exit code from the grep accordingly: +# grep returns 1 if no lines were selected. file_list_tmp=$(git diff --name-only \ -$($ROOT/build-support/get-upstream-commit.sh) | grep -E '\.(c|cc|h)$') +$($ROOT/build-support/get-upstream-commit.sh) | \ +(grep -E '\.(c|cc|h)$' || [ $? -eq 1 ])) if [ -z "$file_list_tmp" ]; then echo "IWYU verification: no updates on related files, declaring success" exit 0 fi +IWYU_LOG=$(mktemp -t kudu-iwyu.XX) +UNFILTERED_IWYU_LOG=${IWYU_LOG}.unfiltered +trap "rm -f $IWYU_LOG $UNFILTERED_IWYU_LOG" EXIT + # Adjust the path for every element in the list. The iwyu_tool.py normalizes # paths (via realpath) to match the records from the compilation database. IWYU_FILE_LIST= @@ -47,13 +54,19 @@ IWYU_ARGS="\ --mapping_file=$IWYU_MAPPINGS_PATH/gtest.imp \ --mapping_file=$IWYU_MAPPINGS_PATH/libstdcpp.imp" -PATH="$PATH:$PWD/../../thirdparty/clang-toolchain/bin" \ +if ! PATH="$PATH:$PWD/../../thirdparty/clang-toolchain/bin" \ python $ROOT/build-support/iwyu/iwyu_tool.py -p . $IWYU_FILE_LIST -- \ -$IWYU_ARGS | awk -f $ROOT/build-support/iwyu/iwyu-filter.awk | \ -tee $IWYU_LOG +$IWYU_ARGS > $UNFILTERED_IWYU_LOG 2>&1; then + echo "IWYU verification: failed to run the tool, see below for details" + cat $UNFILTERED_IWYU_LOG + exit 2 +fi +awk -f $ROOT/build-support/iwyu/iwyu-filter.awk $UNFILTERED_IWYU_LOG | \ +tee $IWYU_LOG if [ -s "$IWYU_LOG" ]; then - # The output is not empty: the changelist needs correction. + # The output is not empty: IWYU finds the set of headers to be inconsistent. + echo "IWYU verification: changelist needs correction, see above for details" exit 1 fi http://git-wip-us.apache.org/repos/asf/kudu/blob/4935b1b2/build-support/iwyu/iwyu_tool.py -- diff --git a/build-support/iwyu/iwyu_tool.py b/build-support/iwyu/iwyu_tool.py index a3dd4a1..8943226 100755 --- a/build-support/iwyu/iwyu_tool.py +++ b/build-support/iwyu/iwyu_tool.py @@ -135,11 +135,10 @@ FORMATTERS = { 'clang': clang_formatter } -def get_outp
kudu git commit: Fix typo in partitioning error message
Repository: kudu Updated Branches: refs/heads/master 4935b1b29 -> 9af2046e6 Fix typo in partitioning error message Change-Id: I5502956e7947a67dba3e68e5add529c7b9e26866 Reviewed-on: http://gerrit.cloudera.org:8080/7997 Tested-by: Kudu Jenkins Reviewed-by: Alexey Serbin Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/9af2046e Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/9af2046e Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/9af2046e Branch: refs/heads/master Commit: 9af2046e610156a9064c87976eec2b5a42e0693b Parents: 4935b1b Author: Dan Burkert Authored: Thu Sep 7 09:16:06 2017 -0700 Committer: Alexey Serbin Committed: Thu Sep 7 20:18:56 2017 + -- src/kudu/common/partition.cc | 2 +- src/kudu/master/master-test.cc | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/9af2046e/src/kudu/common/partition.cc -- diff --git a/src/kudu/common/partition.cc b/src/kudu/common/partition.cc index a09189b..b1388db 100644 --- a/src/kudu/common/partition.cc +++ b/src/kudu/common/partition.cc @@ -285,7 +285,7 @@ Status PartitionSchema::EncodeRangeBounds(const vector= upper) { return Status::InvalidArgument( - "range partition lower bound must be less than or equal to the upper bound", + "range partition lower bound must be less than the upper bound", RangePartitionDebugString(bound.first, bound.second)); } range_partitions->emplace_back(std::move(lower), std::move(upper)); http://git-wip-us.apache.org/repos/asf/kudu/blob/9af2046e/src/kudu/master/master-test.cc -- diff --git a/src/kudu/master/master-test.cc b/src/kudu/master/master-test.cc index 9a19400..7d35e00 100644 --- a/src/kudu/master/master-test.cc +++ b/src/kudu/master/master-test.cc @@ -602,7 +602,7 @@ TEST_F(MasterTest, TestCreateTableCheckRangeInvariants) { ASSERT_TRUE(s.IsInvalidArgument()); ASSERT_STR_CONTAINS(s.ToString(), "Invalid argument: range partition lower bound must be " -"less than or equal to the upper bound"); +"less than the upper bound"); } { // Lower bound equals upper bound. KuduPartialRow bound_lower(&kTableSchema); @@ -614,7 +614,7 @@ TEST_F(MasterTest, TestCreateTableCheckRangeInvariants) { ASSERT_TRUE(s.IsInvalidArgument()); ASSERT_STR_CONTAINS(s.ToString(), "Invalid argument: range partition lower bound must be " -"less than or equal to the upper bound"); +"less than the upper bound"); } { // Split equals lower bound KuduPartialRow bound_lower(&kTableSchema);
kudu git commit: [iwyu] kudu-specific mappings for std::tr1 entities
Repository: kudu Updated Branches: refs/heads/master 9af2046e6 -> 01003ee6b [iwyu] kudu-specific mappings for std::tr1 entities Added Kudu-specific IWYU mappings for std::tr1::shared_ptr and friends from TR1 (those are used in the Kudu C++ client API). Added --max_line_length=256 IWYU option for better reporting on the reason for header files inclusion. If keeping the default setting of 80 characters, IWYU sometimes omits the information which symbols require particular header file to be included. Change-Id: Ia448dedcbb09ee23b30a922c7143df9036490d84 Reviewed-on: http://gerrit.cloudera.org:8080/7998 Reviewed-by: Adar Dembo Tested-by: Kudu Jenkins Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/01003ee6 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/01003ee6 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/01003ee6 Branch: refs/heads/master Commit: 01003ee6b4ece19c5e803dcac9d777a421d7c520 Parents: 9af2046 Author: Alexey Serbin Authored: Wed Sep 6 23:30:56 2017 -0700 Committer: Alexey Serbin Committed: Thu Sep 7 22:16:34 2017 + -- .gitignore | 1 + build-support/iwyu/iwyu-filter.awk | 37 --- build-support/iwyu/iwyu.sh | 2 ++ build-support/iwyu/mappings/kudu.imp | 34 4 files changed, 56 insertions(+), 18 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/01003ee6/.gitignore -- diff --git a/.gitignore b/.gitignore index d009ccc..9271774 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,7 @@ oprofile_data .kdev4/ .metadata/ *.iml +README.html # VIM/emacs/other editors stuff *.swp http://git-wip-us.apache.org/repos/asf/kudu/blob/01003ee6/build-support/iwyu/iwyu-filter.awk -- diff --git a/build-support/iwyu/iwyu-filter.awk b/build-support/iwyu/iwyu-filter.awk index 0d7fa79..6576a72 100644 --- a/build-support/iwyu/iwyu-filter.awk +++ b/build-support/iwyu/iwyu-filter.awk @@ -26,22 +26,34 @@ # https://github.com/include-what-you-use/include-what-you-use/blob/master/docs/IWYUMappings.md # # We are using mappings for the boost library (comes with IWYU) and a few -# custom mappings for gflags, glog, and gtest libraries to address some IWYU -# quirks (hopefully, those should be resolved as IWYU gets better). +# custom mappings for gflags, glog, gtest and other libraries to address some +# IWYU quirks (hopefully, those should be resolved as IWYU gets better). +# The kudu.imp mappings file is used to provide Kudu-specific mappings. +# +# To run the IWYU tool on every C++ source file in the project, +# use the recipe below. # -# Usage: # 1. Run the CMake with -DCMAKE_CXX_INCLUDE_WHAT_YOU_USE= # # The path to the IWYU binary should be absolute. The path to the binary # and the command-line options should be separated by semicolon # (that's for feeding it into CMake list variables). # -# E.g., from the build directory (line breaks are just for readability): +# Below is the set of commands to run from the build directory to configure +# the build accordingly (line breaks are for better readability): # # CC=../../thirdparty/clang-toolchain/bin/clang # CXX=../../thirdparty/clang-toolchain/bin/clang++ -# IWYU="`pwd`../../thirdparty/clang-toolchain/bin/include-what-you-use;\ -# -Xiwyu;--mapping_file=`pwd`../../build-support/iwyu/mappings/map.imp" +# IWYU="`pwd`/../../thirdparty/clang-toolchain/bin/include-what-you-use;\ +# -Xiwyu;--max_line_length=256;\ +# -Xiwyu;--mapping_file=`pwd`/../../build-support/iwyu/mappings/boost-all.imp;\ +# -Xiwyu;--mapping_file=`pwd`/../../build-support/iwyu/mappings/boost-all-private.imp;\ +# -Xiwyu;--mapping_file=`pwd`/../../build-support/iwyu/mappings/boost-extra.imp;\ +# -Xiwyu;--mapping_file=`pwd`/../../build-support/iwyu/mappings/gtest.imp;\ +# -Xiwyu;--mapping_file=`pwd`/../../build-support/iwyu/mappings/glog.imp;\ +# -Xiwyu;--mapping_file=`pwd`/../../build-support/iwyu/mappings/gflags.imp;\ +# -Xiwyu;--mapping_file=`pwd`/../../build-support/iwyu/mappings/kudu.imp;\ +# -Xiwyu;--mapping_file=`pwd`/../../build-support/iwyu/mappings/libstdcpp.imp" # # ../../build-support/enable_devtoolset.sh \ # env CC=$CC CXX=$CXX \ @@ -52,18 +64,7 @@ # NOTE: # Since the Kudu code has some 'ifdef NDEBUG' directives, it's possible # that IWYU would produce different results if run against release, not -# debug build. However, we plan to use the tool only with debug builds. -# -# NOTE: -# As of now, the ap
kudu git commit: [util] fix compilation breakage
Repository: kudu Updated Branches: refs/heads/master d45eb2700 -> 0ed75067c [util] fix compilation breakage On some Linux systems (old ones), there isn't linux/sysinfo.h header file. IWYU might suggest to include linux/sysinfo.h instead of linux/kernel.h if compiling on a new system, so I added corresponding IWYU pragmas. I'll address the IWYU issue in a generic way in a separate change list (so no pragmas would be necessary). This patch is a quick fix for the compilation breakage. Change-Id: Ie9d13815052d8e4c1360db3f94e15ffd77768af5 Reviewed-on: http://gerrit.cloudera.org:8080/8065 Reviewed-by: Adar Dembo Reviewed-by: Andrew Wong Tested-by: Kudu Jenkins Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/0ed75067 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/0ed75067 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/0ed75067 Branch: refs/heads/master Commit: 0ed75067cf805dda8784cca5ea7a3e7a43616ac6 Parents: d45eb27 Author: Alexey Serbin Authored: Wed Sep 13 21:50:18 2017 -0700 Committer: Alexey Serbin Committed: Thu Sep 14 05:25:46 2017 + -- src/kudu/util/env_posix.cc | 9 + 1 file changed, 5 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/0ed75067/src/kudu/util/env_posix.cc -- diff --git a/src/kudu/util/env_posix.cc b/src/kudu/util/env_posix.cc index 9dd5e2b..3352853 100644 --- a/src/kudu/util/env_posix.cc +++ b/src/kudu/util/env_posix.cc @@ -15,9 +15,6 @@ #include #include #include -// IWYU pragma: no_include -// IWYU pragma: no_include -// IWYU pragma: no_include #include #include @@ -73,12 +70,16 @@ #include #include #include +#include // IWYU pragma: keep #include -#include #include #include #include // IWYU pragma: keep #endif // defined(__APPLE__) +// IWYU pragma: no_include +// IWYU pragma: no_include +// IWYU pragma: no_include +// IWYU pragma: no_include using base::subtle::Atomic64; using base::subtle::Barrier_AtomicIncrement;
kudu git commit: [iwyu] introduce Linux system mappings
Repository: kudu Updated Branches: refs/heads/master e2c0e1dbd -> fa8620cc7 [iwyu] introduce Linux system mappings For starters, added the following mappings to achieve better compatibility across different Linus versions and distros: * use even if is available * use even if is available * use as an umbrella header file for various asm types Change-Id: I4e50f9135fa076e429b26e3919ef19eb5d430173 Reviewed-on: http://gerrit.cloudera.org:8080/8071 Reviewed-by: Adar Dembo Reviewed-by: Andrew Wong Tested-by: Kudu Jenkins Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/fa8620cc Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/fa8620cc Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/fa8620cc Branch: refs/heads/master Commit: fa8620cc7b4dd4205a5a58737a7cc3c21957fe05 Parents: e2c0e1d Author: Alexey Serbin Authored: Thu Sep 14 08:37:40 2017 -0700 Committer: Alexey Serbin Committed: Thu Sep 14 19:58:43 2017 + -- build-support/iwyu/iwyu-filter.awk | 3 +- build-support/iwyu/iwyu.sh | 3 +- build-support/iwyu/mappings/system-linux.imp | 35 +++ src/kudu/util/env_posix.cc | 20 ++--- 4 files changed, 48 insertions(+), 13 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/fa8620cc/build-support/iwyu/iwyu-filter.awk -- diff --git a/build-support/iwyu/iwyu-filter.awk b/build-support/iwyu/iwyu-filter.awk index 6576a72..eddcb8c 100644 --- a/build-support/iwyu/iwyu-filter.awk +++ b/build-support/iwyu/iwyu-filter.awk @@ -53,7 +53,8 @@ # -Xiwyu;--mapping_file=`pwd`/../../build-support/iwyu/mappings/glog.imp;\ # -Xiwyu;--mapping_file=`pwd`/../../build-support/iwyu/mappings/gflags.imp;\ # -Xiwyu;--mapping_file=`pwd`/../../build-support/iwyu/mappings/kudu.imp;\ -# -Xiwyu;--mapping_file=`pwd`/../../build-support/iwyu/mappings/libstdcpp.imp" +# -Xiwyu;--mapping_file=`pwd`/../../build-support/iwyu/mappings/libstdcpp.imp\ +# -Xiwyu;--mapping_file=`pwd`/../../build-support/iwyu/mappings/system-linux.imp" # # ../../build-support/enable_devtoolset.sh \ # env CC=$CC CXX=$CXX \ http://git-wip-us.apache.org/repos/asf/kudu/blob/fa8620cc/build-support/iwyu/iwyu.sh -- diff --git a/build-support/iwyu/iwyu.sh b/build-support/iwyu/iwyu.sh index a2c16cf..6ee6287 100755 --- a/build-support/iwyu/iwyu.sh +++ b/build-support/iwyu/iwyu.sh @@ -54,7 +54,8 @@ IWYU_ARGS="\ --mapping_file=$IWYU_MAPPINGS_PATH/glog.imp \ --mapping_file=$IWYU_MAPPINGS_PATH/gtest.imp \ --mapping_file=$IWYU_MAPPINGS_PATH/kudu.imp \ ---mapping_file=$IWYU_MAPPINGS_PATH/libstdcpp.imp" +--mapping_file=$IWYU_MAPPINGS_PATH/libstdcpp.imp\ +--mapping_file=$IWYU_MAPPINGS_PATH/system-linux.imp" if ! PATH="$PATH:$PWD/../../thirdparty/clang-toolchain/bin" \ python $ROOT/build-support/iwyu/iwyu_tool.py -p . $IWYU_FILE_LIST -- \ http://git-wip-us.apache.org/repos/asf/kudu/blob/fa8620cc/build-support/iwyu/mappings/system-linux.imp -- diff --git a/build-support/iwyu/mappings/system-linux.imp b/build-support/iwyu/mappings/system-linux.imp new file mode 100644 index 000..22235f5 --- /dev/null +++ b/build-support/iwyu/mappings/system-linux.imp @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# +# This file contains Linux-specific mappings for IWYU. +# +# For instance, some of these mappings take care of variation in system +# headers between different versions of Linux. From the point of better +# compatibility, it makes sense to favor the most compatible option +# among different system versions and flavors. +# + +[ + { include: ["", private, "", public] }, + { include: ["", private, "", public] },
kudu git commit: standard notation for fixed-size integer type names
Repository: kudu Updated Branches: refs/heads/master 052d1def8 -> 5c8a8a2ab standard notation for fixed-size integer type names Prefer the standard notation for fixed-size integer types in the Kudu code, e.g., use uint32_t instead of uint32. In addition to unification of type naming, this helps to avoid including the kudu/gutil/integral_types.h extra header file. The gutil is left unchanged in that regard. This patch does not contain any functional changes. Change-Id: If8588275d0d81e5915c798d4d244bba560b6d25f Reviewed-on: http://gerrit.cloudera.org:8080/8099 Tested-by: Kudu Jenkins Reviewed-by: Todd Lipcon Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/5c8a8a2a Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/5c8a8a2a Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/5c8a8a2a Branch: refs/heads/master Commit: 5c8a8a2ab9d20ef89e6264cc43b0a3747567434e Parents: 052d1de Author: Alexey Serbin Authored: Mon Sep 18 16:50:56 2017 -0700 Committer: Alexey Serbin Committed: Tue Sep 19 18:26:19 2017 + -- src/kudu/cfile/bloomfile-test.cc| 3 +-- src/kudu/client/client-test.cc | 2 +- src/kudu/client/scan_configuration.h| 3 +-- src/kudu/common/generic_iterators-test.cc | 7 +++-- src/kudu/common/row_changelist-test.cc | 6 ++--- src/kudu/common/types-test.cc | 5 ++-- src/kudu/consensus/raft_consensus.cc| 9 +++ src/kudu/fs/block_id.h | 1 - src/kudu/gutil/ref_counted.h| 2 +- src/kudu/integration-tests/tablet_copy-itest.cc | 5 ++-- src/kudu/integration-tests/ts_itest-base.h | 2 +- src/kudu/rpc/retriable_rpc.h| 2 +- src/kudu/rpc/rpc-test-base.h| 4 +-- src/kudu/security/token_signer.cc | 9 +++ src/kudu/tablet/tablet_metadata.cc | 2 +- src/kudu/tablet/tablet_metadata.h | 3 +-- src/kudu/tools/kudu-admin-test.cc | 9 +++ src/kudu/tools/tool_action_tablet.cc| 4 +-- src/kudu/tserver/scanners.cc| 3 +-- src/kudu/tserver/tablet_copy_client.h | 3 +-- src/kudu/tserver/tablet_copy_source_session.h | 3 +-- src/kudu/util/async_logger.cc | 2 +- src/kudu/util/async_logger.h| 3 +-- src/kudu/util/bloom_filter.h| 5 ++-- src/kudu/util/condition_variable.cc | 4 +-- src/kudu/util/debug/trace_event_impl.cc | 28 ++-- src/kudu/util/debug/trace_event_impl.h | 17 ++-- src/kudu/util/file_cache-stress-test.cc | 2 +- src/kudu/util/logging-test.cc | 4 +-- src/kudu/util/memory/memory.h | 7 +++-- src/kudu/util/os-util.cc| 3 +-- src/kudu/util/pb_util-internal.h| 4 +-- src/kudu/util/spinlock_profiling.cc | 7 +++-- src/kudu/util/test_util.cc | 3 +-- src/kudu/util/thread.cc | 5 ++-- 35 files changed, 80 insertions(+), 101 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/5c8a8a2a/src/kudu/cfile/bloomfile-test.cc -- diff --git a/src/kudu/cfile/bloomfile-test.cc b/src/kudu/cfile/bloomfile-test.cc index 34aead3..afd8171 100644 --- a/src/kudu/cfile/bloomfile-test.cc +++ b/src/kudu/cfile/bloomfile-test.cc @@ -32,7 +32,6 @@ #include "kudu/fs/fs_manager.h" #include "kudu/gutil/endian.h" #include "kudu/gutil/gscoped_ptr.h" -#include "kudu/gutil/integral_types.h" #include "kudu/util/bloom_filter.h" #include "kudu/util/mem_tracker.h" #include "kudu/util/slice.h" @@ -63,7 +62,7 @@ class BloomFileTest : public BloomFileTestBase { int positive_count = 0; // Check that the FP rate for keys we didn't insert is what we expect. -for (uint64 i = 0; i < FLAGS_n_keys; i++) { +for (uint64_t i = 0; i < FLAGS_n_keys; i++) { uint64_t key = random(); Slice s(reinterpret_cast(&key), sizeof(key)); http://git-wip-us.apache.org/repos/asf/kudu/blob/5c8a8a2a/src/kudu/client/client-test.cc -- diff --git a/src/kudu/client/client-test.cc b/src/kudu/client/client-test.cc index cb1ab12..6ed26be 100644 --- a/src/kudu/client/client-test.cc +++ b/src/kudu/client/client-test.cc @@ -3024,7 +3024,7 @@ TEST_F(ClientTest, TestAutoFlushBackgroundPreFlush) { // For details on this behavior please see the diagram in the body of the // KuduSession::Data::ApplyWriteOp() method.
[1/2] kudu git commit: [iwyu] update std::move
Repository: kudu Updated Branches: refs/heads/master 02ba0ec6c -> 3e6e12a9f http://git-wip-us.apache.org/repos/asf/kudu/blob/3e6e12a9/src/kudu/tablet/tablet_bootstrap.cc -- diff --git a/src/kudu/tablet/tablet_bootstrap.cc b/src/kudu/tablet/tablet_bootstrap.cc index 1eb6533..d5c8a3c 100644 --- a/src/kudu/tablet/tablet_bootstrap.cc +++ b/src/kudu/tablet/tablet_bootstrap.cc @@ -17,7 +17,6 @@ #include "kudu/tablet/tablet_bootstrap.h" -#include #include #include #include http://git-wip-us.apache.org/repos/asf/kudu/blob/3e6e12a9/src/kudu/tablet/tablet_history_gc-test.cc -- diff --git a/src/kudu/tablet/tablet_history_gc-test.cc b/src/kudu/tablet/tablet_history_gc-test.cc index d12bba7..7aec88a 100644 --- a/src/kudu/tablet/tablet_history_gc-test.cc +++ b/src/kudu/tablet/tablet_history_gc-test.cc @@ -32,6 +32,7 @@ #include "kudu/clock/clock.h" #include "kudu/clock/hybrid_clock.h" #include "kudu/clock/mock_ntp.h" +#include "kudu/clock/time_service.h" #include "kudu/common/common.pb.h" #include "kudu/common/partial_row.h" #include "kudu/common/schema.h" @@ -39,6 +40,7 @@ #include "kudu/gutil/casts.h" #include "kudu/gutil/move.h" #include "kudu/gutil/port.h" +#include "kudu/gutil/ref_counted.h" #include "kudu/gutil/strings/substitute.h" #include "kudu/gutil/walltime.h" #include "kudu/tablet/compaction.h" @@ -51,7 +53,7 @@ #include "kudu/tablet/tablet.h" #include "kudu/tablet/tablet_metadata.h" #include "kudu/tablet/tablet_metrics.h" -#include "kudu/util/make_shared.h" +#include "kudu/util/metrics.h" #include "kudu/util/monotime.h" #include "kudu/util/status.h" #include "kudu/util/test_macros.h" @@ -65,17 +67,6 @@ using std::string; using std::vector; using strings::Substitute; -// Specify row regex to match on. Empty string means don't match anything. -#define ASSERT_DEBUG_DUMP_ROWS_MATCH(pattern) do { \ - const std::string& _pat = (pattern); \ - vector _rows; \ - ASSERT_OK(tablet()->DebugDump(&_rows)); \ - /* Ignore the non-data (formattting) lines in the output. */ \ - std::string _base_pat = R"(^Dumping|^-|^MRS|^RowSet)"; \ - if (!_pat.empty()) _base_pat += "|"; \ - ASSERT_STRINGS_ALL_MATCH(_rows, _base_pat + _pat); \ -} while (0) - namespace kudu { namespace tablet { @@ -113,6 +104,8 @@ class TabletHistoryGcTest : public TabletTestBase> { uint64_t new_time = now + delta.ToMicroseconds(); SetMockTime(new_time); } + // Specify row regex to match on. Empty string means don't match anything. + void VerifyDebugDumpRowsMatch(const string& pattern) const; int64_t TotalNumRows() const { return num_rowsets_ * rows_per_rowset_; } @@ -126,7 +119,7 @@ class TabletHistoryGcTest : public TabletTestBase> { const int kStartRow = 0; int num_rowsets_ = 3; - int rows_per_rowset_ = 300; + int64_t rows_per_rowset_ = 300; }; void TabletHistoryGcTest::InsertOriginalRows(int64_t num_rowsets, int64_t rows_per_rowset) { @@ -146,6 +139,17 @@ void TabletHistoryGcTest::UpdateOriginalRows(int64_t num_rowsets, int64_t rows_p ASSERT_EQ(num_rowsets, tablet()->num_rowsets()); } +void TabletHistoryGcTest::VerifyDebugDumpRowsMatch(const string& pattern) const { + vector rows; + ASSERT_OK(tablet()->DebugDump(&rows)); \ + // Ignore the non-data (formattting) lines in the output. + std::string base_pattern = R"(^Dumping|^-|^MRS|^RowSet)"; + if (!pattern.empty()) { +base_pattern += "|"; + } + ASSERT_STRINGS_ALL_MATCH(rows, base_pattern + pattern); +} + // Test that we do not generate undos for redo operations that are older than // the AHM during major delta compaction. TEST_F(TabletHistoryGcTest, TestNoGenerateUndoOnMajorDeltaCompaction) { @@ -191,8 +195,9 @@ TEST_F(TabletHistoryGcTest, TestNoGenerateUndoOnMajorDeltaCompaction) { // Now, we should have base data = 2 with no other historical values. // Major delta compaction will not remove UNDOs, so we expect a single UNDO DELETE as well. - ASSERT_DEBUG_DUMP_ROWS_MATCH(R"(int32 val=2\); Undo Mutations: \[@[[:digit:]]+\(DELETE\)\]; )" - R"(Redo Mutations: \[\];$)"); + NO_FATALS(VerifyDebugDumpRowsMatch( + R"(int32 val=2\); Undo Mutations: \[@[[:digit:]]+\(DELETE\)\]; )" + R"(Redo Mutations: \[\];$)")); } // Test that major delta compaction works when run on a subset of columns: @@ -233,8 +238,9 @@ TEST_F(TabletHistoryGcTest, TestMajorDeltaCompactionOnSubsetOfColumns) { tablet()->GetHistoryGcOpts())); } - ASSERT_DEBUG_DUMP_ROWS_MATCH(R"(int32 val=2\); Undo Mutations: \[@[[:digit:]]+\(DELETE\)\]; )" - R"(Redo Mutations: \[@[[:digit:]]+\(SET key_idx=1\)\];$)"); + NO_FATALS(VerifyDebugDumpRowsMatch( + R"(int32 val=2\); Undo Mutations: \[@[[:digit:]]+\(DELETE\)\]; )" + R"(Redo Mutations: \[@[[:digit:]]
[2/2] kudu git commit: [iwyu] update std::move
[iwyu] update std::move Updated IWYU mappings to use for std::move() and std::swap() instead of . Unsorted micro-cleanups in some tests. Also, updated the cpplint.py script from the google-styleguide to reflect std::swap() migration from to . That is to keep things consistent between 'iwyu' and 'lint/ilint' targets. The latest version of the google-styleguide at [1] already contains the corresponding update along with other C++11-related changes, but I would like to bump the google-styleguide version in a separate changelist. References: [1] https://github.com/google/styleguide Change-Id: Ic5972f9a43f4c1db4a0dff5f983f7dba90c8db31 Reviewed-on: http://gerrit.cloudera.org:8080/8088 Reviewed-by: Adar Dembo Tested-by: Alexey Serbin Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/3e6e12a9 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/3e6e12a9 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/3e6e12a9 Branch: refs/heads/master Commit: 3e6e12a9f41abb0f354a81af46698b697c2dcb6f Parents: 02ba0ec Author: Alexey Serbin Authored: Fri Sep 15 20:25:44 2017 -0700 Committer: Alexey Serbin Committed: Wed Sep 20 19:25:59 2017 + -- build-support/iwyu/iwyu-filter.awk | 12 build-support/iwyu/mappings/boost-extra.imp | 1 + build-support/iwyu/mappings/libstdcpp.imp | 6 +- src/kudu/benchmarks/tpch/tpch_real_world.cc | 6 +- src/kudu/cfile/block_cache.h| 2 +- src/kudu/cfile/bloomfile-test.cc| 2 +- src/kudu/cfile/bloomfile.cc | 2 +- src/kudu/cfile/cfile-test.cc| 2 +- src/kudu/cfile/cfile_reader.cc | 1 + src/kudu/cfile/cfile_writer.cc | 3 +- src/kudu/client/batcher.cc | 1 - src/kudu/client/client.cc | 1 - src/kudu/client/meta_cache.h| 2 +- src/kudu/client/partitioner-internal.h | 2 +- src/kudu/client/replica-internal.cc | 2 +- src/kudu/client/scan_configuration.cc | 2 +- src/kudu/client/scan_token-test.cc | 2 +- src/kudu/client/schema.cc | 2 +- src/kudu/client/session-internal.cc | 2 +- src/kudu/client/table-internal.cc | 2 +- src/kudu/client/table_alterer-internal.cc | 2 +- src/kudu/client/tablet_server-internal.cc | 2 +- src/kudu/codegen/jit_wrapper.cc | 2 +- src/kudu/codegen/module_builder.cc | 2 +- src/kudu/common/partial_row.cc | 2 +- src/kudu/common/partition_pruner-test.cc| 1 - src/kudu/common/schema.h| 4 +- src/kudu/consensus/consensus_meta-test.cc | 13 ++-- src/kudu/consensus/consensus_meta.cc| 2 +- .../consensus_meta_manager-stress-test.cc | 2 +- src/kudu/consensus/consensus_meta_manager.cc| 2 +- src/kudu/consensus/consensus_peers-test.cc | 2 +- src/kudu/consensus/leader_election-test.cc | 4 +- src/kudu/consensus/log.cc | 2 +- src/kudu/consensus/log_util.cc | 1 + src/kudu/consensus/raft_consensus.h | 2 +- .../consensus/raft_consensus_quorum-test.cc | 76 ++-- src/kudu/fs/block_manager-stress-test.cc| 2 +- src/kudu/fs/block_manager-test.cc | 1 + src/kudu/fs/block_manager.h | 6 +- src/kudu/fs/block_manager_util-test.cc | 2 +- src/kudu/fs/block_manager_util.cc | 2 +- src/kudu/fs/data_dirs.h | 2 +- src/kudu/fs/file_block_manager.cc | 2 +- src/kudu/fs/fs-test-util.h | 1 + src/kudu/fs/fs_manager.cc | 2 +- src/kudu/gutil/atomicops-internals-x86.cc | 3 +- src/kudu/gutil/cpu.cc | 2 +- src/kudu/gutil/hash/city.cc | 2 +- src/kudu/gutil/ref_counted_memory.cc| 2 +- src/kudu/gutil/strings/split.cc | 1 + src/kudu/gutil/strings/split.h | 3 +- src/kudu/gutil/strings/strcat.h | 3 +- src/kudu/gutil/strings/stringpiece.cc | 10 +-- src/kudu/gutil/strings/strip.cc | 8 --- src/kudu/gutil/strings/util.cc | 11 +-- .../exactly_once_writes-itest.cc| 2 +- .../external_mini_cluster-itest-base.cc | 2 +- .../external_mini_cluster-test.cc | 2 +- .../integration-tests/internal_mini_cluster.cc | 2 +- .../integration-tests/master-stress-test.cc | 4 +- .../integration-tests/open-readonly-fs-itest.cc | 2 +- .../integration-tests/token_signer-itest.cc | 15 ++-- .../integration-tests/version_migration-test.cc | 4 +- src/kudu/m
[1/2] kudu git commit: Fix changes that were left out of KUDU-501 and /table template patches
Repository: kudu Updated Branches: refs/heads/master 9c5d6d4df -> 9c1997ad3 Fix changes that were left out of KUDU-501 and /table template patches Not sure how I did this, but a typo in a comment and some missing substitute args were fixed but then somehow got left out of the final changelist. Change-Id: I97a17f60f23a2ba62f0fa219353b264daeaf7f48 Reviewed-on: http://gerrit.cloudera.org:8080/8125 Reviewed-by: Adar Dembo Tested-by: Kudu Jenkins Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/06a19bd2 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/06a19bd2 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/06a19bd2 Branch: refs/heads/master Commit: 06a19bd25a964664db08ef89ab0ebe26369519ab Parents: 9c5d6d4 Author: Will Berkeley Authored: Fri Sep 22 10:09:30 2017 -0700 Committer: Will Berkeley Committed: Fri Sep 22 22:22:29 2017 + -- src/kudu/master/master-path-handlers.cc | 4 ++-- src/kudu/master/master-path-handlers.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/06a19bd2/src/kudu/master/master-path-handlers.cc -- diff --git a/src/kudu/master/master-path-handlers.cc b/src/kudu/master/master-path-handlers.cc index bf3d12d..e8fa753 100644 --- a/src/kudu/master/master-path-handlers.cc +++ b/src/kudu/master/master-path-handlers.cc @@ -271,13 +271,13 @@ void MasterPathHandlers::HandleTablePage(const Webserver::WebRequest& req, s = SchemaFromPB(l.data().pb.schema(), &schema); if (!s.ok()) { - (*output)["error"] = Substitute("Unable to decode schema: ", s.ToString()); + (*output)["error"] = Substitute("Unable to decode schema: $0", s.ToString()); return; } s = PartitionSchema::FromPB(l.data().pb.partition_schema(), schema, &partition_schema); if (!s.ok()) { (*output)["error"] = - Substitute("Unable to decode partition schema: ", s.ToString()); + Substitute("Unable to decode partition schema: $0", s.ToString()); return; } table->GetAllTablets(&tablets); http://git-wip-us.apache.org/repos/asf/kudu/blob/06a19bd2/src/kudu/master/master-path-handlers.h -- diff --git a/src/kudu/master/master-path-handlers.h b/src/kudu/master/master-path-handlers.h index 6195a74..72cff3f 100644 --- a/src/kudu/master/master-path-handlers.h +++ b/src/kudu/master/master-path-handlers.h @@ -59,7 +59,7 @@ class MasterPathHandlers { // Returns a pair (text, target) given a tserver's TSDescriptor and a tablet id. // - text is the http host and port for the tserver, if available, or the tserver's uuid. - // - target is a url to to the tablet page for the tablet on the tserver's webui, + // - target is a url to the tablet page for the tablet on the tserver's webui, // or an empty string if no http address is available for the tserver. std::pair TSDescToLinkPair(const TSDescriptor& desc, const std::string& tablet_id) const;
[2/2] kudu git commit: Revert "[tests] fix flakiness in catalog_manager_tsk-itest"
Revert "[tests] fix flakiness in catalog_manager_tsk-itest" This reverts commit faa0b14effb6e15f9989d686e5a1f8e1040a1dd6. Since the fix for KUDU-2149 is committed, the catalog_manager_tsk-itest is no longer flaky even with original parameters. We want to use the catalog_manager_tsk-itest with its original settings as a canary for any further regressions in that area. Notes on verification: With this change, and the fix for KUDU-2149 rolled-back, 10 out of 1024 failed in when running the catalog_manager_tsk-itest test with --stress_cpu_threads=16: http://dist-test.cloudera.org//job?job_id=aserbin.1506033479.20652 With this change, no failures were spotted in multiple 1K runs of the test with --stress_cpu_threads=16: http://dist-test.cloudera.org//job?job_id=aserbin.1506109518.28895 Change-Id: Ic7db0eef852a417da5d11480ccdb4f15019ea1da Reviewed-on: http://gerrit.cloudera.org:8080/8129 Reviewed-by: Adar Dembo Tested-by: Kudu Jenkins Reviewed-by: David Ribeiro Alves Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/9c1997ad Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/9c1997ad Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/9c1997ad Branch: refs/heads/master Commit: 9c1997ad3473ab3ce29061c4cd2aa11552967a98 Parents: 06a19bd Author: Alexey Serbin Authored: Fri Sep 22 15:02:23 2017 -0700 Committer: Alexey Serbin Committed: Fri Sep 22 22:46:34 2017 + -- src/kudu/integration-tests/catalog_manager_tsk-itest.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/9c1997ad/src/kudu/integration-tests/catalog_manager_tsk-itest.cc -- diff --git a/src/kudu/integration-tests/catalog_manager_tsk-itest.cc b/src/kudu/integration-tests/catalog_manager_tsk-itest.cc index e573097..acdeba9 100644 --- a/src/kudu/integration-tests/catalog_manager_tsk-itest.cc +++ b/src/kudu/integration-tests/catalog_manager_tsk-itest.cc @@ -61,7 +61,7 @@ class CatalogManagerTskITest : public KuduTest { : num_masters_(3), num_tservers_(1), #if defined(ADDRESS_SANITIZER) || defined(THREAD_SANITIZER) -hb_interval_ms_(64), +hb_interval_ms_(32), run_time_seconds_(5) #else hb_interval_ms_(16),
kudu git commit: [consensus] don't crash in RaftConsensus::ChangeConfig
Repository: kudu Updated Branches: refs/heads/master 26a32fc4b -> 98b66574b [consensus] don't crash in RaftConsensus::ChangeConfig Don't crash if ReplicateConfigChangeUnlocked() returns an error. Instead, just return corresponding error to the caller. This is to avoid crashing a tablet server if an invalid Raft configuration is passed to ChangeConfig() RPC with ChangeConfigRequestPB parameter. Also, use move semantics for all parameters of the RaftConsensus::ReplicateConfigChangeUnlocked() method. Change-Id: Ieac3692adae029d098fea254badf2c53c6ac4fd8 Reviewed-on: http://gerrit.cloudera.org:8080/8124 Tested-by: Kudu Jenkins Reviewed-by: Alexey Serbin Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/98b66574 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/98b66574 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/98b66574 Branch: refs/heads/master Commit: 98b66574b06838347e323b997778f964acba1abd Parents: 26a32fc Author: Alexey Serbin Authored: Thu Sep 21 18:34:08 2017 -0700 Committer: Alexey Serbin Committed: Mon Sep 25 23:24:44 2017 + -- src/kudu/consensus/raft_consensus.cc | 16 +++- src/kudu/consensus/raft_consensus.h | 8 2 files changed, 11 insertions(+), 13 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/98b66574/src/kudu/consensus/raft_consensus.cc -- diff --git a/src/kudu/consensus/raft_consensus.cc b/src/kudu/consensus/raft_consensus.cc index 6b4e968..d70d566 100644 --- a/src/kudu/consensus/raft_consensus.cc +++ b/src/kudu/consensus/raft_consensus.cc @@ -568,9 +568,8 @@ Status RaftConsensus::BecomeLeaderUnlocked() { round.get(), &DoNothingStatusCB, std::placeholders::_1)); - RETURN_NOT_OK(AppendNewRoundToQueueUnlocked(round)); - return Status::OK(); + return AppendNewRoundToQueueUnlocked(round); } Status RaftConsensus::BecomeReplicaUnlocked() { @@ -1643,7 +1642,7 @@ Status RaftConsensus::ChangeConfig(const ChangeConfigRequestPB& req, } RETURN_NOT_OK(ReplicateConfigChangeUnlocked( -committed_config, new_config, std::bind( +std::move(committed_config), std::move(new_config), std::bind( &RaftConsensus::MarkDirtyOnSuccess, this, string("Config change replication complete"), @@ -2080,16 +2079,16 @@ void RaftConsensus::SetLeaderUuidUnlocked(const string& uuid) { } Status RaftConsensus::ReplicateConfigChangeUnlocked( -const RaftConfigPB& old_config, -const RaftConfigPB& new_config, +RaftConfigPB old_config, +RaftConfigPB new_config, StdStatusCallback client_cb) { DCHECK(lock_.is_locked()); auto cc_replicate = new ReplicateMsg(); cc_replicate->set_op_type(CHANGE_CONFIG_OP); ChangeConfigRecordPB* cc_req = cc_replicate->mutable_change_config_record(); cc_req->set_tablet_id(options_.tablet_id); - *cc_req->mutable_old_config() = old_config; - *cc_req->mutable_new_config() = new_config; + *cc_req->mutable_old_config() = std::move(old_config); + *cc_req->mutable_new_config() = std::move(new_config); CHECK_OK(time_manager_->AssignTimestamp(cc_replicate)); scoped_refptr round( @@ -2101,8 +2100,7 @@ Status RaftConsensus::ReplicateConfigChangeUnlocked( std::move(client_cb), std::placeholders::_1)); - CHECK_OK(AppendNewRoundToQueueUnlocked(round)); - return Status::OK(); + return AppendNewRoundToQueueUnlocked(round); } Status RaftConsensus::RefreshConsensusQueueAndPeersUnlocked() { http://git-wip-us.apache.org/repos/asf/kudu/blob/98b66574/src/kudu/consensus/raft_consensus.h -- diff --git a/src/kudu/consensus/raft_consensus.h b/src/kudu/consensus/raft_consensus.h index b82afb3..080bc7e 100644 --- a/src/kudu/consensus/raft_consensus.h +++ b/src/kudu/consensus/raft_consensus.h @@ -415,12 +415,12 @@ class RaftConsensus : public std::enable_shared_from_this, // reporting to the master. void SetLeaderUuidUnlocked(const std::string& uuid); - // Replicate (as leader) a pre-validated config change. This includes - // updating the peers and setting the new_configuration as pending. + // Replicate (as leader) a config change. This includes validating the new + // config and updating the peers and setting the new_configuration as pending. // The old_configuration must be the currently-committed configuration. Status ReplicateConfigChangeUnlocked( - const RaftConfigPB& old_config, - const RaftConfigPB& new_config, + RaftConfigPB old_config, + RaftConfigPB new_config, StdStatusCallback client_cb); // Update the peers and queue to be consistent with a new active configuration.
kudu git commit: [consensus] micro-cleanup on VerifyRaftConfig()
Repository: kudu Updated Branches: refs/heads/master 98b66574b -> bab96a2b2 [consensus] micro-cleanup on VerifyRaftConfig() Removed unused RaftConfigState parameter for VerifyRaftConfig(). In addition, fixed typos in metadata.proto inline documentation. Change-Id: Ic5b4cb6ceacde7a0c0b2f1c75dca4b7d1681ef33 Reviewed-on: http://gerrit.cloudera.org:8080/8078 Tested-by: Kudu Jenkins Reviewed-by: Mike Percy Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/bab96a2b Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/bab96a2b Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/bab96a2b Branch: refs/heads/master Commit: bab96a2b299de4f1f911e23c80167938ed69394d Parents: 98b6657 Author: Alexey Serbin Authored: Thu Sep 14 19:57:43 2017 -0700 Committer: Alexey Serbin Committed: Tue Sep 26 00:56:27 2017 + -- src/kudu/consensus/consensus_meta.cc | 2 +- src/kudu/consensus/metadata.proto| 8 src/kudu/consensus/quorum_util.cc| 10 +- src/kudu/consensus/quorum_util.h | 5 + src/kudu/consensus/raft_consensus.cc | 8 src/kudu/master/sys_catalog.cc | 5 ++--- 6 files changed, 17 insertions(+), 21 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/bab96a2b/src/kudu/consensus/consensus_meta.cc -- diff --git a/src/kudu/consensus/consensus_meta.cc b/src/kudu/consensus/consensus_meta.cc index 75de554..9954c8a 100644 --- a/src/kudu/consensus/consensus_meta.cc +++ b/src/kudu/consensus/consensus_meta.cc @@ -267,7 +267,7 @@ Status ConsensusMetadata::Flush(FlushMode flush_mode) { flush_count_for_tests_++; // Sanity test to ensure we never write out a bad configuration. - RETURN_NOT_OK_PREPEND(VerifyRaftConfig(pb_.committed_config(), COMMITTED_CONFIG), + RETURN_NOT_OK_PREPEND(VerifyRaftConfig(pb_.committed_config()), "Invalid config in ConsensusMetadata, cannot flush to disk"); // Create directories if needed. http://git-wip-us.apache.org/repos/asf/kudu/blob/bab96a2b/src/kudu/consensus/metadata.proto -- diff --git a/src/kudu/consensus/metadata.proto b/src/kudu/consensus/metadata.proto index cc7223d..718d9eb 100644 --- a/src/kudu/consensus/metadata.proto +++ b/src/kudu/consensus/metadata.proto @@ -59,7 +59,7 @@ message RaftPeerPB { // Permanent uuid is optional: RaftPeerPB/RaftConfigPB instances may // be created before the permanent uuid is known (e.g., when // manually specifying a configuration for Master/CatalogManager); - // permament uuid can be retrieved at a later time through RPC. + // permanent uuid can be retrieved at a later time through RPC. optional bytes permanent_uuid = 1; optional MemberType member_type = 2; optional HostPortPB last_known_addr = 3; @@ -97,8 +97,8 @@ message ConsensusStatePB { // to a term check on the UpdateConsensus() RPC request. // // Whenever the local peer sees a new term, the leader flag is cleared until - // a new leader is acknowledged based on the above critera. Simply casting a - // vote for a peer is not sufficient to assume that that peer has won the + // a new leader is acknowledged based on the above criteria. Simply casting a + // vote for a peer is not sufficient to assume that the peer has won the // election, so we do not update this field based on our vote. // // The leader may be a part of the committed or the pending configuration (or both). @@ -125,7 +125,7 @@ message ConsensusMetadataPB { // Whenever a new election is started, the candidate increments this by one // and requests votes from peers. // - // If any RPC or RPC response is received from another node containing a term higher + // If any RPC request or response is received from another node containing a term higher // than this one, the server should step down to FOLLOWER and set its current_term to // match the caller's term. // http://git-wip-us.apache.org/repos/asf/kudu/blob/bab96a2b/src/kudu/consensus/quorum_util.cc -- diff --git a/src/kudu/consensus/quorum_util.cc b/src/kudu/consensus/quorum_util.cc index 6c791b3..ca074b4 100644 --- a/src/kudu/consensus/quorum_util.cc +++ b/src/kudu/consensus/quorum_util.cc @@ -134,9 +134,9 @@ RaftPeerPB::Role GetConsensusRole(const std::string& uuid, const ConsensusStateP return RaftPeerPB::NON_PARTICIPANT; } -Status VerifyRaftConfig(const RaftConfigPB& config, RaftConfigState type) { +Status VerifyRaftConfig(const RaftConfigPB& config) { std::set uuids; - if (config.peers_size() == 0) { + if (config.peers().empty()) {
kudu git commit: [cluster_itest_util] fix typo in format string
Repository: kudu Updated Branches: refs/heads/master 36f4259f4 -> 9b0544ae3 [cluster_itest_util] fix typo in format string Change-Id: Ied30a36d63a331eca7ff96e0ffc3225db5803b22 Reviewed-on: http://gerrit.cloudera.org:8080/8156 Reviewed-by: Adar Dembo Tested-by: Kudu Jenkins Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/9b0544ae Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/9b0544ae Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/9b0544ae Branch: refs/heads/master Commit: 9b0544ae3466235cb5bc699ec1a4def3d2aa1f02 Parents: 36f4259 Author: Alexey Serbin Authored: Wed Sep 27 15:52:23 2017 -0700 Committer: Alexey Serbin Committed: Wed Sep 27 23:18:04 2017 + -- src/kudu/integration-tests/cluster_itest_util.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/9b0544ae/src/kudu/integration-tests/cluster_itest_util.cc -- diff --git a/src/kudu/integration-tests/cluster_itest_util.cc b/src/kudu/integration-tests/cluster_itest_util.cc index 326dd53..fa4c1d4 100644 --- a/src/kudu/integration-tests/cluster_itest_util.cc +++ b/src/kudu/integration-tests/cluster_itest_util.cc @@ -451,8 +451,8 @@ Status WaitForReplicasReportedToMaster( } } if (deadline < MonoTime::Now()) { - return Status::TimedOut(Substitute("Timed out after waiting " - "for tablet $1 expected to report master with $2 replicas, has_leader: $3", + return Status::TimedOut(Substitute("Timed out while waiting " + "for tablet $0 reporting to master with $1 replicas, has_leader: $2", tablet_id, num_replicas, *has_leader)); } SleepFor(MonoDelta::FromMilliseconds(20));
kudu git commit: block_compression: fix arg to Substitute
Repository: kudu Updated Branches: refs/heads/master 4c7d8e040 -> 77dea331d block_compression: fix arg to Substitute Change-Id: If00035e39ba3f837f42e398ca6aa31bbc9afb3a9 Reviewed-on: http://gerrit.cloudera.org:8080/8203 Tested-by: Kudu Jenkins Reviewed-by: Alexey Serbin Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/77dea331 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/77dea331 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/77dea331 Branch: refs/heads/master Commit: 77dea331d8d0919d3b9ba2c8cc7f776b5d5f3a6f Parents: 4c7d8e0 Author: Adar Dembo Authored: Tue Oct 3 16:09:26 2017 -0700 Committer: Alexey Serbin Committed: Wed Oct 4 01:26:20 2017 + -- src/kudu/cfile/block_compression.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/77dea331/src/kudu/cfile/block_compression.cc -- diff --git a/src/kudu/cfile/block_compression.cc b/src/kudu/cfile/block_compression.cc index e7bb5df..aa342bb 100644 --- a/src/kudu/cfile/block_compression.cc +++ b/src/kudu/cfile/block_compression.cc @@ -167,7 +167,7 @@ Status CompressedBlockDecoder::Init() { if (uncompressed_size_ > FLAGS_max_cfile_block_size) { return Status::Corruption( Substitute("uncompressed size $0 overflows the maximum length $1, buffer", - compressed_size, FLAGS_max_cfile_block_size), + uncompressed_size_, FLAGS_max_cfile_block_size), KUDU_REDACT(data_.ToDebugString(50))); }
kudu git commit: [tablet copy] comment on TabletCopyClient lifecycle
Repository: kudu Updated Branches: refs/heads/master 77dea331d -> 3b0ef599b [tablet copy] comment on TabletCopyClient lifecycle Added an explanatory comment on the TabletCopyClient instance lifecycle. That was the result of an attempt to clean-up corresponding code: https://gerrit.cloudera.org/#/c/8133/ Change-Id: I7ae41f12fe75f05c84518c26b6875f34efae0172 Reviewed-on: http://gerrit.cloudera.org:8080/8197 Reviewed-by: Mike Percy Tested-by: Kudu Jenkins Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/3b0ef599 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/3b0ef599 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/3b0ef599 Branch: refs/heads/master Commit: 3b0ef599b2fc3fc9b895937548e98c57552b021f Parents: 77dea33 Author: Alexey Serbin Authored: Mon Oct 2 18:24:45 2017 -0700 Committer: Alexey Serbin Committed: Wed Oct 4 18:30:55 2017 + -- src/kudu/tserver/ts_tablet_manager.cc | 33 ++ 1 file changed, 33 insertions(+) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/3b0ef599/src/kudu/tserver/ts_tablet_manager.cc -- diff --git a/src/kudu/tserver/ts_tablet_manager.cc b/src/kudu/tserver/ts_tablet_manager.cc index 5b6f9e7..2967fe7 100644 --- a/src/kudu/tserver/ts_tablet_manager.cc +++ b/src/kudu/tserver/ts_tablet_manager.cc @@ -626,6 +626,39 @@ void TSTabletManager::RunTabletCopy( LOG(INFO) << init_msg; TRACE(init_msg); + // The TabletCopyClient instance should be kept alive until the tablet + // is successfully copied over and opened/started. This is because we want + // to maintain the LogAnchor until the replica starts up. Upon destruction, + // the TabletCopyClient instance sends an RPC explicitly ending the tablet + // copy session. The source replica then destroys the corresponding + // TabletCopySourceSession object, releasing its LogAnchor and allowing + // the WAL segments being copied to be GCed. + // + // See below for more details on why anchoring of WAL segments is necessary. + // + // * Assume there are WAL segments 0-10 when tablet copy starts. Tablet copy + // will anchor 0 until its destroyed, meaning the source replica wont + // delete it. + // + // * When tablet copy is done the tablet still needs to bootstrap which will + // take some time. + // + // * When tablet bootstrap is done, the new replica will need to continue + // catching up to the leader, this time through consensus. It needs segment + // 11 to be still available. We need the anchor to still be alive at this + // point, otherwise there is nothing preventing the leader from deleting + // segment 11 and thus making the new replica unable to catch up. Yes, it's + // not optimal: we're anchoring 0 and we might only need to anchor 10/11. + // However, having no anchor at all is likely to cause replicas to start + // fail copying. + // + // NOTE: + // Ideally, we should wait until the leader starts tracking of the target + // replica's log watermark. As for current implementation, the intent is + // to at least try preventing GC of logs before the tablet replica connects + // to the leader. + // + // TODO(aserbin): make this robust and more optimal than it is now. TabletCopyClient tc_client(tablet_id, fs_manager_, cmeta_manager_, server_->messenger(), &tablet_copy_metrics_);
[3/3] kudu git commit: tsan: flush TSAN memory every 10 seconds
tsan: flush TSAN memory every 10 seconds I'm seeing a TSAN precommit test fail in a hard-to-diagnose way because the Linux OOM killer is killing tservers underneath it. It doesn't appear to be any leak -- rather, TSAN is just using too much memory due to the test doing a lot of allocation. This enables a TSAN option to flush its memory every 10 seconds. This has the downside of potentially missing races that happen far apart in time (but those are unlikely to actually trigger in practice) but reduces RSS significantly. Change-Id: I721f126eb731c7f35afb732cebe48abd52c3ebd1 Reviewed-on: http://gerrit.cloudera.org:8080/8251 Tested-by: Alexey Serbin Reviewed-by: Alexey Serbin Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/24308aa1 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/24308aa1 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/24308aa1 Branch: refs/heads/master Commit: 24308aa1a5fd05e616dad2e8ee7482a74b9350de Parents: 83b6931 Author: Todd Lipcon Authored: Tue Oct 10 14:08:26 2017 -0700 Committer: Alexey Serbin Committed: Sun Oct 15 04:09:19 2017 + -- build-support/run-test.sh | 3 +++ 1 file changed, 3 insertions(+) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/24308aa1/build-support/run-test.sh -- diff --git a/build-support/run-test.sh b/build-support/run-test.sh index 941912b..6073f19 100755 --- a/build-support/run-test.sh +++ b/build-support/run-test.sh @@ -104,6 +104,9 @@ fi # Configure TSAN (ignored if this isn't a TSAN build). TSAN_OPTIONS="$TSAN_OPTIONS suppressions=$SOURCE_ROOT/build-support/tsan-suppressions.txt" TSAN_OPTIONS="$TSAN_OPTIONS history_size=7" +# Flush TSAN memory every 10 seconds - this prevents RSS blowup in unit tests +# which can cause tests to get killed by the OOM killer. +TSAN_OPTIONS="$TSAN_OPTIONS flush_memory_ms=1" TSAN_OPTIONS="$TSAN_OPTIONS external_symbolizer_path=$ASAN_SYMBOLIZER_PATH" export TSAN_OPTIONS
[1/3] kudu git commit: KUDU-2184. Avoid allocations larger than 1MB
Repository: kudu Updated Branches: refs/heads/master b6f727ea2 -> 24308aa1a KUDU-2184. Avoid allocations larger than 1MB tcmalloc doesn't perform well for allocations larger than 1MB[1]. We currently use large allocations for full arenas (in particular the MRS) which can cause large latency blips when a thread tries to allocate a new arena component. This just changes the max buffer size for all arenas to be 1MB and adds a DCHECK so we avoid this problem in the future. Additionally I increased the buffer size for the /memz output so that we can get more stats on the occupancy of different allocator size classes. [1] https://github.com/gperftools/gperftools/issues/535 Change-Id: I19c8ae9ce268e2623a89624e19db673931a093c0 Reviewed-on: http://gerrit.cloudera.org:8080/8239 Tested-by: Kudu Jenkins Reviewed-by: David Ribeiro Alves Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/eb5c7e2f Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/eb5c7e2f Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/eb5c7e2f Branch: refs/heads/master Commit: eb5c7e2f0c73729a4486bc71512739b0788b1574 Parents: b6f727e Author: Todd Lipcon Authored: Mon Oct 9 14:22:48 2017 -0700 Committer: Todd Lipcon Committed: Fri Oct 13 01:58:04 2017 + -- src/kudu/cfile/binary_dict_block.cc | 2 +- src/kudu/cfile/cfile-test-base.h | 2 +- src/kudu/cfile/cfile-test.cc | 2 +- src/kudu/cfile/cfile_util.cc | 2 +- src/kudu/server/default-path-handlers.cc | 7 --- src/kudu/tablet/concurrent_btree.h | 5 + src/kudu/tablet/deltamemstore.cc | 2 +- src/kudu/tablet/memrowset.cc | 2 +- src/kudu/tablet/tablet-test-base.h | 2 +- src/kudu/tablet/tablet_random_access-test.cc | 2 +- src/kudu/tablet/transactions/transaction.cc | 2 +- src/kudu/util/memory/arena.cc| 9 + 12 files changed, 19 insertions(+), 20 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/eb5c7e2f/src/kudu/cfile/binary_dict_block.cc -- diff --git a/src/kudu/cfile/binary_dict_block.cc b/src/kudu/cfile/binary_dict_block.cc index 9e96163..f00d75d 100644 --- a/src/kudu/cfile/binary_dict_block.cc +++ b/src/kudu/cfile/binary_dict_block.cc @@ -52,7 +52,7 @@ namespace cfile { BinaryDictBlockBuilder::BinaryDictBlockBuilder(const WriterOptions* options) : options_(options), dict_block_(options_), - dictionary_strings_arena_(1024, 32*1024*1024), + dictionary_strings_arena_(1024, 1024*1024), mode_(kCodeWordMode) { data_builder_.reset(new BShufBlockBuilder(options_)); // We use this invalid StringPiece for the "empty key". It's safe to build such http://git-wip-us.apache.org/repos/asf/kudu/blob/eb5c7e2f/src/kudu/cfile/cfile-test-base.h -- diff --git a/src/kudu/cfile/cfile-test-base.h b/src/kudu/cfile/cfile-test-base.h index f4f0c8f..64fd35d 100644 --- a/src/kudu/cfile/cfile-test-base.h +++ b/src/kudu/cfile/cfile-test-base.h @@ -467,7 +467,7 @@ void TimeReadFile(FsManager* fs_manager, const BlockId& block_id, size_t *count_ ASSERT_OK(reader->NewIterator(&iter, CFileReader::CACHE_BLOCK)); ASSERT_OK(iter->SeekToOrdinal(0)); - Arena arena(8192, 8*1024*1024); + Arena arena(8192, 1024 * 1024); int count = 0; switch (reader->type_info()->physical_type()) { case UINT8: http://git-wip-us.apache.org/repos/asf/kudu/blob/eb5c7e2f/src/kudu/cfile/cfile-test.cc -- diff --git a/src/kudu/cfile/cfile-test.cc b/src/kudu/cfile/cfile-test.cc index 364fb93..f7ffe85 100644 --- a/src/kudu/cfile/cfile-test.cc +++ b/src/kudu/cfile/cfile-test.cc @@ -200,7 +200,7 @@ class TestCFile : public CFileTestBase { gscoped_ptr iter; ASSERT_OK(reader->NewIterator(&iter, CFileReader::CACHE_BLOCK)); -Arena arena(8192, 8*1024*1024); +Arena arena(8192, 1024 * 1024); ScopedColumnBlock cb(10); SelectionVector sel(10); http://git-wip-us.apache.org/repos/asf/kudu/blob/eb5c7e2f/src/kudu/cfile/cfile_util.cc -- diff --git a/src/kudu/cfile/cfile_util.cc b/src/kudu/cfile/cfile_util.cc index 2995407..31c5af3 100644 --- a/src/kudu/cfile/cfile_util.cc +++ b/src/kudu/cfile/cfile_util.cc @@ -46,7 +46,7 @@ Status DumpIterator(const CFileReader& reader, int num_rows, int indent) { - Arena arena(8192, 8*1024*1024); + Arena arena(8192, 1024 * 1024); uint8_t buf[kBufSize]; const TypeInfo *type = reader.type_info(); size_t max_rows = kBufSize/type->size(); http://git-wip-u
[2/3] kudu git commit: Reduce log spew from rpcz_store.cc
Reduce log spew from rpcz_store.cc The sampled RPC call statement has been filing the log. Let's dial down its log level. This particularly impacts Impala as its default log level is 1. If it turns out that this log statement is useful for Kudu deployment, we can consider doing this change in Impala only. Testing done: ctest -R rpc-test; Built debug and release builds; Change-Id: I8efe68be0ae7f9ab05937d5c81fb53a709a881f1 Reviewed-on: http://gerrit.cloudera.org:8080/8273 Reviewed-by: Todd Lipcon Tested-by: Kudu Jenkins Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/83b69315 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/83b69315 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/83b69315 Branch: refs/heads/master Commit: 83b693150d0c5772ae52cf9bc2bad58a46c25a92 Parents: eb5c7e2 Author: Michael Ho Authored: Thu Oct 12 16:36:27 2017 -0700 Committer: Todd Lipcon Committed: Sat Oct 14 00:46:29 2017 + -- src/kudu/rpc/rpcz_store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/83b69315/src/kudu/rpc/rpcz_store.cc -- diff --git a/src/kudu/rpc/rpcz_store.cc b/src/kudu/rpc/rpcz_store.cc index ffbbf2f..8c8c04a 100644 --- a/src/kudu/rpc/rpcz_store.cc +++ b/src/kudu/rpc/rpcz_store.cc @@ -159,7 +159,7 @@ void MethodSampler::SampleCall(InboundCall* call) { std::swap(bucket->sample, new_sample); bucket->last_sample_time.Store(now); } -VLOG(1) << "Sampled call " << call->ToString(); +VLOG(2) << "Sampled call " << call->ToString(); } }
kudu git commit: [security] fix typo on rpc_tls_ciphers
Repository: kudu Updated Branches: refs/heads/master 24308aa1a -> 78843dd81 [security] fix typo on rpc_tls_ciphers Fixed typo in the default list of TLS ciphers. Due to the typo, the prior revisions didn't use the following ciphers: ECDHE-RSA-AES128-SHA256 TLSv1.2 Kx=ECDH Au=RSA Enc=AES(128) Mac=SHA256 AES256-GCM-SHA384 TLSv1.2 Kx=RSA Au=RSA Enc=AESGCM(256) Mac=AEAD Change-Id: I8508bf9a9002e77cb38911837b9c3881acc01cf4 Reviewed-on: http://gerrit.cloudera.org:8080/8281 Tested-by: Kudu Jenkins Reviewed-by: Dan Burkert Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/78843dd8 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/78843dd8 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/78843dd8 Branch: refs/heads/master Commit: 78843dd81c628239a5fdd962208f949c63273384 Parents: 24308aa Author: Alexey Serbin Authored: Mon Oct 16 09:41:19 2017 -0700 Committer: Alexey Serbin Committed: Mon Oct 16 18:21:49 2017 + -- src/kudu/security/tls_context.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/78843dd8/src/kudu/security/tls_context.cc -- diff --git a/src/kudu/security/tls_context.cc b/src/kudu/security/tls_context.cc index 3ab0131..1cea173 100644 --- a/src/kudu/security/tls_context.cc +++ b/src/kudu/security/tls_context.cc @@ -67,7 +67,7 @@ DEFINE_string(rpc_tls_ciphers, "ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:" "ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:" "ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:" - "ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256" + "ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:" "AES256-GCM-SHA384:AES128-GCM-SHA256:" "AES256-SHA256:AES128-SHA256:" "AES256-SHA:AES128-SHA",
[1/2] kudu git commit: [ts_itest-base] split declaration and definition
Repository: kudu Updated Branches: refs/heads/master 92064c023 -> 3a7342a31 http://git-wip-us.apache.org/repos/asf/kudu/blob/3a7342a3/src/kudu/tserver/tablet_server-test-base.h -- diff --git a/src/kudu/tserver/tablet_server-test-base.h b/src/kudu/tserver/tablet_server-test-base.h index ec015dc..a34c9b5 100644 --- a/src/kudu/tserver/tablet_server-test-base.h +++ b/src/kudu/tserver/tablet_server-test-base.h @@ -15,447 +15,109 @@ // specific language governing permissions and limitations // under the License. -#ifndef KUDU_TSERVER_TABLET_SERVER_TEST_BASE_H_ -#define KUDU_TSERVER_TABLET_SERVER_TEST_BASE_H_ +#pragma once -#include -#include -#include -#include -#include - -#include -#include +#include #include #include #include #include -#include - -#include "kudu/common/wire_protocol-test-util.h" +#include "kudu/common/row.h" +#include "kudu/common/schema.h" #include "kudu/consensus/consensus.proxy.h" -#include "kudu/consensus/log_reader.h" -#include "kudu/gutil/atomicops.h" #include "kudu/gutil/gscoped_ptr.h" -#include "kudu/gutil/stl_util.h" -#include "kudu/gutil/strings/substitute.h" -#include "kudu/rpc/messenger.h" -#include "kudu/rpc/rpc_controller.h" +#include "kudu/gutil/ref_counted.h" #include "kudu/server/server_base.proxy.h" -#include "kudu/tablet/local_tablet_writer.h" -#include "kudu/tablet/tablet.h" #include "kudu/tablet/tablet_replica.h" #include "kudu/tserver/mini_tablet_server.h" #include "kudu/tserver/tablet_copy.proxy.h" -#include "kudu/tserver/scanners.h" -#include "kudu/tserver/tablet_server.h" -#include "kudu/tserver/tablet_server_test_util.h" +#include "kudu/tserver/tserver.pb.h" #include "kudu/tserver/tserver_admin.proxy.h" #include "kudu/tserver/tserver_service.proxy.h" -#include "kudu/tserver/ts_tablet_manager.h" -#include "kudu/util/maintenance_manager.h" #include "kudu/util/metrics.h" -#include "kudu/util/pb_util.h" -#include "kudu/util/test_graph.h" -#include "kudu/util/test_macros.h" +#include "kudu/util/status.h" #include "kudu/util/test_util.h" -DEFINE_int32(rpc_timeout, 1000, "Timeout for RPC calls, in seconds"); -DEFINE_int32(num_updater_threads, 1, "Number of updating threads to launch"); -DECLARE_bool(enable_maintenance_manager); -DECLARE_int32(heartbeat_rpc_timeout_ms); +namespace kudu { -METRIC_DEFINE_entity(test); +class KuduPartialRow; +class TimeSeries; + +namespace rpc { +class Messenger; +class RpcController; +} // namespace rpc -namespace kudu { namespace tserver { class TabletServerTestBase : public KuduTest { public: typedef std::pair KeyValue; - TabletServerTestBase() -: schema_(GetSimpleTestSchema()), - ts_test_metric_entity_(METRIC_ENTITY_test.Instantiate( - &ts_test_metric_registry_, "ts_server-test")) { + TabletServerTestBase(); + + // Starts the tablet server, override to start it later. + void SetUp() override; -// Disable the maintenance ops manager since we want to trigger our own -// maintenance operations at predetermined times. -FLAGS_enable_maintenance_manager = false; + virtual void StartTabletServer(int num_data_dirs); -// Decrease heartbeat timeout: we keep re-trying heartbeats when a -// single master server fails due to a network error. Decreasing -// the hearbeat timeout to 1 second speeds up unit tests which -// purposefully specify non-running Master servers. -FLAGS_heartbeat_rpc_timeout_ms = 1000; - } + Status WaitForTabletRunning(const char *tablet_id); - // Starts the tablet server, override to start it later. - virtual void SetUp() OVERRIDE { -KuduTest::SetUp(); - -key_schema_ = schema_.CreateKeyProjection(); -rb_.reset(new RowBuilder(schema_)); - -rpc::MessengerBuilder bld("Client"); -ASSERT_OK(bld.Build(&client_messenger_)); - } - - virtual void StartTabletServer(int num_data_dirs) { -CHECK(!mini_server_); - -// Start server with an invalid master address, so it never successfully -// heartbeats, even if there happens to be a master running on this machine. -mini_server_.reset(new MiniTabletServer(GetTestPath("TabletServerTest-fsroot"), -HostPort("127.0.0.1", 0), num_data_dirs)); -mini_server_->options()->master_addresses.clear(); -mini_server_->options()->master_addresses.emplace_back("255.255.255.255", 1); -ASSERT_OK(mini_server_->Start()); - -// Set up a tablet inside the server. -ASSERT_OK(mini_server_->AddTestTablet(kTableId, kTabletId, schema_)); - ASSERT_TRUE(mini_server_->server()->tablet_manager()->LookupTablet(kTabletId, - &tablet_replica_)); - -// Creating a tablet is async, we wait here instead of having to handle errors later. -ASSERT_OK(WaitForTabletRunning(kTabletId)); - -// Connect to it. -ResetClientProxies(); - } - -
[2/2] kudu git commit: [ts_itest-base] split declaration and definition
[ts_itest-base] split declaration and definition Splitted declaration and definition of methods in TabletServerIntegrationTestBase and TabletServerTestBase test classes. In addition to shorter compilation times (no need to compile the same code multiple times), this is also necessary for upcoming changelist which splits set of tests in raft_consensus-itest into separate files. This changelist does not contain any functional changes. Change-Id: I31085ad2a6df0df795dad70e9e1c90d34cdd3716 Reviewed-on: http://gerrit.cloudera.org:8080/8277 Tested-by: Kudu Jenkins Reviewed-by: Adar Dembo Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/3a7342a3 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/3a7342a3 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/3a7342a3 Branch: refs/heads/master Commit: 3a7342a31b2c0e30f5175a2b26b6fdb6acac0dff Parents: 92064c0 Author: Alexey Serbin Authored: Sat Oct 14 12:09:20 2017 -0700 Committer: Alexey Serbin Committed: Mon Oct 16 20:42:25 2017 + -- src/kudu/integration-tests/CMakeLists.txt | 1 + .../exactly_once_writes-itest.cc| 10 +- src/kudu/integration-tests/linked_list-test.cc | 19 +- .../integration-tests/raft_consensus-itest.cc | 96 ++- src/kudu/integration-tests/ts_itest-base.cc | 580 +++ src/kudu/integration-tests/ts_itest-base.h | 496 ++-- src/kudu/tools/kudu-admin-test.cc | 57 +- src/kudu/tserver/CMakeLists.txt | 4 +- src/kudu/tserver/tablet_copy-test-base.h| 3 +- src/kudu/tserver/tablet_server-stress-test.cc | 2 +- src/kudu/tserver/tablet_server-test-base.cc | 467 +++ src/kudu/tserver/tablet_server-test-base.h | 436 ++ src/kudu/tserver/tablet_server-test.cc | 68 ++- 13 files changed, 1271 insertions(+), 968 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/3a7342a3/src/kudu/integration-tests/CMakeLists.txt -- diff --git a/src/kudu/integration-tests/CMakeLists.txt b/src/kudu/integration-tests/CMakeLists.txt index 968ef19..d124662 100644 --- a/src/kudu/integration-tests/CMakeLists.txt +++ b/src/kudu/integration-tests/CMakeLists.txt @@ -28,6 +28,7 @@ set(INTEGRATION_TESTS_SRCS internal_mini_cluster-itest-base.cc log_verifier.cc test_workload.cc + ts_itest-base.cc ) add_library(itest_util ${INTEGRATION_TESTS_SRCS}) http://git-wip-us.apache.org/repos/asf/kudu/blob/3a7342a3/src/kudu/integration-tests/exactly_once_writes-itest.cc -- diff --git a/src/kudu/integration-tests/exactly_once_writes-itest.cc b/src/kudu/integration-tests/exactly_once_writes-itest.cc index 1595432..b291bc9 100644 --- a/src/kudu/integration-tests/exactly_once_writes-itest.cc +++ b/src/kudu/integration-tests/exactly_once_writes-itest.cc @@ -22,6 +22,7 @@ #include #include +#include #include #include @@ -53,7 +54,12 @@ #include "kudu/util/test_util.h" #include "kudu/util/thread.h" +DECLARE_int32(consensus_rpc_timeout_ms); +DECLARE_int32(num_replicas); +DECLARE_int32(num_tablet_servers); + using std::string; +using std::unique_ptr; using std::vector; namespace kudu { @@ -112,7 +118,7 @@ void ExactlyOnceSemanticsITest::WriteRowsAndCollectResponses(int thread_idx, rpc::MessengerBuilder bld("Client"); ASSERT_OK(bld.Build(&client_messenger)); - std::unique_ptr proxy(new TabletServerServiceProxy( + unique_ptr proxy(new TabletServerServiceProxy( client_messenger, address, address.host())); for (int i = 0; i < num_batches; i++) { // Wait for all of the other writer threads to finish their attempts of the prior @@ -146,7 +152,7 @@ void ExactlyOnceSemanticsITest::WriteRowsAndCollectResponses(int thread_idx, controller.Reset(); WriteResponsePB response; - std::unique_ptr request_id(new rpc::RequestIdPB()); + unique_ptr request_id(new rpc::RequestIdPB()); request_id->set_client_id("test_client"); request_id->set_seq_no(i); request_id->set_attempt_no(base_attempt_idx * kMaxAttempts + num_attempts); http://git-wip-us.apache.org/repos/asf/kudu/blob/3a7342a3/src/kudu/integration-tests/linked_list-test.cc -- diff --git a/src/kudu/integration-tests/linked_list-test.cc b/src/kudu/integration-tests/linked_list-test.cc index f7479d7..b12eb14 100644 --- a/src/kudu/integration-tests/linked_list-test.cc +++ b/src/kudu/integration-tests/linked_list-test.cc @@ -39,6 +39,7 @@ #include #include +#include #include #include @@ -57,6 +58,9 @@
kudu git commit: [consensus] do not call EnableFailureDetector twice
Repository: kudu Updated Branches: refs/heads/master 3a7342a31 -> 441408908 [consensus] do not call EnableFailureDetector twice This minor clean-up removes second call to EnableFailureDetector() upon starting the Raft consensus state machine. Change-Id: I87dac245b20d85dcda991ea02b51d6ee0fe41822 Reviewed-on: http://gerrit.cloudera.org:8080/8271 Reviewed-by: Adar Dembo Tested-by: Alexey Serbin Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/44140890 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/44140890 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/44140890 Branch: refs/heads/master Commit: 4414089085876b6acfc4025391cd2b96bf833944 Parents: 3a7342a Author: Alexey Serbin Authored: Thu Oct 12 23:38:50 2017 -0700 Committer: Alexey Serbin Committed: Mon Oct 16 21:50:29 2017 + -- src/kudu/consensus/raft_consensus.cc | 15 ++- src/kudu/consensus/raft_consensus.h | 9 + 2 files changed, 11 insertions(+), 13 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/44140890/src/kudu/consensus/raft_consensus.cc -- diff --git a/src/kudu/consensus/raft_consensus.cc b/src/kudu/consensus/raft_consensus.cc index 53268db..50b60bf 100644 --- a/src/kudu/consensus/raft_consensus.cc +++ b/src/kudu/consensus/raft_consensus.cc @@ -298,7 +298,7 @@ Status RaftConsensus::Start(const ConsensusBootstrapInfo& info, // If this is the first term expire the FD immediately so that we have a // fast first election, otherwise we just let the timer expire normally. -boost::optional initial_delta; +boost::optional fd_initial_delta; if (CurrentTermUnlocked() == 0) { // The failure detector is initialized to a low value to trigger an early // election (unless someone else requested a vote from us first, which @@ -311,14 +311,13 @@ Status RaftConsensus::Start(const ConsensusBootstrapInfo& info, if (PREDICT_TRUE(FLAGS_enable_leader_failure_detection)) { LOG_WITH_PREFIX_UNLOCKED(INFO) << "Consensus starting up: Expiring failure detector timer " "to make a prompt election more likely"; -initial_delta = MonoDelta::FromMilliseconds( +fd_initial_delta = MonoDelta::FromMilliseconds( rng_.Uniform(FLAGS_raft_heartbeat_interval_ms)); } } -EnableFailureDetector(initial_delta); // Now assume "follower" duties. -RETURN_NOT_OK(BecomeReplicaUnlocked()); +RETURN_NOT_OK(BecomeReplicaUnlocked(fd_initial_delta)); SetStateUnlocked(kRunning); } @@ -572,16 +571,15 @@ Status RaftConsensus::BecomeLeaderUnlocked() { return AppendNewRoundToQueueUnlocked(round); } -Status RaftConsensus::BecomeReplicaUnlocked() { +Status RaftConsensus::BecomeReplicaUnlocked(boost::optional fd_delta) { DCHECK(lock_.is_locked()); LOG_WITH_PREFIX_UNLOCKED(INFO) << "Becoming Follower/Learner. State: " << ToStringUnlocked(); - ClearLeaderUnlocked(); // FD should be running while we are a follower. - EnableFailureDetector(); + EnableFailureDetector(std::move(fd_delta)); // Now that we're a replica, we can allow voting for other nodes. withhold_votes_until_ = MonoTime::Min(); @@ -1496,8 +1494,7 @@ Status RaftConsensus::RequestVote(const VoteRequestPB* request, // // See also https://ramcloud.stanford.edu/~ongaro/thesis.pdf // section 4.2.3. - MonoTime now = MonoTime::Now(); - if (!request->ignore_live_leader() && now < withhold_votes_until_) { + if (!request->ignore_live_leader() && MonoTime::Now() < withhold_votes_until_) { return RequestVoteRespondLeaderIsAlive(request, response); } http://git-wip-us.apache.org/repos/asf/kudu/blob/44140890/src/kudu/consensus/raft_consensus.h -- diff --git a/src/kudu/consensus/raft_consensus.h b/src/kudu/consensus/raft_consensus.h index 080bc7e..5ab93b8 100644 --- a/src/kudu/consensus/raft_consensus.h +++ b/src/kudu/consensus/raft_consensus.h @@ -435,9 +435,10 @@ class RaftConsensus : public std::enable_shared_from_this, Status BecomeLeaderUnlocked(); // Makes the peer become a replica, i.e. a FOLLOWER or a LEARNER. + // See EnableFailureDetector() for description of the 'fd_delta' parameter. // // 'lock_' must be held for configuration change before calling. - Status BecomeReplicaUnlocked(); + Status BecomeReplicaUnlocked(boost::optional fd_delta = boost::none); // Updates the state in a replica by storing the received operations in the log // and tr
[3/3] kudu git commit: [raft_consensus-itest] separate Raft election tests
[raft_consensus-itest] separate Raft election tests Separated election-related tests from raft_consensus-itest into raft_consensus-election-itest. This changelist does not contain any functional changes. Change-Id: Ic5289b3de097f9a2ca152034fa2635b67ccddb93 Reviewed-on: http://gerrit.cloudera.org:8080/8278 Reviewed-by: Adar Dembo Tested-by: Alexey Serbin Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/ab77ce02 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/ab77ce02 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/ab77ce02 Branch: refs/heads/master Commit: ab77ce025d9fe68c993d26e15b604c4188e36860 Parents: 4414089 Author: Alexey Serbin Authored: Sat Oct 14 23:18:59 2017 -0700 Committer: Alexey Serbin Committed: Tue Oct 17 01:09:23 2017 + -- build-support/dist_test.py |2 +- src/kudu/integration-tests/CMakeLists.txt |2 + .../raft_consensus-itest-base.cc| 273 .../raft_consensus-itest-base.h | 68 + .../integration-tests/raft_consensus-itest.cc | 1507 +- .../raft_consensus_election-itest.cc| 532 +++ src/kudu/tablet/mt-tablet-test.cc |2 +- src/kudu/tserver/tablet_server-test-base.cc |1 - 8 files changed, 1316 insertions(+), 1071 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/ab77ce02/build-support/dist_test.py -- diff --git a/build-support/dist_test.py b/build-support/dist_test.py index a91d7d3..328d2b8 100755 --- a/build-support/dist_test.py +++ b/build-support/dist_test.py @@ -89,7 +89,7 @@ NUM_SHARDS_BY_TEST = { 'delete_table-test': 8, 'flex_partitioning-itest': 8, 'mt-tablet-test': 4, - 'raft_consensus-itest': 8 + 'raft_consensus-itest': 6 } http://git-wip-us.apache.org/repos/asf/kudu/blob/ab77ce02/src/kudu/integration-tests/CMakeLists.txt -- diff --git a/src/kudu/integration-tests/CMakeLists.txt b/src/kudu/integration-tests/CMakeLists.txt index d124662..ecdda20 100644 --- a/src/kudu/integration-tests/CMakeLists.txt +++ b/src/kudu/integration-tests/CMakeLists.txt @@ -27,6 +27,7 @@ set(INTEGRATION_TESTS_SRCS external_mini_cluster_fs_inspector.cc internal_mini_cluster-itest-base.cc log_verifier.cc + raft_consensus-itest-base.cc test_workload.cc ts_itest-base.cc ) @@ -86,6 +87,7 @@ ADD_KUDU_TEST(multidir_cluster-itest) ADD_KUDU_TEST(open-readonly-fs-itest) ADD_KUDU_TEST(raft_config_change-itest) ADD_KUDU_TEST(raft_consensus-itest RUN_SERIAL true) +ADD_KUDU_TEST(raft_consensus_election-itest) ADD_KUDU_TEST(registration-test RESOURCE_LOCK "master-web-port") ADD_KUDU_TEST(security-faults-itest) ADD_KUDU_TEST(security-itest) http://git-wip-us.apache.org/repos/asf/kudu/blob/ab77ce02/src/kudu/integration-tests/raft_consensus-itest-base.cc -- diff --git a/src/kudu/integration-tests/raft_consensus-itest-base.cc b/src/kudu/integration-tests/raft_consensus-itest-base.cc new file mode 100644 index 000..afd8c64 --- /dev/null +++ b/src/kudu/integration-tests/raft_consensus-itest-base.cc @@ -0,0 +1,273 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "kudu/integration-tests/raft_consensus-itest-base.h" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "kudu/client/client-test-util.h" +#include "kudu/client/client.h" +#include "kudu/client/shared_ptr.h" +#include "kudu/client/write_op.h" +#include "kudu/common/partial_row.h" +#include "kudu/common/wire_protocol.h" +#include "kudu/consensus/consensus.pb.h" +#include "kudu/consensus/opid.pb.h&
[1/3] kudu git commit: [raft_consensus-itest] separate Raft election tests
Repository: kudu Updated Branches: refs/heads/master 441408908 -> ab77ce025 http://git-wip-us.apache.org/repos/asf/kudu/blob/ab77ce02/src/kudu/integration-tests/raft_consensus_election-itest.cc -- diff --git a/src/kudu/integration-tests/raft_consensus_election-itest.cc b/src/kudu/integration-tests/raft_consensus_election-itest.cc new file mode 100644 index 000..ba4d538 --- /dev/null +++ b/src/kudu/integration-tests/raft_consensus_election-itest.cc @@ -0,0 +1,532 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "kudu/common/wire_protocol.pb.h" +#include "kudu/consensus/metadata.pb.h" +#include "kudu/gutil/gscoped_ptr.h" +#include "kudu/gutil/map-util.h" +#include "kudu/gutil/strings/substitute.h" +#include "kudu/integration-tests/cluster_itest_util.h" +#include "kudu/integration-tests/cluster_verifier.h" +#include "kudu/integration-tests/external_mini_cluster_fs_inspector.h" +#include "kudu/integration-tests/raft_consensus-itest-base.h" +#include "kudu/integration-tests/test_workload.h" +#include "kudu/mini-cluster/external_mini_cluster.h" +#include "kudu/tablet/metadata.pb.h" +#include "kudu/tserver/tablet_server-test-base.h" +#include "kudu/tserver/tserver.pb.h" +#include "kudu/util/metrics.h" +#include "kudu/util/monotime.h" +#include "kudu/util/pb_util.h" +#include "kudu/util/status.h" +#include "kudu/util/stopwatch.h" +#include "kudu/util/test_macros.h" +#include "kudu/util/test_util.h" + +DECLARE_int64(client_inserts_per_thread); +DECLARE_int64(client_num_batches_per_thread); +DECLARE_int32(consensus_rpc_timeout_ms); +DECLARE_int32(num_client_threads); +DECLARE_int32(num_replicas); +DECLARE_int32(num_tablet_servers); + +METRIC_DECLARE_entity(tablet); +METRIC_DECLARE_counter(transaction_memory_pressure_rejections); +METRIC_DECLARE_gauge_int64(raft_term); + +using kudu::cluster::ExternalTabletServer; +using kudu::consensus::RaftPeerPB; +using kudu::itest::AddServer; +using kudu::itest::GetReplicaStatusAndCheckIfLeader; +using kudu::itest::LeaderStepDown; +using kudu::itest::RemoveServer; +using kudu::itest::StartElection; +using kudu::itest::TabletServerMap; +using kudu::itest::TServerDetails; +using kudu::itest::WaitUntilLeader; +using kudu::itest::WriteSimpleTestRow; +using kudu::pb_util::SecureShortDebugString; +using kudu::tablet::TABLET_DATA_COPYING; +using std::string; +using std::unordered_set; +using std::vector; +using strings::Substitute; + +namespace kudu { +namespace tserver { + +static const int kTestRowKey = 1234; +static const int kTestRowIntVal = 5678; + +class RaftConsensusElectionITest : public RaftConsensusITestBase { + protected: + void CreateClusterForChurnyElectionsTests(const vector& extra_ts_flags); + void DoTestChurnyElections(TestWorkload* workload, int max_rows_to_insert); +}; + +void RaftConsensusElectionITest::CreateClusterForChurnyElectionsTests( +const vector& extra_ts_flags) { + vector ts_flags; + +#ifdef THREAD_SANITIZER + // On TSAN builds, we need to be a little bit less churny in order to make + // any progress at all. + ts_flags.push_back("--raft_heartbeat_interval_ms=5"); +#else + ts_flags.emplace_back("--raft_heartbeat_interval_ms=1"); +#endif + + ts_flags.insert(ts_flags.end(), extra_ts_flags.cbegin(), extra_ts_flags.cend()); + + CreateCluster("raft_consensus-itest-cluster", ts_flags, {}); +} + +void RaftConsensusElectionITest::DoTestChurnyElections(TestWorkload* workload, + int max_rows_to_insert) { + workload->set_num_replicas(FLAGS_num_replicas); + // Set a really high write timeout so that even in the presence of many failures we + // can verify an exact number of rows in the end, thanks to exactly once semantics. + workload->set_write_timeout_millis(60 * 1000 /* 60 seconds */); + workload->set_num_write_threads(2); + workload->set_write_batch_size(1); + workload->Setup(); + workload->Start(); + + // Run for either a prescribed number of writes, or 30 seconds, + // whichever comes first. This prevents test timeouts on slower + // buil
[2/3] kudu git commit: [raft_consensus-itest] separate Raft election tests
http://git-wip-us.apache.org/repos/asf/kudu/blob/ab77ce02/src/kudu/integration-tests/raft_consensus-itest.cc -- diff --git a/src/kudu/integration-tests/raft_consensus-itest.cc b/src/kudu/integration-tests/raft_consensus-itest.cc index b3b63f9..4d030c5 100644 --- a/src/kudu/integration-tests/raft_consensus-itest.cc +++ b/src/kudu/integration-tests/raft_consensus-itest.cc @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -#include #include #include #include @@ -27,13 +26,11 @@ #include #include -#include #include #include #include #include -#include "kudu/client/client-test-util.h" #include "kudu/client/client.h" #include "kudu/client/shared_ptr.h" #include "kudu/client/write_op.h" @@ -52,10 +49,8 @@ #include "kudu/gutil/basictypes.h" #include "kudu/gutil/gscoped_ptr.h" #include "kudu/gutil/map-util.h" -#include "kudu/gutil/port.h" #include "kudu/gutil/ref_counted.h" #include "kudu/gutil/stl_util.h" -#include "kudu/gutil/stringprintf.h" #include "kudu/gutil/strings/strcat.h" #include "kudu/gutil/strings/substitute.h" #include "kudu/gutil/strings/util.h" @@ -63,8 +58,8 @@ #include "kudu/integration-tests/cluster_verifier.h" #include "kudu/integration-tests/external_mini_cluster_fs_inspector.h" #include "kudu/integration-tests/log_verifier.h" +#include "kudu/integration-tests/raft_consensus-itest-base.h" #include "kudu/integration-tests/test_workload.h" -#include "kudu/integration-tests/ts_itest-base.h" #include "kudu/master/master.pb.h" #include "kudu/mini-cluster/external_mini_cluster.h" #include "kudu/rpc/messenger.h" @@ -81,23 +76,18 @@ #include "kudu/util/monotime.h" #include "kudu/util/pb_util.h" #include "kudu/util/random.h" -#include "kudu/util/slice.h" #include "kudu/util/status.h" -#include "kudu/util/stopwatch.h" #include "kudu/util/test_macros.h" #include "kudu/util/test_util.h" #include "kudu/util/thread.h" +DECLARE_int64(client_inserts_per_thread); +DECLARE_int64(client_num_batches_per_thread); DECLARE_int32(consensus_rpc_timeout_ms); +DECLARE_int32(num_client_threads); DECLARE_int32(num_replicas); DECLARE_int32(num_tablet_servers); DECLARE_int32(rpc_timeout); -DEFINE_int32(num_client_threads, 8, - "Number of client threads to launch"); -DEFINE_int64(client_inserts_per_thread, 50, - "Number of rows inserted by each client thread"); -DEFINE_int64(client_num_batches_per_thread, 5, - "In how many batches to group the rows, for each client"); METRIC_DECLARE_entity(tablet); METRIC_DECLARE_counter(transaction_memory_pressure_rejections); @@ -148,50 +138,9 @@ static const int kTestRowIntVal = 5678; // Integration test for the raft consensus implementation. // Uses the whole tablet server stack with ExternalMiniCluster. -class RaftConsensusITest : public TabletServerIntegrationTestBase { +class RaftConsensusITest : public RaftConsensusITestBase { public: - RaftConsensusITest() - : inserters_(FLAGS_num_client_threads) { - } - - virtual void SetUp() OVERRIDE { -TabletServerIntegrationTestBase::SetUp(); -FLAGS_consensus_rpc_timeout_ms = kConsensusRpcTimeoutForTests; - } - - void ScanReplica(TabletServerServiceProxy* replica_proxy, - vector* results) { - -ScanRequestPB req; -ScanResponsePB resp; -RpcController rpc; -rpc.set_timeout(MonoDelta::FromSeconds(10)); // Squelch warnings. - -NewScanRequestPB* scan = req.mutable_new_scan_request(); -scan->set_tablet_id(tablet_id_); -ASSERT_OK(SchemaToColumnPBs(schema_, scan->mutable_projected_columns())); - -// Send the call -{ - req.set_batch_size_bytes(0); - SCOPED_TRACE(SecureDebugString(req)); - ASSERT_OK(replica_proxy->Scan(req, &resp, &rpc)); - SCOPED_TRACE(SecureDebugString(resp)); - if (resp.has_error()) { -ASSERT_OK(StatusFromPB(resp.error().status())); - } -} - -if (!resp.has_more_results()) - return; - -// Drain all the rows from the scanner. -NO_FATALS(DrainScannerToStrings(resp.scanner_id(), -schema_, -results, -replica_proxy)); - -std::sort(results->begin(), results->end()); + RaftConsensusITest() { } // Scan the given replica in a loop until the number of rows @@ -199,28 +148,7 @@ class RaftConsensusITest : public TabletServerIntegrationTestBase { // fails the test. void WaitForRowCount(TabletServerServiceProxy* replica_proxy, int expected_count, - vector* results) { -LOG(INFO) << "Waiting for row count " << expected_count << "..."; -MonoTime start = MonoTime::Now(); -MonoTime deadline = start + MonoDelta::FromSeconds(90); -while (true) { - results->clear(); - NO_FATALS(ScanReplica(replica_proxy, results)); -
[2/5] kudu git commit: [webui] Add templates for master web ui.
[webui] Add templates for master web ui. This adds templates to the rest of the master webui endpoints for which it makes sense: - /tablet-servers - /masters Change-Id: I4520261dd617a160da51cdd089f42f078a0da728 Reviewed-on: http://gerrit.cloudera.org:8080/8227 Tested-by: Kudu Jenkins Reviewed-by: Todd Lipcon Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/33c06692 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/33c06692 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/33c06692 Branch: refs/heads/master Commit: 33c06692e60d0347063fb92a0affddd471e06b3f Parents: c18815f Author: Will Berkeley Authored: Sat Sep 16 23:35:21 2017 -0700 Committer: Todd Lipcon Committed: Tue Oct 17 04:36:24 2017 + -- src/kudu/master/master-path-handlers.cc | 140 +++ src/kudu/master/master-path-handlers.h | 11 +-- www/masters.mustache| 45 + www/tables.mustache | 2 +- www/tablet-servers.mustache | 81 5 files changed, 185 insertions(+), 94 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/33c06692/src/kudu/master/master-path-handlers.cc -- diff --git a/src/kudu/master/master-path-handlers.cc b/src/kudu/master/master-path-handlers.cc index 28b604a..397776b 100644 --- a/src/kudu/master/master-path-handlers.cc +++ b/src/kudu/master/master-path-handlers.cc @@ -86,63 +86,53 @@ MasterPathHandlers::~MasterPathHandlers() { } void MasterPathHandlers::HandleTabletServers(const Webserver::WebRequest& /*req*/, - Webserver::PrerenderedWebResponse* resp) { - ostringstream* output = resp->output; - vector > descs; + Webserver::WebResponse* resp) { + EasyJson* output = resp->output; + vector> descs; master_->ts_manager()->GetAllDescriptors(&descs); - *output << "Tablet Servers\n"; - *output << Substitute("There are $0 registered tablet servers.", descs.size()); - + (*output)["num_ts"] = std::to_string(descs.size()); + + // In mustache, when conditionally rendering a section of the template based + // on a key, the value becomes the context. In the subcontext, searches for + // keys used to display a value (e.g. {{my_key}}) recurse into the parent + // context-- but not when used to define a subsection (e.g. {{#my_list}}. + // Thus, we use a negative test {{^has_no_live_ts}} to decide if we + // should render the live and dead tablet server tables. + bool has_no_live_ts = true; + bool has_no_dead_ts = true; + output->Set("live_tservers", EasyJson::kArray); + output->Set("dead_tservers", EasyJson::kArray); map> version_counts; - vector live_tserver_rows; - vector dead_tserver_rows; for (const std::shared_ptr& desc : descs) { -const string time_since_hb = StringPrintf("%.1fs", desc->TimeSinceHeartbeat().ToSeconds()); +string ts_key = desc->PresumedDead() ? "dead_tservers" : "live_tservers"; +EasyJson ts_json = (*output)[ts_key].PushBack(EasyJson::kObject); + ServerRegistrationPB reg; desc->GetRegistration(®); - -string row = Substitute("$0$1$2\n", -RegistrationToHtml(reg, desc->permanent_uuid()), -time_since_hb, - EscapeForHtmlToString(pb_util::SecureShortDebugString(reg))); - -if (desc->PresumedDead()) { - version_counts[reg.software_version()][1]++; - dead_tserver_rows.push_back(row); -} else { - version_counts[reg.software_version()][0]++; - live_tserver_rows.push_back(row); +ts_json["uuid"] = desc->permanent_uuid(); +if (!reg.http_addresses().empty()) { + ts_json["target"] = Substitute("$0://$1:$2/", + reg.https_enabled() ? "https" : "http", + reg.http_addresses(0).host(), + reg.http_addresses(0).port()); } - +ts_json["time_since_hb"] = StringPrintf("%.1fs", desc->TimeSinceHeartbeat().ToSeconds()); +ts_json["registration"] = pb_util::SecureShortDebugString(reg); +version_counts[reg.software_version()][desc->PresumedDead() ? 1 : 0]++; +has_no_live_ts &= desc->PresumedDead(); +has_no_dead_ts &= !desc->PresumedDead(); } + (*output)["has_no_live_ts"] = has_no_live_ts; + (*output)["has_no_dead_ts"] = has_no_dead_ts; - *output << "Version Summary"; - *output << "\n"; - *output << "VersionCount (Live)Count (Dead)\n"; - *output << "\n"; + output->Set("version_counts", EasyJson::kArray); for (const auto& entry : version_counts) { -*output << Substitute("$0$1$2\n", - entry.first, entry.se
[5/5] kudu git commit: [consensus] adding/removing NON_VOTER members
[consensus] adding/removing NON_VOTER members Added ability to add and remove NON_VOTER member replicas. Updated the kudu CLI accordingly. Also, added new integration test: * RaftConsensusITest.AddNonVoterReplica * RaftConsensusITest.AddThenRemoveNonVoterReplica * RaftConsensusITest.NonVoterReplicasDoNotVote Change-Id: I2662d45ad9bb6a4bf325d4202c2ee619ffad02b7 Reviewed-on: http://gerrit.cloudera.org:8080/8138 Reviewed-by: Todd Lipcon Reviewed-by: Mike Percy Tested-by: Alexey Serbin Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/fe23710c Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/fe23710c Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/fe23710c Branch: refs/heads/master Commit: fe23710c63c90e561b2622f6f67563572d10dbf9 Parents: 8c23c97 Author: Alexey Serbin Authored: Mon Oct 16 16:23:57 2017 -0700 Committer: Alexey Serbin Committed: Tue Oct 17 15:20:31 2017 + -- src/kudu/consensus/leader_election-test.cc | 5 +- src/kudu/consensus/leader_election.cc | 22 +- src/kudu/consensus/leader_election.h| 4 +- src/kudu/consensus/quorum_util-test.cc | 47 +- src/kudu/consensus/quorum_util.cc | 10 +- src/kudu/consensus/quorum_util.h| 4 + src/kudu/consensus/raft_consensus.cc| 22 +- src/kudu/integration-tests/CMakeLists.txt | 1 + .../integration-tests/cluster_itest_util.cc | 4 +- .../raft_consensus-itest-base.cc| 6 + .../raft_consensus-itest-base.h | 12 +- .../integration-tests/raft_consensus-itest.cc | 82 ++- .../raft_consensus_nonvoter-itest.cc| 534 +++ 13 files changed, 671 insertions(+), 82 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/fe23710c/src/kudu/consensus/leader_election-test.cc -- diff --git a/src/kudu/consensus/leader_election-test.cc b/src/kudu/consensus/leader_election-test.cc index cec04c3..1dc26d9 100644 --- a/src/kudu/consensus/leader_election-test.cc +++ b/src/kudu/consensus/leader_election-test.cc @@ -161,7 +161,8 @@ void LeaderElectionTest::ElectionCallback(const ElectionResult& result) { void LeaderElectionTest::InitUUIDs(int num_voters) { voter_uuids_ = GenVoterUUIDs(num_voters); - candidate_uuid_ = voter_uuids_[num_voters - 1]; + CHECK(!voter_uuids_.empty()); + candidate_uuid_ = voter_uuids_.back(); voter_uuids_.pop_back(); } @@ -170,6 +171,7 @@ void LeaderElectionTest::InitNoOpPeerProxies() { for (const string& uuid : voter_uuids_) { RaftPeerPB* peer_pb = config_.add_peers(); peer_pb->set_permanent_uuid(uuid); +peer_pb->set_member_type(RaftPeerPB::VOTER); PeerProxy* proxy = new NoOpTestPeerProxy(pool_.get(), *peer_pb); InsertOrDie(&proxies_, uuid, proxy); } @@ -180,6 +182,7 @@ void LeaderElectionTest::InitDelayableMockedProxies(bool enable_delay) { for (const string& uuid : voter_uuids_) { RaftPeerPB* peer_pb = config_.add_peers(); peer_pb->set_permanent_uuid(uuid); +peer_pb->set_member_type(RaftPeerPB::VOTER); auto proxy = new DelayablePeerProxy(pool_.get(), new MockedPeerProxy(pool_.get())); if (enable_delay) { http://git-wip-us.apache.org/repos/asf/kudu/blob/fe23710c/src/kudu/consensus/leader_election.cc -- diff --git a/src/kudu/consensus/leader_election.cc b/src/kudu/consensus/leader_election.cc index 07b8a01..b8cbddc 100644 --- a/src/kudu/consensus/leader_election.cc +++ b/src/kudu/consensus/leader_election.cc @@ -161,8 +161,18 @@ LeaderElection::LeaderElection(const RaftConfigPB& config, decision_callback_(std::move(decision_callback)), highest_voter_term_(0) { for (const RaftPeerPB& peer : config.peers()) { -if (request.candidate_uuid() == peer.permanent_uuid()) continue; -follower_uuids_.push_back(peer.permanent_uuid()); +if (request.candidate_uuid() == peer.permanent_uuid()) { + DCHECK_EQ(peer.member_type(), RaftPeerPB::VOTER) + << Substitute("non-voter member $0 tried to start an election; " +"Raft config {$1}", +peer.permanent_uuid(), +pb_util::SecureShortDebugString(config)); + continue; +} +if (peer.member_type() != RaftPeerPB::VOTER) { + continue; +} +other_voter_uuids_.push_back(peer.permanent_uuid()); gscoped_ptr state(new VoterState()); state->proxy_status = proxy_factory->NewProxy(peer, &state->proxy); @@ -173,10 +183,10 @@ LeaderElecti
[4/5] kudu git commit: KUDU-2044 Tombstoned tablets show up in /metrics
KUDU-2044 Tombstoned tablets show up in /metrics This patch stops tombstoned tablets from showing up in /metrics. When a tablet replica is shut down, the tablet's metric entity is marked as unpublished. All its child metrics will be marked for retirement, then retired after the retirement interval has passed. Then the entity itself is unregistered, i.e. removed from the metric registry's map of entities. If a new replica of the same tablet is added to the server, it will create a new entity that will be registered with the metric_registry, either as a new insertion or overwriting the old replica's entity if the entity had been unpublished but not yet unregistered. The tombstoned tablet's metric entity is not destroyed when it's deregistered, since there are still refs to it and its metrics in the TabletReplica class, Tablet class, and many other classes associated with a TabletReplica. The entity will be destroyed when the TabletReplica is deleted (since it either contains or holds final references to all the other classes), which happens if the tablet is replaced or deleted. While it's not ideal to hold the memory for the entity when it's no longer used, the reason this occurs is because the TabletReplica instance mostly stays alive. Releasing all the metric references would require specifically dropping refs to those metrics in all the surviving subcomponents of a TabletReplica instance that has been shut down; I think this problem would be better solved by more completely cleaning up a shut down TabletReplica instance, but that's a much larger scope than suppressing the metrics. Change-Id: I21b52262203410ded1e514502b59a79be12f8fb3 Reviewed-on: http://gerrit.cloudera.org:8080/7981 Tested-by: Kudu Jenkins Reviewed-by: David Ribeiro Alves Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/8c23c97b Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/8c23c97b Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/8c23c97b Branch: refs/heads/master Commit: 8c23c97b82708b96e163e75e0002ce96adf8d8fc Parents: 60ea4a6 Author: Will Berkeley Authored: Wed Sep 6 10:57:39 2017 -0700 Committer: Will Berkeley Committed: Tue Oct 17 14:54:51 2017 + -- src/kudu/integration-tests/tablet_copy-itest.cc | 72 ++-- src/kudu/tablet/tablet.cc | 3 + src/kudu/tserver/tablet_server-test.cc | 47 + src/kudu/util/metrics.cc| 18 +++-- src/kudu/util/metrics.h | 17 - 5 files changed, 146 insertions(+), 11 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/8c23c97b/src/kudu/integration-tests/tablet_copy-itest.cc -- diff --git a/src/kudu/integration-tests/tablet_copy-itest.cc b/src/kudu/integration-tests/tablet_copy-itest.cc index 88a9350..a36e65e 100644 --- a/src/kudu/integration-tests/tablet_copy-itest.cc +++ b/src/kudu/integration-tests/tablet_copy-itest.cc @@ -124,10 +124,12 @@ using std::unordered_map; using std::vector; using strings::Substitute; +METRIC_DECLARE_entity(tablet); METRIC_DECLARE_histogram(handler_latency_kudu_consensus_ConsensusService_UpdateConsensus); METRIC_DECLARE_counter(glog_info_messages); METRIC_DECLARE_counter(glog_warning_messages); METRIC_DECLARE_counter(glog_error_messages); +METRIC_DECLARE_counter(rows_inserted); METRIC_DECLARE_counter(tablet_copy_bytes_fetched); METRIC_DECLARE_counter(tablet_copy_bytes_sent); METRIC_DECLARE_gauge_int32(tablet_copy_open_client_sessions); @@ -1624,6 +1626,17 @@ int64_t CountBootstrappingTablets(ExternalTabletServer *ets) { &ret)); return ret; } + +int64_t CountRowsInserted(ExternalTabletServer *ets) { + int64_t ret; + CHECK_OK(ets->GetInt64Metric( + &METRIC_ENTITY_tablet, + nullptr, + &METRIC_rows_inserted, + "value", + &ret)); + return ret; +} } // anonymous namespace // Check that state metrics work during tablet copy. @@ -1676,10 +1689,9 @@ TEST_F(TabletCopyITest, TestTabletStateMetricsDuringTabletCopy) { }); // Wait for the tablet to be revived via tablet copy. - ASSERT_OK(inspect_->WaitForTabletDataStateOnTS( - follower_index, tablet_id, - { tablet::TABLET_DATA_READY }, - kTimeout)); + ASSERT_OK(inspect_->WaitForTabletDataStateOnTS(follower_index, tablet_id, + { tablet::TABLET_DATA_READY }, + kTimeout)); // State: 1 running or bootstrapping tablet. ASSERT_EVENTUALLY([&] { @@ -1688,6 +1700,58 @@ TEST_F(TabletCopyITest, TestTabletStateMetricsDuringTabletCopy) { }); } +TEST_F(TabletCopyITest, TestMetricsResetAfterRevival) { + MonoDelta kTimeout = MonoDelta::FromSeconds(3
[3/5] kudu git commit: Implement per-tablet active scanner metric
Implement per-tablet active scanner metric The active_scanners metric would be useful scoped per-tablet. This patch exposes it as tablet_active_scanners. It also adds coverage of scanner expiration in tablet_server-test. Change-Id: I3b2d85f78c655e02654f8581e9521452b149b7d5 Reviewed-on: http://gerrit.cloudera.org:8080/8228 Tested-by: Kudu Jenkins Reviewed-by: Todd Lipcon Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/60ea4a64 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/60ea4a64 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/60ea4a64 Branch: refs/heads/master Commit: 60ea4a64366d46a7b2e86aef5c9b38c32e6af15b Parents: 33c0669 Author: Will Berkeley Authored: Mon Oct 2 15:50:33 2017 -0700 Committer: Todd Lipcon Committed: Tue Oct 17 04:39:03 2017 + -- src/kudu/tablet/tablet_metrics.cc | 22 ++ src/kudu/tablet/tablet_metrics.h | 4 ++ src/kudu/tserver/scanner_metrics.cc| 4 +- src/kudu/tserver/scanners.cc | 16 ++- src/kudu/tserver/tablet_server-test.cc | 67 +++-- 5 files changed, 96 insertions(+), 17 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/60ea4a64/src/kudu/tablet/tablet_metrics.cc -- diff --git a/src/kudu/tablet/tablet_metrics.cc b/src/kudu/tablet/tablet_metrics.cc index b5469e9..a3b6b41 100644 --- a/src/kudu/tablet/tablet_metrics.cc +++ b/src/kudu/tablet/tablet_metrics.cc @@ -42,6 +42,15 @@ METRIC_DEFINE_counter(tablet, rows_deleted, "Rows Deleted", kudu::MetricUnit::kRows, "Number of row delete operations performed on this tablet since service start"); +METRIC_DEFINE_counter(tablet, insertions_failed_dup_key, "Duplicate Key Inserts", + kudu::MetricUnit::kRows, + "Number of inserts which failed because the key already existed"); + +METRIC_DEFINE_counter(tablet, upserts_as_updates, "Upserts converted into updates", + kudu::MetricUnit::kRows, + "Number of upserts which were applied as updates because the key already " + "existed."); + METRIC_DEFINE_counter(tablet, scanner_rows_returned, "Scanner Rows Returned", kudu::MetricUnit::kRows, "Number of rows returned by scanners to clients. This count " @@ -59,7 +68,6 @@ METRIC_DEFINE_counter(tablet, scanner_bytes_returned, "Scanner Bytes Returned", "for consumption by clients, and thus is not " "a reflection of the amount of work being done by scanners."); - METRIC_DEFINE_counter(tablet, scanner_rows_scanned, "Scanner Rows Scanned", kudu::MetricUnit::kRows, "Number of rows processed by scan requests. This is measured " @@ -90,17 +98,12 @@ METRIC_DEFINE_counter(tablet, scanner_bytes_scanned_from_disk, "Scanner Bytes Sc "and does not include data read from in-memory stores. However, it" "includes both cache misses and cache hits."); - -METRIC_DEFINE_counter(tablet, insertions_failed_dup_key, "Duplicate Key Inserts", - kudu::MetricUnit::kRows, - "Number of inserts which failed because the key already existed"); -METRIC_DEFINE_counter(tablet, upserts_as_updates, "Upserts converted into updates", - kudu::MetricUnit::kRows, - "Number of upserts which were applied as updates because the key already " - "existed."); METRIC_DEFINE_counter(tablet, scans_started, "Scans Started", kudu::MetricUnit::kScanners, "Number of scanners which have been started on this tablet"); +METRIC_DEFINE_gauge_size(tablet, tablet_active_scanners, "Active Scanners", + kudu::MetricUnit::kScanners, + "Number of scanners that are currently active on this tablet"); METRIC_DEFINE_counter(tablet, bloom_lookups, "Bloom Filter Lookups", kudu::MetricUnit::kProbes, @@ -275,6 +278,7 @@ TabletMetrics::TabletMetrics(const scoped_refptr& entity) MINIT(scanner_cells_scanned_from_disk), MINIT(scanner_bytes_scanned_from_disk), MINIT(scans_started), +GINIT(tablet_active_scanners), MINIT(bloom_lookups), MINIT(key_file_lookups), MINIT(delta_file_lookups), http://git-wip-us.apache.org/repos/asf/kudu/blob/60ea4a64/src/kudu/tablet/tablet_metrics.h -- diff --git a/src/kudu/tablet/tablet_metrics.h b/src/kudu/tablet/tablet_metrics.h index 6fa5a64..9fe87b7 100644 --- a/src/kudu/tablet/tabl
[1/5] kudu git commit: Allow configuration of values passed into kerberos env vars
Repository: kudu Updated Branches: refs/heads/master f5e1203a3 -> fe23710c6 Allow configuration of values passed into kerberos env vars We always used hardcoded constants for the following kerberos environment variables: KRB5CCNAME and KRB5RCACHETYPE. This patch allows for the configuration of these variables by taking arguments to InitKerberosForServer(). Callsites within Kudu have not been changed as all the parameters have default values. The motivation for this patch is that, Impala as a user of the KuduRPC and Kudu security libraries, needs to have a file based credential cache since the kinit happens on the C++ side and this cache needs to be read by the Java side too. Hence, we cannot have it in memory. Also, Impala still requires replay protection, since some Impala services use Thrift which lacks the nonce mechanism that KRPC uses for replay protection. Change-Id: Iab4ce72c04ec4056dc89fb4c1c540a6fdaca4404 Reviewed-on: http://gerrit.cloudera.org:8080/8247 Reviewed-by: Todd Lipcon Tested-by: Todd Lipcon Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/c18815f1 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/c18815f1 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/c18815f1 Branch: refs/heads/master Commit: c18815f1edd3fc23abc874ffe9fe564c3114fc8e Parents: f5e1203 Author: Sailesh Mukil Authored: Mon Oct 9 23:20:39 2017 -0700 Committer: Todd Lipcon Committed: Tue Oct 17 04:21:34 2017 + -- src/kudu/security/init.cc | 20 +--- src/kudu/security/init.h | 11 ++- 2 files changed, 19 insertions(+), 12 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/c18815f1/src/kudu/security/init.cc -- diff --git a/src/kudu/security/init.cc b/src/kudu/security/init.cc index ded848b..58dd48d 100644 --- a/src/kudu/security/init.cc +++ b/src/kudu/security/init.cc @@ -459,21 +459,19 @@ boost::optional GetLoggedInUsernameFromKeytab() { return g_kinit_ctx->username_str(); } -Status InitKerberosForServer() { +Status InitKerberosForServer(const std::string& krb5ccname, bool disable_krb5_replay_cache) { if (FLAGS_keytab_file.empty()) return Status::OK(); - // Have the daemons use an in-memory ticket cache, so they don't accidentally - // pick up credentials from test cases or any other daemon. - // TODO(todd): extract these krb5 env vars into some constants since they're - // typo-prone. - setenv("KRB5CCNAME", "MEMORY:kudu", 1); + setenv("KRB5CCNAME", krb5ccname.c_str(), 1); setenv("KRB5_KTNAME", FLAGS_keytab_file.c_str(), 1); - // KUDU-1897: disable the Kerberos replay cache. The KRPC protocol includes a - // per-connection server-generated nonce to protect against replay attacks - // when authenticating via Kerberos. The replay cache has many performance and - // implementation issues. - setenv("KRB5RCACHETYPE", "none", 1); + if (disable_krb5_replay_cache) { +// KUDU-1897: disable the Kerberos replay cache. The KRPC protocol includes a +// per-connection server-generated nonce to protect against replay attacks +// when authenticating via Kerberos. The replay cache has many performance and +// implementation issues. +setenv("KRB5RCACHETYPE", "none", 1); + } g_kinit_ctx = new KinitContext(); string principal; http://git-wip-us.apache.org/repos/asf/kudu/blob/c18815f1/src/kudu/security/init.h -- diff --git a/src/kudu/security/init.h b/src/kudu/security/init.h index 913585f..0832256 100644 --- a/src/kudu/security/init.h +++ b/src/kudu/security/init.h @@ -30,9 +30,18 @@ class Status; namespace security { +// The default kerberos credential cache name. +// Have the daemons use an in-memory ticket cache, so they don't accidentally +// pick up credentials from test cases or any other daemon. +static const std::string kKrb5CCName = "MEMORY:kudu"; + // Initializes Kerberos for a server. In particular, this processes // the '--keytab_file' command line flag. -Status InitKerberosForServer(); +// 'krb5ccname' is passed into the KRB5CCNAME env var. +// 'disable_krb5_replay_cache' if set to true, disables the kerberos replay cache by setting +// the KRB5RCACHETYPE env var to "none". +Status InitKerberosForServer(const std::string& krb5ccname = kKrb5CCName, + bool disable_krb5_replay_cache = true); // Returns the process lock 'kerberos_reinit_lock' // This lock is taken in write mode while the ticket is being reacquired, and
kudu git commit: consensus: Rename CHANGE_ROLE op => CHANGE_REPLICA_TYPE
Repository: kudu Updated Branches: refs/heads/master 827c35bc3 -> abbfa49de consensus: Rename CHANGE_ROLE op => CHANGE_REPLICA_TYPE "Role" is currently a pretty ambiguous term in Kudu because role is currently defined as LEADER, FOLLOWER, LEARNER, etc. "member_type" is the standard type name but "replica" is more descriptive. Let's rename this operation CHANGE_REPLICA_TYPE. This is a wire-compatible change. Change-Id: I29c8bed1f6b5c84b86be3997b76ac4f825fa2f0e Reviewed-on: http://gerrit.cloudera.org:8080/8295 Tested-by: Kudu Jenkins Reviewed-by: Alexey Serbin Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/abbfa49d Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/abbfa49d Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/abbfa49d Branch: refs/heads/master Commit: abbfa49dee52d49dc4658c1ef57ed8467e43bb5d Parents: 827c35b Author: Mike Percy Authored: Tue Oct 10 12:56:13 2017 -0700 Committer: Alexey Serbin Committed: Tue Oct 17 21:50:28 2017 + -- src/kudu/consensus/consensus.proto | 2 +- src/kudu/consensus/raft_consensus.cc | 4 ++-- src/kudu/tools/tool_action_tablet.cc | 7 --- 3 files changed, 7 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/abbfa49d/src/kudu/consensus/consensus.proto -- diff --git a/src/kudu/consensus/consensus.proto b/src/kudu/consensus/consensus.proto index c2acf7b..c8abd5d 100644 --- a/src/kudu/consensus/consensus.proto +++ b/src/kudu/consensus/consensus.proto @@ -120,7 +120,7 @@ enum ChangeConfigType { UNKNOWN_CHANGE = 0; ADD_SERVER = 1; REMOVE_SERVER = 2; - CHANGE_ROLE = 3; + CHANGE_REPLICA_TYPE = 3; } // A configuration change request for the tablet with 'tablet_id'. http://git-wip-us.apache.org/repos/asf/kudu/blob/abbfa49d/src/kudu/consensus/raft_consensus.cc -- diff --git a/src/kudu/consensus/raft_consensus.cc b/src/kudu/consensus/raft_consensus.cc index 66a3c81..581bace 100644 --- a/src/kudu/consensus/raft_consensus.cc +++ b/src/kudu/consensus/raft_consensus.cc @@ -1646,8 +1646,8 @@ Status RaftConsensus::ChangeConfig(const ChangeConfigRequestPB& req, } break; - // TODO: Support role change. - case CHANGE_ROLE: + // TODO(mpercy): Support changing between VOTER and NON_VOTER. + case CHANGE_REPLICA_TYPE: default: return Status::NotSupported("Role change is not yet implemented."); } http://git-wip-us.apache.org/repos/asf/kudu/blob/abbfa49d/src/kudu/tools/tool_action_tablet.cc -- diff --git a/src/kudu/tools/tool_action_tablet.cc b/src/kudu/tools/tool_action_tablet.cc index 3677cf9..8faa214 100644 --- a/src/kudu/tools/tool_action_tablet.cc +++ b/src/kudu/tools/tool_action_tablet.cc @@ -166,7 +166,8 @@ Status DoChangeConfig(const vector& master_addresses, if (cc_type == consensus::REMOVE_SERVER && member_type) { return Status::InvalidArgument("cannot supply Raft member type when removing a server"); } - if ((cc_type == consensus::ADD_SERVER || cc_type == consensus::CHANGE_ROLE) && !member_type) { + if ((cc_type == consensus::ADD_SERVER || cc_type == consensus::CHANGE_REPLICA_TYPE) && + !member_type) { return Status::InvalidArgument( "must specify member type when adding a server or changing member type"); } @@ -219,7 +220,7 @@ Status ChangeConfig(const RunnerContext& context, ChangeConfigType cc_type) { const string& tablet_id = FindOrDie(context.required_args, kTabletIdArg); const string& replica_uuid = FindOrDie(context.required_args, kTsUuidArg); boost::optional member_type; - if (cc_type == consensus::ADD_SERVER || cc_type == consensus::CHANGE_ROLE) { + if (cc_type == consensus::ADD_SERVER || cc_type == consensus::CHANGE_REPLICA_TYPE) { const string& replica_type = FindOrDie(context.required_args, kReplicaTypeArg); string uppercase_peer_type; ToUpperCase(replica_type, &uppercase_peer_type); @@ -238,7 +239,7 @@ Status AddReplica(const RunnerContext& context) { } Status ChangeReplicaType(const RunnerContext& context) { - return ChangeConfig(context, consensus::CHANGE_ROLE); + return ChangeConfig(context, consensus::CHANGE_REPLICA_TYPE); } Status RemoveReplica(const RunnerContext& context) {
kudu git commit: consensus: Make it easier to swap in a role change
Repository: kudu Updated Branches: refs/heads/master 57d5a3151 -> aa7d6a97b consensus: Make it easier to swap in a role change This patch simply updates some helper functions to make it easier in a follow-on patch to modify the role of a member in a copied Raft config. Change-Id: Icca40d5ee0baeb542360b7047cb88102ff0432bd Reviewed-on: http://gerrit.cloudera.org:8080/8296 Tested-by: Kudu Jenkins Reviewed-by: Alexey Serbin Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/aa7d6a97 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/aa7d6a97 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/aa7d6a97 Branch: refs/heads/master Commit: aa7d6a97bc8561ae4bc96a5c923736cb4676ecfa Parents: 57d5a31 Author: Mike Percy Authored: Tue Oct 10 13:39:55 2017 -0700 Committer: Alexey Serbin Committed: Wed Oct 18 02:12:48 2017 + -- src/kudu/consensus/quorum_util-test.cc | 14 +++--- src/kudu/consensus/quorum_util.cc| 14 +++--- src/kudu/consensus/quorum_util.h | 6 +++--- src/kudu/consensus/raft_consensus_quorum-test.cc | 4 ++-- src/kudu/tserver/tablet_copy_client-test.cc | 6 +- 5 files changed, 24 insertions(+), 20 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/aa7d6a97/src/kudu/consensus/quorum_util-test.cc -- diff --git a/src/kudu/consensus/quorum_util-test.cc b/src/kudu/consensus/quorum_util-test.cc index 9982c74..4b2ad0c 100644 --- a/src/kudu/consensus/quorum_util-test.cc +++ b/src/kudu/consensus/quorum_util-test.cc @@ -47,20 +47,20 @@ TEST(QuorumUtilTest, TestMemberExtraction) { AddPeer(&config, "C", RaftPeerPB::VOTER); // Basic test for GetRaftConfigMember(). - RaftPeerPB peer_pb; - Status s = GetRaftConfigMember(config, "invalid", &peer_pb); + RaftPeerPB* peer_pb; + Status s = GetRaftConfigMember(&config, "invalid", &peer_pb); ASSERT_TRUE(s.IsNotFound()) << s.ToString(); - ASSERT_OK(GetRaftConfigMember(config, "A", &peer_pb)); - ASSERT_EQ("A", peer_pb.permanent_uuid()); + ASSERT_OK(GetRaftConfigMember(&config, "A", &peer_pb)); + ASSERT_EQ("A", peer_pb->permanent_uuid()); // Basic test for GetRaftConfigLeader(). ConsensusStatePB cstate; *cstate.mutable_committed_config() = config; - s = GetRaftConfigLeader(cstate, &peer_pb); + s = GetRaftConfigLeader(&cstate, &peer_pb); ASSERT_TRUE(s.IsNotFound()) << s.ToString(); cstate.set_leader_uuid("B"); - ASSERT_OK(GetRaftConfigLeader(cstate, &peer_pb)); - ASSERT_EQ("B", peer_pb.permanent_uuid()); + ASSERT_OK(GetRaftConfigLeader(&cstate, &peer_pb)); + ASSERT_EQ("B", peer_pb->permanent_uuid()); } TEST(QuorumUtilTest, TestDiffConsensusStates) { http://git-wip-us.apache.org/repos/asf/kudu/blob/aa7d6a97/src/kudu/consensus/quorum_util.cc -- diff --git a/src/kudu/consensus/quorum_util.cc b/src/kudu/consensus/quorum_util.cc index 55c5bb9..f81f2a7 100644 --- a/src/kudu/consensus/quorum_util.cc +++ b/src/kudu/consensus/quorum_util.cc @@ -64,23 +64,23 @@ bool IsVoterRole(RaftPeerPB::Role role) { return role == RaftPeerPB::LEADER || role == RaftPeerPB::FOLLOWER; } -Status GetRaftConfigMember(const RaftConfigPB& config, +Status GetRaftConfigMember(RaftConfigPB* config, const std::string& uuid, - RaftPeerPB* peer_pb) { - for (const RaftPeerPB& peer : config.peers()) { + RaftPeerPB** peer_pb) { + for (RaftPeerPB& peer : *config->mutable_peers()) { if (peer.permanent_uuid() == uuid) { - *peer_pb = peer; + *peer_pb = &peer; return Status::OK(); } } return Status::NotFound(Substitute("Peer with uuid $0 not found in consensus config", uuid)); } -Status GetRaftConfigLeader(const ConsensusStatePB& cstate, RaftPeerPB* peer_pb) { - if (cstate.leader_uuid().empty()) { +Status GetRaftConfigLeader(ConsensusStatePB* cstate, RaftPeerPB** peer_pb) { + if (cstate->leader_uuid().empty()) { return Status::NotFound("Consensus config has no leader"); } - return GetRaftConfigMember(cstate.committed_config(), cstate.leader_uuid(), peer_pb); + return GetRaftConfigMember(cstate->mutable_committed_config(), cstate->leader_uuid(), peer_pb); } bool RemoveFromRaftConfig(RaftConfigPB* config, const string& uuid) { http://git-wip-us.apache.org/repos/asf/kudu/blob/aa7d6a97/src/kudu/consensus/quorum_util.h ---
kudu git commit: [itests] fix flake in TombstonedVoteAfterFailedTabletCopy
Repository: kudu Updated Branches: refs/heads/master 20118c7e4 -> bba425e8d [itests] fix flake in TombstonedVoteAfterFailedTabletCopy Fixed flakiness in the TombstonedVoteAfterFailedTabletCopy scenario of the raft_consensus_election-itest. A race between adding a new replica and DeleteTablet was possible if adding a new server before the catalog manager acknowledges eviction of the former replica and sends DeleteTablet request. In other words, prior to this fix, a delayed DeleteTablet request might be processed _after_ a new server has been added. The race was easily reproducible by running the test scenario with high number of stress-cpu-threads: ./bin/raft_consensus_election-itest \ --stress_cpu_threads=128 \ --gtest_filter=\ 'RaftConsensusElectionITest.TombstonedVoteAfterFailedTabletCopy' Prior to this change, the test failed almost 100% if running as described above. After the fix, no failures observed in about 100 runs. Change-Id: Ifaba16d1acfee6ff01e475525482f21ff64133f4 Reviewed-on: http://gerrit.cloudera.org:8080/8298 Reviewed-by: Todd Lipcon Tested-by: Kudu Jenkins Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/bba425e8 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/bba425e8 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/bba425e8 Branch: refs/heads/master Commit: bba425e8de0f6fcbd78013960faa327fa0be7b85 Parents: 20118c7 Author: Alexey Serbin Authored: Tue Oct 17 08:21:52 2017 -0700 Committer: Alexey Serbin Committed: Wed Oct 18 22:16:22 2017 + -- .../integration-tests/raft_consensus-itest.cc | 3 ++- .../raft_consensus_election-itest.cc| 21 ++-- 2 files changed, 17 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/bba425e8/src/kudu/integration-tests/raft_consensus-itest.cc -- diff --git a/src/kudu/integration-tests/raft_consensus-itest.cc b/src/kudu/integration-tests/raft_consensus-itest.cc index d012867..4c45849 100644 --- a/src/kudu/integration-tests/raft_consensus-itest.cc +++ b/src/kudu/integration-tests/raft_consensus-itest.cc @@ -113,6 +113,7 @@ using kudu::itest::TServerDetails; using kudu::itest::TabletServerMap; using kudu::itest::WAIT_FOR_LEADER; using kudu::itest::WaitForReplicasReportedToMaster; +using kudu::itest::WaitUntilCommittedOpIdIndexIs; using kudu::itest::WaitUntilLeader; using kudu::itest::WriteSimpleTestRow; using kudu::master::TabletLocationsPB; @@ -1708,7 +1709,7 @@ TEST_F(RaftConsensusITest, TestMasterNotifiedOnConfigChange) { // Wait for initial NO_OP to be committed by the leader. TServerDetails* leader_ts; ASSERT_OK(FindTabletLeader(tablet_servers_, tablet_id, timeout, &leader_ts)); - ASSERT_OK(itest::WaitUntilCommittedOpIdIndexIs(1, leader_ts, tablet_id, timeout)); + ASSERT_OK(WaitUntilCommittedOpIdIndexIs(1, leader_ts, tablet_id, timeout)); // Change the config. TServerDetails* tserver_to_add = tablet_servers_[uuid_to_add]; http://git-wip-us.apache.org/repos/asf/kudu/blob/bba425e8/src/kudu/integration-tests/raft_consensus_election-itest.cc -- diff --git a/src/kudu/integration-tests/raft_consensus_election-itest.cc b/src/kudu/integration-tests/raft_consensus_election-itest.cc index ba4d538..23f5010 100644 --- a/src/kudu/integration-tests/raft_consensus_election-itest.cc +++ b/src/kudu/integration-tests/raft_consensus_election-itest.cc @@ -72,6 +72,7 @@ using kudu::itest::WaitUntilLeader; using kudu::itest::WriteSimpleTestRow; using kudu::pb_util::SecureShortDebugString; using kudu::tablet::TABLET_DATA_COPYING; +using kudu::tablet::TABLET_DATA_TOMBSTONED; using std::string; using std::unordered_set; using std::vector; @@ -491,11 +492,19 @@ TEST_F(RaftConsensusElectionITest, TombstonedVoteAfterFailedTabletCopy) { ASSERT_OK(WaitUntilCommittedConfigNumVotersIs(2, tablet_servers_[leader_uuid], tablet_id_, kTimeout)); ASSERT_EQ(1, active_tablet_servers.erase(follower_uuid)); - ASSERT_OK(WaitForServersToAgree(kTimeout, active_tablet_servers, tablet_id_, 2)); - // Now the follower is evicted and will be tombstoned. - // We'll add it back to the config and then crash the follower during the - // resulting tablet copy. + // Wait for the deleted tablet to be tombstoned, meaning the catalog manager + // has already sent DeleteTablet request and it has been processed. A race + // between adding a new server and processing DeleteTablet request is + // possible: the DeleteTablet request might be sent/processed _after_ + // a new server has been added. + const int follower_idx = + cluster_->ta
[2/2] kudu git commit: tool: speed up test runtime a bit
tool: speed up test runtime a bit We're good about setting --never_fsync in ExternalMiniClusters and in all gtests, but test-based invocations of the Kudu CLI don't get it. This patch changes that by centralizing all test-based CLI invocations and ensuring they pass --never_fsync. Change-Id: I27c67368b736f864f7cfa3482fdae72acafce851 Reviewed-on: http://gerrit.cloudera.org:8080/8337 Tested-by: Kudu Jenkins Reviewed-by: Alexey Serbin Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/a6f2322b Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/a6f2322b Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/a6f2322b Branch: refs/heads/master Commit: a6f2322b241189d0a32d1d07165dee8268e4dcaa Parents: 4d84eee Author: Adar Dembo Authored: Thu Oct 19 11:55:01 2017 -0700 Committer: Alexey Serbin Committed: Thu Oct 19 21:47:22 2017 + -- src/kudu/tools/kudu-admin-test.cc | 80 - src/kudu/tools/kudu-tool-test.cc | 40 +++-- src/kudu/tools/kudu-ts-cli-test.cc | 16 +++ src/kudu/tools/tool_test_util.cc | 16 ++- src/kudu/tools/tool_test_util.h| 13 +- 5 files changed, 89 insertions(+), 76 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/a6f2322b/src/kudu/tools/kudu-admin-test.cc -- diff --git a/src/kudu/tools/kudu-admin-test.cc b/src/kudu/tools/kudu-admin-test.cc index f49e9bd..696ba89 100644 --- a/src/kudu/tools/kudu-admin-test.cc +++ b/src/kudu/tools/kudu-admin-test.cc @@ -58,7 +58,6 @@ #include "kudu/util/monotime.h" #include "kudu/util/net/sockaddr.h" #include "kudu/util/status.h" -#include "kudu/util/subprocess.h" #include "kudu/util/test_macros.h" #include "kudu/util/test_util.h" @@ -85,6 +84,8 @@ using kudu::itest::WaitUntilCommittedOpIdIndexIs; using kudu::itest::WaitUntilTabletInState; using kudu::itest::WaitUntilTabletRunning; using kudu::pb_util::SecureDebugString; +using std::back_inserter; +using std::copy; using std::deque; using std::string; using std::vector; @@ -152,8 +153,7 @@ TEST_F(AdminCliTest, TestChangeConfig) { ASSERT_EQ(leader->uuid(), master_observed_leader->uuid()); LOG(INFO) << "Adding tserver with uuid " << new_node->uuid() << " as VOTER..."; - ASSERT_OK(Subprocess::Call({ -GetKuduCtlAbsolutePath(), + ASSERT_OK(RunKuduTool({ "tablet", "change_config", "add_replica", @@ -190,8 +190,7 @@ TEST_F(AdminCliTest, TestChangeConfig) { // Now remove the server once again. LOG(INFO) << "Removing tserver with uuid " << new_node->uuid() << " from the config..."; - ASSERT_OK(Subprocess::Call({ -GetKuduCtlAbsolutePath(), + ASSERT_OK(RunKuduTool({ "tablet", "change_config", "remove_replica", @@ -250,16 +249,15 @@ TEST_F(AdminCliTest, TestMoveTablet) { for (int i = 0; i < num_moves; i++) { const string remove = active_tservers.front(); const string add = inactive_tservers.front(); -ASSERT_OK(Subprocess::Call({ - GetKuduCtlAbsolutePath(), - "tablet", - "change_config", - "move_replica", - cluster_->master()->bound_rpc_addr().ToString(), - tablet_id_, - remove, - add - })); +ASSERT_OK(RunKuduTool({ + "tablet", + "change_config", + "move_replica", + cluster_->master()->bound_rpc_addr().ToString(), + tablet_id_, + remove, + add +})); active_tservers.pop_front(); active_tservers.push_back(add); inactive_tservers.pop_front(); @@ -283,12 +281,14 @@ TEST_F(AdminCliTest, TestMoveTablet) { Status RunUnsafeChangeConfig(const string& tablet_id, const string& dst_host, vector peer_uuid_list) { - vector command_args = { GetKuduCtlAbsolutePath(), "remote_replica", - "unsafe_change_config", dst_host, tablet_id }; - for (const auto& peer_uuid : peer_uuid_list) { -command_args.push_back(peer_uuid); - } - return Subprocess::Call(command_args); + vector command_args = { + "remote_replica", + "unsafe_change_config", + dst_host, + tablet_id + }; + copy(peer_uuid_list.begin(), peer_uuid_lis
[1/2] kudu git commit: fs: adjust directory sync behavior
Repository: kudu Updated Branches: refs/heads/master 741bdfa41 -> a6f2322b2 fs: adjust directory sync behavior This patch centralizes the synchronization of fs roots and data directories in a new env_util helper method, which uses the set of all created files and directories as input to synchronization. There are a few related fixes: - DataDirManager::Create wasn't considering enable_data_block_fsync when synchronizing. It does now. - It was also synchronizing the directories themselves instead of their parent directories; the latter is what's needed for durability. Change-Id: I27e7bf6a5a31720cf80a002c3553536ee0975758 Reviewed-on: http://gerrit.cloudera.org:8080/8290 Reviewed-by: Andrew Wong Tested-by: Kudu Jenkins Reviewed-by: Todd Lipcon Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/4d84eeea Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/4d84eeea Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/4d84eeea Branch: refs/heads/master Commit: 4d84eeea1695734f2723af0e200f134b9fe72005 Parents: 741bdfa Author: Adar Dembo Authored: Fri Oct 13 17:38:57 2017 -0700 Committer: Todd Lipcon Committed: Thu Oct 19 18:49:33 2017 + -- src/kudu/fs/data_dirs.cc | 11 --- src/kudu/fs/fs_manager.cc | 13 - src/kudu/util/env_util.cc | 22 +- src/kudu/util/env_util.h | 7 +++ 4 files changed, 36 insertions(+), 17 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/4d84eeea/src/kudu/fs/data_dirs.cc -- diff --git a/src/kudu/fs/data_dirs.cc b/src/kudu/fs/data_dirs.cc index 9ece90d..635a4a8 100644 --- a/src/kudu/fs/data_dirs.cc +++ b/src/kudu/fs/data_dirs.cc @@ -28,7 +28,6 @@ #include #include #include -#include #include #include @@ -104,6 +103,7 @@ METRIC_DEFINE_gauge_uint64(server, data_dirs_full, kudu::MetricUnit::kDataDirectories, "Number of data directories whose disks are currently full"); +DECLARE_bool(enable_data_block_fsync); DECLARE_string(block_manager); namespace kudu { @@ -117,7 +117,6 @@ using std::set; using std::shuffle; using std::string; using std::unique_ptr; -using std::unordered_set; using std::vector; using strings::Substitute; @@ -393,7 +392,6 @@ Status DataDirManager::Create() { int idx = 0; // Ensure the data dirs exist and create the instance files. - unordered_set to_sync; for (const auto& root : canonicalized_data_fs_roots_) { RETURN_NOT_OK_PREPEND(root.status, "Could not create directory manager with disks failed"); string data_dir = JoinPathSegments(root.path, kDataDirName); @@ -402,7 +400,6 @@ Status DataDirManager::Create() { Substitute("Could not create directory $0", data_dir)); if (created) { dirs_to_delete.emplace_back(data_dir); - to_sync.insert(root.path); } if (block_manager_type_ == "log") { @@ -419,9 +416,9 @@ Status DataDirManager::Create() { } // Ensure newly created directories are synchronized to disk. - for (const string& dir : to_sync) { -RETURN_NOT_OK_PREPEND(env_->SyncDir(dir), - Substitute("Unable to synchronize directory $0", dir)); + if (FLAGS_enable_data_block_fsync) { +RETURN_NOT_OK_PREPEND(env_util::SyncAllParentDirs( +env_, dirs_to_delete, files_to_delete), "could not sync data directories"); } // Success: don't delete any files. http://git-wip-us.apache.org/repos/asf/kudu/blob/4d84eeea/src/kudu/fs/fs_manager.cc -- diff --git a/src/kudu/fs/fs_manager.cc b/src/kudu/fs/fs_manager.cc index bf88c63..5f62858 100644 --- a/src/kudu/fs/fs_manager.cc +++ b/src/kudu/fs/fs_manager.cc @@ -22,7 +22,6 @@ #include #include #include -#include #include #include @@ -107,7 +106,6 @@ using std::ostream; using std::set; using std::string; using std::unique_ptr; -using std::unordered_set; using std::vector; using strings::Substitute; @@ -400,7 +398,6 @@ Status FsManager::CreateInitialFileSystemLayout(boost::optional uuid) { InstanceMetadataPB metadata; RETURN_NOT_OK(CreateInstanceMetadata(std::move(uuid), &metadata)); - unordered_set to_sync; for (const auto& root : canonicalized_all_fs_roots_) { if (!root.status.ok()) { continue; @@ -411,7 +408,6 @@ Status FsManager::CreateInitialFileSystemLayout(boost::optional uuid) { "Unable to create FSManager root"); if (created) { dirs_to_delete.emplace_back(root_name); - to_sync.insert(DirName(root_name)); } RETURN_NOT_OK_PREPEND(WriteInstanceMetadata(metadata, root_name), "Unable to write instance
kudu git commit: [tests] clean-up on cluster_itest_util
Repository: kudu Updated Branches: refs/heads/master ac18aca75 -> 407f55327 [tests] clean-up on cluster_itest_util Updated signature of the AddServer(), RemoveServer(), and DeleteTablet() methods to accommodate the majority of their use cases. Also, changed the signature of the TSTabletManager::DeleteTablet() to be more lightweight. This changelist does not contain any functional modifications. Change-Id: Icc5c85fb58750dec286a8ae546db955e3bd4073c Reviewed-on: http://gerrit.cloudera.org:8080/8318 Tested-by: Kudu Jenkins Reviewed-by: Dan Burkert Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/407f5532 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/407f5532 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/407f5532 Branch: refs/heads/master Commit: 407f55327e34008929edabbceda3a16a349e9045 Parents: ac18aca Author: Alexey Serbin Authored: Wed Oct 18 12:29:44 2017 -0700 Committer: Alexey Serbin Committed: Mon Oct 23 17:49:43 2017 + -- .../integration-tests/client_failover-itest.cc | 11 +- .../integration-tests/cluster_itest_util.cc | 68 src/kudu/integration-tests/cluster_itest_util.h | 19 ++- .../integration-tests/delete_table-itest.cc | 59 +++ .../integration-tests/delete_tablet-itest.cc| 8 +- .../raft_config_change-itest.cc | 6 +- .../integration-tests/raft_consensus-itest.cc | 41 ++--- .../raft_consensus_election-itest.cc| 7 +- .../raft_consensus_nonvoter-itest.cc| 6 +- src/kudu/integration-tests/tablet_copy-itest.cc | 165 +-- .../tablet_copy_client_session-itest.cc | 9 +- .../tablet_replacement-itest.cc | 21 ++- .../tombstoned_voting-imc-itest.cc | 18 +- .../tombstoned_voting-stress-test.cc| 4 +- src/kudu/tools/kudu-admin-test.cc | 9 +- src/kudu/tserver/tablet_service.cc | 2 +- src/kudu/tserver/ts_tablet_manager.cc | 24 ++- src/kudu/tserver/ts_tablet_manager.h| 10 +- 18 files changed, 227 insertions(+), 260 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/407f5532/src/kudu/integration-tests/client_failover-itest.cc -- diff --git a/src/kudu/integration-tests/client_failover-itest.cc b/src/kudu/integration-tests/client_failover-itest.cc index afbc786..47cedf4 100644 --- a/src/kudu/integration-tests/client_failover-itest.cc +++ b/src/kudu/integration-tests/client_failover-itest.cc @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -#include #include #include #include @@ -23,7 +22,6 @@ #include #include -#include #include #include @@ -58,6 +56,7 @@ using kudu::client::sp::shared_ptr; using kudu::cluster::ExternalTabletServer; using kudu::cluster::ScopedResumeExternalDaemon; using kudu::itest::TServerDetails; +using kudu::itest::DeleteTablet; using kudu::tablet::TABLET_DATA_TOMBSTONED; using std::set; using std::string; @@ -167,8 +166,7 @@ TEST_P(ClientFailoverParamITest, TestDeleteLeaderWhileScanning) { // Delete the leader replica. This will cause the next scan to the same // leader to get a TABLET_NOT_FOUND error. - ASSERT_OK(itest::DeleteTablet(leader, tablet_id, TABLET_DATA_TOMBSTONED, -boost::none, kTimeout)); + ASSERT_OK(DeleteTablet(leader, tablet_id, TABLET_DATA_TOMBSTONED, kTimeout)); int old_leader_index = leader_index; TServerDetails* old_leader = leader; @@ -185,14 +183,13 @@ TEST_P(ClientFailoverParamITest, TestDeleteLeaderWhileScanning) { // Do a config change to remove the old replica and add a new one. // Cause the new replica to become leader, then do the scan again. - ASSERT_OK(RemoveServer(leader, tablet_id, old_leader, boost::none, kTimeout)); + ASSERT_OK(RemoveServer(leader, tablet_id, old_leader, kTimeout)); // Wait until the config is committed, otherwise AddServer() will fail. ASSERT_OK(WaitUntilCommittedConfigOpIdIndexIs(workload.batches_completed() + 4, leader, tablet_id, kTimeout)); TServerDetails* to_add = ts_map_[cluster_->tablet_server(missing_replica_index)->uuid()]; - ASSERT_OK(AddServer(leader, tablet_id, to_add, consensus::RaftPeerPB::VOTER, - boost::none, kTimeout)); + ASSERT_OK(AddServer(leader, tablet_id, to_add, consensus::RaftPeerPB::VOTER, kTimeout)); HostPort hp; ASSERT_OK(HostPortFromPB(leader->registration.rpc_addresses(0), &hp)); ASSERT_OK(StartTabletCopy(to_add, tablet_id, leader->uuid(), hp, 1, kTimeout)); http://git-wip-us.apache.org/repos/asf/kudu/blob/407f5532/src/kud
kudu git commit: consensus: support changing between VOTER and NON_VOTER
Repository: kudu Updated Branches: refs/heads/master e938aaa21 -> 28a671365 consensus: support changing between VOTER and NON_VOTER This patch implements changing from VOTER to NON_VOTER and vice-versa. Added an integration test scenario for NON_VOTER --> VOTER, VOTER --> NON_VOTER changes and other use cases. Change-Id: I978cbed683b0e95a903f2bd2b57496aee71cb33a Reviewed-on: http://gerrit.cloudera.org:8080/8297 Tested-by: Kudu Jenkins Reviewed-by: Todd Lipcon Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/28a67136 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/28a67136 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/28a67136 Branch: refs/heads/master Commit: 28a671365d6d38da966481daf937b3776e3d4852 Parents: e938aaa Author: Mike Percy Authored: Wed Oct 18 01:12:26 2017 +0900 Committer: Todd Lipcon Committed: Tue Oct 24 18:16:36 2017 + -- src/kudu/consensus/quorum_util-test.cc | 10 + src/kudu/consensus/quorum_util.cc | 6 + src/kudu/consensus/quorum_util.h| 4 + src/kudu/consensus/raft_consensus.cc| 61 - src/kudu/consensus/raft_consensus.h | 8 + .../integration-tests/cluster_itest_util.cc | 35 ++- src/kudu/integration-tests/cluster_itest_util.h | 10 + .../raft_consensus_nonvoter-itest.cc| 225 ++- 8 files changed, 345 insertions(+), 14 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/28a67136/src/kudu/consensus/quorum_util-test.cc -- diff --git a/src/kudu/consensus/quorum_util-test.cc b/src/kudu/consensus/quorum_util-test.cc index 4b2ad0c..8864977 100644 --- a/src/kudu/consensus/quorum_util-test.cc +++ b/src/kudu/consensus/quorum_util-test.cc @@ -179,6 +179,16 @@ TEST(QuorumUtilTest, TestIsRaftConfigVoter) { ASSERT_FALSE(IsRaftConfigVoter("B", config)); ASSERT_FALSE(IsRaftConfigVoter("C", config)); ASSERT_FALSE(IsRaftConfigVoter(no_member_type_peer_uuid, config)); + + RaftPeerPB* peer_a; + ASSERT_OK(GetRaftConfigMember(&config, "A", &peer_a)); + RaftPeerPB* peer_b; + ASSERT_OK(GetRaftConfigMember(&config, "B", &peer_b)); + ASSERT_FALSE(ReplicaTypesEqual(*peer_a, *peer_b)); + ASSERT_TRUE(ReplicaTypesEqual(*peer_b, *peer_b)); + RaftPeerPB* peer_c; + ASSERT_OK(GetRaftConfigMember(&config, "C", &peer_c)); + ASSERT_FALSE(ReplicaTypesEqual(*peer_b, *peer_c)); } } // namespace consensus http://git-wip-us.apache.org/repos/asf/kudu/blob/28a67136/src/kudu/consensus/quorum_util.cc -- diff --git a/src/kudu/consensus/quorum_util.cc b/src/kudu/consensus/quorum_util.cc index f81f2a7..90d9671 100644 --- a/src/kudu/consensus/quorum_util.cc +++ b/src/kudu/consensus/quorum_util.cc @@ -98,6 +98,12 @@ bool RemoveFromRaftConfig(RaftConfigPB* config, const string& uuid) { return true; } +bool ReplicaTypesEqual(const RaftPeerPB& peer1, const RaftPeerPB& peer2) { + // TODO(mpercy): Include comparison of replica intentions once they are + // implemented. + return peer1.member_type() == peer2.member_type(); +} + int CountVoters(const RaftConfigPB& config) { int voters = 0; for (const RaftPeerPB& peer : config.peers()) { http://git-wip-us.apache.org/repos/asf/kudu/blob/28a67136/src/kudu/consensus/quorum_util.h -- diff --git a/src/kudu/consensus/quorum_util.h b/src/kudu/consensus/quorum_util.h index 26b6e78..3bd0afc 100644 --- a/src/kudu/consensus/quorum_util.h +++ b/src/kudu/consensus/quorum_util.h @@ -56,6 +56,10 @@ Status GetRaftConfigLeader(ConsensusStatePB* cstate, RaftPeerPB** peer_pb); // Returns true on success. bool RemoveFromRaftConfig(RaftConfigPB* config, const std::string& uuid); +// Returns true iff the two peers have equivalent replica types and associated +// options. +bool ReplicaTypesEqual(const RaftPeerPB& peer1, const RaftPeerPB& peer2); + // Counts the number of voters in the configuration. int CountVoters(const RaftConfigPB& config); http://git-wip-us.apache.org/repos/asf/kudu/blob/28a67136/src/kudu/consensus/raft_consensus.cc -- diff --git a/src/kudu/consensus/raft_consensus.cc b/src/kudu/consensus/raft_consensus.cc index 581bace..5f7f392 100644 --- a/src/kudu/consensus/raft_consensus.cc +++ b/src/kudu/consensus/raft_consensus.cc @@ -316,7 +316,7 @@ Status RaftConsensus::Start(const ConsensusBootstrapInfo& info, } } -// Now assume "follower" duties. +// Now assume non-leader replica duties. RETURN_NOT_OK(BecomeReplicaUnlocked(fd_initial_delta)); SetStateUnlocked(kRunning); @@ -585,15 +585,9 @@ S
[1/2] kudu git commit: thirdparty: treat cc the same as gcc when building boost
Repository: kudu Updated Branches: refs/heads/master c8603ab77 -> 6097e2d1a thirdparty: treat cc the same as gcc when building boost Change-Id: I6c26723b2bcbcba524aad441e39b2d3fbe580079 Reviewed-on: http://gerrit.cloudera.org:8080/8391 Tested-by: Adar Dembo Reviewed-by: Todd Lipcon Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/49a7d971 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/49a7d971 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/49a7d971 Branch: refs/heads/master Commit: 49a7d9712bf367b89946e0b81f5db26a85ef00aa Parents: c8603ab Author: Adar Dembo Authored: Wed Oct 25 16:21:43 2017 -0700 Committer: Todd Lipcon Committed: Thu Oct 26 20:39:40 2017 + -- thirdparty/build-definitions.sh | 6 ++ 1 file changed, 6 insertions(+) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/49a7d971/thirdparty/build-definitions.sh -- diff --git a/thirdparty/build-definitions.sh b/thirdparty/build-definitions.sh index acfdf30..e5b8045 100644 --- a/thirdparty/build-definitions.sh +++ b/thirdparty/build-definitions.sh @@ -686,6 +686,12 @@ build_boost() { # and clang, even when they're called through ccache. local COMPILER=$($CC --version | awk 'NR==1 {print $1;}') +# If the compiler binary used was 'cc' and not 'gcc', it will also report +# itself as 'cc'. Coerce it to gcc. +if [ "$COMPILER" = "cc" ]; then + COMPILER=gcc +fi + TOOLSET="toolset=${COMPILER}" echo "Using $TOOLSET" echo "using ${COMPILER} : : $CXX ;" > $USER_JAMFILE
[2/2] kudu git commit: [tools] limit per-session error buffer space
[tools] limit per-session error buffer space Introduced --error-buffer-size-bytes flag to control the size of the per-session error buffer. By default, the limit is set to 16MB per session (i.e. per worker thread). Prior to this patch, the per-session error buffer had no size limit and could grow very large because of huge number of errors during the run (e.g., because of duplicate value errors). I also updated the in-line documentation to add an example of using the tool for subsequent runs against the same table using the --seq-start flag. Change-Id: I2b7663c52e57b2d3a1e11b22301b804da784fa74 Reviewed-on: http://gerrit.cloudera.org:8080/8396 Reviewed-by: Adar Dembo Tested-by: Kudu Jenkins Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/6097e2d1 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/6097e2d1 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/6097e2d1 Branch: refs/heads/master Commit: 6097e2d1a4287563ef22a4f6e32e6f4ee2604031 Parents: 49a7d97 Author: Alexey Serbin Authored: Wed Oct 25 17:47:29 2017 -0700 Committer: Alexey Serbin Committed: Thu Oct 26 20:43:51 2017 + -- src/kudu/tools/tool_action_perf.cc | 52 ++--- 1 file changed, 41 insertions(+), 11 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/6097e2d1/src/kudu/tools/tool_action_perf.cc -- diff --git a/src/kudu/tools/tool_action_perf.cc b/src/kudu/tools/tool_action_perf.cc index 568b0a0..1e4e30c 100644 --- a/src/kudu/tools/tool_action_perf.cc +++ b/src/kudu/tools/tool_action_perf.cc @@ -30,7 +30,7 @@ // of length 64 for binary and string fields // with Kudu master server listening on the default port at localhost: // -// kudu test loadgen \ +// kudu perf loadgen \ // --num_threads=1 \ // --num_rows_per_thread=800 \ // --string_len=64 \ @@ -46,7 +46,7 @@ // using the specified pre-set string for binary and string fields // with Kudu master server listening on the default port at localhost: // -// kudu test loadgen \ +// kudu perf loadgen \ // --num_threads=2 \ // --num_rows_per_thread=400 \ // --string_fixed=012345678901234567890123456789012 \ @@ -62,7 +62,7 @@ // using the specified pre-set string for binary and string fields // with Kudu master server listening at 127.0.0.1:8765 // -// kudu test loadgen \ +// kudu perf loadgen \ // --num_threads=4 \ // --num_rows_per_thread=200 \ // --string_fixed=0123456789 \ @@ -79,10 +79,31 @@ // plus run post-insertion row scan to verify // that the count of the inserted rows matches the expected number: // -// kudu test loadgen \ +// kudu perf loadgen \ // --run_scan=true \ // 127.0.0.1 // +// +// If running the tool against already existing table multiple times, +// use the '--seq_start' flag to avoid errors on duplicate values in subsequent +// runs. For example: an already existing table 't3' has 5 columns. +// The sequence below contains 3 runs which insert 6000 rows in total +// (3 runs * 1000 rows per thread * 2 threads) +// with no duplicate values across all columns: +// +// kudu perf loadgen 127.0.0.1 --table_name=t3 --num_threads=2 \ +// --num_rows_per_thread=1000 --seq_start=0 +// +// kudu perf loadgen 127.0.0.1 --table_name=t3 --num_threads=2 \ +// --num_rows_per_thread=1000 --seq_start=1 +// +// perf perf loadgen 127.0.0.1 --table_name=t3 --num_threads=2 \ +// --num_rows_per_thread=1000 --seq_start=2 +// +// The single sequence number is used to generate values for all table columns, +// so for the example above each run increments the sequence number by 1: +// 1000 rows per thread * 2 threads * 5 columns +// #include "kudu/tools/tool_action.h" @@ -103,7 +124,6 @@ #include #include "kudu/client/client.h" -#include "kudu/client/row_result.h" #include "kudu/client/scan_batch.h" #include "kudu/client/schema.h" #include "kudu/client/shared_ptr.h" @@ -131,11 +151,10 @@ using kudu::client::KuduClientBuilder; using kudu::client::KuduColumnSchema; using kudu::client::KuduError; using kudu::client::KuduInsert; -using kudu::client::KuduRowResult; -using kudu::client::KuduSchema; -using kudu::client::KuduSchemaBuilder; using kudu::client::KuduScanBatch; using kudu::client::KuduScanner; +using kudu::client::KuduSchema; +using kudu::client::KuduSchemaBuilder; using kudu::client::KuduSession; using kudu::client::KuduTable; using kudu::client::KuduTableCreator; @@ -150,8 +169,8 @@ using std::numeric_limits; using std::ostringstream; using std::string; using std::thread; -using std::vector; using std::unique_ptr; +using std::vector; usi
kudu git commit: [build] fix OPENSSL_ROOT_DIR override on RH/CentOS
Repository: kudu Updated Branches: refs/heads/master c08f27fc5 -> 9f85f04a9 [build] fix OPENSSL_ROOT_DIR override on RH/CentOS Due to an extra level of indirection in the condition for one if() expression, the OpenSSL location was not overridden on RedHat/CentOS machines where the OpenSSL workaround build existed in $KUDU_ROOT/thirdparty/installed/openssl-el6-workaround, even if the OPENSSL_ROOT_DIR was overridden for cmake invocation (i.e. -DOPENSSL_ROOT_DIR=/path/to/alt-openssl-location was specified). Removing the level of indirection for the OPENSSL_ROOT_DIR variable in the expression fixed the issue, which seems to be in sync with https://cmake.org/cmake/help/v3.9/command/if.html At least, with this patch the issue is gone if running cmake 3.9.0. Change-Id: Ibe69b5d62ef4054e60138a154a85569d92166aff Reviewed-on: http://gerrit.cloudera.org:8080/8407 Tested-by: Alexey Serbin Reviewed-by: Todd Lipcon Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/9f85f04a Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/9f85f04a Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/9f85f04a Branch: refs/heads/master Commit: 9f85f04a91c38229649ccb4aa4773a45fc742dc3 Parents: c08f27f Author: Alexey Serbin Authored: Fri Oct 27 14:58:55 2017 -0700 Committer: Alexey Serbin Committed: Mon Oct 30 19:33:07 2017 + -- CMakeLists.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/9f85f04a/CMakeLists.txt -- diff --git a/CMakeLists.txt b/CMakeLists.txt index 7e1350c..59542cf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -912,8 +912,7 @@ ADD_THIRDPARTY_LIB(mustache ## binary in thirdparty (deposited there by thirdparty/install-openssl-el6-workaround.sh) ## then we'll use that one. See that script for more information. set(CENTOS_6_4_OPENSSL_DIR "${THIRDPARTY_INSTALL_DIR}/openssl-el6-workaround/usr/") -if (NOT "${OPENSSL_ROOT_DIR}" AND -EXISTS "${CENTOS_6_4_OPENSSL_DIR}") +if (NOT OPENSSL_ROOT_DIR AND EXISTS "${CENTOS_6_4_OPENSSL_DIR}") set(OPENSSL_ROOT_DIR "${CENTOS_6_4_OPENSSL_DIR}") endif() find_package(OpenSSL 1.0.0 REQUIRED)
kudu git commit: Update auth_token_validity_seconds description
Repository: kudu Updated Branches: refs/heads/master 563f41b5c -> 354f4c445 Update auth_token_validity_seconds description The caveats are no longer relevant since KUDU-2013 landed. Change-Id: I050e39d0377049cdaac0afb5085a8a9a19d620a5 Reviewed-on: http://gerrit.cloudera.org:8080/8413 Tested-by: Kudu Jenkins Reviewed-by: Alexey Serbin Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/354f4c44 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/354f4c44 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/354f4c44 Branch: refs/heads/master Commit: 354f4c44562de03e20064a5349c306f3dd05fa19 Parents: 563f41b Author: Dan Burkert Authored: Mon Oct 30 13:48:08 2017 -0400 Committer: Alexey Serbin Committed: Tue Oct 31 16:02:53 2017 + -- src/kudu/master/master.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/354f4c44/src/kudu/master/master.cc -- diff --git a/src/kudu/master/master.cc b/src/kudu/master/master.cc index 0b2d7d9..0bb45cf 100644 --- a/src/kudu/master/master.cc +++ b/src/kudu/master/master.cc @@ -75,9 +75,8 @@ TAG_FLAG(tsk_rotation_seconds, experimental); DEFINE_int64(authn_token_validity_seconds, 60 * 60 * 24 * 7, "Period of time for which an issued authentication token is valid. " - "It's not possible to renew a token, hence the token validity " - "interval defines the longest possible lifetime of an external " - "job which uses a token for authentication."); + "Clients will automatically attempt to reacquire a token after the " + "validity period expires."); TAG_FLAG(authn_token_validity_seconds, experimental); using std::min;
kudu git commit: KUDU-2209. HybridClock doesn't handle changes in STA_NANO flag
Repository: kudu Updated Branches: refs/heads/master b418e88b6 -> 10f6164b1 KUDU-2209. HybridClock doesn't handle changes in STA_NANO flag Users have occasionally reported spurious crashes due to Kudu thinking that another node has a time stamp from the future. After some debugging I realized that the issue is that we currently capture the flag 'STA_NANO' from the kernel only at startup. This flag indicates whether the kernel's sub-second timestamp is in nanoseconds or microseconds. We initially assumed this was a static property of the kernel. However it turns out that this flag can get toggled at runtime by ntp in certain circumstances. Given this, it was possible for us to interpret a number of nanoseconds as if it were microseconds, resulting in a timestamp up to 1000 seconds in the future. This patch changes the SystemNtp time source to always use the 'adjtime' call to fetch the clock, and looks at the STA_NANO flag on every such call rather than only at startup. I checked the source for the ntp_gettime call that we used to use, and it turns out it was implemented in terms of the same adjtime() call, so there should be no performance penalty in this change. This patch doesn't include tests since it's based on some kernel functionality. However, I was able to test it as follows: - wrote a simple program to print the time once a second - stopped ntp, ran chrony, stopped chrony, and started ntpd -- this has the side effect of clearing STA_NANO - waited 10 minutes or so, and eventually ntp reset back to STA_NANO -- this caused my test program to start printing incorrect timestamps as it was interpreting nanosecond values from the kernel as if they were microseconds Change-Id: Iaa9241dea1304076ed5cfaec78779d15b11293ff Reviewed-on: http://gerrit.cloudera.org:8080/8450 Reviewed-by: Alexey Serbin Tested-by: Alexey Serbin Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/10f6164b Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/10f6164b Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/10f6164b Branch: refs/heads/master Commit: 10f6164b1217e0299bcfedc061d2c57581c389bd Parents: b418e88 Author: Todd Lipcon Authored: Wed Nov 1 18:41:07 2017 -0700 Committer: Alexey Serbin Committed: Fri Nov 3 05:12:00 2017 + -- src/kudu/clock/system_ntp.cc | 67 --- src/kudu/clock/system_ntp.h | 7 +--- 2 files changed, 21 insertions(+), 53 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/10f6164b/src/kudu/clock/system_ntp.cc -- diff --git a/src/kudu/clock/system_ntp.cc b/src/kudu/clock/system_ntp.cc index 9cdac05..d1e8d56 100644 --- a/src/kudu/clock/system_ntp.cc +++ b/src/kudu/clock/system_ntp.cc @@ -22,12 +22,9 @@ #include #include -#include #include -#include "kudu/gutil/port.h" -#include "kudu/gutil/strings/substitute.h" #include "kudu/util/errno.h" #include "kudu/util/logging.h" #include "kudu/util/status.h" @@ -35,37 +32,21 @@ namespace kudu { namespace clock { -using strings::Substitute; - const double SystemNtp::kAdjtimexScalingFactor = 65536; -const uint64_t SystemNtp::kNanosPerSec = 100; +const uint64_t SystemNtp::kMicrosPerSec = 100; namespace { -// Returns the clock modes and checks if the clock is synchronized. -Status GetClockModes(timex* timex) { - // this makes ntp_adjtime a read-only call - timex->modes = 0; - int rc = ntp_adjtime(timex); - if (PREDICT_FALSE(rc == TIME_ERROR)) { -return Status::ServiceUnavailable( -Substitute("Error reading clock. Clock considered unsynchronized. Return code: $0", rc)); - } - // TODO what to do about leap seconds? see KUDU-146 - if (PREDICT_FALSE(rc != TIME_OK)) { -LOG(ERROR) << Substitute("TODO Server undergoing leap second. Return code: $0", rc); - } - return Status::OK(); -} - // Returns the current time/max error and checks if the clock is synchronized. -Status GetClockTime(ntptimeval* timeval) { - int rc = ntp_gettime(timeval); +Status CallAdjTime(timex* tx) { + // Set mode to 0 to query the current time. + tx->modes = 0; + int rc = ntp_adjtime(tx); switch (rc) { case TIME_OK: return Status::OK(); case -1: // generic error - return Status::ServiceUnavailable("Error reading clock. ntp_gettime() failed", + return Status::ServiceUnavailable("Error reading clock. ntp_adjtime() failed", ErrnoToString(errno)); case TIME_ERROR: return Status::ServiceUnavailable("Error reading clock. Clock considered unsynchronized"); @@
[1/4] kudu git commit: rpc: fix service lookup to use the proper lock type
Repository: kudu Updated Branches: refs/heads/master b167b98e4 -> ed45d7207 rpc: fix service lookup to use the proper lock type Looking up the RPC service in the service map was using an exclusive lock rather than a per-CPU read-lock. Fixing this yielded a ~5% improvement on the average throughput of rpc-bench. I ran: rpc-bench --server-reactors 96 --async-call-concurrency 100 \ -worker-threads 40 -client-threads 48 -run-seconds 1 \ --gtest_filter=\*Async --gtest_repeat=100 2>&1 | tee /tmp/log before and after the change. Running 100 short benchmarks is preferable to a long one since this benchmark has quite a bit of variability depending on reactor thread hashing, etc. I grepped out the reqs/sec number and then ran a t-test using R: data: d.before and d.after t = -4.538, df = 165.27, p-value = 1.088e-05 alternative hypothesis: true difference in means is not equal to 0 95 percent confidence interval: -59048.87 -23244.07 sample estimates: mean of x mean of y 785314.9 826461.4 Special thanks to Sergei Politov from Yugabyte for finding this. Change-Id: I871765dffc5208dbdfc6b7f9f7fa2160611631b9 Reviewed-on: http://gerrit.cloudera.org:8080/8478 Tested-by: Kudu Jenkins Reviewed-by: Alexey Serbin Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/10ed76cc Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/10ed76cc Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/10ed76cc Branch: refs/heads/master Commit: 10ed76cc1e31e0d226ee5411e738d6675072956a Parents: b167b98 Author: Todd Lipcon Authored: Sun Nov 5 20:22:20 2017 -0800 Committer: Todd Lipcon Committed: Mon Nov 6 05:26:15 2017 + -- src/kudu/rpc/messenger.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/10ed76cc/src/kudu/rpc/messenger.cc -- diff --git a/src/kudu/rpc/messenger.cc b/src/kudu/rpc/messenger.cc index 46b144f..1c8b5b2 100644 --- a/src/kudu/rpc/messenger.cc +++ b/src/kudu/rpc/messenger.cc @@ -567,7 +567,7 @@ void Messenger::ScheduleOnReactor(const boost::function& fu const scoped_refptr Messenger::rpc_service(const string& service_name) const { scoped_refptr service; { -std::lock_guard guard(lock_); +shared_lock guard(lock_.get_lock()); if (!FindCopy(rpc_services_, service_name, &service)) { return scoped_refptr(nullptr); }
[4/4] kudu git commit: [tools] simplify code in DumpRowSet
[tools] simplify code in DumpRowSet This changelist does not contain any functional modifications. Change-Id: Id5347732023ea13e7275927be8ef346cd03e45f1 Reviewed-on: http://gerrit.cloudera.org:8080/8475 Tested-by: Alexey Serbin Reviewed-by: Todd Lipcon Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/ed45d720 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/ed45d720 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/ed45d720 Branch: refs/heads/master Commit: ed45d72070bb4c3f70357d4c67077023c6d186fa Parents: f7b54d8 Author: Alexey Serbin Authored: Sat Nov 4 22:44:29 2017 -0700 Committer: Alexey Serbin Committed: Mon Nov 6 05:45:30 2017 + -- src/kudu/tools/tool_action_local_replica.cc | 26 ++-- 1 file changed, 11 insertions(+), 15 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/ed45d720/src/kudu/tools/tool_action_local_replica.cc -- diff --git a/src/kudu/tools/tool_action_local_replica.cc b/src/kudu/tools/tool_action_local_replica.cc index 94b3ae7..784def5 100644 --- a/src/kudu/tools/tool_action_local_replica.cc +++ b/src/kudu/tools/tool_action_local_replica.cc @@ -849,27 +849,23 @@ Status DumpRowSet(const RunnerContext& context) { // If rowset index is provided, only dump that rowset. if (FLAGS_rowset_index != -1) { -bool found = false; for (const shared_ptr& rs_meta : meta->rowsets()) { if (rs_meta->id() == FLAGS_rowset_index) { -found = true; return DumpRowSetInternal(fs_manager.get(), meta->schema(), rs_meta, 0); } } -if (!found) { - return Status::InvalidArgument( - Substitute("Could not find rowset $0 in tablet id $1", - FLAGS_rowset_index, tablet_id)); -} - } else { -// Rowset index not provided, dump all rowsets -size_t idx = 0; -for (const shared_ptr& rs_meta : meta->rowsets()) { - cout << endl << "Dumping rowset " << idx++ << endl << kSeparatorLine; - RETURN_NOT_OK(DumpRowSetInternal(fs_manager.get(), meta->schema(), - rs_meta, 2)); -} +return Status::InvalidArgument( +Substitute("Could not find rowset $0 in tablet id $1", + FLAGS_rowset_index, tablet_id)); + } + + // Rowset index not provided, dump all rowsets + size_t idx = 0; + for (const shared_ptr& rs_meta : meta->rowsets()) { +cout << endl << "Dumping rowset " << idx++ << endl << kSeparatorLine; +RETURN_NOT_OK(DumpRowSetInternal(fs_manager.get(), meta->schema(), + rs_meta, 2)); } return Status::OK(); }
[3/4] kudu git commit: [TabletServerTestBase] fix typo in DeleteTestRowsRemote
[TabletServerTestBase] fix typo in DeleteTestRowsRemote Change-Id: I80638320b312060b303fdea83b5f83c93a1b9fc4 Reviewed-on: http://gerrit.cloudera.org:8080/8474 Tested-by: Alexey Serbin Reviewed-by: Todd Lipcon Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/f7b54d85 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/f7b54d85 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/f7b54d85 Branch: refs/heads/master Commit: f7b54d85b28e59a065fb6d644a9364a69ea578b4 Parents: 7f51045 Author: Alexey Serbin Authored: Sat Nov 4 22:39:26 2017 -0700 Committer: Todd Lipcon Committed: Mon Nov 6 05:27:32 2017 + -- src/kudu/tserver/tablet_server-test-base.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/f7b54d85/src/kudu/tserver/tablet_server-test-base.cc -- diff --git a/src/kudu/tserver/tablet_server-test-base.cc b/src/kudu/tserver/tablet_server-test-base.cc index 0ab114e..5107934 100644 --- a/src/kudu/tserver/tablet_server-test-base.cc +++ b/src/kudu/tserver/tablet_server-test-base.cc @@ -278,7 +278,7 @@ void TabletServerTestBase::DeleteTestRowsRemote(int64_t first_row, } SCOPED_TRACE(SecureDebugString(req)); - ASSERT_OK(proxy_->Write(req, &resp, &controller)); + ASSERT_OK(DCHECK_NOTNULL(proxy)->Write(req, &resp, &controller)); SCOPED_TRACE(SecureDebugString(resp)); ASSERT_FALSE(resp.has_error()) << SecureShortDebugString(resp); }
[2/4] kudu git commit: KUDU-1578. Hybrid clock should ride over brief interruptions in NTP service
KUDU-1578. Hybrid clock should ride over brief interruptions in NTP service This changes HybridClock so that, if the clock loses synchronization for a brief period of time, it will continue to use the most recent clock reading, adjusting it forward based on the local monotonic clock and a conservative estimate for max error. In the case that the clock is out of sync for a significantly long time, the max error will grow large enough to eclipse the 10-second default, at which point it will still crash as before. But, if NTP is properly restored within a few minutes, the server should remain operational. A simple test is included which injects a fake error into the NTP source. I also ran a kudu-master while briefly disabling NTP and ensured that the log messages showed up as expected and then went away when I resumed NTP synchronization. Change-Id: I0d03a77033070e6bcdfdefc4341981f28a1477a0 Reviewed-on: http://gerrit.cloudera.org:8080/8451 Reviewed-by: Alexey Serbin Tested-by: Todd Lipcon Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/7f510458 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/7f510458 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/7f510458 Branch: refs/heads/master Commit: 7f5104586fa381347c582260df539b8cbb02f08b Parents: 10ed76c Author: Todd Lipcon Authored: Wed Nov 1 19:13:05 2017 -0700 Committer: Todd Lipcon Committed: Mon Nov 6 05:26:38 2017 + -- src/kudu/clock/hybrid_clock-test.cc | 38 + src/kudu/clock/hybrid_clock.cc | 71 src/kudu/clock/hybrid_clock.h | 12 -- src/kudu/clock/system_ntp.cc| 10 + 4 files changed, 119 insertions(+), 12 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/7f510458/src/kudu/clock/hybrid_clock-test.cc -- diff --git a/src/kudu/clock/hybrid_clock-test.cc b/src/kudu/clock/hybrid_clock-test.cc index e3d2afd..0524a84 100644 --- a/src/kudu/clock/hybrid_clock-test.cc +++ b/src/kudu/clock/hybrid_clock-test.cc @@ -41,6 +41,7 @@ #include "kudu/util/test_util.h" #include "kudu/util/thread.h" +DECLARE_bool(inject_adjtimex_errors); DECLARE_string(time_source); namespace kudu { @@ -309,5 +310,42 @@ TEST_F(HybridClockTest, TestGetPhysicalComponentDifference) { ASSERT_EQ(-100, negative_delta.ToMicroseconds()); } + +TEST_F(HybridClockTest, TestRideOverNtpInterruption) { + Timestamp timestamps[3]; + uint64_t max_error_usec[3]; + + // Get the clock once, with a working NTP. + clock_->NowWithError(×tamps[0], &max_error_usec[0]); + + // Try to read the clock again a second later, but with an error + // injected. It should extrapolate from the first read. + SleepFor(MonoDelta::FromSeconds(1)); + FLAGS_inject_adjtimex_errors = true; + clock_->NowWithError(×tamps[1], &max_error_usec[1]); + + // The new clock reading should be a second or longer from the + // first one, since SleepFor guarantees sleeping at least as long + // as specified. + MonoDelta phys_diff = clock_->GetPhysicalComponentDifference( + timestamps[1], timestamps[0]); + ASSERT_GE(phys_diff.ToSeconds(), 1); + + // The new clock reading should have higher error than the first. + // The error should have increased based on the clock skew. + int64_t error_diff = max_error_usec[1] - max_error_usec[0]; + ASSERT_NEAR(error_diff, clock_->time_service()->skew_ppm() * phys_diff.ToSeconds(), + 10); + + // Now restore the ability to read the system clock, and + // read it again. + FLAGS_inject_adjtimex_errors = false; + clock_->NowWithError(×tamps[2], &max_error_usec[2]); + + ASSERT_LT(timestamps[0].ToUint64(), timestamps[1].ToUint64()); + ASSERT_LT(timestamps[1].ToUint64(), timestamps[2].ToUint64()); +} + + } // namespace clock } // namespace kudu http://git-wip-us.apache.org/repos/asf/kudu/blob/7f510458/src/kudu/clock/hybrid_clock.cc -- diff --git a/src/kudu/clock/hybrid_clock.cc b/src/kudu/clock/hybrid_clock.cc index 6ee7129..0518afe 100644 --- a/src/kudu/clock/hybrid_clock.cc +++ b/src/kudu/clock/hybrid_clock.cc @@ -34,6 +34,7 @@ #include "kudu/gutil/strings/substitute.h" #include "kudu/util/debug/trace_event.h" #include "kudu/util/flag_tags.h" +#include "kudu/util/logging.h" #include "kudu/util/metrics.h" #include "kudu/util/monotime.h" #include "kudu/util/status.h" @@ -170,11 +171,8 @@ void HybridClock::NowWithError(Timestamp* timestamp, uint64_t* max_error_usec) { uint64_t now_usec; uint64_t error_usec; - Status s = WalltimeWithError(&now_usec, &e
kudu git commit: [stopwatch] update Stopwatch::GetTimes() on OS X
Repository: kudu Updated Branches: refs/heads/master 26b06c5c5 -> 1520b3958 [stopwatch] update Stopwatch::GetTimes() on OS X Updated implementation of Stopwatch::GetTimes() for OS X to output relevant numbers corresponding to getrusage(..., RUSAGE_THREAD). The new implementation is modeled after OS X-specific code from https://codereview.chromium.org/16818003 Example of output prior to the fix: Time spent creating directory manager: \ real 0.009s user 18446744046.974s sys 0.005s Example of output after the fix: Time spent creating directory manager: \ real 0.008s user 0.002s sys 0.006s Change-Id: I13584e12342e3548c50a38c890b3f2cb18fd3f07 Reviewed-on: http://gerrit.cloudera.org:8080/8473 Tested-by: Alexey Serbin Reviewed-by: Dan Burkert Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/1520b395 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/1520b395 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/1520b395 Branch: refs/heads/master Commit: 1520b39585e0bdc4a9b64bf1b900f3ecb05e0276 Parents: 26b06c5 Author: Alexey Serbin Authored: Sat Nov 4 20:40:38 2017 -0700 Committer: Alexey Serbin Committed: Mon Nov 6 18:18:11 2017 + -- src/kudu/gutil/walltime.h | 2 +- src/kudu/util/stopwatch.h | 20 +++- 2 files changed, 12 insertions(+), 10 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/1520b395/src/kudu/gutil/walltime.h -- diff --git a/src/kudu/gutil/walltime.h b/src/kudu/gutil/walltime.h index 40af48e..02338a8 100644 --- a/src/kudu/gutil/walltime.h +++ b/src/kudu/gutil/walltime.h @@ -74,7 +74,7 @@ extern void InitializeTimebaseInfo(); inline void GetCurrentTime(mach_timespec_t* ts) { clock_serv_t cclock; host_get_clock_service(mach_host_self(), CALENDAR_CLOCK, &cclock); - clock_get_time(cclock, ts); + CHECK_EQ(KERN_SUCCESS, clock_get_time(cclock, ts)); mach_port_deallocate(mach_task_self(), cclock); } http://git-wip-us.apache.org/repos/asf/kudu/blob/1520b395/src/kudu/util/stopwatch.h -- diff --git a/src/kudu/util/stopwatch.h b/src/kudu/util/stopwatch.h index 66d54d1..617ee8b 100644 --- a/src/kudu/util/stopwatch.h +++ b/src/kudu/util/stopwatch.h @@ -25,6 +25,7 @@ #if defined(__APPLE__) #include #include +#include #endif // defined(__APPLE__) #include "kudu/gutil/macros.h" @@ -167,8 +168,8 @@ class Stopwatch { // Construct a new stopwatch. The stopwatch is initially stopped. explicit Stopwatch(Mode mode = THIS_THREAD) -: stopped_(true), - mode_(mode) { + : mode_(mode), +stopped_(true) { times_.clear(); } @@ -240,15 +241,17 @@ class Stopwatch { #if defined(__APPLE__) if (mode_ == THIS_THREAD) { - //Adapted from http://blog.kuriositaet.de/?p=257. - struct task_basic_info t_info; - mach_msg_type_number_t t_info_count = TASK_BASIC_INFO_COUNT; - CHECK_EQ(KERN_SUCCESS, task_info(mach_task_self(), TASK_THREAD_TIMES_INFO, - (task_info_t)&t_info, &t_info_count)); + // Adapted from https://codereview.chromium.org/16818003 + thread_basic_info_data_t t_info; + mach_msg_type_number_t count = THREAD_BASIC_INFO_COUNT; + CHECK_EQ(KERN_SUCCESS, thread_info(mach_thread_self(), THREAD_BASIC_INFO, + (thread_info_t)&t_info, &count)); usage.ru_utime.tv_sec = t_info.user_time.seconds; usage.ru_utime.tv_usec = t_info.user_time.microseconds; usage.ru_stime.tv_sec = t_info.system_time.seconds; usage.ru_stime.tv_usec = t_info.system_time.microseconds; + usage.ru_nivcsw = t_info.suspend_count; + usage.ru_nvcsw = 0; } else { CHECK_EQ(0, getrusage(RUSAGE_SELF, &usage)); } @@ -267,10 +270,9 @@ class Stopwatch { times->context_switches = usage.ru_nvcsw + usage.ru_nivcsw; } + const Mode mode_; bool stopped_; - CpuTimes times_; - Mode mode_; };
kudu git commit: [security] test and fixes for TLS socket EINTR issues
Repository: kudu Updated Branches: refs/heads/master 1520b3958 -> 18e024cf8 [security] test and fixes for TLS socket EINTR issues SSL_{read,write}() can return SSL_ERROR_WANT_{READ,WRITE} correspondingly when signal interrupts recv()/send() calls even if SSL_MODE_AUTO_RETRY is set in the TLS context. To handle that properly in Socket::Blocking{Recv,Write}() methods, return NetworkError() with appropriate POSIX error code from TlsSocket::{Recv,Write}(). As a by-product, this changelist fixes flakiness in TestUniqueClientIds scenario of the ClientStressTest test and other flaky tests which failed with errors like below: Bad status: IO error: Could not connect to the cluster: \ Client connection negotiation failed: client connection to \ IP:port: Read zero bytes on a blocking Recv() call: \ Transferred 0 of 4 bytes Prior to this fix, the test failure ratio observed with dist-test for TSAN builds was about 6% in multiple 1K runs. After the fix, no failures observed. Change-Id: Ibec9049186f79f1c43295e4735538ed7ba4e516e Reviewed-on: http://gerrit.cloudera.org:8080/8462 Tested-by: Kudu Jenkins Reviewed-by: Dan Burkert Reviewed-by: Alexey Serbin Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/18e024cf Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/18e024cf Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/18e024cf Branch: refs/heads/master Commit: 18e024cf8bcaea192efb63780802cc4c799bbb9c Parents: 1520b39 Author: Todd Lipcon Authored: Fri Nov 3 12:39:25 2017 -0700 Committer: Alexey Serbin Committed: Mon Nov 6 19:20:09 2017 + -- src/kudu/security/CMakeLists.txt | 1 + src/kudu/security/tls_handshake.cc | 1 - src/kudu/security/tls_socket-test.cc | 196 ++ src/kudu/security/tls_socket.cc | 9 ++ src/kudu/util/net/socket.cc | 27 +++- 5 files changed, 227 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/18e024cf/src/kudu/security/CMakeLists.txt -- diff --git a/src/kudu/security/CMakeLists.txt b/src/kudu/security/CMakeLists.txt index c669e25..b6ae528 100644 --- a/src/kudu/security/CMakeLists.txt +++ b/src/kudu/security/CMakeLists.txt @@ -126,5 +126,6 @@ if (NOT NO_TESTS) ADD_KUDU_TEST(crypto-test) ADD_KUDU_TEST(test/mini_kdc-test) ADD_KUDU_TEST(tls_handshake-test) + ADD_KUDU_TEST(tls_socket-test) ADD_KUDU_TEST(token-test) endif() http://git-wip-us.apache.org/repos/asf/kudu/blob/18e024cf/src/kudu/security/tls_handshake.cc -- diff --git a/src/kudu/security/tls_handshake.cc b/src/kudu/security/tls_handshake.cc index 57f9c7c..6daea6b 100644 --- a/src/kudu/security/tls_handshake.cc +++ b/src/kudu/security/tls_handshake.cc @@ -116,7 +116,6 @@ Status TlsHandshake::Continue(const string& recv, string* send) { DCHECK_GE(send->size(), 0); return Status::OK(); } - DCHECK_GT(send->size(), 0); return Status::Incomplete("TLS Handshake incomplete"); } http://git-wip-us.apache.org/repos/asf/kudu/blob/18e024cf/src/kudu/security/tls_socket-test.cc -- diff --git a/src/kudu/security/tls_socket-test.cc b/src/kudu/security/tls_socket-test.cc new file mode 100644 index 000..a978e68 --- /dev/null +++ b/src/kudu/security/tls_socket-test.cc @@ -0,0 +1,196 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "kudu/security/tls_handshake.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "kudu/gutil/macros.h" +#include "kudu/security/tls_context.h" +#include "kudu/util/countdown_latch.h" +#include "kudu/util/monotime.h" +#include "kudu/util/net/sockaddr.h" +#include "kudu/util/net/socket
kudu git commit: [hybrid_clock-test] adapt the test for OS X
Repository: kudu Updated Branches: refs/heads/master 9748fd2c8 -> 28fdec405 [hybrid_clock-test] adapt the test for OS X Fixed linkage breakage on OS X: adapted the hybrid_clock-test for OS X. Change-Id: I50a4b139eb6b0b5c79bce48eedb24bc7c445970c Reviewed-on: http://gerrit.cloudera.org:8080/8488 Tested-by: Alexey Serbin Reviewed-by: Todd Lipcon Reviewed-by: Dan Burkert Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/28fdec40 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/28fdec40 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/28fdec40 Branch: refs/heads/master Commit: 28fdec4053c86f08f410d0ef460e4c82a78f9e1c Parents: 9748fd2 Author: Alexey Serbin Authored: Mon Nov 6 19:08:57 2017 -0800 Committer: Alexey Serbin Committed: Wed Nov 8 18:34:40 2017 + -- src/kudu/clock/hybrid_clock-test.cc | 8 +++- src/kudu/clock/system_ntp.cc | 9 +++-- src/kudu/clock/system_unsync_time.cc | 14 ++ 3 files changed, 20 insertions(+), 11 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/28fdec40/src/kudu/clock/hybrid_clock-test.cc -- diff --git a/src/kudu/clock/hybrid_clock-test.cc b/src/kudu/clock/hybrid_clock-test.cc index 0524a84..8857b32 100644 --- a/src/kudu/clock/hybrid_clock-test.cc +++ b/src/kudu/clock/hybrid_clock-test.cc @@ -41,7 +41,7 @@ #include "kudu/util/test_util.h" #include "kudu/util/thread.h" -DECLARE_bool(inject_adjtimex_errors); +DECLARE_bool(inject_unsync_time_errors); DECLARE_string(time_source); namespace kudu { @@ -310,7 +310,6 @@ TEST_F(HybridClockTest, TestGetPhysicalComponentDifference) { ASSERT_EQ(-100, negative_delta.ToMicroseconds()); } - TEST_F(HybridClockTest, TestRideOverNtpInterruption) { Timestamp timestamps[3]; uint64_t max_error_usec[3]; @@ -321,7 +320,7 @@ TEST_F(HybridClockTest, TestRideOverNtpInterruption) { // Try to read the clock again a second later, but with an error // injected. It should extrapolate from the first read. SleepFor(MonoDelta::FromSeconds(1)); - FLAGS_inject_adjtimex_errors = true; + FLAGS_inject_unsync_time_errors = true; clock_->NowWithError(×tamps[1], &max_error_usec[1]); // The new clock reading should be a second or longer from the @@ -339,13 +338,12 @@ TEST_F(HybridClockTest, TestRideOverNtpInterruption) { // Now restore the ability to read the system clock, and // read it again. - FLAGS_inject_adjtimex_errors = false; + FLAGS_inject_unsync_time_errors = false; clock_->NowWithError(×tamps[2], &max_error_usec[2]); ASSERT_LT(timestamps[0].ToUint64(), timestamps[1].ToUint64()); ASSERT_LT(timestamps[1].ToUint64(), timestamps[2].ToUint64()); } - } // namespace clock } // namespace kudu http://git-wip-us.apache.org/repos/asf/kudu/blob/28fdec40/src/kudu/clock/system_ntp.cc -- diff --git a/src/kudu/clock/system_ntp.cc b/src/kudu/clock/system_ntp.cc index ff2c2a6..770a3fa 100644 --- a/src/kudu/clock/system_ntp.cc +++ b/src/kudu/clock/system_ntp.cc @@ -23,18 +23,15 @@ #include #include -#include +#include #include #include "kudu/gutil/port.h" #include "kudu/util/errno.h" -#include "kudu/util/flag_tags.h" #include "kudu/util/logging.h" #include "kudu/util/status.h" -DEFINE_bool(inject_adjtimex_errors, false, -"If true, will return a fake 'unsynchronized' status from NTP."); -TAG_FLAG(inject_adjtimex_errors, unsafe); +DECLARE_bool(inject_unsync_time_errors); namespace kudu { namespace clock { @@ -49,7 +46,7 @@ Status CallAdjTime(timex* tx) { // Set mode to 0 to query the current time. tx->modes = 0; int rc = ntp_adjtime(tx); - if (PREDICT_FALSE(FLAGS_inject_adjtimex_errors)) { + if (PREDICT_FALSE(FLAGS_inject_unsync_time_errors)) { rc = TIME_ERROR; } switch (rc) { http://git-wip-us.apache.org/repos/asf/kudu/blob/28fdec40/src/kudu/clock/system_unsync_time.cc -- diff --git a/src/kudu/clock/system_unsync_time.cc b/src/kudu/clock/system_unsync_time.cc index 453a2b2..12da925 100644 --- a/src/kudu/clock/system_unsync_time.cc +++ b/src/kudu/clock/system_unsync_time.cc @@ -19,9 +19,17 @@ #include +#include #include +#include "kudu/gutil/port.h" #include "kudu/gutil/walltime.h" +#include "kudu/util/flag_tags.h" + +DEFINE_bool(inject_unsync_time_errors, false, +"If true, both Init() and WalltimeWithError() return " +"'clock considered unsynchronized' error."); +TAG_FLAG(in
kudu git commit: [tools] Fix doc'd endianness for kudu test mini_cluster
Repository: kudu Updated Branches: refs/heads/master 04c23bc84 -> b8e7dd423 [tools] Fix doc'd endianness for kudu test mini_cluster See ControlShellProtocol::ReceiveMessage in tool_action_common.cc. Change-Id: Ida22188ed5b477888e2ae88d6bab9e935db72066 Reviewed-on: http://gerrit.cloudera.org:8080/8504 Tested-by: Kudu Jenkins Reviewed-by: Alexey Serbin Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/b8e7dd42 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/b8e7dd42 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/b8e7dd42 Branch: refs/heads/master Commit: b8e7dd423a66aa19958470a51be74768ef4103a1 Parents: 04c23bc Author: Will Berkeley Authored: Wed Nov 8 12:55:48 2017 -0800 Committer: Alexey Serbin Committed: Thu Nov 9 17:18:21 2017 + -- src/kudu/tools/tool.proto | 2 +- src/kudu/tools/tool_action_common.h | 2 +- src/kudu/tools/tool_action_test.cc | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/b8e7dd42/src/kudu/tools/tool.proto -- diff --git a/src/kudu/tools/tool.proto b/src/kudu/tools/tool.proto index 8a30bf1..6107c26 100644 --- a/src/kudu/tools/tool.proto +++ b/src/kudu/tools/tool.proto @@ -76,7 +76,7 @@ enum DaemonType { // Identifier for a cluster daemon, unique to the cluster. message DaemonIdentifierPB { - // Whether the daemon is amaster, tserver, or whatever. + // Whether the daemon is a master, tserver, or whatever. optional DaemonType type = 1; // Index of the daemon in the cluster, if the cluster has multiple daemons http://git-wip-us.apache.org/repos/asf/kudu/blob/b8e7dd42/src/kudu/tools/tool_action_common.h -- diff --git a/src/kudu/tools/tool_action_common.h b/src/kudu/tools/tool_action_common.h index ea08643..9edf079 100644 --- a/src/kudu/tools/tool_action_common.h +++ b/src/kudu/tools/tool_action_common.h @@ -177,7 +177,7 @@ class LeaderMasterProxy { class ControlShellProtocol { public: enum class SerializationMode { -// Each message is serialized as a four byte little-endian size followed by +// Each message is serialized as a four byte big-endian size followed by // the protobuf-encoded message itself. PB, http://git-wip-us.apache.org/repos/asf/kudu/blob/b8e7dd42/src/kudu/tools/tool_action_test.cc -- diff --git a/src/kudu/tools/tool_action_test.cc b/src/kudu/tools/tool_action_test.cc index fd8a7cf..504b4e1 100644 --- a/src/kudu/tools/tool_action_test.cc +++ b/src/kudu/tools/tool_action_test.cc @@ -48,7 +48,7 @@ DEFINE_string(serialization, "json", "Serialization method to be used by the " "control shell. Valid values are 'json' (protobuf serialized " "into JSON and terminated with a newline character) or 'pb' " - "(four byte protobuf message length in little endian followed by " + "(four byte protobuf message length in big endian followed by " "the protobuf message itself)."); DEFINE_validator(serialization, [](const char* /*n*/, const std::string& v) { return boost::iequals(v, "pb") ||
kudu git commit: [tests] fix flake in RaftConsensusNonVoterITest
Repository: kudu Updated Branches: refs/heads/master b8e7dd423 -> bf1fcb0df [tests] fix flake in RaftConsensusNonVoterITest Fixed flakiness in the RaftConsensusNonVoterITest::PromoteAndDemote scenario. The problematic scenario was when the tablet leader had been paused before the tablet copy completed. That resulted in the new leader being unable to commit its own NO_OP because the tablet copying server could not commit any new messages before completing its (hung) tablet copy. Meanwhile, the log was flooded with messages like below: Couldn't send request to peer 95f63392a59643d3b15bfbb97662d24e for \ tablet bb1c77bc65544659bfa24d852e2fc728. \ Error code: TABLET_NOT_RUNNING (12). \ Status: Illegal state: Tablet not RUNNING: INITIALIZED. \ Retrying in the next heartbeat period. Already tried 1210 times. Other issues were related to tablet leader re-elections and also cases where a newly promoted replica has almost instantly became a new leader. To deal with those issues and preserve the coverage, I separated part of the former PromoteAndDemote scenario into a new scenario called PromotedReplicaCanVote. Change-Id: I177e45cdfc883b6cec074877fea034374b00c677 Reviewed-on: http://gerrit.cloudera.org:8080/8477 Reviewed-by: Mike Percy Tested-by: Kudu Jenkins Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/bf1fcb0d Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/bf1fcb0d Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/bf1fcb0d Branch: refs/heads/master Commit: bf1fcb0df5f2fbb6d1557b6b932bc67045fff531 Parents: b8e7dd4 Author: Alexey Serbin Authored: Sun Nov 5 18:24:32 2017 -0800 Committer: Alexey Serbin Committed: Fri Nov 10 06:12:45 2017 + -- src/kudu/consensus/raft_consensus.cc| 5 +- .../integration-tests/cluster_itest_util.cc | 1 - .../raft_consensus_nonvoter-itest.cc| 355 +-- src/kudu/integration-tests/ts_itest-base.cc | 52 ++- src/kudu/integration-tests/ts_itest-base.h | 10 + 5 files changed, 294 insertions(+), 129 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/bf1fcb0d/src/kudu/consensus/raft_consensus.cc -- diff --git a/src/kudu/consensus/raft_consensus.cc b/src/kudu/consensus/raft_consensus.cc index 5f7f392..99ecefe 100644 --- a/src/kudu/consensus/raft_consensus.cc +++ b/src/kudu/consensus/raft_consensus.cc @@ -385,7 +385,7 @@ string ReasonString(RaftConsensus::ElectionReason reason, StringPiece leader_uui } // anonymous namespace Status RaftConsensus::StartElection(ElectionMode mode, ElectionReason reason) { - const char* mode_str = ModeString(mode); + const char* const mode_str = ModeString(mode); TRACE_EVENT2("consensus", "RaftConsensus::StartElection", "peer", LogPrefixThreadSafe(), @@ -398,7 +398,8 @@ Status RaftConsensus::StartElection(ElectionMode mode, ElectionReason reason) { RaftPeerPB::Role active_role = cmeta_->active_role(); if (active_role == RaftPeerPB::LEADER) { - LOG_WITH_PREFIX_UNLOCKED(INFO) << "Not starting " << mode << " -- already leader"; + LOG_WITH_PREFIX_UNLOCKED(INFO) << Substitute( + "Not starting $0 -- already a leader", mode_str); return Status::OK(); } if (PREDICT_FALSE(!consensus::IsVoterRole(active_role))) { http://git-wip-us.apache.org/repos/asf/kudu/blob/bf1fcb0d/src/kudu/integration-tests/cluster_itest_util.cc -- diff --git a/src/kudu/integration-tests/cluster_itest_util.cc b/src/kudu/integration-tests/cluster_itest_util.cc index cac682d..114c239 100644 --- a/src/kudu/integration-tests/cluster_itest_util.cc +++ b/src/kudu/integration-tests/cluster_itest_util.cc @@ -1071,7 +1071,6 @@ Status BeginTabletCopySession(const TServerDetails* ts, return Status::OK(); } - Status GetInt64Metric(const HostPort& http_hp, const MetricEntityPrototype* entity_proto, const char* entity_id, http://git-wip-us.apache.org/repos/asf/kudu/blob/bf1fcb0d/src/kudu/integration-tests/raft_consensus_nonvoter-itest.cc -- diff --git a/src/kudu/integration-tests/raft_consensus_nonvoter-itest.cc b/src/kudu/integration-tests/raft_consensus_nonvoter-itest.cc index 4722185..a190ebe 100644 --- a/src/kudu/integration-tests/raft_consensus_nonvoter-itest.cc +++ b/src/kudu/integration-tests/raft_consensus_nonvoter-itest.cc @@ -26,10 +26,13 @@ #include #include -#include "kudu/consensus/consensus.pb.h" +#include "kudu/cl
kudu git commit: Fix compiler warnings in tablet_replacement-itest
Repository: kudu Updated Branches: refs/heads/master 338cf61cb -> 471feccd0 Fix compiler warnings in tablet_replacement-itest This patch removes the following compiler warnings: [879/1023] Building CXX object src/kudu/inte...ment-itest.dir/tablet_replacement-itest.cc.o ../../src/kudu/integration-tests/tablet_replacement-itest.cc:370:7: warning: ignoring return value of function declared with 'warn_unused_result' attribute [-Wunused-result] cluster_->tablet_server(kFollower1Index)->Pause(); ^~~ ../../src/kudu/integration-tests/tablet_replacement-itest.cc:371:7: warning: ignoring return value of function declared with 'warn_unused_result' attribute [-Wunused-result] cluster_->tablet_server(kFollower2Index)->Pause(); ^~~ Change-Id: I5f9f65a73d2fe0e4b6d75010628fd36a57f98def Reviewed-on: http://gerrit.cloudera.org:8080/8584 Reviewed-by: Alexey Serbin Tested-by: Alexey Serbin Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/471feccd Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/471feccd Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/471feccd Branch: refs/heads/master Commit: 471feccd073e90ac767ce6d6339b4fda71f72ff0 Parents: 338cf61 Author: Mike Percy Authored: Thu Nov 16 18:22:08 2017 -0800 Committer: Alexey Serbin Committed: Fri Nov 17 07:40:36 2017 + -- src/kudu/integration-tests/tablet_replacement-itest.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/471feccd/src/kudu/integration-tests/tablet_replacement-itest.cc -- diff --git a/src/kudu/integration-tests/tablet_replacement-itest.cc b/src/kudu/integration-tests/tablet_replacement-itest.cc index c012f02..eb724f4 100644 --- a/src/kudu/integration-tests/tablet_replacement-itest.cc +++ b/src/kudu/integration-tests/tablet_replacement-itest.cc @@ -367,8 +367,8 @@ void TabletReplacementITest::TestDontEvictIfRemainingConfigIsUnstable(Instabilit cluster_->tablet_server(kFollower2Index)->Shutdown(); break; case NODE_STOPPED: - cluster_->tablet_server(kFollower1Index)->Pause(); - cluster_->tablet_server(kFollower2Index)->Pause(); + ASSERT_OK(cluster_->tablet_server(kFollower1Index)->Pause()); + ASSERT_OK(cluster_->tablet_server(kFollower2Index)->Pause()); break; }
kudu git commit: KUDU-2218. tls_socket: properly handle temporary socket errors in Writev
Repository: kudu Updated Branches: refs/heads/master 471feccd0 -> 64eb9f37b KUDU-2218. tls_socket: properly handle temporary socket errors in Writev This fixes a bug which caused RaftConsensusITest.TestLargeBatches to fail when run under stress, as in the following command line: taskset -c 0-4 \ build/latest/bin/raft_consensus-itest \ --gtest_filter=\*LargeBat\* \ --stress-cpu-threads=8 This would produce an error like: Network error: failed to write to TLS socket: error:1409F07F:SSL routines:SSL3_WRITE_PENDING:bad write retry:s3_pkt.c:878 This means that we were retrying a write after getting EAGAIN, but with a different buffer than the first time. I tracked this down to mishandling of temporary socket errors in TlsSocket::Writev(). In the case that we successfully write part of the io vector but hit such an error trying to write a later element in the vector, we were still propagating the error back up to the caller. The caller didn't realize that part of the write was successful, and thus it would retry the write from the beginning. The fix is to fix the above, but also to enable partial writes in TlsContext. The new test fails if either of the above two changes are backed out. Change-Id: If797f220f42bfb2e6f452b66f15e7a758e883472 Reviewed-on: http://gerrit.cloudera.org:8080/8570 Tested-by: Kudu Jenkins Reviewed-by: Alexey Serbin Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/64eb9f37 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/64eb9f37 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/64eb9f37 Branch: refs/heads/master Commit: 64eb9f37b171419ed12a3795efe28faf2fd33b3d Parents: 471fecc Author: Todd Lipcon Authored: Wed Nov 15 22:55:44 2017 -0800 Committer: Alexey Serbin Committed: Fri Nov 17 17:42:14 2017 + -- src/kudu/security/tls_context.cc | 2 +- src/kudu/security/tls_socket-test.cc | 277 ++ src/kudu/security/tls_socket.cc | 10 +- 3 files changed, 219 insertions(+), 70 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/64eb9f37/src/kudu/security/tls_context.cc -- diff --git a/src/kudu/security/tls_context.cc b/src/kudu/security/tls_context.cc index 1cea173..28e180b 100644 --- a/src/kudu/security/tls_context.cc +++ b/src/kudu/security/tls_context.cc @@ -115,7 +115,7 @@ Status TlsContext::Init() { if (!ctx_) { return Status::RuntimeError("failed to create TLS context", GetOpenSSLErrors()); } - SSL_CTX_set_mode(ctx_.get(), SSL_MODE_AUTO_RETRY); + SSL_CTX_set_mode(ctx_.get(), SSL_MODE_AUTO_RETRY | SSL_MODE_ENABLE_PARTIAL_WRITE); // Disable SSLv2 and SSLv3 which are vulnerable to various issues such as POODLE. // We support versions back to TLSv1.0 since OpenSSL on RHEL 6.4 and earlier does not http://git-wip-us.apache.org/repos/asf/kudu/blob/64eb9f37/src/kudu/security/tls_socket-test.cc -- diff --git a/src/kudu/security/tls_socket-test.cc b/src/kudu/security/tls_socket-test.cc index a978e68..214d2bf 100644 --- a/src/kudu/security/tls_socket-test.cc +++ b/src/kudu/security/tls_socket-test.cc @@ -17,7 +17,11 @@ #include "kudu/security/tls_handshake.h" +#include #include +#include +#include +#include #include #include @@ -28,6 +32,7 @@ #include #include #include +#include #include #include @@ -38,6 +43,8 @@ #include "kudu/util/monotime.h" #include "kudu/util/net/sockaddr.h" #include "kudu/util/net/socket.h" +#include "kudu/util/random.h" +#include "kudu/util/random_util.h" #include "kudu/util/scoped_cleanup.h" #include "kudu/util/status.h" #include "kudu/util/test_macros.h" @@ -46,25 +53,27 @@ using std::string; using std::thread; using std::unique_ptr; - +using std::vector; namespace kudu { namespace security { +const MonoDelta kTimeout = MonoDelta::FromSeconds(10); + +// Size is big enough to not fit into output socket buffer of default size +// (controlled by setsockopt() with SO_SNDBUF). +constexpr size_t kEchoChunkSize = 32 * 1024 * 1024; class TlsSocketTest : public KuduTest { public: void SetUp() override { KuduTest::SetUp(); - ASSERT_OK(client_tls_.Init()); -ASSERT_OK(server_tls_.Init()); -ASSERT_OK(server_tls_.GenerateSelfSignedCertAndKey()); } protected: + void ConnectClient(const Sockaddr& addr, unique_ptr* sock); TlsContext client_tls_; - TlsContext server_tls_; }; Status DoNegotiationSide(Socket* sock, TlsHandshake* tls, const char* side) { @@ -101,19 +110,112 @@ Status DoNegotiationSide(Socket* sock, TlsHandshake* tls, const char*
[2/2] kudu git commit: catalog_manager_tsk-itest: ensure that test eventually makes progress
catalog_manager_tsk-itest: ensure that test eventually makes progress This test previously tried to introduce a lot of master leader elections by setting a very low heartbeat and failure interval. This worked, but sometimes worked so well that the test never made progress and couldn't obtain a stable leader long enough to create a table. This patch changes the test to instead use a separate thread which triggers elections manually on all the leaders. The elections start off very frequent and then back off as the test progresses to ensure that by the end, the leaders do actually make progress. I verified that this still covers the case of a failed write when writing TSKs by changing the RETURN_NOT_OK to a CHECK_OK when storing the TSK. With the CHECK_OK, the test failed nearly immediately. Change-Id: I3ecda0c269225e7674bc384fee652576b110ae7b Reviewed-on: http://gerrit.cloudera.org:8080/8567 Tested-by: Kudu Jenkins Reviewed-by: Alexey Serbin Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/8e6bfa9f Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/8e6bfa9f Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/8e6bfa9f Branch: refs/heads/master Commit: 8e6bfa9fb8aa7292e75a9fd3dcf9c50c8ea5191e Parents: c4006ae Author: Todd Lipcon Authored: Wed Nov 15 21:49:17 2017 -0800 Committer: Alexey Serbin Committed: Fri Nov 17 23:43:13 2017 + -- .../catalog_manager_tsk-itest.cc| 66 ++-- 1 file changed, 46 insertions(+), 20 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/8e6bfa9f/src/kudu/integration-tests/catalog_manager_tsk-itest.cc -- diff --git a/src/kudu/integration-tests/catalog_manager_tsk-itest.cc b/src/kudu/integration-tests/catalog_manager_tsk-itest.cc index 0a0a0e4..9812102 100644 --- a/src/kudu/integration-tests/catalog_manager_tsk-itest.cc +++ b/src/kudu/integration-tests/catalog_manager_tsk-itest.cc @@ -16,12 +16,17 @@ // under the License. #include +#include +#include #include #include #include +#include #include +#include #include +#include #include #include "kudu/client/client-test-util.h" @@ -30,11 +35,18 @@ #include "kudu/client/shared_ptr.h" #include "kudu/client/write_op.h" #include "kudu/common/partial_row.h" +#include "kudu/consensus/consensus.pb.h" +#include "kudu/consensus/consensus.proxy.h" #include "kudu/gutil/gscoped_ptr.h" #include "kudu/gutil/strings/substitute.h" +#include "kudu/master/sys_catalog.h" #include "kudu/mini-cluster/external_mini_cluster.h" +#include "kudu/rpc/rpc_controller.h" #include "kudu/tablet/key_value_test_schema.h" #include "kudu/util/monotime.h" +#include "kudu/util/net/sockaddr.h" +#include "kudu/util/scoped_cleanup.h" +#include "kudu/util/status.h" #include "kudu/util/test_macros.h" #include "kudu/util/test_util.h" @@ -47,6 +59,7 @@ using kudu::client::KuduTable; using kudu::client::KuduTableCreator; using kudu::cluster::ExternalMiniCluster; using kudu::cluster::ExternalMiniClusterOptions; +using std::atomic; using std::back_inserter; using std::copy; using std::string; @@ -74,22 +87,10 @@ class CatalogManagerTskITest : public KuduTest { cluster_opts_.master_rpc_ports = { 11030, 11031, 11032 }; cluster_opts_.num_tablet_servers = num_tservers_; -// Add common flags for both masters and tservers. -const vector common_flags = { - Substitute("--raft_heartbeat_interval_ms=$0", hb_interval_ms_), -}; -copy(common_flags.begin(), common_flags.end(), -back_inserter(cluster_opts_.extra_master_flags)); -copy(common_flags.begin(), common_flags.end(), -back_inserter(cluster_opts_.extra_tserver_flags)); - // Add master-only flags. const vector master_flags = { "--catalog_manager_inject_latency_prior_tsk_write_ms=1000", "--raft_enable_pre_election=false", - Substitute("--leader_failure_exp_backoff_max_delta_ms=$0", - hb_interval_ms_ * 4), - "--leader_failure_max_missed_heartbeat_periods=1.0", "--master_non_leader_masters_propagate_tsk", "--tsk_rotation_seconds=2", }; @@ -113,7 +114,7 @@ class CatalogManagerTskITest : public KuduTest { using ::kudu::client::sp::shared_ptr; static const char* kTableName = "test-table"; // Using the setting for both RPC and admin operation timeout. -const MonoDelta timeout = MonoDelta::FromSeconds(600); +const MonoDelta timeout = MonoDelta::FromSeconds(120);
[1/2] kudu git commit: dist_test: enable sharding of a few more tests
Repository: kudu Updated Branches: refs/heads/master 64eb9f37b -> 8e6bfa9fb dist_test: enable sharding of a few more tests Adds sharding for rowset_tree-test, tablet_copy-itest, and delete_tablet-itest which are the longest-running non-sharded tests in recent builds such as [1] [1] http://dist-test.cloudera.org/job?job_id=jenkins-slave.1510801703.18407 Change-Id: Iff1e0ac39f1834e8ac22283e1dc0c336d328ae35 Reviewed-on: http://gerrit.cloudera.org:8080/8565 Reviewed-by: Dan Burkert Tested-by: Todd Lipcon Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/c4006ae0 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/c4006ae0 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/c4006ae0 Branch: refs/heads/master Commit: c4006ae028b44e9b7f8f006d64ed7a3a3f34fe9a Parents: 64eb9f3 Author: Todd Lipcon Authored: Wed Nov 15 20:58:02 2017 -0800 Committer: Todd Lipcon Committed: Fri Nov 17 21:26:12 2017 + -- build-support/dist_test.py | 5 - 1 file changed, 4 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/c4006ae0/build-support/dist_test.py -- diff --git a/build-support/dist_test.py b/build-support/dist_test.py index a6f97d4..78fe5cb 100755 --- a/build-support/dist_test.py +++ b/build-support/dist_test.py @@ -90,10 +90,13 @@ DEPS_FOR_ALL = \ NUM_SHARDS_BY_TEST = { 'cfile-test': 4, 'client-test': 8, + 'delete_table-itest': 4, 'delete_table-test': 8, 'flex_partitioning-itest': 8, 'mt-tablet-test': 4, - 'raft_consensus-itest': 6 + 'raft_consensus-itest': 6, + 'rowset_tree-test': 6, + 'tablet_copy-itest': 6 }
kudu git commit: tablet_copy_service: fix Status message with misplaced $0
Repository: kudu Updated Branches: refs/heads/master f9d85f53b -> a4642a41e tablet_copy_service: fix Status message with misplaced $0 Change-Id: I0443d58f32b306a0d59dd76fa901ad384ceac6e0 Reviewed-on: http://gerrit.cloudera.org:8080/8598 Reviewed-by: Alexey Serbin Tested-by: Kudu Jenkins Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/a4642a41 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/a4642a41 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/a4642a41 Branch: refs/heads/master Commit: a4642a41e5db7a7a887b39c1cb661a6de1dca47e Parents: f9d85f5 Author: Todd Lipcon Authored: Mon Nov 20 10:50:54 2017 -0800 Committer: Alexey Serbin Committed: Tue Nov 21 01:07:35 2017 + -- src/kudu/tserver/tablet_copy_service.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/a4642a41/src/kudu/tserver/tablet_copy_service.cc -- diff --git a/src/kudu/tserver/tablet_copy_service.cc b/src/kudu/tserver/tablet_copy_service.cc index 6cfb9e7..2d10eb7 100644 --- a/src/kudu/tserver/tablet_copy_service.cc +++ b/src/kudu/tserver/tablet_copy_service.cc @@ -170,8 +170,8 @@ void TabletCopyServiceImpl::BeginTabletCopySession( if (!new_session && !session->IsInitialized()) { RPC_RETURN_NOT_OK( -Status::ServiceUnavailable("tablet copy session for tablet $0 is initializing", - tablet_id), +Status::ServiceUnavailable( +Substitute("tablet copy session for tablet $0 is initializing", tablet_id)), TabletCopyErrorPB::UNKNOWN_ERROR, "try again later", context);
kudu git commit: consensus: Add gflag to enable improved re-replication
Repository: kudu Updated Branches: refs/heads/master 8a81f4ff1 -> 6c02bea97 consensus: Add gflag to enable improved re-replication This gflag will be used to control whether improved re-replication is enabled on a cluster. Change-Id: I247842af7d0172d6e6b167a29613f6b261990a01 Reviewed-on: http://gerrit.cloudera.org:8080/8626 Tested-by: Kudu Jenkins Reviewed-by: Alexey Serbin Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/6c02bea9 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/6c02bea9 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/6c02bea9 Branch: refs/heads/master Commit: 6c02bea97db29ef178b2f907c6d7fb561d9ac82c Parents: 8a81f4f Author: Mike Percy Authored: Tue Nov 21 14:55:36 2017 -0800 Committer: Alexey Serbin Committed: Wed Nov 22 19:35:30 2017 + -- src/kudu/consensus/raft_consensus.cc | 7 +++ 1 file changed, 7 insertions(+) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/6c02bea9/src/kudu/consensus/raft_consensus.cc -- diff --git a/src/kudu/consensus/raft_consensus.cc b/src/kudu/consensus/raft_consensus.cc index ce4ba54..58cc087 100644 --- a/src/kudu/consensus/raft_consensus.cc +++ b/src/kudu/consensus/raft_consensus.cc @@ -120,6 +120,13 @@ DEFINE_bool(raft_enable_tombstoned_voting, true, TAG_FLAG(raft_enable_tombstoned_voting, experimental); TAG_FLAG(raft_enable_tombstoned_voting, runtime); +// Enable improved re-replication (KUDU-1097). +DEFINE_bool(raft_prepare_replacement_before_eviction, false, +"When enabled, failed replicas will only be evicted after a " +"replacement has been prepared for them."); +TAG_FLAG(raft_prepare_replacement_before_eviction, advanced); +TAG_FLAG(raft_prepare_replacement_before_eviction, experimental); + DECLARE_int32(memory_limit_warn_threshold_percentage); // Metrics
kudu git commit: KUDU-1097: 3-4-3 behavior for catalog manager
Repository: kudu Updated Branches: refs/heads/master 88e39bad1 -> 87dcaf34d KUDU-1097: 3-4-3 behavior for catalog manager Updated the catalog_manager to behave as specified in the 3-4-3 re-replication improvement v1 design implementation plan. This update includes changes referred as 'patch 2' and 'patch 4'. Change-Id: I6f0469ac641bf7a03dbef01eaa3f1b58a5bf5d27 Reviewed-on: http://gerrit.cloudera.org:8080/8619 Tested-by: Kudu Jenkins Reviewed-by: Mike Percy Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/87dcaf34 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/87dcaf34 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/87dcaf34 Branch: refs/heads/master Commit: 87dcaf34d3495f87164b80e03b757efc12be2fe0 Parents: 88e39ba Author: Alexey Serbin Authored: Mon Nov 20 12:04:40 2017 -0800 Committer: Alexey Serbin Committed: Mon Nov 27 06:17:45 2017 + -- src/kudu/consensus/quorum_util-test.cc | 442 ++- src/kudu/consensus/quorum_util.cc | 187 src/kudu/consensus/quorum_util.h| 12 + .../raft_consensus_nonvoter-itest.cc| 153 ++- src/kudu/master/catalog_manager.cc | 343 +- src/kudu/master/catalog_manager.h | 2 +- 6 files changed, 1015 insertions(+), 124 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/87dcaf34/src/kudu/consensus/quorum_util-test.cc -- diff --git a/src/kudu/consensus/quorum_util-test.cc b/src/kudu/consensus/quorum_util-test.cc index 8864977..cede1db 100644 --- a/src/kudu/consensus/quorum_util-test.cc +++ b/src/kudu/consensus/quorum_util-test.cc @@ -15,8 +15,12 @@ // specific language governing permissions and limitations // under the License. +#include #include +#include +#include +#include #include #include "kudu/common/common.pb.h" @@ -25,26 +29,68 @@ #include "kudu/util/status.h" #include "kudu/util/test_macros.h" +using std::string; +using std::unique_ptr; +using std::vector; + namespace kudu { namespace consensus { -using std::string; +// Handy notation of membership types used by AddPeer(), etc. +constexpr auto N = RaftPeerPB::NON_VOTER; +constexpr auto U = RaftPeerPB::UNKNOWN_MEMBER_TYPE; +constexpr auto V = RaftPeerPB::VOTER; + +typedef std::pair Attr; // Add a consensus peer into the specified configuration. static void AddPeer(RaftConfigPB* config, const string& uuid, -RaftPeerPB::MemberType type) { +RaftPeerPB::MemberType type, +boost::optional overall_health = boost::none, +vector attrs = {}) { RaftPeerPB* peer = config->add_peers(); peer->set_permanent_uuid(uuid); peer->mutable_last_known_addr()->set_host(uuid + ".example.com"); peer->set_member_type(type); + if (overall_health) { +unique_ptr health_report(new HealthReportPB); +switch (*overall_health) { + case '+': +health_report->set_overall_health(HealthReportPB::HEALTHY); +break; + case '-': +health_report->set_overall_health(HealthReportPB::FAILED); +break; + case '?': +health_report->set_overall_health(HealthReportPB::UNKNOWN); +break; + default: +FAIL() << *overall_health << ": unexpected replica health status"; +break; +} +peer->set_allocated_health_report(health_report.release()); + } + if (!attrs.empty()) { +unique_ptr attrs_pb(new RaftPeerAttrsPB); +for (const auto& attr : attrs) { + if (attr.first == "PROMOTE") { +attrs_pb->set_promote(attr.second); + } else if (attr.first == "REPLACE") { +attrs_pb->set_replace(attr.second); + } else { +FAIL() << attr.first << ": unexpected attribute to set"; + } +} +peer->set_allocated_attrs(attrs_pb.release()); + } } TEST(QuorumUtilTest, TestMemberExtraction) { RaftConfigPB config; - AddPeer(&config, "A", RaftPeerPB::VOTER); - AddPeer(&config, "B", RaftPeerPB::VOTER); - AddPeer(&config, "C", RaftPeerPB::VOTER); + AddPeer(&config, "A", V); + AddPeer(&config, "B", V); + AddPeer(&config, "C", V); // Basic test for GetRaftConfigMember(). RaftPeerPB* peer_pb; @@ -65,9 +111,9 @@ TEST(QuorumUtilTest, TestMemberExtraction) { TEST(QuorumUtilTest, TestDiffConsensusStates) { ConsensusStatePB old_cs; - AddPeer(old_cs.mutable_committed_config(), &q
kudu git commit: KUDU-1097 (patch1): test for replica health reporting
Repository: kudu Updated Branches: refs/heads/master 87dcaf34d -> 37b89924e KUDU-1097 (patch1): test for replica health reporting Added a test to verify that the leader tablet replica reports on the replica health changes. The tests verifies that the health reports are present in the Raft consensus state reported by the leader replica. The test also verifies that the incremental tablet reports contain appropriate information once the replica health status changes. Change-Id: Ie62b49efebad9a123eec51dd302e375e46e0682d Reviewed-on: http://gerrit.cloudera.org:8080/8642 Tested-by: Kudu Jenkins Reviewed-by: Mike Percy Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/37b89924 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/37b89924 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/37b89924 Branch: refs/heads/master Commit: 37b89924ee807f39ae30343b84e837daa7a979b3 Parents: 87dcaf3 Author: Alexey Serbin Authored: Wed Nov 22 22:03:59 2017 -0800 Committer: Alexey Serbin Committed: Mon Nov 27 06:33:39 2017 + -- .../ts_tablet_manager-itest.cc | 340 --- 1 file changed, 291 insertions(+), 49 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/37b89924/src/kudu/integration-tests/ts_tablet_manager-itest.cc -- diff --git a/src/kudu/integration-tests/ts_tablet_manager-itest.cc b/src/kudu/integration-tests/ts_tablet_manager-itest.cc index 98b0e26..ee5a957 100644 --- a/src/kudu/integration-tests/ts_tablet_manager-itest.cc +++ b/src/kudu/integration-tests/ts_tablet_manager-itest.cc @@ -16,10 +16,12 @@ // under the License. #include +#include #include #include #include #include +#include #include #include @@ -56,37 +58,43 @@ #include "kudu/util/test_macros.h" #include "kudu/util/test_util.h" -DECLARE_bool(enable_leader_failure_detection); -DECLARE_bool(catalog_manager_wait_for_new_tablets_to_elect_leader); DECLARE_bool(allow_unsafe_replication_factor); +DECLARE_bool(catalog_manager_wait_for_new_tablets_to_elect_leader); +DECLARE_bool(enable_leader_failure_detection); +DECLARE_bool(raft_prepare_replacement_before_eviction); DEFINE_int32(num_election_test_loops, 3, "Number of random EmulateElection() loops to execute in " "TestReportNewLeaderOnLeaderChange"); -namespace kudu { -namespace tserver { - -using client::KuduClient; -using client::KuduSchema; -using client::KuduTable; -using client::KuduTableCreator; -using cluster::InternalMiniCluster; -using cluster::InternalMiniClusterOptions; -using consensus::GetConsensusRole; -using consensus::RaftPeerPB; -using itest::SimpleIntKeyKuduSchema; -using master::MasterServiceProxy; -using master::ReportedTabletPB; -using master::TabletReportPB; -using rpc::Messenger; -using rpc::MessengerBuilder; +using kudu::client::KuduClient; +using kudu::client::KuduSchema; +using kudu::client::KuduTable; +using kudu::client::KuduTableCreator; +using kudu::cluster::InternalMiniCluster; +using kudu::cluster::InternalMiniClusterOptions; +using kudu::consensus::ConsensusStatePB; +using kudu::consensus::GetConsensusRole; +using kudu::consensus::HealthReportPB; +using kudu::consensus::RaftConfigPB; +using kudu::consensus::RaftConsensus; +using kudu::consensus::RaftPeerPB; +using kudu::itest::SimpleIntKeyKuduSchema; +using kudu::master::MasterServiceProxy; +using kudu::master::ReportedTabletPB; +using kudu::master::TabletReportPB; +using kudu::rpc::Messenger; +using kudu::rpc::MessengerBuilder; +using kudu::tablet::TabletReplica; +using kudu::tserver::MiniTabletServer; +using std::map; using std::shared_ptr; using std::string; -using std::vector; using std::unique_ptr; +using std::vector; using strings::Substitute; -using tablet::TabletReplica; -using tserver::MiniTabletServer; + +namespace kudu { +namespace tserver { static const char* const kTableName = "test-table"; @@ -95,7 +103,7 @@ class TsTabletManagerITest : public KuduTest { TsTabletManagerITest() : schema_(SimpleIntKeyKuduSchema()) { } - virtual void SetUp() override { + void SetUp() override { KuduTest::SetUp(); MessengerBuilder bld("client"); @@ -111,6 +119,21 @@ class TsTabletManagerITest : public KuduTest { } protected: + void DisableHeartbeatingToMaster(); + + // Populate the 'replicas' container with corresponding objects representing + // tablet replicas running at tablet servers in the test cluster. It's assumed + // there is at least one tablet replica per tablet server. Also, this utility + // method awaits up to the specified timeout for the consensus to be running + // before adding an elemen
kudu git commit: consensus: Rename ChangeConfigType enum constants
Repository: kudu Updated Branches: refs/heads/master 79a255bbf -> fc143c01d consensus: Rename ChangeConfigType enum constants Since we now have attributes in the RaftConfigPeerPB message, it would be more useful to allow a peer to be modified such that only its attributes change, but not its member type. In order to provide meaningful semantics, that means we should change the constant name for the "modify peer" operation as well. This patch changes CHANGE_REPLICA_TYPE to MODIFY_PEER. While we're in there, let's also rename the other constants to more accurately reflect the terminology we tend to use in this part of the code. There are no functional or wire-protocol changes in this patch. The functional changes to go along with this patch will come in a follow-up patch. Change-Id: I77b838f1b7ddcf176d1a5146e04fa1fddc1042fa Reviewed-on: http://gerrit.cloudera.org:8080/8643 Tested-by: Kudu Jenkins Reviewed-by: Mike Percy Reviewed-by: Alexey Serbin Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/fc143c01 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/fc143c01 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/fc143c01 Branch: refs/heads/master Commit: fc143c01dd9091ef212f9b572752875f11524567 Parents: 79a255b Author: Mike Percy Authored: Sun Nov 26 21:17:29 2017 -0800 Committer: Alexey Serbin Committed: Mon Nov 27 19:41:44 2017 + -- src/kudu/consensus/consensus.proto| 8 src/kudu/consensus/raft_consensus.cc | 14 +++--- src/kudu/integration-tests/cluster_itest_util.cc | 6 +++--- src/kudu/integration-tests/cluster_itest_util.h | 4 ++-- .../raft_consensus_nonvoter-itest.cc | 4 ++-- src/kudu/master/catalog_manager.cc| 8 src/kudu/tools/tool_action_tablet.cc | 18 +- 7 files changed, 31 insertions(+), 31 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/fc143c01/src/kudu/consensus/consensus.proto -- diff --git a/src/kudu/consensus/consensus.proto b/src/kudu/consensus/consensus.proto index c8abd5d..6ff4186 100644 --- a/src/kudu/consensus/consensus.proto +++ b/src/kudu/consensus/consensus.proto @@ -118,9 +118,9 @@ message ChangeConfigRecordPB { enum ChangeConfigType { UNKNOWN_CHANGE = 0; - ADD_SERVER = 1; - REMOVE_SERVER = 2; - CHANGE_REPLICA_TYPE = 3; + ADD_PEER = 1; + REMOVE_PEER = 2; + MODIFY_PEER = 3; } // A configuration change request for the tablet with 'tablet_id'. @@ -139,7 +139,7 @@ message ChangeConfigRequestPB { optional ChangeConfigType type = 2; // The peer to add or remove. - // When 'type' == ADD_SERVER, both the permanent_uuid and last_known_addr + // When 'type' == ADD_PEER, both the permanent_uuid and last_known_addr // fields must be set. Otherwise, only the permanent_uuid field is required. optional RaftPeerPB server = 3; http://git-wip-us.apache.org/repos/asf/kudu/blob/fc143c01/src/kudu/consensus/raft_consensus.cc -- diff --git a/src/kudu/consensus/raft_consensus.cc b/src/kudu/consensus/raft_consensus.cc index f1e0650..cdfb0fb 100644 --- a/src/kudu/consensus/raft_consensus.cc +++ b/src/kudu/consensus/raft_consensus.cc @@ -802,7 +802,7 @@ void RaftConsensus::TryRemoveFollowerTask(const string& uuid, ChangeConfigRequestPB req; req.set_tablet_id(options_.tablet_id); req.mutable_server()->set_permanent_uuid(uuid); - req.set_type(REMOVE_SERVER); + req.set_type(REMOVE_PEER); req.set_cas_config_opid_index(committed_config.opid_index()); LOG(INFO) << LogPrefixThreadSafe() << "Attempting to remove follower " << uuid << " from the Raft config. Reason: " << reason; @@ -849,7 +849,7 @@ void RaftConsensus::TryPromoteNonVoterTask(const std::string& peer_uuid, ChangeConfigRequestPB req; req.set_tablet_id(options_.tablet_id); - req.set_type(CHANGE_REPLICA_TYPE); + req.set_type(MODIFY_PEER); req.mutable_server()->set_permanent_uuid(peer_uuid); req.mutable_server()->set_member_type(RaftPeerPB::VOTER); req.mutable_server()->mutable_attrs()->set_promote(false); @@ -1682,7 +1682,7 @@ Status RaftConsensus::ChangeConfig(const ChangeConfigRequestPB& req, new_config.clear_opid_index(); const string& server_uuid = server.permanent_uuid(); switch (type) { - case ADD_SERVER: + case ADD_PEER: // Ensure the server we are adding is not already a member of the configuration. if (IsRaftConfigMember(server_uuid, committed_config)) {
kudu git commit: [tablet_metadata] protect pre_flush_callback_ by flush_lock_
Repository: kudu Updated Branches: refs/heads/master b29c93734 -> fa65d647a [tablet_metadata] protect pre_flush_callback_ by flush_lock_ This patch makes the pre_flush_callback_ protected by the flush_lock_ to address the case of concurrent calls to TabletMetadata::DeleteTabletData() and Tablet::Shutdown(). The call to Tablet::Shutdown() was originated from TabletReplica::OnDiskSize() when the local shared_ptr variable ended up keeping the last reference to the object corresponding to the tablet being deleted. The issue contributed to the flakiness at least of the following tests: * CreateTableStressTest.CreateAndDeleteBigTable * DeleteTableWhileScanInProgressParamTest Prior to this patch, TSAN reported about read/write race for the callback with the traces like the following: Read of size 8 at 0x7b5703d0 by thread T56 (mutexes: write M2161): #0 kudu::Callback::Run() const /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/gutil/callback.h:394:45 (libtablet.so+0x1caa81) #1 kudu::tablet::TabletMetadata::Flush() /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tablet/tablet_metadata.cc:549:23 (libtablet.so+0x1c30b8) #2 kudu::tablet::TabletMetadata::DeleteTabletData(kudu::tablet::TabletDataState, boost::optional const&) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tablet/tablet_metadata.cc:232:3 (libtablet.so+0x1c411a) #3 kudu::tserver::TSTabletManager::DeleteTabletData(scoped_refptr const&, scoped_refptr const&, kudu::tablet::TabletDataState, boost::optional) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tserver/ts_tablet_manager.cc:1281:3 (libtserver.so+0xfad26) #4 kudu::tserver::TSTabletManager::DeleteTablet(std::__1::basic_string, std::__1::allocator > const&, kudu::tablet::TabletDataState, boost::optional const&, kudu::tserver::TabletServerErrorPB_Code*) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tserver/ts_tablet_manager.cc:826:14 (libtserver.so+0xfc365) #5 kudu::tserver::TabletServiceAdminImpl::DeleteTablet(kudu::tserver::DeleteTabletRequestPB const*, kudu::tserver::DeleteTabletResponsePB*, kudu::rpc::RpcContext*) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tserver/tablet_service.cc:804:41 (libtserver.so+0xd5dfd) ... Previous write of size 8 at 0x7b5703d0 by thread T12 (mutexes: write M90770251150680224, write M623321216325058272): #0 kudu::internal::CallbackBase::operator=(kudu::internal::CallbackBase const&) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/gutil/callback_internal.h:34:7 (libtserver.so+0xb0596) #1 kudu::Callback::operator=(kudu::Callback&&) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/gutil/callback.h:358:7 (libtablet.so+0x115cb0) #2 kudu::tablet::TabletMetadata::SetPreFlushCallback(kudu::Callback) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tablet/tablet_metadata.h:229:74 (libtablet.so+0x10e437) #3 kudu::tablet::Tablet::Shutdown() /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tablet/tablet.cc:352:14 (libtablet.so+0xf7fdc) #4 kudu::tablet::Tablet::~Tablet() /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tablet/tablet.cc:248:3 (libtablet.so+0xf7c9e) ... #10 kudu::tablet::TabletReplica::OnDiskSize() const /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tablet/tablet_replica.cc:742:1 (libtablet.so+0x143af7) ... Change-Id: I21d3195183584d1a51aeec64b049ac49994f69be Reviewed-on: http://gerrit.cloudera.org:8080/8649 Reviewed-by: Mike Percy Reviewed-by: Andrew Wong Tested-by: Alexey Serbin Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/fa65d647 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/fa65d647 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/fa65d647 Branch: refs/heads/master Commit: fa65d647a450c6895bac279dd5e5ad5fb6f8d228 Parents: b29c937 Author: Alexey Serbin Authored: Mon Nov 27 12:16:49 2017 -0800 Committer: Alexey Serbin Committed: Tue Nov 28 02:00:39 2017 + -- src/kudu/tablet/tablet_metadata.cc | 5 + src/kudu/tablet/tablet_metadata.h | 5 ++--- 2 files changed, 7 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/fa65d647/src/kudu/tablet/tablet_metadata.cc -- diff --git a/src/kudu/tablet/tablet_metadata.cc b/src/kudu/tablet/tablet_metadata.cc index d848c1f..92eb8f8 100644 --- a/src/kudu/tablet/tablet_metadata.cc +++ b/src/kudu/tablet/tablet_metadata.cc @@ -617,6 +617,11 @@ Status TabletMetadata::ReplaceSuperBlockUnlocked(const TabletSuperBlockPB &pb) { return Status::OK(); } +void TabletMetadata::SetPreFlushCallb
kudu git commit: KUDU-1097: scenario for fallen-behind-the-WAL replica
Repository: kudu Updated Branches: refs/heads/master 5c87c22be -> aafed6d1d KUDU-1097: scenario for fallen-behind-the-WAL replica Added end-to-end test scenario for the case when an existing follower replica falls behind the WAL GC threshold. First, the system attempts to replace it with a non-voter where tablet copying fails. As soon as that detected, a new non-voter replica is added and the replacement succeeds, so the tablet configuration eventually gets rid of failed replicas and contains exactly 3 healthy replicas. Change-Id: Ifcaff45c9fa3beb51d99dc53d88138da7dc120af Reviewed-on: http://gerrit.cloudera.org:8080/8675 Reviewed-by: Mike Percy Tested-by: Kudu Jenkins Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/aafed6d1 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/aafed6d1 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/aafed6d1 Branch: refs/heads/master Commit: aafed6d1d749978899a78dae84dd25c028a1763a Parents: 5c87c22 Author: Alexey Serbin Authored: Tue Nov 28 15:56:31 2017 -0800 Committer: Alexey Serbin Committed: Wed Nov 29 10:28:17 2017 + -- src/kudu/consensus/quorum_util-test.cc | 18 +++ src/kudu/consensus/quorum_util.cc | 13 +- .../raft_consensus-itest-base.cc| 12 +- .../raft_consensus-itest-base.h | 4 +- .../integration-tests/raft_consensus-itest.cc | 9 +- .../raft_consensus_election-itest.cc| 3 +- .../raft_consensus_nonvoter-itest.cc| 147 +++ src/kudu/master/catalog_manager.cc | 4 + src/kudu/tserver/tablet_copy_service.cc | 7 +- 9 files changed, 202 insertions(+), 15 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/aafed6d1/src/kudu/consensus/quorum_util-test.cc -- diff --git a/src/kudu/consensus/quorum_util-test.cc b/src/kudu/consensus/quorum_util-test.cc index cede1db..461ae7a 100644 --- a/src/kudu/consensus/quorum_util-test.cc +++ b/src/kudu/consensus/quorum_util-test.cc @@ -382,6 +382,24 @@ TEST(QuorumUtilTest, IsUnderReplicated) { EXPECT_FALSE(IsUnderReplicated(config, 2)); EXPECT_TRUE(IsUnderReplicated(config, 3)); } + { +RaftConfigPB config; +AddPeer(&config, "A", V, '-'); +AddPeer(&config, "B", V, '+'); +AddPeer(&config, "C", V, '+'); +AddPeer(&config, "D", N, '-', {{"PROMOTE", true}}); +AddPeer(&config, "E", N, '+', {{"PROMOTE", true}}); +EXPECT_FALSE(IsUnderReplicated(config, 3)); + } + { +RaftConfigPB config; +AddPeer(&config, "A", V, '-'); +AddPeer(&config, "B", V, '+'); +AddPeer(&config, "C", V, '+'); +AddPeer(&config, "D", N, '-', {{"PROMOTE", true}}); +AddPeer(&config, "E", N, '+', {{"PROMOTE", false}}); +EXPECT_TRUE(IsUnderReplicated(config, 3)); + } } // Verify logic of the kudu::consensus::CanEvictReplica(), anticipating http://git-wip-us.apache.org/repos/asf/kudu/blob/aafed6d1/src/kudu/consensus/quorum_util.cc -- diff --git a/src/kudu/consensus/quorum_util.cc b/src/kudu/consensus/quorum_util.cc index e002070..8632e3c 100644 --- a/src/kudu/consensus/quorum_util.cc +++ b/src/kudu/consensus/quorum_util.cc @@ -34,6 +34,7 @@ using google::protobuf::RepeatedPtrField; using kudu::pb_util::SecureShortDebugString; +using kudu::pb_util::SecureDebugString; using std::map; using std::pair; using std::set; @@ -408,6 +409,7 @@ bool IsUnderReplicated(const RaftConfigPB& config, int replication_factor) { // While working with the optional fields related to per-replica health status // and attributes, has_a_field()-like methods are not called because of // the appropriate default values of those fields. + VLOG(2) << "config to evaluate: " << SecureDebugString(config); for (const RaftPeerPB& peer : config.peers()) { switch (peer.member_type()) { case RaftPeerPB::VOTER: @@ -428,8 +430,11 @@ bool IsUnderReplicated(const RaftConfigPB& config, int replication_factor) { break; } } - return replication_factor > (num_voters_total - num_voters_need_replacement) + - num_non_voters_to_promote; + const bool is_under_replicated = replication_factor > + (num_voters_total - num_voters_need_replacement) + num_non_voters_to_promote; + VLOG(2) << "decision: the config is" << (is_under_replicated
kudu git commit: KUDU-2230: the leader is always a viable voter
Repository: kudu Updated Branches: refs/heads/master 3f4253753 -> dc497fec2 KUDU-2230: the leader is always a viable voter While iterating over peers in PeerMessageQueue::SafeToEvictUnlocked(), assume that the leader is always a viable voter. Prior to this patch, the leader itself was not counted as a viable voter since the leader does not update its own last_communication_time. Both test scenarios TestDontEvictIfRemainingConfigIsUnstable_NodesDown and TestDontEvictIfRemainingConfigIsUnstable_NodesStopped of the TabletReplacementITest test have been modified to cover the fixed issue. I verified that commenting out the fix in consensus_queue.cc makes them fail. Change-Id: Id200ec8e562e2b21c8eef09e7b38a8d85d23239c Reviewed-on: http://gerrit.cloudera.org:8080/8709 Reviewed-by: Mike Percy Reviewed-by: Andrew Wong Tested-by: Alexey Serbin Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/dc497fec Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/dc497fec Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/dc497fec Branch: refs/heads/master Commit: dc497fec2cb1cabec17eb250ca7d8a7e3d6a9e8f Parents: 3f42537 Author: Alexey Serbin Authored: Thu Nov 30 19:44:22 2017 -0800 Committer: Alexey Serbin Committed: Fri Dec 1 22:47:21 2017 + -- src/kudu/consensus/consensus_queue.cc | 37 + .../tablet_replacement-itest.cc | 82 +++- 2 files changed, 84 insertions(+), 35 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/dc497fec/src/kudu/consensus/consensus_queue.cc -- diff --git a/src/kudu/consensus/consensus_queue.cc b/src/kudu/consensus/consensus_queue.cc index 008b488..6d09b33 100644 --- a/src/kudu/consensus/consensus_queue.cc +++ b/src/kudu/consensus/consensus_queue.cc @@ -401,6 +401,7 @@ OpId PeerMessageQueue::GetNextOpId() const { bool PeerMessageQueue::SafeToEvictUnlocked(const string& evict_uuid) const { DCHECK(queue_lock_.is_locked()); + DCHECK_EQ(LEADER, queue_state_.mode); auto now = MonoTime::Now(); int remaining_voters = 0; @@ -418,24 +419,26 @@ bool PeerMessageQueue::SafeToEvictUnlocked(const string& evict_uuid) const { remaining_voters++; bool viable = true; +// Being alive, the local peer itself (the leader) is always a viable +// voter: the criteria below apply only to non-local peers. +if (uuid != local_peer_pb_.permanent_uuid()) { + // Only consider a peer to be a viable voter if... + // ...its last exchange was successful + viable &= peer->last_exchange_status == PeerStatus::OK; + + // ...the peer is up to date with the latest majority. + // + //This indicates that it's actively participating in majorities and likely to + //replicate a config change immediately when we propose it. + viable &= peer->last_received.index() >= queue_state_.majority_replicated_index; -// Only consider a peer to be a viable voter if... -// ...its last exchange was successful -viable &= peer->last_exchange_status == PeerStatus::OK; - -// ...the peer is up to date with the latest majority. -// -//This indicates that it's actively participating in majorities and likely to -//replicate a config change immediately when we propose it. -viable &= peer->last_received.index() >= queue_state_.majority_replicated_index; - -// ...we have communicated successfully with it recently. -// -//This handles the case where the tablet has had no recent writes and therefore -//even a replica that is down would have participated in the latest majority. -auto unreachable_time = now - peer->last_communication_time; -viable &= unreachable_time.ToMilliseconds() < FLAGS_consensus_rpc_timeout_ms; - + // ...we have communicated successfully with it recently. + // + //This handles the case where the tablet has had no recent writes and therefore + //even a replica that is down would have participated in the latest majority. + auto unreachable_time = now - peer->last_communication_time; + viable &= unreachable_time.ToMilliseconds() < FLAGS_consensus_rpc_timeout_ms; +} if (viable) { remaining_viable_voters++; } http://git-wip-us.apache.org/repos/asf/kudu/blob/dc497fec/src/kudu/integration-tests/tablet_replacement-itest.cc -- diff --git a/src/kudu/integration-tests/tablet_replacement-itest.cc b/src/kudu/integration-tests/tablet_replacement-itest.cc index eb724f4..c0665b4 100644 --- a/src/kudu/integration-tests/tablet_replacement-ite
kudu git commit: KUDU-2229. consensus: Leader should not start FD
Repository: kudu Updated Branches: refs/heads/master dc497fec2 -> 1c9e424c2 KUDU-2229. consensus: Leader should not start FD This patch fixes a log spam issue caused by commit 28a671365d6d38da966481daf937b3776e3d4852 that erroneously enables the failure detector on the leader when a configuration takes place. That bug had no consensus safety implications, but it did generate a large volume of confusing log messages in a neverending loop. This patch also includes some minor updates to InternalMiniCluster for convenience purposes. Change-Id: Ie2ec9c5499e8e4c1659333bd53dd2d7f6dae81f5 Reviewed-on: http://gerrit.cloudera.org:8080/8711 Reviewed-by: Alexey Serbin Tested-by: Kudu Jenkins Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/1c9e424c Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/1c9e424c Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/1c9e424c Branch: refs/heads/master Commit: 1c9e424c251d6b47fa8a161a94358634fbd13549 Parents: dc497fe Author: Mike Percy Authored: Thu Nov 30 19:40:23 2017 -0800 Committer: Alexey Serbin Committed: Fri Dec 1 23:15:12 2017 + -- src/kudu/consensus/raft_consensus.cc| 21 ++- src/kudu/consensus/raft_consensus.h | 19 ++- src/kudu/integration-tests/CMakeLists.txt | 3 +- ...raft_consensus_failure_detector-imc-itest.cc | 144 +++ src/kudu/mini-cluster/external_mini_cluster.h | 4 +- src/kudu/mini-cluster/internal_mini_cluster.cc | 18 +++ src/kudu/mini-cluster/internal_mini_cluster.h | 5 + src/kudu/mini-cluster/mini_cluster.h| 4 + src/kudu/rpc/periodic.cc| 5 + src/kudu/rpc/periodic.h | 5 +- 10 files changed, 206 insertions(+), 22 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/1c9e424c/src/kudu/consensus/raft_consensus.cc -- diff --git a/src/kudu/consensus/raft_consensus.cc b/src/kudu/consensus/raft_consensus.cc index 291c146..cb3118e 100644 --- a/src/kudu/consensus/raft_consensus.cc +++ b/src/kudu/consensus/raft_consensus.cc @@ -599,7 +599,7 @@ Status RaftConsensus::BecomeReplicaUnlocked(boost::optional fd_delta) // Enable/disable leader failure detection if becoming VOTER/NON_VOTER replica // correspondingly. - ToggleFailureDetector(std::move(fd_delta)); + UpdateFailureDetectorState(std::move(fd_delta)); // Now that we're a replica, we can allow voting for other nodes. withhold_votes_until_ = MonoTime::Min(); @@ -2610,7 +2610,7 @@ void RaftConsensus::CompleteConfigChangeRoundUnlocked(ConsensusRound* round, con // Disable leader failure detection if transitioning from VOTER to // NON_VOTER and vice versa. - ToggleFailureDetector(); + UpdateFailureDetectorState(); } else { LOG_WITH_PREFIX_UNLOCKED(INFO) << "Skipping abort of non-pending config change with OpId " @@ -2670,14 +2670,15 @@ void RaftConsensus::DisableFailureDetector() { } } -void RaftConsensus::ToggleFailureDetector(boost::optional delta) { - if (IsRaftConfigVoter(peer_uuid(), cmeta_->ActiveConfig())) { -// A non-leader voter replica should run failure detector. +void RaftConsensus::UpdateFailureDetectorState(boost::optional delta) { + const auto& uuid = peer_uuid(); + if (uuid != cmeta_->leader_uuid() && + cmeta_->IsVoterInConfig(uuid, ACTIVE_CONFIG)) { +// A voter that is not the leader should run the failure detector. EnableFailureDetector(std::move(delta)); } else { -// A non-voter should not start leader elections. The leader failure -// detector should be re-enabled once the non-voter replica is promoted -// to voter replica. +// Otherwise, the local peer should not start leader elections +// (e.g. if it is the leader, a non-voter, a non-participant, etc). DisableFailureDetector(); } } @@ -2805,9 +2806,7 @@ Status RaftConsensus::SetPendingConfigUnlocked(const RaftConfigPB& new_config) { } cmeta_->set_pending_config(new_config); - // Disable leader failure detection if transitioning from VOTER to NON_VOTER - // and vice versa. - ToggleFailureDetector(); + UpdateFailureDetectorState(); return Status::OK(); } http://git-wip-us.apache.org/repos/asf/kudu/blob/1c9e424c/src/kudu/consensus/raft_consensus.h -- diff --git a/src/kudu/consensus/raft_consensus.h b/src/kudu/consensus/raft_consensus.h index 2c438d1..5ace791 100644 --- a/src/kudu/consensus/raft_consensus.h +++ b/src/kudu/consensus/raft_consensus.h @@ -158,6 +158,11 @@ class RaftConsensus : public std::enable_shared_from_this, // Retur
kudu git commit: [client] expose non-voter replicas for kudu CLI tool
Repository: kudu Updated Branches: refs/heads/master 1c9e424c2 -> d9754746b [client] expose non-voter replicas for kudu CLI tool Added private interface to expose non-voter replica in the kudu CLI tools. Also, use that functionality to list table's replicas. Updated output of ksck to include information on non-voters replicas: * renamed column 'Voters' into 'Replicas' * a non-voter is displayed with '~' extra suffix, e.g. in the example below two non-voter replicas, B and C, are in the config along with leader replica A: Config source | Replicas | Current term | Config index | Committed? ---+--+--+--+--- master| A* B~ C~ | | | Yes A | A* B~ C~ | 4| 5| Yes B | A* B~ C~ | 4| 5| Yes C | A* B~ C~ | 4| 5| Yes I also updated the format of output to make it more readable. An example of new output for 'kudu table list --list_tablets': loadgen_auto_1e5f57a73dd34fa7bde7465219bd1866 T 69e22ffecb5b459db8a7fbf2f751a76f L 5acf108a9d364178b1aa672e52214ab8 127.0.0.1:9872 N f2e726948d3143e184097cd953dd2230 127.0.0.1:9878 N 3ae7fdf107cf424dbf9c5e33c33eafd0 127.0.0.1:9876 T 662bae40e4ec46588e90670bec16b2b5 L 5acf108a9d364178b1aa672e52214ab8 127.0.0.1:9872 T 5ba95da9b2f9455db9d690bdd35ae5cb L df8296cf97df42eaacd78e794c028c79 127.0.0.1:9874 T 528ae88cce1e4415a0d075dead5c983c L 20e66417713446039c22987fec13b800 127.0.0.1:9870 T 157ea108357e469e8fee001aec521d7e L 5acf108a9d364178b1aa672e52214ab8 127.0.0.1:9872 T 9c6a4d493fff4bda9bfe8f158cb08e69 L 5acf108a9d364178b1aa672e52214ab8 127.0.0.1:9872 T 36d03adab6ad494ea4e769ef92ef5577 L df8296cf97df42eaacd78e794c028c79 127.0.0.1:9874 T 7ce6d42984734614bd627d872e99c833 L 20e66417713446039c22987fec13b800 127.0.0.1:9870 Change-Id: I19317fdf5a2d5c8bb5f37b27bb83067a4df4ea20 Reviewed-on: http://gerrit.cloudera.org:8080/8586 Reviewed-by: Mike Percy Tested-by: Kudu Jenkins Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/d9754746 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/d9754746 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/d9754746 Branch: refs/heads/master Commit: d9754746b64098fdd4fe0e35d9344ba62d8c7a25 Parents: 1c9e424 Author: Alexey Serbin Authored: Thu Nov 16 18:03:51 2017 -0800 Committer: Alexey Serbin Committed: Fri Dec 1 23:28:22 2017 + -- src/kudu/client/CMakeLists.txt | 1 + src/kudu/client/client.cc | 10 +- src/kudu/client/client.h| 4 + src/kudu/client/client_builder-internal.cc | 6 +- src/kudu/client/client_builder-internal.h | 2 + src/kudu/client/meta_cache.cc | 41 -- src/kudu/client/meta_cache.h| 6 +- src/kudu/client/replica-internal.cc | 3 +- src/kudu/client/replica-internal.h | 3 +- src/kudu/client/replica_controller-internal.cc | 40 ++ src/kudu/client/replica_controller-internal.h | 57 src/kudu/client/scan_token-internal.cc | 3 +- .../raft_consensus_nonvoter-itest.cc| 86 src/kudu/tools/ksck-test.cc | 30 ++-- src/kudu/tools/ksck.cc | 140 --- src/kudu/tools/ksck.h | 51 --- src/kudu/tools/ksck_remote.cc | 9 +- src/kudu/tools/kudu-admin-test.cc | 2 +- src/kudu/tools/tool_action_table.cc | 83 ++- 19 files changed, 428 insertions(+), 149 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/d9754746/src/kudu/client/CMakeLists.txt -- diff --git a/src/kudu/client/CMakeLists.txt b/src/kudu/client/CMakeLists.txt index a78d300..92a1c7e 100644 --- a/src/kudu/client/CMakeLists.txt +++ b/src/kudu/client/CMakeLists.txt @@ -44,6 +44,7 @@ set(CLIENT_SRCS scan_token-internal.cc scanner-internal.cc replica-internal.cc + replica_controller-internal.cc resource_metrics.cc schema.cc session-internal.cc http://git-wip-us.apache.org/repos/asf/kudu/blob/d9754746/src/kudu/client/client.cc -- diff --git a/src/kudu/client/client.cc b/src/kudu/client/client.cc index aed362d..93431b7 100644 --- a/src/kudu/client/client.cc +++ b/src/kudu/client/client.cc @@ -324,7 +324,7 @@ Status KuduClientBuilder::Build(shared_ptr* client) { RETURN_NOT_OK(ImportAuthnCred
kudu git commit: KUDU-1097: more robust criteria for replica eviction
Repository: kudu Updated Branches: refs/heads/master ff535b521 -> 2a6de8ea4 KUDU-1097: more robust criteria for replica eviction Updated the replica eviction criteria to address scenarios where newly added replicas fail (before or after promotion) one after another prior to eviction of the original failed voter replica. With this patch, the replacement process should not end up with placing failed replicas at every available tablet server, deadlocking the replacement process. Added corresponding unit tests to cover the updated behavior of the consensus::CanEvictReplica() utility function. Updated the replica eviction criteria for more robust handling of the 'replace' attribute. In this context, the case when the leader replica itself is marked with the 'replace' attribute is now handled properly. Added unit tests to cover appropriate cases. Also, enabled the AdminCliTest.TestMoveTablet_KUDU_1097 test scenario since now it should not be flaky anymore. As an additional touch, updated the FailedTabletCopy scenario of the RaftConsensusNonVoterITest to provide more comprehensive coverage. Change-Id: I2062c2963fface43062b3154bf5ffa61b4fa684f Reviewed-on: http://gerrit.cloudera.org:8080/8679 Tested-by: Alexey Serbin Reviewed-by: Mike Percy Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/2a6de8ea Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/2a6de8ea Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/2a6de8ea Branch: refs/heads/master Commit: 2a6de8ea46ae699bffb0435119fc62a15200eab7 Parents: ff535b5 Author: Alexey Serbin Authored: Wed Nov 29 02:15:06 2017 -0800 Committer: Alexey Serbin Committed: Tue Dec 12 19:20:25 2017 + -- src/kudu/consensus/quorum_util-test.cc | 682 +-- src/kudu/consensus/quorum_util.cc | 183 - .../raft_consensus_nonvoter-itest.cc| 66 +- src/kudu/master/catalog_manager.cc | 10 +- src/kudu/tools/kudu-admin-test.cc | 2 +- 5 files changed, 847 insertions(+), 96 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/2a6de8ea/src/kudu/consensus/quorum_util-test.cc -- diff --git a/src/kudu/consensus/quorum_util-test.cc b/src/kudu/consensus/quorum_util-test.cc index cd038fc..3f525c0 100644 --- a/src/kudu/consensus/quorum_util-test.cc +++ b/src/kudu/consensus/quorum_util-test.cc @@ -26,23 +26,46 @@ #include "kudu/common/common.pb.h" #include "kudu/consensus/metadata.pb.h" #include "kudu/consensus/quorum_util.h" +#include "kudu/gutil/strings/substitute.h" #include "kudu/util/status.h" #include "kudu/util/test_macros.h" using std::string; using std::unique_ptr; using std::vector; +using strings::Substitute; namespace kudu { namespace consensus { // Handy notation of membership types used by AddPeer(), etc. -constexpr auto N = RaftPeerPB::NON_VOTER; -constexpr auto U = RaftPeerPB::UNKNOWN_MEMBER_TYPE; -constexpr auto V = RaftPeerPB::VOTER; +constexpr auto N = RaftPeerPB::NON_VOTER; // NOLINT(readability-identifier-naming) +constexpr auto U = RaftPeerPB::UNKNOWN_MEMBER_TYPE; // NOLINT(readability-identifier-naming) +constexpr auto V = RaftPeerPB::VOTER; // NOLINT(readability-identifier-naming) + +// The various possible health statuses. +constexpr auto kHealthStatuses = { '?', '-', '+' }; typedef std::pair Attr; +static void SetOverallHealth(HealthReportPB* health_report, + char overall_health) { + switch (overall_health) { +case '+': + health_report->set_overall_health(HealthReportPB::HEALTHY); + break; +case '-': + health_report->set_overall_health(HealthReportPB::FAILED); + break; +case '?': + health_report->set_overall_health(HealthReportPB::UNKNOWN); + break; +default: + FAIL() << overall_health << ": unexpected replica health status"; + break; + } +} + // Add a consensus peer into the specified configuration. static void AddPeer(RaftConfigPB* config, const string& uuid, @@ -55,20 +78,7 @@ static void AddPeer(RaftConfigPB* config, peer->set_member_type(type); if (overall_health) { unique_ptr health_report(new HealthReportPB); -switch (*overall_health) { - case '+': -health_report->set_overall_health(HealthReportPB::HEALTHY); -break; - case '-': -health_report->set_overall_health(HealthReportPB::FAILED); -break; - case '?': -health_report->set_overall_he
kudu git commit: [itests] fix build
Repository: kudu Updated Branches: refs/heads/master bb2620299 -> dbc84a2d7 [itests] fix build This is a follow-up for 7c8fc625e72ec2bf6ef614df2ec25346e1a0f7b7, which became a bit outdated after bb2620299c45b3db90590236298a497a619c0bda was merged. This patch does not contain any functional changes. Change-Id: I466d941f8634e37b68dd85f29d39f1eb02b26173 Reviewed-on: http://gerrit.cloudera.org:8080/8844 Reviewed-by: Mike Percy Tested-by: Alexey Serbin Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/dbc84a2d Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/dbc84a2d Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/dbc84a2d Branch: refs/heads/master Commit: dbc84a2d7469e2de97fe9e35f5b5443e94f2a790 Parents: bb26202 Author: Alexey Serbin Authored: Thu Dec 14 10:55:14 2017 -0800 Committer: Alexey Serbin Committed: Thu Dec 14 19:01:37 2017 + -- .../integration-tests/raft_consensus_nonvoter-itest.cc | 11 +-- 1 file changed, 5 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/dbc84a2d/src/kudu/integration-tests/raft_consensus_nonvoter-itest.cc -- diff --git a/src/kudu/integration-tests/raft_consensus_nonvoter-itest.cc b/src/kudu/integration-tests/raft_consensus_nonvoter-itest.cc index 11fbbbe..022fa4f 100644 --- a/src/kudu/integration-tests/raft_consensus_nonvoter-itest.cc +++ b/src/kudu/integration-tests/raft_consensus_nonvoter-itest.cc @@ -1585,14 +1585,13 @@ TEST_F(RaftConsensusNonVoterITest, RestartClusterWithNonVoter) { } workload.StopAndJoin(); - vector servers_without_replica; - NO_FATALS(GetServersWithoutReplica(&servers_without_replica)); - ASSERT_EQ(4, servers_without_replica.size()); - - ExternalTabletServer* ts_with_replica = GetServerWithReplica(); + const auto server_uuids_with_replica = GetServersWithReplica(tablet_id_); + ASSERT_FALSE(server_uuids_with_replica.empty()); + const string& failed_replica_uuid = server_uuids_with_replica.front(); + ExternalTabletServer* ts_with_replica = + cluster_->tablet_server_by_uuid(failed_replica_uuid); ASSERT_NE(nullptr, ts_with_replica); ts_with_replica->Shutdown(); - const string& failed_replica_uuid = ts_with_replica->uuid(); bool has_leader = false; TabletLocationsPB tablet_locations;
kudu git commit: KUDU-2236 pt 2: fix flake in TestKuduClient
Repository: kudu Updated Branches: refs/heads/master 5356334ef -> a89c8f395 KUDU-2236 pt 2: fix flake in TestKuduClient The original patch d78b2727d1246069b2006ee652c3ff9a2005601c was too loose in what exceptions it would allow. A more elegant way to uphold the original intention of the test (i.e. to avoid log spew caused by an unexpected disconnection) is to verify that there are no messages complaining about unexpectedly-disconnected connections. Change-Id: I7aafdc0eca00e743048ecc099dcb3241ce7ac8ad Reviewed-on: http://gerrit.cloudera.org:8080/8908 Tested-by: Kudu Jenkins Reviewed-by: Alexey Serbin Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/a89c8f39 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/a89c8f39 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/a89c8f39 Branch: refs/heads/master Commit: a89c8f39544337c6ff36cdd28edec4fadac8427c Parents: 5356334 Author: Andrew Wong Authored: Thu Dec 21 16:55:52 2017 -0800 Committer: Alexey Serbin Committed: Sat Dec 23 02:00:02 2017 + -- .../src/test/java/org/apache/kudu/client/TestKuduClient.java | 8 +++- 1 file changed, 3 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/a89c8f39/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java -- diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java index 2e0ff7a..1081cc6 100644 --- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java +++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java @@ -836,12 +836,10 @@ public class TestKuduClient extends BaseKuduTest { localClient.exportAuthenticationCredentials(); } } -// Allow "connection disconnected" exceptions, which can come from threads -// that don't get synchronously joined by client.close(). +// Ensure there is no log spew due to an unexpected lost connection. String exception_text = cla.getAppendedText(); -assertTrue("Unexpected exception:\n" + exception_text, - !exception_text.contains("Exception") || - exception_text.contains("connection disconnected")); +assertFalse("Unexpected exception:\n" + exception_text, + exception_text.contains("lost connection to peer")); } @Test(timeout = 10)
kudu git commit: [rpc] micro clean-up on RpcContext
Repository: kudu Updated Branches: refs/heads/master ae62b8697 -> 07e4cbd4d [rpc] micro clean-up on RpcContext Reuse the code from RpcContext::RespondRpcFailure() in RpcContext::RespondFailure() implementation. This changelist does not contain any functional changes. Change-Id: I0ef93200a8491e9cdecd5ebf3f146d1f986977ef Reviewed-on: http://gerrit.cloudera.org:8080/8919 Tested-by: Kudu Jenkins Reviewed-by: Will Berkeley Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/07e4cbd4 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/07e4cbd4 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/07e4cbd4 Branch: refs/heads/master Commit: 07e4cbd4d1edb0d1ce9bd0e9c2dac7b8ce321539 Parents: ae62b86 Author: Alexey Serbin Authored: Fri Dec 22 17:55:29 2017 -0800 Committer: Alexey Serbin Committed: Thu Dec 28 00:03:21 2017 + -- src/kudu/rpc/rpc_context.cc | 13 + 1 file changed, 1 insertion(+), 12 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/07e4cbd4/src/kudu/rpc/rpc_context.cc -- diff --git a/src/kudu/rpc/rpc_context.cc b/src/kudu/rpc/rpc_context.cc index 9c3d154..7af981b 100644 --- a/src/kudu/rpc/rpc_context.cc +++ b/src/kudu/rpc/rpc_context.cc @@ -96,18 +96,7 @@ void RpcContext::RespondNoCache() { } void RpcContext::RespondFailure(const Status &status) { - if (AreResultsTracked()) { -result_tracker_->FailAndRespond(call_->header().request_id(), -ErrorStatusPB::ERROR_APPLICATION, status); - } else { -VLOG(4) << call_->remote_method().service_name() << ": Sending RPC failure response for " -<< call_->ToString() << ": " << status.ToString(); -TRACE_EVENT_ASYNC_END2("rpc_call", "RPC", this, - "status", status.ToString(), - "trace", trace()->DumpToString()); -call_->RespondFailure(ErrorStatusPB::ERROR_APPLICATION, status); -delete this; - } + return RespondRpcFailure(ErrorStatusPB::ERROR_APPLICATION, status); } void RpcContext::RespondRpcFailure(ErrorStatusPB_RpcErrorCodePB err, const Status& status) {
kudu git commit: [thirdparty] fix lib link in PREFIX_DIR
Repository: kudu Updated Branches: refs/heads/master cc236542e -> 9bdeddf33 [thirdparty] fix lib link in PREFIX_DIR Avoid creating the unintended 'lib' soft link in already soft-linked directory. Change-Id: I9ff2d57414eb048a97b5494bebab5826a9d83fdf Reviewed-on: http://gerrit.cloudera.org:8080/8943 Reviewed-by: Todd Lipcon Tested-by: Kudu Jenkins Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/9bdeddf3 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/9bdeddf3 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/9bdeddf3 Branch: refs/heads/master Commit: 9bdeddf3373369e1406972861682f2a61889040d Parents: cc23654 Author: Alexey Serbin Authored: Thu Jan 4 11:50:43 2018 -0800 Committer: Alexey Serbin Committed: Thu Jan 4 20:33:20 2018 + -- thirdparty/build-thirdparty.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/9bdeddf3/thirdparty/build-thirdparty.sh -- diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index e239984..c570f9e 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -125,7 +125,7 @@ for PREFIX_DIR in $PREFIX_COMMON $PREFIX_DEPS $PREFIX_DEPS_TSAN; do # On some systems, autotools installs libraries to lib64 rather than lib. Fix # this by setting up lib64 as a symlink to lib. We have to do this step first # to handle cases where one third-party library depends on another. -ln -sf "$PREFIX_DIR/lib" "$PREFIX_DIR/lib64" +ln -nsf "$PREFIX_DIR/lib" "$PREFIX_DIR/lib64" fi done
kudu git commit: [cmake_modules] one more OpenJDK location
Repository: kudu Updated Branches: refs/heads/master 9bdeddf33 -> bddbb34e2 [cmake_modules] one more OpenJDK location Added the location of java-7-openjdk at Ubuntu 14.04 LTS (amd64). Change-Id: Ia595e6ce03708066421b59ea074ac9f3acd276c4 Reviewed-on: http://gerrit.cloudera.org:8080/8952 Reviewed-by: Dan Burkert Tested-by: Alexey Serbin Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/bddbb34e Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/bddbb34e Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/bddbb34e Branch: refs/heads/master Commit: bddbb34e2e28fd4affae3d4d37d6509922ffb4fb Parents: 9bdeddf Author: Alexey Serbin Authored: Fri Jan 5 12:32:23 2018 -0800 Committer: Alexey Serbin Committed: Fri Jan 5 21:34:23 2018 + -- cmake_modules/FindJavaHome.cmake | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/bddbb34e/cmake_modules/FindJavaHome.cmake -- diff --git a/cmake_modules/FindJavaHome.cmake b/cmake_modules/FindJavaHome.cmake index 4cce3bf..0463cd8 100644 --- a/cmake_modules/FindJavaHome.cmake +++ b/cmake_modules/FindJavaHome.cmake @@ -49,6 +49,7 @@ set(JAVA_HOME_CANDIDATES # OpenJDK 7 Candidates /usr/lib/jvm/java-1.7.0-openjdk +/usr/lib/jvm/java-7-openjdk-amd64 /usr/lib/jvm/java-7-openjdk # Misc. Candidates
kudu git commit: [tests] update for 3-4-3 replica management scheme (3/3)
Repository: kudu Updated Branches: refs/heads/master 36995e260 -> 391e3255d [tests] update for 3-4-3 replica management scheme (3/3) Updated scenarios of the following tests to run with 3-4-3 replica management scheme: * DiskFailureITest * TabletCopyFailureITest * TabletCopyITest * TsTabletManagerITest This patch is a part of the work done in the context of KUDU-1097. Change-Id: Id68e29fcee4e6dbd166ada9f18dac2d8fe351a3b Reviewed-on: http://gerrit.cloudera.org:8080/8858 Reviewed-by: Mike Percy Tested-by: Kudu Jenkins Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/391e3255 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/391e3255 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/391e3255 Branch: refs/heads/master Commit: 391e3255d6f0ae59f375e5532cea985384f636f8 Parents: 36995e2 Author: Alexey Serbin Authored: Fri Dec 15 15:09:24 2017 -0800 Committer: Alexey Serbin Committed: Sat Jan 6 03:22:24 2018 + -- .../integration-tests/disk_failure-itest.cc | 61 + src/kudu/integration-tests/tablet_copy-itest.cc | 18 +++- .../ts_tablet_manager-itest.cc | 90 +--- 3 files changed, 120 insertions(+), 49 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/391e3255/src/kudu/integration-tests/disk_failure-itest.cc -- diff --git a/src/kudu/integration-tests/disk_failure-itest.cc b/src/kudu/integration-tests/disk_failure-itest.cc index 3657846..8d79884 100644 --- a/src/kudu/integration-tests/disk_failure-itest.cc +++ b/src/kudu/integration-tests/disk_failure-itest.cc @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -29,6 +30,7 @@ #include "kudu/integration-tests/external_mini_cluster-itest-base.h" #include "kudu/integration-tests/test_workload.h" #include "kudu/mini-cluster/external_mini_cluster.h" +#include "kudu/tserver/tserver.pb.h" #include "kudu/util/metrics.h" #include "kudu/util/monotime.h" #include "kudu/util/path_util.h" @@ -38,19 +40,19 @@ METRIC_DECLARE_gauge_uint64(data_dirs_failed); -namespace kudu { - -using cluster::ExternalMiniClusterOptions; -using cluster::ExternalTabletServer; -using fs::BlockManager; +using kudu::cluster::ExternalMiniClusterOptions; +using kudu::cluster::ExternalTabletServer; +using kudu::fs::BlockManager; using std::string; using std::vector; using strings::Substitute; +namespace kudu { + const MonoDelta kAgreementTimeout = MonoDelta::FromSeconds(30); class DiskFailureITest : public ExternalMiniClusterITestBase, - public ::testing::WithParamInterface { + public ::testing::WithParamInterface> { public: // Waits for 'ext_tserver' to experience 'target_failed_disks' disk failures. @@ -70,14 +72,24 @@ class DiskFailureITest : public ExternalMiniClusterITestBase, // shut down and restarted. Errors are injected to one of the directories while // it is shut down. TEST_P(DiskFailureITest, TestFailDuringServerStartup) { - // Set up a cluster with three servers with five disks each. + const string block_manager_type = std::get<0>(GetParam()); + const bool is_3_4_3_mode = std::get<1>(GetParam()); + // Set up a cluster with five disks at each tablet server. In case of 3-4-3 + // replication scheme one more tablet server is needed to put the replacement + // non-voter replica there. + const auto kNumTabletServers = is_3_4_3_mode ? 4 : 3; + const auto kNumTablets = 5; + const auto kNumRows = 100; + ExternalMiniClusterOptions opts; - opts.num_tablet_servers = 3; + opts.num_tablet_servers = kNumTabletServers; opts.num_data_dirs = 5; - opts.block_manager_type = GetParam(); + opts.block_manager_type = block_manager_type; + opts.extra_master_flags.push_back( + Substitute("--raft_prepare_replacement_before_eviction=$0", is_3_4_3_mode)); + opts.extra_tserver_flags.push_back( + Substitute("--raft_prepare_replacement_before_eviction=$0", is_3_4_3_mode)); NO_FATALS(StartClusterWithOpts(opts)); - const int kNumTablets = 5; - const int kNumRows = 100; // Write some data to a tablet. This will spread blocks across all // directories. @@ -90,14 +102,25 @@ TEST_P(DiskFailureITest, TestFailDuringServerStartup) { }); write_workload.StopAndJoin(); - // Ensure the tablets get to a running state. - ExternalTabletServer* ts = cluster_->tablet_server(0); - ASSERT_OK(cluster_->WaitForTabletsRunning(ts, kNumTablets, kAgreementTimeout)); + // Arbitrarily select one tablet server which hosts a replica of the tablet. + ExternalTabletServer* ts = nu
kudu git commit: KUDU-1097: turn on 3-4-3 replica management scheme
Repository: kudu Updated Branches: refs/heads/master 72c83c571 -> 1ac10d536 KUDU-1097: turn on 3-4-3 replica management scheme This patch enables 3-4-3 replica management scheme for tablets: now the 3-4-3 scheme is used by default. To switch back to 3-2-3 scheme, set --raft_prepare_replacement_before_eviction=false for both masters and tablet servers. Change-Id: I781d02fe455140ed3a4693438eece74f07700946 Reviewed-on: http://gerrit.cloudera.org:8080/8862 Tested-by: Kudu Jenkins Reviewed-by: Mike Percy Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/1ac10d53 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/1ac10d53 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/1ac10d53 Branch: refs/heads/master Commit: 1ac10d5364db507e5052f5dc13c19d32f62dc4f6 Parents: 72c83c5 Author: Alexey Serbin Authored: Sun Dec 17 22:06:16 2017 -0800 Committer: Alexey Serbin Committed: Tue Jan 9 01:13:07 2018 + -- src/kudu/consensus/raft_consensus.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/1ac10d53/src/kudu/consensus/raft_consensus.cc -- diff --git a/src/kudu/consensus/raft_consensus.cc b/src/kudu/consensus/raft_consensus.cc index 0892f9f..09f66d9 100644 --- a/src/kudu/consensus/raft_consensus.cc +++ b/src/kudu/consensus/raft_consensus.cc @@ -123,7 +123,7 @@ TAG_FLAG(raft_enable_tombstoned_voting, experimental); TAG_FLAG(raft_enable_tombstoned_voting, runtime); // Enable improved re-replication (KUDU-1097). -DEFINE_bool(raft_prepare_replacement_before_eviction, false, +DEFINE_bool(raft_prepare_replacement_before_eviction, true, "When enabled, failed replicas will only be evicted after a " "replacement has been prepared for them."); TAG_FLAG(raft_prepare_replacement_before_eviction, advanced);
[2/3] kudu git commit: [mvcc] Fix watermark advancement in the absence of committed transactions
[mvcc] Fix watermark advancement in the absence of committed transactions MvccSnapshot's 'none_committed_at_or_after_' (ncaoa) watermark indicates the point after which no transactions are committed. This is used to cull redo deltas that cannot contain any committed transactions under a snapshot, thus reducing work when applying deltas. By definition, this watermark is never supposed to be lower than the 'all_committed_before_' watermark. If this invariant is broken delta selection might be wrong by skipping whole delta files. Normally the ncaoa watermark is advanced when transactions are marked as committed, in which case this is done correctly. However there is problem when the 'all_committed_before_' watermark is advanced and there are no committed or in flight transactions. In this case the ncaoa watermark might be left behind, manifesting as incorrect delta application/skipping or even crashes. This patch includes a fix and a very directed test that makes sure it is correct. A follow up patch includes a new test in fuzz-itest that would trigger a crash without the fix. Change-Id: Icf94ea289204adf2eed797c3623cdea1b11b09db Reviewed-on: http://gerrit.cloudera.org:8080/8964 Tested-by: David Ribeiro Alves Reviewed-by: Todd Lipcon Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/ad3b6a63 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/ad3b6a63 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/ad3b6a63 Branch: refs/heads/master Commit: ad3b6a63a41c5b3d2f9907654f505bf76dba14ab Parents: 8b873a1 Author: David Alves Authored: Mon Jan 8 12:36:35 2018 -0800 Committer: Todd Lipcon Committed: Tue Jan 9 20:31:02 2018 + -- src/kudu/tablet/mvcc-test.cc | 42 +++ src/kudu/tablet/mvcc.cc | 7 +++ src/kudu/tablet/mvcc.h | 2 +- 3 files changed, 50 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/ad3b6a63/src/kudu/tablet/mvcc-test.cc -- diff --git a/src/kudu/tablet/mvcc-test.cc b/src/kudu/tablet/mvcc-test.cc index 9bcea5e..8458922 100644 --- a/src/kudu/tablet/mvcc-test.cc +++ b/src/kudu/tablet/mvcc-test.cc @@ -740,5 +740,47 @@ TEST_F(MvccTest, TestWaitUntilCleanDeadline) { ASSERT_TRUE(s.IsTimedOut()) << s.ToString(); } +// Test for a bug related to the initialization of the MvccManager without +// any pending transactions, i.e. when there are only calls to AdvanceSafeTime(). +// Prior to the fix we would advance safe/clean time but not the +// 'none_committed_at_or_after_' watermark, meaning the latter would become lower +// than safe/clean time. This had the effect on compaction of culling delta files +// even though they shouldn't be culled. +// This test makes sure that watermarks are advanced correctly and that delta +// files are culled correctly. +TEST_F(MvccTest, TestCorrectInitWithNoTxns) { + MvccManager mgr; + + MvccSnapshot snap; + mgr.TakeSnapshot(&snap); + EXPECT_EQ(snap.all_committed_before_, Timestamp::kInitialTimestamp); + EXPECT_EQ(snap.none_committed_at_or_after_, Timestamp::kInitialTimestamp); + EXPECT_EQ(snap.committed_timestamps_.size(), 0); + + // Read the clock a few times to advance the timestamp + for (int i = 0; i < 10; i++) clock_->Now(); + + // Advance the safe timestamp. + Timestamp new_safe_time = clock_->Now(); + mgr.AdjustSafeTime(new_safe_time); + + // Test that the snapshot reports that a timestamp lower than the safe time + // may have be committed transaction before that timestamp. + // Conversely, test that the snapshot reports that a timestamp greater + // than the safe time (and thus greater than none_committed_at_or_after_' + // as there are no other transactions committed) cannot have any committed + // transactions after that timestamp. + MvccSnapshot snap2; + mgr.TakeSnapshot(&snap2); + Timestamp before_safe_time(new_safe_time.value() - 1); + Timestamp after_safe_time(new_safe_time.value() + 1); + EXPECT_TRUE(snap2.MayHaveCommittedTransactionsAtOrAfter(before_safe_time)); + EXPECT_FALSE(snap2.MayHaveCommittedTransactionsAtOrAfter(after_safe_time)); + + EXPECT_EQ(snap2.all_committed_before_, new_safe_time); + EXPECT_EQ(snap2.none_committed_at_or_after_, new_safe_time); + EXPECT_EQ(snap2.committed_timestamps_.size(), 0); +} + } // namespace tablet } // namespace kudu http://git-wip-us.apache.org/repos/asf/kudu/blob/ad3b6a63/src/kudu/tablet/mvcc.cc -- diff --git a/src/kudu/tablet/mvcc.cc b/src/kudu/tablet/mvcc.cc index 87786b9..62d3d43 100644 --- a/src/kudu/tablet/mvcc.cc +++ b/src/kudu/tablet/mvcc.cc @@ -238,6 +238,13 @@ void MvccManager::AdjustCleanTime() { // Filter out any committed timestamp
[3/3] kudu git commit: [tests] update tests for replication scheme consistency
[tests] update tests for replication scheme consistency Fixed and updated some scenarios once the re-replication scheme consistency is enforced: * MasterTest.TestRegisterAndHeartbeat scenario * MasterReplicationTest.TestHeartbeatAcceptedByAnyMaster * MasterCertAuthorityTest.RefuseToSignInvalidCSR * MasterCertAuthorityTest.MasterLeaderSignsCSR This is a follow-up for 1769eed53ee2c21a88a766cb72bf8c9ae622099d. Change-Id: I8f5bbb581abeb3a781b522dff90711b2ffc988df Reviewed-on: http://gerrit.cloudera.org:8080/8979 Reviewed-by: Adar Dembo Tested-by: Alexey Serbin Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/746d0a5c Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/746d0a5c Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/746d0a5c Branch: refs/heads/master Commit: 746d0a5cd53c8252af0c0b6ad294cdb6b39a0319 Parents: ad3b6a6 Author: Alexey Serbin Authored: Tue Jan 9 11:11:43 2018 -0800 Committer: Alexey Serbin Committed: Tue Jan 9 21:17:26 2018 + -- .../master_cert_authority-itest.cc | 32 +++ .../master_replication-itest.cc | 33 +++ src/kudu/master/master-test.cc | 59 +++- 3 files changed, 101 insertions(+), 23 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/746d0a5c/src/kudu/integration-tests/master_cert_authority-itest.cc -- diff --git a/src/kudu/integration-tests/master_cert_authority-itest.cc b/src/kudu/integration-tests/master_cert_authority-itest.cc index 109f3f0..806b870 100644 --- a/src/kudu/integration-tests/master_cert_authority-itest.cc +++ b/src/kudu/integration-tests/master_cert_authority-itest.cc @@ -20,12 +20,13 @@ #include #include +#include #include #include "kudu/common/common.pb.h" #include "kudu/common/wire_protocol.pb.h" +#include "kudu/consensus/replica_management.pb.h" #include "kudu/gutil/gscoped_ptr.h" -#include "kudu/gutil/port.h" #include "kudu/master/master.h" #include "kudu/master/master.pb.h" #include "kudu/master/master.proxy.h" @@ -45,19 +46,21 @@ #include "kudu/util/test_util.h" #include "kudu/util/user.h" +DECLARE_bool(raft_prepare_replacement_before_eviction); + +using kudu::cluster::InternalMiniCluster; +using kudu::cluster::InternalMiniClusterOptions; +using kudu::consensus::ReplicaManagementInfoPB; +using kudu::security::ca::CertRequestGenerator; +using kudu::security::Cert; +using kudu::security::CertSignRequest; +using kudu::security::DataFormat; +using kudu::security::PrivateKey; using std::string; using std::shared_ptr; -namespace kudu { - -using cluster::InternalMiniCluster; -using cluster::InternalMiniClusterOptions; -using security::ca::CertRequestGenerator; -using security::Cert; -using security::CertSignRequest; -using security::DataFormat; -using security::PrivateKey; +namespace kudu { namespace master { class MasterCertAuthorityTest : public KuduTest { @@ -71,7 +74,7 @@ class MasterCertAuthorityTest : public KuduTest { opts_.num_masters = num_masters_ = opts_.master_rpc_ports.size(); } - virtual void SetUp() OVERRIDE { + void SetUp() override { KuduTest::SetUp(); cluster_.reset(new InternalMiniCluster(env_, opts_)); ASSERT_OK(cluster_->Start()); @@ -117,6 +120,12 @@ class MasterCertAuthorityTest : public KuduTest { pb->set_host("localhost"); pb->set_port(2000); +// Information on the replica management scheme. +ReplicaManagementInfoPB rmi; +rmi.set_replacement_scheme(FLAGS_raft_prepare_replacement_before_eviction +? ReplicaManagementInfoPB::PREPARE_REPLACEMENT_BEFORE_EVICTION +: ReplicaManagementInfoPB::EVICT_FIRST); + for (int i = 0; i < cluster_->num_masters(); ++i) { TSHeartbeatRequestPB req; TSHeartbeatResponsePB resp; @@ -124,6 +133,7 @@ class MasterCertAuthorityTest : public KuduTest { req.mutable_common()->CopyFrom(common); req.mutable_registration()->CopyFrom(fake_reg); + req.mutable_replica_management_info()->CopyFrom(rmi); MiniMaster* m = cluster_->mini_master(i); if (!m->is_started()) { http://git-wip-us.apache.org/repos/asf/kudu/blob/746d0a5c/src/kudu/integration-tests/master_replication-itest.cc -- diff --git a/src/kudu/integration-tests/master_replication-itest.cc b/src/kudu/integration-tests/master_replication-itest.cc index f75bca5..a521ee0 100644 --- a/src/kudu/integration-tests/master_replication-itest.cc +++ b/src/kudu/integration-tests/master_replication-itest.cc @@ -21,6 +21,7 @@ #includ
[1/3] kudu git commit: Add some DVLOG statements to help in debugging compaction issues
Repository: kudu Updated Branches: refs/heads/master f932547b2 -> 746d0a5cd Add some DVLOG statements to help in debugging compaction issues This adds some DVLOG statemenst that were crucial in debugging KUDU-2233, particularly aroung merge compactions, clock advancement and log gc. Change-Id: I958045a75feceee19c09961de106bbacd343c739 Reviewed-on: http://gerrit.cloudera.org:8080/8886 Tested-by: Kudu Jenkins Reviewed-by: Todd Lipcon Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/8b873a12 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/8b873a12 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/8b873a12 Branch: refs/heads/master Commit: 8b873a1257d11510cdbf80fe4c39ae26398e364a Parents: f932547 Author: David Alves Authored: Tue Dec 19 13:30:35 2017 -0800 Committer: Todd Lipcon Committed: Tue Jan 9 20:30:55 2018 + -- src/kudu/tablet/compaction.cc | 12 src/kudu/tablet/mvcc.cc | 4 +++- src/kudu/tablet/tablet_replica.cc | 7 ++- 3 files changed, 21 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/8b873a12/src/kudu/tablet/compaction.cc -- diff --git a/src/kudu/tablet/compaction.cc b/src/kudu/tablet/compaction.cc index 89c2a3f..62b6302 100644 --- a/src/kudu/tablet/compaction.cc +++ b/src/kudu/tablet/compaction.cc @@ -477,17 +477,23 @@ class MergeCompactionInput : public CompactionInput { if (smallest_idx < 0) { smallest_idx = i; smallest = state->next(); + DVLOG(4) << "Set (initial) smallest from state: " << i << " smallest: " + << CompactionInputRowToString(*smallest); continue; } int row_comp = schema_->Compare(state->next()->row, smallest->row); if (row_comp < 0) { smallest_idx = i; smallest = state->next(); + DVLOG(4) << "Set (by comp) smallest from state: " << i << " smallest: " + << CompactionInputRowToString(*smallest); continue; } // If we found two rows with the same key, we want to make the newer one point to the older // one, which must be a ghost. if (PREDICT_FALSE(row_comp == 0)) { + DVLOG(4) << "Duplicate row.\nLeft: " << CompactionInputRowToString(*state->next()) + << "\nRight: " << CompactionInputRowToString(*smallest); int mutation_comp = CompareDuplicatedRows(*state->next(), *smallest); CHECK_NE(mutation_comp, 0); if (mutation_comp > 0) { @@ -498,11 +504,15 @@ class MergeCompactionInput : public CompactionInput { states_[smallest_idx]->pop_front(); smallest_idx = i; smallest = state->next(); +DVLOG(4) << "Set smallest to right duplicate: " + << CompactionInputRowToString(*smallest); continue; } // .. otherwise copy and pop the other one. RETURN_NOT_OK(SetPreviousGhost(smallest, state->next(), true /* clone */, smallest->row.row_block()->arena())); + DVLOG(4) << "Updated left duplicate smallest: " + << CompactionInputRowToString(*smallest); states_[i]->pop_front(); continue; } @@ -510,6 +520,7 @@ class MergeCompactionInput : public CompactionInput { DCHECK_GE(smallest_idx, 0); states_[smallest_idx]->pop_front(); + DVLOG(4) << "Pushing smallest to block: " << CompactionInputRowToString(*smallest); block->push_back(*smallest); } @@ -1227,6 +1238,7 @@ Status ReupdateMissedDeltas(const string &tablet_name, // But was this row GCed at flush time? if (decoder.is_delete() && history_gc_opts.IsAncientHistory(mut->timestamp())) { +DVLOG(3) << "Marking for garbage collection row: " << schema->DebugRow(row.row); is_garbage_collected = true; } continue; http://git-wip-us.apache.org/repos/asf/kudu/blob/8b873a12/src/kudu/tablet/mvcc.cc -- diff --git a/src/kudu/tablet/mvcc.cc b/src/kudu/tablet/mvcc.cc index eb2fcf5..87786b9 100644 --- a/src/kudu/tablet/mvcc.cc +++ b/src/kudu/tablet/mvcc.cc @@ -170,10 +170,10 @@ void MvccManager::AdvanceEarliestInFlightTimestamp() { void MvccManager::AdjustSafeTime(Timestamp safe_time) { std::lock_guard l(lock_); - // No more transactions will start with a ts that is lower than or equal // to 'safe_time', so we adjust the snapshot accordingly. if (PREDICT_TRUE(safe_time_ <= safe_time)) { +DVLOG(4) << "Adjusting safe time to: " << safe_time;
kudu git commit: [raft_consensus-itest] fix for Test_KUDU_1735
Repository: kudu Updated Branches: refs/heads/master 746d0a5cd -> 8bcc80eec [raft_consensus-itest] fix for Test_KUDU_1735 Quick fir for Test_KUDU_1735 after enabling 3-4-3 replication mode by default. For some reason, this test slipped under radar and I discovered that it fails only now (the other relevant tests were already updated). I tried to make this test work for 3-4-3, but it was not successful yet. So, to fix the build, let's push this easy fix first. This is a follow-up for 1ac10d5364db507e5052f5dc13c19d32f62dc4f6. Change-Id: I8d96b66d528f90f7611ef32726401e3208b0e47d Reviewed-on: http://gerrit.cloudera.org:8080/8987 Reviewed-by: Adar Dembo Tested-by: Alexey Serbin Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/8bcc80ee Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/8bcc80ee Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/8bcc80ee Branch: refs/heads/master Commit: 8bcc80eec075321faef26b2a0ccac12ac9d7 Parents: 746d0a5 Author: Alexey Serbin Authored: Tue Jan 9 14:55:18 2018 -0800 Committer: Alexey Serbin Committed: Tue Jan 9 23:19:05 2018 + -- src/kudu/integration-tests/raft_consensus-itest.cc | 6 ++ 1 file changed, 6 insertions(+) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/8bcc80ee/src/kudu/integration-tests/raft_consensus-itest.cc -- diff --git a/src/kudu/integration-tests/raft_consensus-itest.cc b/src/kudu/integration-tests/raft_consensus-itest.cc index fdda2da..281ef57 100644 --- a/src/kudu/integration-tests/raft_consensus-itest.cc +++ b/src/kudu/integration-tests/raft_consensus-itest.cc @@ -2402,10 +2402,16 @@ TEST_F(RaftConsensusITest, TestChangeConfigRejectedUnlessNoopReplicated) { TEST_F(RaftConsensusITest, Test_KUDU_1735) { const MonoDelta kTimeout = MonoDelta::FromSeconds(10); const vector kTsFlags = { +// This scenario uses 'manual election' mode and assumes no leader change. "--enable_leader_failure_detection=false", +// This test is specific for the 3-2-3 replica management scheme. +"--raft_prepare_replacement_before_eviction=false", }; const vector kMasterFlags = { +// This scenario uses 'manual election' mode and assumes no leader change. "--catalog_manager_wait_for_new_tablets_to_elect_leader=false", +// This test is specific for the 3-2-3 replica management scheme. +"--raft_prepare_replacement_before_eviction=false", }; NO_FATALS(BuildAndStart(kTsFlags, kMasterFlags));
kudu git commit: [tls_socket-test] fix test to pass on Ubuntu 16.04
Repository: kudu Updated Branches: refs/heads/master 35fc7b7e5 -> 26fd153b0 [tls_socket-test] fix test to pass on Ubuntu 16.04 Prior to this fix, the TlsSocketTest.TestNonBlockingWritev test failed on Ubuntu 16.04 (kernel 4.4.0). After some investigation, it turned out that setting socket send buffer size for something lower than the MTU size (which is 64kB for localhost) triggered the delayed acknowledgement logic. Since the test passes big chunks of data (32MB on every iteration), with 40ms delay for almost every 16KB sent, the test eventually timed out. I verified that disabling the delayed TCP ack for TLS sockets fixed the issue. I think that the proper fix is to remove the custom setting for the socket send buffer: a 32MB chunk of data is big enough to not fit into the socket buffer of the default size. Also, it seems this issue is not going to bite us in real life since we don't set the size of the socket send buffer anywhere. NOTE: however, some mystery is left since the failing test would pass if running under strace. Change-Id: I94170d5bdbe17a5952a82faf74d8b868b42460aa Reviewed-on: http://gerrit.cloudera.org:8080/8996 Tested-by: Kudu Jenkins Reviewed-by: Todd Lipcon Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/26fd153b Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/26fd153b Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/26fd153b Branch: refs/heads/master Commit: 26fd153b055b10bee90275d4f4c9e5f7088d4e83 Parents: 35fc7b7 Author: Alexey Serbin Authored: Wed Jan 10 13:39:05 2018 -0800 Committer: Todd Lipcon Committed: Fri Jan 12 05:28:47 2018 + -- src/kudu/security/tls_socket-test.cc | 4 1 file changed, 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/26fd153b/src/kudu/security/tls_socket-test.cc -- diff --git a/src/kudu/security/tls_socket-test.cc b/src/kudu/security/tls_socket-test.cc index 214d2bf..4f332a5 100644 --- a/src/kudu/security/tls_socket-test.cc +++ b/src/kudu/security/tls_socket-test.cc @@ -20,7 +20,6 @@ #include #include #include -#include #include #include @@ -282,9 +281,6 @@ TEST_F(TlsSocketTest, TestNonBlockingWritev) { unique_ptr client_sock; NO_FATALS(ConnectClient(server.listen_addr(), &client_sock)); - int sndbuf = 16 * 1024; - CHECK_ERR(setsockopt(client_sock->GetFd(), SOL_SOCKET, SO_SNDBUF, &sndbuf, sizeof(sndbuf))); - unique_ptr buf(new uint8_t[kEchoChunkSize]); unique_ptr rbuf(new uint8_t[kEchoChunkSize]); RandomString(buf.get(), kEchoChunkSize, &rng);
[2/2] kudu git commit: [catalog_manager] more info if unable to replace a replica
[catalog_manager] more info if unable to replace a replica Output actionable warning message when the catalog manager is unable to find a spot for a replacement replica. Since the 3-4-3 replication scheme is now enabled by default, this might be useful in case if running a cluster with just 3 tablet servers when tables have replication factor of 3. Change-Id: Id5f562c6d1ff526daa785ea535e440598c03cd37 Reviewed-on: http://gerrit.cloudera.org:8080/9040 Reviewed-by: Mike Percy Tested-by: Kudu Jenkins Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/c40e0587 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/c40e0587 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/c40e0587 Branch: refs/heads/master Commit: c40e0587bf6a6aa55e5bd72dd2dd9356b1507f2e Parents: 5806b80 Author: Alexey Serbin Authored: Tue Jan 16 15:01:34 2018 -0800 Committer: Alexey Serbin Committed: Wed Jan 17 03:18:11 2018 + -- src/kudu/master/catalog_manager.cc | 38 ++--- src/kudu/master/ts_manager.cc | 4 ++-- src/kudu/master/ts_manager.h | 8 +++ 3 files changed, 41 insertions(+), 9 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/c40e0587/src/kudu/master/catalog_manager.cc -- diff --git a/src/kudu/master/catalog_manager.cc b/src/kudu/master/catalog_manager.cc index d6b369b..1248144 100644 --- a/src/kudu/master/catalog_manager.cc +++ b/src/kudu/master/catalog_manager.cc @@ -3133,11 +3133,43 @@ bool AsyncAddReplicaTask::SendRequest(int attempt) { InsertOrDie(&excluded, ts_desc); } } + auto replacement_replica = SelectReplica(ts_descs, excluded, rng_); if (PREDICT_FALSE(!replacement_replica)) { -KLOG_EVERY_N(WARNING, 100) << LogPrefix() - << "No candidate replacement replica found for tablet " - << tablet_->ToString(); +auto msg = Substitute("no candidate replacement replica found for tablet $0", + tablet_->ToString()); +// Check whether it's a situation when a replacement replica cannot be found +// due to an inconsistency in cluster configuration. If the tablet has the +// replication factor of N, and the cluster is configured to use N->(N+1)->N +// replication scheme (see --raft_prepare_replacement_before_eviction flag), +// at least N+1 tablet servers should be registered to find a place +// for a replacement replica. +TSDescriptorVector all_descriptors; +master_->ts_manager()->GetAllDescriptors(&all_descriptors); +const auto num_tservers_registered = all_descriptors.size(); + +auto replication_factor = 0; +{ + TableMetadataLock l(tablet_->table().get(), LockMode::READ); + replication_factor = tablet_->table()->metadata().state().pb.num_replicas(); +} +DCHECK_GE(replication_factor, 0); +const auto num_tservers_needed = +FLAGS_raft_prepare_replacement_before_eviction ? replication_factor + 1 + : replication_factor; +if (num_tservers_registered < num_tservers_needed) { + msg += Substitute( + "; the total number of registered tablet servers ($0) does not allow " + "for replacement of the failed replica: at least $1 tablet servers " + "are required", num_tservers_registered, num_tservers_needed); + if (FLAGS_raft_prepare_replacement_before_eviction && + num_tservers_registered == replication_factor) { +msg += + "; consider either adding an additional tablet server or running " + "the cluster with --raft_prepare_replacement_before_eviction=false"; + } +} +KLOG_EVERY_N_SECS(WARNING, 60) << LogPrefix() << msg; return false; } http://git-wip-us.apache.org/repos/asf/kudu/blob/c40e0587/src/kudu/master/ts_manager.cc -- diff --git a/src/kudu/master/ts_manager.cc b/src/kudu/master/ts_manager.cc index 65faad8..8b1aa1c 100644 --- a/src/kudu/master/ts_manager.cc +++ b/src/kudu/master/ts_manager.cc @@ -92,13 +92,13 @@ Status TSManager::RegisterTS(const NodeInstancePB& instance, return Status::OK(); } -void TSManager::GetAllDescriptors(vector > *descs) const { +void TSManager::GetAllDescriptors(vector> *descs) const { descs->clear(); shared_lock l(lock_); AppendValuesFromMap(servers_by_id_, descs); } -void TSManager::GetAllLiveDescriptors(vector > *descs) const { +void TSManager::GetAllLiveDes
[1/2] kudu git commit: mini-cluster: rename data_root to cluster_root
Repository: kudu Updated Branches: refs/heads/master b0d27d3f5 -> c40e0587b mini-cluster: rename data_root to cluster_root I found that the name data_root for the root directory in which to place the cluster's data and logs was a confusing name because: 1. we use "data" here loosely to describe any old bytes, encompassing not only data blocks, but WALs and logs as well, and 2. the concept of a "data root" already exists to mean a user-specified location to place data blocks (i.e. an entry in fs_data_dirs). As such, this patch renames External- and InternalMiniClusterOptions' "data_root" members to "cluster_root". Change-Id: Id8ccc9e232f47a684d6b9226fd84639c4c94d0c3 Reviewed-on: http://gerrit.cloudera.org:8080/9033 Reviewed-by: Adar Dembo Tested-by: Andrew Wong Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/5806b807 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/5806b807 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/5806b807 Branch: refs/heads/master Commit: 5806b807a712116e79e8a67dc4dcc55d51ea6b2a Parents: b0d27d3 Author: Andrew Wong Authored: Tue Jan 16 08:44:24 2018 -0800 Committer: Andrew Wong Committed: Wed Jan 17 01:06:54 2018 + -- src/kudu/benchmarks/tpch/tpch1.cc | 2 +- src/kudu/benchmarks/tpch/tpch_real_world.cc| 2 +- src/kudu/integration-tests/ts_itest-base.cc| 4 ++-- src/kudu/mini-cluster/external_mini_cluster.cc | 22 ++--- src/kudu/mini-cluster/external_mini_cluster.h | 10 +- src/kudu/mini-cluster/internal_mini_cluster.cc | 14 ++--- src/kudu/mini-cluster/internal_mini_cluster.h | 4 ++-- src/kudu/tools/kudu-tool-test.cc | 4 ++-- src/kudu/tools/tool.proto | 2 +- src/kudu/tools/tool_action_test.cc | 20 +-- 10 files changed, 42 insertions(+), 42 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/5806b807/src/kudu/benchmarks/tpch/tpch1.cc -- diff --git a/src/kudu/benchmarks/tpch/tpch1.cc b/src/kudu/benchmarks/tpch/tpch1.cc index d8d4a11..2feb823 100644 --- a/src/kudu/benchmarks/tpch/tpch1.cc +++ b/src/kudu/benchmarks/tpch/tpch1.cc @@ -261,7 +261,7 @@ int main(int argc, char **argv) { kudu::Status s = env->CreateDir(FLAGS_mini_cluster_base_dir); CHECK(s.IsAlreadyPresent() || s.ok()) << s.ToString(); kudu::cluster::InternalMiniClusterOptions options; -options.data_root = FLAGS_mini_cluster_base_dir; +options.cluster_root = FLAGS_mini_cluster_base_dir; cluster.reset(new kudu::cluster::InternalMiniCluster(env, options)); CHECK_OK(cluster->StartSync()); master_address = cluster->mini_master()->bound_rpc_addr_str(); http://git-wip-us.apache.org/repos/asf/kudu/blob/5806b807/src/kudu/benchmarks/tpch/tpch_real_world.cc -- diff --git a/src/kudu/benchmarks/tpch/tpch_real_world.cc b/src/kudu/benchmarks/tpch/tpch_real_world.cc index ce65501..03a25f1 100644 --- a/src/kudu/benchmarks/tpch/tpch_real_world.cc +++ b/src/kudu/benchmarks/tpch/tpch_real_world.cc @@ -194,7 +194,7 @@ Status TpchRealWorld::Init() { ExternalMiniClusterOptions opts; opts.num_tablet_servers = 1; -opts.data_root = FLAGS_tpch_mini_cluster_base_dir; +opts.cluster_root = FLAGS_tpch_mini_cluster_base_dir; if (!FLAGS_tpch_path_to_ts_flags_file.empty()) { opts.extra_tserver_flags.push_back("--flagfile=" + FLAGS_tpch_path_to_ts_flags_file); } http://git-wip-us.apache.org/repos/asf/kudu/blob/5806b807/src/kudu/integration-tests/ts_itest-base.cc -- diff --git a/src/kudu/integration-tests/ts_itest-base.cc b/src/kudu/integration-tests/ts_itest-base.cc index f8344f0..26f4010 100644 --- a/src/kudu/integration-tests/ts_itest-base.cc +++ b/src/kudu/integration-tests/ts_itest-base.cc @@ -105,7 +105,7 @@ void TabletServerIntegrationTestBase::AddExtraFlags( } void TabletServerIntegrationTestBase::CreateCluster( -const string& data_root_path, +const string& cluster_root_path, const vector& non_default_ts_flags, const vector& non_default_master_flags, uint32_t num_data_dirs) { @@ -118,7 +118,7 @@ void TabletServerIntegrationTestBase::CreateCluster( cluster::ExternalMiniClusterOptions opts; opts.num_tablet_servers = FLAGS_num_tablet_servers; - opts.data_root = GetTestPath(data_root_path); + opts.cluster_root = GetTestPath(cluster_root_path); opts.num_data_dirs = num_data_dirs; // Enable exactly once semantics for tests. http://git-wip-us.apache.org/repos/asf/kudu/blob/5806b807/src/kudu/mini-cluster/external_mini_cluster.cc --
kudu git commit: [rpc] don't issue authn tokens over non-confidential connections
Repository: kudu Updated Branches: refs/heads/master d3165ed2b -> 1f346a1be [rpc] don't issue authn tokens over non-confidential connections With this patch, master will not issue authn tokens to clients over a non-confidential connection. The 'confidentiality' property is based on the features of the negotiated connection and stored into the RpcContext object, accessible via the RpcContext::is_confidential() method. Change-Id: Ie31aa492bcc460dbd43975bccfe571354f3bf885 Reviewed-on: http://gerrit.cloudera.org:8080/9025 Tested-by: Kudu Jenkins Reviewed-by: Alexey Serbin Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/1f346a1b Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/1f346a1b Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/1f346a1b Branch: refs/heads/master Commit: 1f346a1bec3db99877a4ace0d3682e5341bec9c6 Parents: d3165ed Author: Alexey Serbin Authored: Thu Jan 11 14:55:20 2018 -0800 Committer: Alexey Serbin Committed: Thu Jan 18 05:31:15 2018 + -- src/kudu/integration-tests/security-itest.cc | 105 ++ src/kudu/master/master_service.cc| 7 +- src/kudu/rpc/connection.cc | 5 ++ src/kudu/rpc/connection.h| 12 +++ src/kudu/rpc/negotiation.cc | 4 + src/kudu/rpc/rpc_context.cc | 5 ++ src/kudu/rpc/rpc_context.h | 7 ++ 7 files changed, 143 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/1f346a1b/src/kudu/integration-tests/security-itest.cc -- diff --git a/src/kudu/integration-tests/security-itest.cc b/src/kudu/integration-tests/security-itest.cc index 16c0175..6fb76bf 100644 --- a/src/kudu/integration-tests/security-itest.cc +++ b/src/kudu/integration-tests/security-itest.cc @@ -22,11 +22,13 @@ #include #include +#include #include #include #include "kudu/client/client-test-util.h" #include "kudu/client/client.h" +#include "kudu/client/client.pb.h" #include "kudu/client/schema.h" #include "kudu/client/shared_ptr.h" #include "kudu/client/write_op.h" @@ -37,9 +39,11 @@ #include "kudu/master/master.pb.h" #include "kudu/master/master.proxy.h" #include "kudu/mini-cluster/external_mini_cluster.h" +#include "kudu/mini-cluster/mini_cluster.h" #include "kudu/rpc/messenger.h" #include "kudu/rpc/rpc_controller.h" #include "kudu/security/test/mini_kdc.h" +#include "kudu/security/token.pb.h" #include "kudu/server/server_base.pb.h" #include "kudu/server/server_base.proxy.h" #include "kudu/tablet/key_value_test_schema.h" @@ -52,6 +56,8 @@ #include "kudu/util/test_macros.h" #include "kudu/util/test_util.h" +DECLARE_string(local_ip_for_outbound_sockets); + using kudu::client::KuduClient; using kudu::client::KuduInsert; using kudu::client::KuduSchema; @@ -304,4 +310,103 @@ TEST_F(SecurityITest, TestWorldReadablePrivateKey) { credentials_name)); } +struct AuthTokenIssuingTestParams { + const ExternalMiniCluster::BindMode bind_mode; + const string rpc_authentication; + const string rpc_encryption; + const bool rpc_encrypt_loopback_connections; + const bool authn_token_present; +}; +class AuthTokenIssuingTest : +public SecurityITest, +public ::testing::WithParamInterface { +}; +INSTANTIATE_TEST_CASE_P(, AuthTokenIssuingTest, ::testing::ValuesIn( +vector{ + { ExternalMiniCluster::BindMode::LOOPBACK, "required", "required", true, true, }, + { ExternalMiniCluster::BindMode::LOOPBACK, "required", "required", false, true, }, + //ExternalMiniCluster::BindMode::LOOPBACK, "required", "disabled": non-acceptable + //ExternalMiniCluster::BindMode::LOOPBACK, "required", "disabled": non-acceptable + { ExternalMiniCluster::BindMode::LOOPBACK, "disabled", "required", true, true, }, + { ExternalMiniCluster::BindMode::LOOPBACK, "disabled", "required", false, true, }, + { ExternalMiniCluster::BindMode::LOOPBACK, "disabled", "disabled", true, false, }, + { ExternalMiniCluster::BindMode::LOOPBACK, "disabled", "disabled", false, true, }, +#if defined(__linux__) + { ExternalMiniCluster::BindMode::UNIQUE_LOOPBACK, "required", "required", true, true, }, + { ExternalMiniCluster::BindMode::UNIQUE_LOOPBACK, "required", "required&qu
[1/3] kudu git commit: consensus: avoid extra thread wakeups for Peer::SignalRequest
Repository: kudu Updated Branches: refs/heads/master 3b51bf337 -> c497c69bb consensus: avoid extra thread wakeups for Peer::SignalRequest Looking at a profile on a YCSB workload I see a significant amount of CPU in submitting tasks to the raft threadpool for requests. My guess is that, in a workload like this, we often call SignalRequest while the request was already pending. This patch avoids submitting the threadpool task if there is already a task pending. Change-Id: I62ec4ea91e976590ce2de4fa34db61eaf9aab92a Reviewed-on: http://gerrit.cloudera.org:8080/9092 Tested-by: Kudu Jenkins Reviewed-by: David Ribeiro Alves Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/9961902d Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/9961902d Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/9961902d Branch: refs/heads/master Commit: 9961902d8b4f18a5e5d68aeb624e27167651b837 Parents: 3b51bf3 Author: Todd Lipcon Authored: Mon Jan 22 08:13:15 2018 -0800 Committer: Todd Lipcon Committed: Wed Jan 24 19:07:17 2018 + -- src/kudu/consensus/consensus_peers.cc | 6 ++ 1 file changed, 6 insertions(+) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/9961902d/src/kudu/consensus/consensus_peers.cc -- diff --git a/src/kudu/consensus/consensus_peers.cc b/src/kudu/consensus/consensus_peers.cc index 6d68528..187ed6f 100644 --- a/src/kudu/consensus/consensus_peers.cc +++ b/src/kudu/consensus/consensus_peers.cc @@ -167,6 +167,12 @@ Status Peer::SignalRequest(bool even_if_queue_empty) { return Status::IllegalState("Peer was closed."); } + // Only allow one request at a time. No sense waking up the + // raft thread pool if the task will just abort anyway. + if (request_pending_) { +return Status::OK(); + } + // Capture a weak_ptr reference into the submitted functor so that we can // safely handle the functor outliving its peer. weak_ptr w_this = shared_from_this();
[3/3] kudu git commit: KUDU-2265 CA-signed server certs for non-leader masters
KUDU-2265 CA-signed server certs for non-leader masters This changelist addresses KUDU-2265. Prior to this fix, a non-leader master had a self-signed server TLS certificates if it hasn't ever become a leader. An integration test to verify the new behavior is added as well. Change-Id: Ia3539d58d10ed319ad1d8686c1259c92822fb710 Reviewed-on: http://gerrit.cloudera.org:8080/9076 Tested-by: Kudu Jenkins Reviewed-by: Todd Lipcon Reviewed-by: Dan Burkert Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/c497c69b Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/c497c69b Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/c497c69b Branch: refs/heads/master Commit: c497c69bb2128a8a62738b9b6bb97a5b83da0323 Parents: e0a743d Author: Alexey Serbin Authored: Thu Jan 18 23:52:03 2018 -0800 Committer: Alexey Serbin Committed: Wed Jan 24 19:24:32 2018 + -- src/kudu/integration-tests/CMakeLists.txt | 1 + .../security-master-certificates-itest.cc | 100 +++ src/kudu/master/catalog_manager.cc | 46 +++-- src/kudu/master/catalog_manager.h | 30 +- src/kudu/security/tls_context.cc| 10 +- src/kudu/util/rw_mutex.cc | 6 ++ src/kudu/util/rw_mutex.h| 2 + 7 files changed, 177 insertions(+), 18 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/c497c69b/src/kudu/integration-tests/CMakeLists.txt -- diff --git a/src/kudu/integration-tests/CMakeLists.txt b/src/kudu/integration-tests/CMakeLists.txt index 1a10a7d..a6a6938 100644 --- a/src/kudu/integration-tests/CMakeLists.txt +++ b/src/kudu/integration-tests/CMakeLists.txt @@ -94,6 +94,7 @@ ADD_KUDU_TEST(raft_consensus-itest RUN_SERIAL true) ADD_KUDU_TEST(registration-test RESOURCE_LOCK "master-web-port") ADD_KUDU_TEST(security-faults-itest) ADD_KUDU_TEST(security-itest) +ADD_KUDU_TEST(security-master-certificates-itest RESOURCE_LOCK "master-rpc-ports") ADD_KUDU_TEST(security-unknown-tsk-itest) ADD_KUDU_TEST(stop_tablet-itest) ADD_KUDU_TEST(table_locations-itest) http://git-wip-us.apache.org/repos/asf/kudu/blob/c497c69b/src/kudu/integration-tests/security-master-certificates-itest.cc -- diff --git a/src/kudu/integration-tests/security-master-certificates-itest.cc b/src/kudu/integration-tests/security-master-certificates-itest.cc new file mode 100644 index 000..b521d65 --- /dev/null +++ b/src/kudu/integration-tests/security-master-certificates-itest.cc @@ -0,0 +1,100 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include + +#include +#include + +#include "kudu/consensus/raft_consensus.h" +#include "kudu/gutil/ref_counted.h" +#include "kudu/master/catalog_manager.h" +#include "kudu/master/master.h" +#include "kudu/master/mini_master.h" +#include "kudu/master/sys_catalog.h" +#include "kudu/mini-cluster/internal_mini_cluster.h" +#include "kudu/security/tls_context.h" +#include "kudu/tablet/tablet_replica.h" +#include "kudu/util/status.h" +#include "kudu/util/test_macros.h" +#include "kudu/util/test_util.h" + +DECLARE_bool(enable_leader_failure_detection); + +using kudu::cluster::InternalMiniCluster; +using kudu::cluster::InternalMiniClusterOptions; +using kudu::consensus::RaftConsensus; + +using std::unique_ptr; + + +namespace kudu { + +class SecurityMasterCertsTest : public KuduTest { + public: + void SetUp() override { +KuduTest::SetUp(); + +// This test requires manual system catalog leader election. +FLAGS_enable_leader_failure_detection = false; + +InternalMiniClusterOptions opts; +opts.master_rpc_ports = { 11010, 11011, 11012, 11013, 11014, }; +opts.num_masters = opts.master_rpc_ports.size
[2/3] kudu git commit: Upgrade to tcmalloc 2.6.3 and enabled sized-deallocation
Upgrade to tcmalloc 2.6.3 and enabled sized-deallocation This bumps to a new gperftools/tcmalloc version which ought to be a bit faster. It also enables sized-deallocation support on compilers that support it. The upgrade is relatively straigthforward but for a few items: - for whatever reason, we developed a conflict between our own base::SpinLock from gutil and tcmalloc's. This conflict has always existed, but apparently tcmalloc's implementation changed sizes slightly, meaning that now the linker complains about the duplicate symbol definition. I created a simple find-replace patch to change tcmalloc to use a 'tcmalloc::' namespace instead of 'base::'. - our previous patches are all now included in tcmalloc upstream and no longer necessary. - for unknown reasons, gperftools now fails to build with TSAN instrumentation: it shows duplicate symbols between the tcmalloc 'operator new' and clang_rt's 'operator new'. We never actually needed to use gperftools in TSAN builds anyway, so this just removes the instrumented build and properly ifdefs out a few of the places where we'd included gperftools headers unconditionally. Sized deallocation is a C++14 feature (which can be selectively enabled on C++11) which adds a new 'operator delete(void*, size_t)'. In the common case where we delete a Foo*, the compiler already knows sizeof(Foo) and thus can pass this second argument to 'operator delete'. This goes through an optimized code path inside tcmalloc -- namely, it can skip the expensive lookup which otherwise would be required to map the pointer back to its size class. Unfortuantely, we can't always rely on the system allocator to provide sized deallocation, and thus we can't use this in the exported variants of our code (i.e. the client library). So, this patch only enables it when building the Kudu binaries and not the client library. I ran a quick performance check using full_stack-insert-scan-test twice each before and after: Before: Performance counter stats for 'build/latest/bin/full_stack-insert-scan-test --gtest_filter=*MRSOnlyStressTest* --inserts_per_client=20 --concurrent_inserts=10 --rows_per_batch=1 --skip_scans': 322623.135418 task-clock (msec) #4.964 CPUs utilized 11,751,640 context-switches #0.036 M/sec 3,619,010 cpu-migrations#0.011 M/sec 117,706 page-faults #0.365 K/sec 1,057,656,296,542 cycles#3.278 GHz 695,668,728,662 instructions #0.66 insn per cycle 127,565,245,327 branches # 395.400 M/sec 1,480,987,998 branch-misses #1.16% of all branches 64.996506196 seconds time elapsed 330206.351604 task-clock (msec) #5.330 CPUs utilized 12,518,370 context-switches #0.038 M/sec 3,695,790 cpu-migrations#0.011 M/sec 113,580 page-faults #0.344 K/sec 1,059,061,436,907 cycles#3.207 GHz 697,782,051,928 instructions #0.66 insn per cycle 127,949,774,596 branches # 387.484 M/sec 1,371,967,917 branch-misses #1.07% of all branches 61.952217532 seconds time elapsed After: 310788.334202 task-clock (msec) #5.315 CPUs utilized 12,486,276 context-switches #0.040 M/sec 3,690,660 cpu-migrations#0.012 M/sec 113,988 page-faults #0.367 K/sec 1,027,033,932,375 cycles#3.305 GHz 663,508,168,985 instructions #0.65 insn per cycle 125,864,966,052 branches # 404.986 M/sec 1,442,683,495 branch-misses #1.15% of all branches 58.479033228 seconds time elapsed Performance counter stats for 'build/latest/bin/full_stack-insert-scan-test.263 --gtest_filter=*MRSOnlyStressTest* --inserts_per_client=20 --concurrent_inserts=10 --rows_per_batch=1 --skip_scans': 317505.548908 task-clock (msec) #5.323 CPUs utilized 12,461,003 context-switches #0.039 M/sec 3,688,491 cpu-migrations#0.012 M/sec 113,952 page-faults #0.359 K/sec 1,029,595,155,391 cycles#3.243 GHz 663,514,269,656 instructions #0.64 insn per cycle 125,865,138,975 branches # 396.419 M/sec 1,419,717,877 branch-misses #1.13% of all branches 59.642651558 seconds time elapsed Seems like a small improvement in cycle count, wall clock, etc. Change-Id: I3b76c9d71b29d58f0b6dade3fc33b32d0c3de23b Reviewed-on: http://gerrit.cloudera.org:8080/9108 Tested-b
kudu git commit: [gradle] fetch gradle-wrapper.jar from raw.githubusercontent.com
Repository: kudu Updated Branches: refs/heads/master 37a81b82c -> 4ee1c4fd7 [gradle] fetch gradle-wrapper.jar from raw.githubusercontent.com This patch switches the URL to fetch gradle-wrapper.jar from github.com to raw.githubusercontent.com. That's to be able to build the Java part of the project at older Linux distros (RH < 7.4). The github.com servers dropped TLSv1/TLSv1.1 support some time ago: see [1] for details. [1] https://githubengineering.com/crypto-removal-notice/ Change-Id: Ia745ef4d69726852fbe60be0d1778705d5580f14 Reviewed-on: http://gerrit.cloudera.org:8080/11262 Reviewed-by: Grant Henke Tested-by: Alexey Serbin Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/4ee1c4fd Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/4ee1c4fd Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/4ee1c4fd Branch: refs/heads/master Commit: 4ee1c4fd7a716182abbe4bab7a8a20089158a894 Parents: 37a81b8 Author: Alexey Serbin Authored: Fri Aug 17 13:21:14 2018 -0700 Committer: Alexey Serbin Committed: Sat Aug 18 00:17:05 2018 + -- java/gradle/wrapper.gradle | 8 ++-- java/gradlew | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/4ee1c4fd/java/gradle/wrapper.gradle -- diff --git a/java/gradle/wrapper.gradle b/java/gradle/wrapper.gradle index 56071a5..10171d5 100644 --- a/java/gradle/wrapper.gradle +++ b/java/gradle/wrapper.gradle @@ -51,8 +51,12 @@ task bootstrapWrapper() { // Add a trailing zero to the version if needed. def fullVersion = versions.gradle.count(".") == 1 ? "${versions.gradle}.0" : versions.gradle -// Leverages the wrapper jar checked into the gradle project on github because the jar isn't available elsewhere. -def wrapperBaseUrl = "https://github.com/gradle/gradle/raw/v$fullVersion/gradle/wrapper"; +// Leverages the wrapper jar checked into the gradle project on github because the jar isn't +// available elsewhere. Using raw.githubusercontent.com instead of github.com because +// github.com servers deprecated TLSv1/TLSv1.1 support some time ago, so older versions +// of curl (built against OpenSSL library that doesn't support TLSv1.2) would fail to +// fetch the jar. +def wrapperBaseUrl = "https://raw.githubusercontent.com/gradle/gradle/v$fullVersion/gradle/wrapper"; def wrapperJarUrl = wrapperBaseUrl + "/gradle-wrapper.jar" def bootstrapString = """ http://git-wip-us.apache.org/repos/asf/kudu/blob/4ee1c4fd/java/gradlew -- diff --git a/java/gradlew b/java/gradlew index 3807ff4..f40bb25 100755 --- a/java/gradlew +++ b/java/gradlew @@ -82,7 +82,7 @@ esac # Loop in case we encounter an error. for attempt in 1 2 3; do if [ ! -e $APP_HOME/gradle/wrapper/gradle-wrapper.jar ]; then -if ! curl -s -S --retry 3 -L -o "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" "https://github.com/gradle/gradle/raw/v4.9.0/gradle/wrapper/gradle-wrapper.jar";; then +if ! curl -s -S --retry 3 -L -o "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" "https://raw.githubusercontent.com/gradle/gradle/v4.9.0/gradle/wrapper/gradle-wrapper.jar";; then rm -f "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" # Pause for a bit before looping in case the server throttled us. sleep 5
kudu git commit: [gutil] add libunwind along with libglog to fix build
Repository: kudu Updated Branches: refs/heads/master be360bf01 -> 0dc19bfb3 [gutil] add libunwind along with libglog to fix build Since libglog depends on libunwind (at least on older Linux systems), it's necessary to add libunwind into the list of libraries when linking in libglog. Otherwise, in case of a static linkage, there are errors like below: ./../../lib/../src/kudu/util/CMakeFiles/kudu_util.dir/debug-util.cc.o: In function `PrimeLibunwind': src/kudu/src/kudu/util/debug-util.cc:349: undefined reference to `_Ux86_64_getcontext' src/kudu/src/kudu/util/debug-util.cc:350: undefined reference to `_ULx86_64_init_local' ../../../lib/../src/kudu/util/CMakeFiles/kudu_util.dir/debug-util.cc.o: In function `kudu::StackTrace::Collect(int)': src/kudu/src/kudu/util/debug-util.cc:591: undefined reference to `_Ux86_64_getcontext' src/kudu/src/kudu/util/debug-util.cc:592: undefined reference to `_ULx86_64_init_local' src/kudu/src/kudu/util/debug-util.cc:607: undefined reference to `_ULx86_64_step' src/kudu/src/kudu/util/debug-util.cc:598: undefined reference to `_ULx86_64_get_reg' src/kudu/src/kudu/util/debug-util.cc:607: undefined reference to `_ULx86_64_step' ../../../../../thirdparty/installed/uninstrumented/lib/libglog.a(libglog_la-utilities.o): In function `GetStackTrace': src/kudu/thirdparty/src/glog-0.3.5/src/stacktrace_libunwind-inl.h:65: undefined reference to `_Ux86_64_getcontext' src/kudu/thirdparty/src/glog-0.3.5/src/stacktrace_libunwind-inl.h:66: undefined reference to `_ULx86_64_init_local' src/kudu/thirdparty/src/glog-0.3.5/src/stacktrace_libunwind-inl.h:78: undefined reference to `_ULx86_64_step' src/kudu/thirdparty/src/glog-0.3.5/src/stacktrace_libunwind-inl.h:70: undefined reference to `_ULx86_64_get_reg' src/kudu/thirdparty/src/glog-0.3.5/src/stacktrace_libunwind-inl.h:78: undefined reference to `_ULx86_64_step' collect2: error: ld returned 1 exit status This is a follow-up for a8d088e11b8b901a91e4820dcfdca9537a1e801a: in that changelist KUDU_BASE_LIBS were removed from the linkage of libraries/binaries where libgutil was linked in (KUDU_BASE_LIBS contains both libglog and libunwind). Change-Id: Ia91824b257fbc41a442b484bbc5a5f926db708ec Reviewed-on: http://gerrit.cloudera.org:8080/11327 Reviewed-by: Andrew Wong Tested-by: Kudu Jenkins Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/0dc19bfb Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/0dc19bfb Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/0dc19bfb Branch: refs/heads/master Commit: 0dc19bfb37cf9a20b9e1d7122084c199720d5ca4 Parents: be360bf Author: Alexey Serbin Authored: Sat Aug 25 00:05:52 2018 -0700 Committer: Alexey Serbin Committed: Sat Aug 25 08:06:19 2018 + -- src/kudu/gutil/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/0dc19bfb/src/kudu/gutil/CMakeLists.txt -- diff --git a/src/kudu/gutil/CMakeLists.txt b/src/kudu/gutil/CMakeLists.txt index 3d487eb..093cc1b 100644 --- a/src/kudu/gutil/CMakeLists.txt +++ b/src/kudu/gutil/CMakeLists.txt @@ -54,6 +54,7 @@ set(GUTIL_SRCS set(GUTIL_LIBS glog + unwind protobuf) if (NOT APPLE)
kudu git commit: [gutil] change libunwind to Linux specific to fix build
Repository: kudu Updated Branches: refs/heads/master f5ffcb4d7 -> bb78520a7 [gutil] change libunwind to Linux specific to fix build This is a follow-up for 0dc19bfb37cf9a20b9e1d7122084c199720d5ca4: that change breaks the build on macOS. Change-Id: I879564cc4e6f233bc234890d401d7cb068c3eaf4 Reviewed-on: http://gerrit.cloudera.org:8080/11338 Tested-by: Kudu Jenkins Reviewed-by: Alexey Serbin Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/bb78520a Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/bb78520a Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/bb78520a Branch: refs/heads/master Commit: bb78520a7a636926bba3c68fc85bdca157524ef0 Parents: f5ffcb4 Author: Kiyoshi Mizumaru Authored: Tue Aug 28 04:32:41 2018 +0900 Committer: Alexey Serbin Committed: Mon Aug 27 20:06:41 2018 + -- src/kudu/gutil/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/bb78520a/src/kudu/gutil/CMakeLists.txt -- diff --git a/src/kudu/gutil/CMakeLists.txt b/src/kudu/gutil/CMakeLists.txt index 093cc1b..787e6db 100644 --- a/src/kudu/gutil/CMakeLists.txt +++ b/src/kudu/gutil/CMakeLists.txt @@ -54,12 +54,12 @@ set(GUTIL_SRCS set(GUTIL_LIBS glog - unwind protobuf) if (NOT APPLE) set(GUTIL_LIBS ${GUTIL_LIBS} +unwind rt) # clock_gettime() requires -lrt endif()
kudu git commit: KUDU-2560: exported thirdparty library variants should use exported deps
Repository: kudu Updated Branches: refs/heads/master b3eb06a01 -> 1d9ce5625 KUDU-2560: exported thirdparty library variants should use exported deps This addresses a longstanding bug in ADD_THIRDPARTY_LIB that caused "exported" thirdparty library targets to express their own thirdparty dependencies in terms of regular (i.e. non-exported) targets. The first time this mattered was in commit d55df3c6e, where libunwind was added as a dependency of glog. This caused the exported client library, by virtue of its glog dependency, to depend on 'unwind' (shared object) instead of on 'unwind_exported' (static archive). In most systems the linker elided that dependency because it was unnecessary, but it is preserved by the linker in devtoolset3. That causes client_examples-test to fail if the system doesn't have libunwind installed. While I was there I removed some usages of DEPS that were not actually used. Change-Id: I28dd499b152d4eadd9b5cf6575650a1a3db163f1 Reviewed-on: http://gerrit.cloudera.org:8080/11354 Reviewed-by: Dan Burkert Tested-by: Kudu Jenkins Reviewed-by: Andrew Wong Reviewed-by: Alexey Serbin Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/1d9ce562 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/1d9ce562 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/1d9ce562 Branch: refs/heads/master Commit: 1d9ce56251bb41e21d58cb5b8904ba0aa35110be Parents: b3eb06a Author: Adar Dembo Authored: Wed Aug 29 15:14:35 2018 -0700 Committer: Alexey Serbin Committed: Thu Aug 30 01:16:30 2018 + -- CMakeLists.txt | 16 cmake_modules/FindPmem.cmake | 1 - 2 files changed, 8 insertions(+), 9 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/1d9ce562/CMakeLists.txt -- diff --git a/CMakeLists.txt b/CMakeLists.txt index 0cd6bc1..d7821e8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -942,7 +942,7 @@ function(ADD_THIRDPARTY_LIB LIB_NAME) # A shared object exported variant will force any "leaf" library that # transitively depends on it to also depend on it at runtime; this is # desirable for some libraries (e.g. cyrus_sasl). - set(LIB_NAME_EXPORTED ${LIB_NAME}_exported) + set(LIB_NAME_EXPORTED "${LIB_NAME}_exported") if(ARG_STATIC_LIB) add_library(${LIB_NAME_EXPORTED} STATIC IMPORTED) set_target_properties(${LIB_NAME_EXPORTED} @@ -953,8 +953,11 @@ function(ADD_THIRDPARTY_LIB LIB_NAME) PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}") endif() if(ARG_DEPS) +foreach(DEP ${ARG_DEPS}) + list(APPEND EXPORTED_DEPS "${DEP}_exported") +endforeach() set_target_properties(${LIB_NAME_EXPORTED} - PROPERTIES IMPORTED_LINK_INTERFACE_LIBRARIES "${ARG_DEPS}") + PROPERTIES IMPORTED_LINK_INTERFACE_LIBRARIES "${EXPORTED_DEPS}") endif() endfunction() @@ -974,15 +977,13 @@ set(CMAKE_PREFIX_PATH find_package(CyrusSASL REQUIRED) include_directories(SYSTEM ${CYRUS_SASL_INCLUDE_DIR}) ADD_THIRDPARTY_LIB(cyrus_sasl - SHARED_LIB "${CYRUS_SASL_SHARED_LIB}" - DEPS ${CYRUS_SASL_LIB_DEPS}) + SHARED_LIB "${CYRUS_SASL_SHARED_LIB}") ## GSSAPI find_package(GSSAPI REQUIRED) include_directories(SYSTEM ${GSSAPI_INCLUDE_DIR}) ADD_THIRDPARTY_LIB(gssapi_krb5 - SHARED_LIB "${GSSAPI_SHARED_LIB}" - DEPS ${GSSAPI_LIB_DEPS}) + SHARED_LIB "${GSSAPI_SHARED_LIB}") ## GLog (depends on libunwind) find_package(GLog REQUIRED) @@ -1157,8 +1158,7 @@ ADD_THIRDPARTY_LIB(pmem SHARED_LIB "${PMEM_SHARED_LIB}") ADD_THIRDPARTY_LIB(pmemobj STATIC_LIB "${PMEMOBJ_STATIC_LIB}" - SHARED_LIB "${PMEMOBJ_SHARED_LIB}" - DEPS ${PMEMOBJ_DEPS}) + SHARED_LIB "${PMEMOBJ_SHARED_LIB}") endif() if (EXISTS ${VMEM_SHARED_LIB}) http://git-wip-us.apache.org/repos/asf/kudu/blob/1d9ce562/cmake_modules/FindPmem.cmake -- diff --git a/cmake_modules/FindPmem.cmake b/cmake_modules/FindPmem.cmake index 2270e65..a628e2f 100644 --- a/cmake_modules/FindPmem.cmake +++ b/cmake_modules/FindPmem.cmake @@ -21,7 +21,6 @@ # XXX_STATIC_LIBS, path to *.a # XXX_SHARED_LIBS, path to *.so shared library # PMEM_FOUND, whether PMEM libraries have been found -# PMEMOBJ_DEPS, dependencies required for using libpmemobj find_path(PMEM_INCLUDE_DIR libpmem.h NO_CMAKE_SYSTEM_PATH
kudu git commit: [tests] make master-stress-test more stable
Repository: kudu Updated Branches: refs/heads/master 1d9ce5625 -> eb15beabd [tests] make master-stress-test more stable The master-stress-test has been flaky for some time. After looking at those failure closely, I found at least five different issues. This patch addresses the most prominent one: failures of the test scenario because of timeout errors in case of TSAN builds. About 9 out of 10 failures were due to the issue fixed by this patch. The timeout errors were triggered by RPC queue overflow and the timing of master restarts wrt the retry/back-off pattern used by KuduClient and other test utility code. The rest of issues behind the flakiness will be addressed separately. This patch also introduces rpc_negotiation_timeout as a member for ExternalMiniClusterOptions: that's to customize connection negotiation timeout for the cluster's utility messenger. Some statistics about the flakiness: before the fix: 37 out of 256 failed in TSAN build, where almost all failures were due to the issues fixed by this patch: http://dist-test.cloudera.org//job?job_id=aserbin.1535666928.86597 after the fix: 2 out of 256 failed in TSAN build, where the failure was due to [2], which will be addressed separately: http://dist-test.cloudera.org/job?job_id=aserbin.1535665784.64065 A few of other issues due to which the test is still a bit flaky: [1] https://issues.apache.org/jira/browse/KUDU-2561 [2] https://issues.apache.org/jira/browse/KUDU-2564 [3] https://issues.apache.org/jira/browse/HIVE-19874 By my understanding, Dan found [3] to be the reason behind one type of HMS-related failures; and there two more to evaluate. Change-Id: I6b30d8afd4a24acdbd96481cadeaf8f6a9475adf Reviewed-on: http://gerrit.cloudera.org:8080/11364 Tested-by: Kudu Jenkins Reviewed-by: Adar Dembo Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/eb15beab Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/eb15beab Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/eb15beab Branch: refs/heads/master Commit: eb15beabd2d4d86649a35dcd027e1749cc695866 Parents: 1d9ce56 Author: Alexey Serbin Authored: Wed Aug 29 16:52:09 2018 -0700 Committer: Alexey Serbin Committed: Fri Aug 31 18:16:44 2018 + -- .../integration-tests/master-stress-test.cc | 117 +-- src/kudu/mini-cluster/external_mini_cluster.cc | 5 +- src/kudu/mini-cluster/external_mini_cluster.h | 6 + 3 files changed, 93 insertions(+), 35 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/eb15beab/src/kudu/integration-tests/master-stress-test.cc -- diff --git a/src/kudu/integration-tests/master-stress-test.cc b/src/kudu/integration-tests/master-stress-test.cc index 38aa71f..86d37cf 100644 --- a/src/kudu/integration-tests/master-stress-test.cc +++ b/src/kudu/integration-tests/master-stress-test.cc @@ -26,6 +26,7 @@ #include #include +#include #include #include @@ -58,6 +59,7 @@ #include "kudu/util/test_macros.h" #include "kudu/util/test_util.h" +DECLARE_int64(timeout_ms); DEFINE_int32(num_create_table_threads, 4, "Number of threads that should create tables"); DEFINE_int32(num_alter_table_threads, 2, @@ -66,36 +68,40 @@ DEFINE_int32(num_delete_table_threads, 2, "Number of threads that should delete tables"); DEFINE_int32(num_replace_tablet_threads, 2, "Number of threads that should replace tablets"); -DEFINE_int32(num_seconds_to_run, 5, +DEFINE_int32(num_seconds_to_run, 10, "Number of seconds that the test should run"); -namespace kudu { - -using client::KuduClient; -using client::KuduClientBuilder; -using client::KuduColumnSchema; -using client::KuduSchema; -using client::KuduSchemaBuilder; -using client::KuduTableAlterer; -using client::KuduTableCreator; -using cluster::ExternalMaster; -using cluster::ExternalMiniCluster; -using cluster::ExternalMiniClusterOptions; -using master::MasterServiceProxy; -using master::ListTablesRequestPB; -using master::ListTablesResponsePB; -using rpc::Messenger; -using rpc::MessengerBuilder; -using rpc::RpcController; +using kudu::client::KuduClient; +using kudu::client::KuduClientBuilder; +using kudu::client::KuduColumnSchema; +using kudu::client::KuduSchema; +using kudu::client::KuduSchemaBuilder; +using kudu::client::KuduTableAlterer; +using kudu::client::KuduTableCreator; +using kudu::cluster::ExternalMaster; +using kudu::cluster::ExternalMiniCluster; +using kudu::cluster::ExternalMiniClusterOptions; +using kudu::itest::ListTablets; +using kudu::master::ListTablesRequestPB; +using kudu::master::ListTablesResponsePB; +using kudu::master::ReplaceTabletReq
[2/2] kudu git commit: test: quell warning in disk_failure-itest
test: quell warning in disk_failure-itest disk_failure-itest will yield the following warning when built: ../../src/kudu/integration-tests/disk_failure-itest.cc:246:3: warning: control may reach end of non-void function [-Wreturn-type] Change-Id: If9cd6beb9855b0d22cf5fbb74c4d253ff84f4bca Reviewed-on: http://gerrit.cloudera.org:8080/11385 Tested-by: Kudu Jenkins Reviewed-by: Alexey Serbin Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/a58867c5 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/a58867c5 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/a58867c5 Branch: refs/heads/master Commit: a58867c59c0ecbdd2c74d8ac6744b7b3cd304a36 Parents: 060d7b0 Author: Andrew Wong Authored: Tue Sep 4 12:27:04 2018 -0700 Committer: Alexey Serbin Committed: Tue Sep 4 20:06:18 2018 + -- src/kudu/integration-tests/disk_failure-itest.cc | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/a58867c5/src/kudu/integration-tests/disk_failure-itest.cc -- diff --git a/src/kudu/integration-tests/disk_failure-itest.cc b/src/kudu/integration-tests/disk_failure-itest.cc index 111ccf4..7123820 100644 --- a/src/kudu/integration-tests/disk_failure-itest.cc +++ b/src/kudu/integration-tests/disk_failure-itest.cc @@ -243,6 +243,7 @@ class DiskErrorITest : public ExternalMiniClusterITestBase, }; return SetFlags(cluster_->tablet_server(i), recovery_flags); } +return Status::OK(); } // Waits for the number of failed tablets on the tablet server to reach
[1/2] kudu git commit: [tablet] Add tablet metric for number of diskrowsets
Repository: kudu Updated Branches: refs/heads/master cf6927cb1 -> a58867c59 [tablet] Add tablet metric for number of diskrowsets Change-Id: Ifb1ca202c281da07007eff1f555af2eef9b2393e Reviewed-on: http://gerrit.cloudera.org:8080/11378 Reviewed-by: Adar Dembo Tested-by: Kudu Jenkins Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/060d7b0f Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/060d7b0f Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/060d7b0f Branch: refs/heads/master Commit: 060d7b0f59b287c18c0690a7795ae225a1f534ee Parents: cf6927c Author: Will Berkeley Authored: Fri Aug 31 17:06:54 2018 -0700 Committer: Will Berkeley Committed: Tue Sep 4 18:23:11 2018 + -- src/kudu/tablet/tablet.cc | 8 +++- src/kudu/tserver/tablet_server-test.cc | 19 +++ 2 files changed, 26 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/060d7b0f/src/kudu/tablet/tablet.cc -- diff --git a/src/kudu/tablet/tablet.cc b/src/kudu/tablet/tablet.cc index f3a5143..7a44b91 100644 --- a/src/kudu/tablet/tablet.cc +++ b/src/kudu/tablet/tablet.cc @@ -170,6 +170,9 @@ METRIC_DEFINE_gauge_size(tablet, memrowset_size, "MemRowSet Memory Usage", METRIC_DEFINE_gauge_size(tablet, on_disk_data_size, "Tablet Data Size On Disk", kudu::MetricUnit::kBytes, "Space used by this tablet's data blocks."); +METRIC_DEFINE_gauge_size(tablet, num_rowsets_on_disk, "Tablet Number of Rowsets on Disk", + kudu::MetricUnit::kUnits, + "Number of diskrowsets in this tablet"); using kudu::MaintenanceManager; using kudu::clock::HybridClock; @@ -236,6 +239,9 @@ Tablet::Tablet(scoped_refptr metadata, METRIC_on_disk_data_size.InstantiateFunctionGauge( metric_entity_, Bind(&Tablet::OnDiskDataSize, Unretained(this))) ->AutoDetach(&metric_detacher_); +METRIC_num_rowsets_on_disk.InstantiateFunctionGauge( + metric_entity_, Bind(&Tablet::num_rowsets, Unretained(this))) + ->AutoDetach(&metric_detacher_); } if (FLAGS_tablet_throttler_rpc_per_sec > 0 || FLAGS_tablet_throttler_bytes_per_sec > 0) { @@ -2250,7 +2256,7 @@ int64_t Tablet::CountRedoDeltasForTests() const { size_t Tablet::num_rowsets() const { shared_lock l(component_lock_); - return components_->rowsets->all_rowsets().size(); + return components_ ? components_->rowsets->all_rowsets().size() : 0; } void Tablet::PrintRSLayout(ostream* o) { http://git-wip-us.apache.org/repos/asf/kudu/blob/060d7b0f/src/kudu/tserver/tablet_server-test.cc -- diff --git a/src/kudu/tserver/tablet_server-test.cc b/src/kudu/tserver/tablet_server-test.cc index 671949f..084da03 100644 --- a/src/kudu/tserver/tablet_server-test.cc +++ b/src/kudu/tserver/tablet_server-test.cc @@ -169,6 +169,7 @@ METRIC_DECLARE_gauge_uint64(log_block_manager_containers); METRIC_DECLARE_counter(log_block_manager_holes_punched); METRIC_DECLARE_gauge_size(active_scanners); METRIC_DECLARE_gauge_size(tablet_active_scanners); +METRIC_DECLARE_gauge_size(num_rowsets_on_disk); namespace kudu { @@ -3323,6 +3324,24 @@ TEST_F(TabletServerTest, TestNoMetricsForTombstonedTablet) { } } +TEST_F(TabletServerTest, TestTabletNumberOfDiskRowSetsMetric) { + scoped_refptr tablet; + ASSERT_TRUE(mini_server_->server()->tablet_manager()->LookupTablet(kTabletId, &tablet)); + ASSERT_TRUE(tablet->tablet()->GetMetricEntity()); + + // We don't care what the function is, since the metric is already instantiated. + auto num_diskrowsets = METRIC_num_rowsets_on_disk.InstantiateFunctionGauge( + tablet->tablet()->GetMetricEntity(), Callback()); + + // No data, no diskrowsets. + ASSERT_EQ(0, num_diskrowsets->value()); + + // Insert a row and flush. There should be 1 diskrowset. + ASSERT_NO_FATAL_FAILURE(InsertTestRowsRemote(0, 1, 1)); + ASSERT_OK(tablet->tablet()->Flush()); + ASSERT_EQ(1, num_diskrowsets->value()); +} + // Test ensuring that when rowset min/max keys are stored with and read from // the rowset metadata, the tablet server doesn't read any blocks when // bootstrapping.
[1/2] kudu git commit: [tools] Fix help text for replica_type
Repository: kudu Updated Branches: refs/heads/master 257dd585a -> 885d8bdaf [tools] Fix help text for replica_type 'kudu tablet change_config [add_replica|change_replica_type]' take an argument for 'replica type', which the help text said had to be "VOTER" or "NON-VOTER", but actually it has to be "VOTER" or "NON_VOTER", because it is parsed into an enum by a generated PB function, and the PB enum names are "VOTER" and "NON_VOTER". This fixes the help text. Change-Id: Icb43e0eaffbfa17967103c7aebe15ea0a06c5598 Reviewed-on: http://gerrit.cloudera.org:8080/11400 Tested-by: Kudu Jenkins Reviewed-by: Alexey Serbin Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/0653dcde Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/0653dcde Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/0653dcde Branch: refs/heads/master Commit: 0653dcdebb8a5dbe0463341c8f12bbc5ae14fec8 Parents: 257dd58 Author: Will Berkeley Authored: Fri Sep 7 11:34:44 2018 -0700 Committer: Alexey Serbin Committed: Fri Sep 7 20:29:31 2018 + -- src/kudu/tools/tool_action_tablet.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/0653dcde/src/kudu/tools/tool_action_tablet.cc -- diff --git a/src/kudu/tools/tool_action_tablet.cc b/src/kudu/tools/tool_action_tablet.cc index bda4637..da0209b 100644 --- a/src/kudu/tools/tool_action_tablet.cc +++ b/src/kudu/tools/tool_action_tablet.cc @@ -242,7 +242,7 @@ unique_ptr BuildTabletMode() { .AddRequiredParameter({ kTsUuidArg, "UUID of the tablet server that should host the new replica" }) .AddRequiredParameter( - { kReplicaTypeArg, "New replica's type. Must be VOTER or NON-VOTER." + { kReplicaTypeArg, "New replica's type. Must be VOTER or NON_VOTER." }) .Build(); @@ -255,7 +255,7 @@ unique_ptr BuildTabletMode() { .AddRequiredParameter({ kTsUuidArg, "UUID of the tablet server hosting the existing replica" }) .AddRequiredParameter( - { kReplicaTypeArg, "Existing replica's new type. Must be VOTER or NON-VOTER." + { kReplicaTypeArg, "Existing replica's new type. Must be VOTER or NON_VOTER." }) .Build();
[2/2] kudu git commit: bitmap: add equality method
bitmap: add equality method This is useful for tests. Change-Id: If53e41250435501e158bdbc076dc8e89ea153256 Reviewed-on: http://gerrit.cloudera.org:8080/11266 Tested-by: Kudu Jenkins Reviewed-by: Alexey Serbin Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/885d8bda Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/885d8bda Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/885d8bda Branch: refs/heads/master Commit: 885d8bdafd46b7a4d48cabb8ae929431b78b7cab Parents: 0653dcd Author: Adar Dembo Authored: Fri Aug 17 17:03:22 2018 -0700 Committer: Alexey Serbin Committed: Fri Sep 7 21:07:38 2018 + -- src/kudu/util/bitmap-test.cc | 49 +-- src/kudu/util/bitmap.h | 21 + 2 files changed, 68 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/885d8bda/src/kudu/util/bitmap-test.cc -- diff --git a/src/kudu/util/bitmap-test.cc b/src/kudu/util/bitmap-test.cc index 089ed3b..18ccb27 100644 --- a/src/kudu/util/bitmap-test.cc +++ b/src/kudu/util/bitmap-test.cc @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +#include "kudu/util/bitmap.h" + #include #include #include @@ -22,7 +24,6 @@ #include #include "kudu/gutil/strings/join.h" -#include "kudu/util/bitmap.h" namespace kudu { @@ -73,7 +74,7 @@ TEST(TestBitMap, TestIteration2) { ASSERT_EQ("1", JoinElements(read_back, ",")); } -TEST(TestBitmap, TestSetAndTestBits) { +TEST(TestBitMap, TestSetAndTestBits) { uint8_t bm[1]; memset(bm, 0, sizeof(bm)); @@ -227,4 +228,48 @@ TEST(TestBitMap, TestBitmapIteration) { ASSERT_EQ(expected_sizes[i], size); } +TEST(TestBitMap, TestEquals) { + uint8_t bm1[8] = { 0 }; + uint8_t bm2[8] = { 0 }; + size_t num_bits = sizeof(bm1) * 8; + ASSERT_TRUE(BitmapEquals(bm1, bm2, num_bits)); + + // Loop over each bit starting from the end and going to the beginning. In + // each iteration, set the bit in one bitmap and verify that although the two + // bitmaps aren't equal, if we were to ignore the changed bits, they are still equal. + for (int i = num_bits - 1; i >= 0; i--) { +SCOPED_TRACE(i); +BitmapChange(bm1, i, true); +ASSERT_FALSE(BitmapEquals(bm1, bm2, num_bits)); +ASSERT_TRUE(BitmapEquals(bm1, bm2, i)); + } + + // Now loop in the other direction, setting the second bitmap bit by bit. + // As before, if we consider the bitmaps in their entirety, they're not equal, + // but if we consider just the sequences where both are set, they are equal. + for (int i = 0; i < num_bits - 1; i++) { +SCOPED_TRACE(i); +BitmapChange(bm2, i, true); +ASSERT_FALSE(BitmapEquals(bm1, bm2, num_bits)); +ASSERT_TRUE(BitmapEquals(bm1, bm2, i + 1)); + } + + // If we set the very last bit, both bitmaps are now equal in their entirety. + BitmapChange(bm2, num_bits - 1, true); + ASSERT_TRUE(BitmapEquals(bm1, bm2, num_bits)); + + // Test equality on overlapped bitmaps (i.e. a single underlying bitmap, two + // subsequences of which are considered to be two separate bitmaps). + + // Set every third bit; the rest are unset. + uint8_t bm3[8] = { 0 }; + for (int i = 0; i < num_bits; i += 3) { +BitmapChange(bm3, i, true); + } + + ASSERT_TRUE(BitmapEquals(bm3, bm3, num_bits)); // fully overlapped + ASSERT_FALSE(BitmapEquals(bm3, bm3 + 1, num_bits - 8)); // off by one byte + ASSERT_TRUE(BitmapEquals(bm3, bm3 + 3, num_bits - 24)); // off by three bytes +} + } // namespace kudu http://git-wip-us.apache.org/repos/asf/kudu/blob/885d8bda/src/kudu/util/bitmap.h -- diff --git a/src/kudu/util/bitmap.h b/src/kudu/util/bitmap.h index d9f5260..947a6c6 100644 --- a/src/kudu/util/bitmap.h +++ b/src/kudu/util/bitmap.h @@ -28,6 +28,7 @@ #include "kudu/gutil/bits.h" #include "kudu/gutil/port.h" +#include "kudu/gutil/strings/fastmem.h" namespace kudu { @@ -98,6 +99,26 @@ inline bool BitmapIsAllZero(const uint8_t *bitmap, size_t offset, size_t bitmap_ return !BitmapFindFirstSet(bitmap, offset, bitmap_size, &idx); } +// Returns true if the two bitmaps are equal. +// +// It is assumed that both bitmaps have 'bitmap_size' number of bits. +inline bool BitmapEquals(const uint8_t* bm1, const uint8_t* bm2, size_t bitmap_size) { + // Use memeq() to check all of the full bytes. + size_t num_full_bytes = bitmap_size >> 3; + if (!strings::memeq(bm1, bm2, num_full_bytes)) { +return false; + } + + // Check any remaining bits in one extra operation. + size_t num_rema
[2/2] kudu git commit: [catalog_manager] updated warning message
[catalog_manager] updated warning message Updated the warning message logged upon a failure to allocate an extra replica for a tablet: don't call the new replica a replacement because AsyncAddReplicaTask is used not only in re-replication scenarios, but in replica movement scenarios as well. Also, do not mention unsetting --raft_prepare_replacement_before_eviction since the 3-2-3 replica management scheme is deprecated at this point and is no better in the case of a whole tablet server failure. This is a follow-up to 1fcce42200d22597e7e69baa7232b4de93d5e2a3. There are no functional changes in this patch. Change-Id: Ifb6905dc1870acd34553187594008ba34781ce6d Reviewed-on: http://gerrit.cloudera.org:8080/11402 Tested-by: Kudu Jenkins Reviewed-by: Adar Dembo Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/953a09b8 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/953a09b8 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/953a09b8 Branch: refs/heads/master Commit: 953a09b826b4a0ccd379480f7aac441186f8bacb Parents: e357051 Author: Alexey Serbin Authored: Fri Sep 7 16:35:33 2018 -0700 Committer: Alexey Serbin Committed: Mon Sep 10 20:35:04 2018 + -- src/kudu/master/catalog_manager.cc | 27 +++ 1 file changed, 11 insertions(+), 16 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/953a09b8/src/kudu/master/catalog_manager.cc -- diff --git a/src/kudu/master/catalog_manager.cc b/src/kudu/master/catalog_manager.cc index 0dd8003..27e9722 100644 --- a/src/kudu/master/catalog_manager.cc +++ b/src/kudu/master/catalog_manager.cc @@ -3475,14 +3475,14 @@ bool AsyncAddReplicaTask::SendRequest(int attempt) { auto replacement_replica = SelectReplica(ts_descs, excluded, rng_); if (PREDICT_FALSE(!replacement_replica)) { -auto msg = Substitute("no candidate replacement replica found for tablet $0", +auto msg = Substitute("no extra replica candidate found for tablet $0", tablet_->ToString()); -// Check whether it's a situation when a replacement replica cannot be found +// Check whether it's a situation when an extra replica cannot be found // due to an inconsistency in cluster configuration. If the tablet has the -// replication factor of N, and the cluster is configured to use N->(N+1)->N -// replication scheme (see --raft_prepare_replacement_before_eviction flag), -// at least N+1 tablet servers should be registered to find a place -// for a replacement replica. +// replication factor of N, and the cluster is using the N->(N+1)->N +// replica management scheme (see --raft_prepare_replacement_before_eviction +// flag), at least N+1 tablet servers should be registered to find a place +// for an extra replica. TSDescriptorVector all_descriptors; master_->ts_manager()->GetAllDescriptors(&all_descriptors); const auto num_tservers_registered = all_descriptors.size(); @@ -3492,21 +3492,16 @@ bool AsyncAddReplicaTask::SendRequest(int attempt) { TableMetadataLock l(tablet_->table().get(), LockMode::READ); replication_factor = tablet_->table()->metadata().state().pb.num_replicas(); } -DCHECK_GE(replication_factor, 0); +DCHECK_GE(replication_factor, 1); const auto num_tservers_needed = FLAGS_raft_prepare_replacement_before_eviction ? replication_factor + 1 : replication_factor; if (num_tservers_registered < num_tservers_needed) { msg += Substitute( - "; the total number of registered tablet servers ($0) does not allow " - "for replacement of the failed replica: at least $1 tablet servers " - "are required", num_tservers_registered, num_tservers_needed); - if (FLAGS_raft_prepare_replacement_before_eviction && - num_tservers_registered == replication_factor) { -msg += - "; consider either adding an additional tablet server or running " - "the cluster with --raft_prepare_replacement_before_eviction=false"; - } + ": the total number of registered tablet servers ($0) does not allow " + "for adding an extra replica; consider bringing up more " + "to have at least $1 tablet servers up and running", + num_tservers_registered, num_tservers_needed); } KLOG_EVERY_N_SECS(WARNING, 60) << LogPrefix() << msg; return false;
[1/2] kudu git commit: [docs] Add "one client only" best practice for kudu-spark
Repository: kudu Updated Branches: refs/heads/master b552d9118 -> 953a09b82 [docs] Add "one client only" best practice for kudu-spark Change-Id: Ibaf369315b8627674ba64e6418d153568ded6fe8 Reviewed-on: http://gerrit.cloudera.org:8080/11409 Tested-by: Will Berkeley Reviewed-by: Alexey Serbin Tested-by: Kudu Jenkins Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/e3570519 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/e3570519 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/e3570519 Branch: refs/heads/master Commit: e3570519b200a0ffbd713798bc8aabd6f36ed3b7 Parents: b552d91 Author: Will Berkeley Authored: Mon Sep 10 10:45:30 2018 -0700 Committer: Will Berkeley Committed: Mon Sep 10 18:43:43 2018 + -- docs/developing.adoc | 17 + 1 file changed, 17 insertions(+) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/e3570519/docs/developing.adoc -- diff --git a/docs/developing.adoc b/docs/developing.adoc index 98db2ba..49d8c7e 100644 --- a/docs/developing.adoc +++ b/docs/developing.adoc @@ -217,6 +217,23 @@ mode, the submitting user must have an active Kerberos ticket granted through name and keytab location must be provided through the `--principal` and `--keytab` arguments to `spark2-submit`. +=== Spark Integration Best Practices + + Avoid multiple Kudu clients per cluster. + +One common Kudu-Spark coding error is instantiating extra `KuduClient` objects. +In kudu-spark, a `KuduClient` is owned by the `KuduContext`. Spark application code +should not create another `KuduClient` connecting to the same cluster. Instead, +application code should use the `KuduContext` to access a `KuduClient` using +`KuduContext#syncClient`. + +To diagnose multiple `KuduClient` instances in a Spark job, look for signs in +the logs of the master being overloaded by many `GetTableLocations` or +`GetTabletLocations` requests coming from different clients, usually around the +same time. This symptom is especially likely in Spark Streaming code, +where creating a `KuduClient` per task will result in periodic waves of master +requests from new clients. + === Spark Integration Known Issues and Limitations - Spark 2.2+ requires Java 8 at runtime even though Kudu Spark 2.x integration
[2/2] kudu git commit: [jepsen] set default admin operation timeout to 2 minutes
[jepsen] set default admin operation timeout to 2 minutes Increased the default admin operation timeout for the Kudu client used in Jepsen tests up to 2 minutes. That's to work around slow DNS resolutions (sometimes up to 30 seconds) that happen intermittently while running the tests at VMs provisioned in GCE. Change-Id: Iaa9ade5cf38b4f3247749fa33bd40ada9616f21f Reviewed-on: http://gerrit.cloudera.org:8080/11405 Reviewed-by: Hao Hao Tested-by: Kudu Jenkins Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/edde4beb Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/edde4beb Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/edde4beb Branch: refs/heads/master Commit: edde4bebea8e309a43e3316f78c45df6e6c8c9fe Parents: 38a4847 Author: Alexey Serbin Authored: Fri Sep 7 18:33:30 2018 -0700 Committer: Alexey Serbin Committed: Mon Sep 10 22:18:21 2018 + -- java/kudu-jepsen/src/main/clojure/jepsen/kudu/client.clj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/edde4beb/java/kudu-jepsen/src/main/clojure/jepsen/kudu/client.clj -- diff --git a/java/kudu-jepsen/src/main/clojure/jepsen/kudu/client.clj b/java/kudu-jepsen/src/main/clojure/jepsen/kudu/client.clj index 65b6e5a..c50201c 100644 --- a/java/kudu-jepsen/src/main/clojure/jepsen/kudu/client.clj +++ b/java/kudu-jepsen/src/main/clojure/jepsen/kudu/client.clj @@ -41,9 +41,9 @@ (defn sync-client "Builds and returns a new synchronous Kudu client." [master-addresses] - (let [builder (new KuduClient$KuduClientBuilder master-addresses) -client (. builder build)] -client)) + (let [builder (new KuduClient$KuduClientBuilder master-addresses)] +(.defaultAdminOperationTimeoutMs builder 12) +(.build builder))) (defn close-client [sync-client]
[1/2] kudu git commit: [gutil] add LookupOrEmplace()
Repository: kudu Updated Branches: refs/heads/master 953a09b82 -> edde4bebe [gutil] add LookupOrEmplace() Added LookupOrEmplace() which is similar to LookupOrInsert() but uses the r-value mechanics. Also, added unit tests for new functionality. Change-Id: I3a177f410c1d4ed33e6f44cb7d6c33d6141f84fc Reviewed-on: http://gerrit.cloudera.org:8080/11401 Reviewed-by: Adar Dembo Tested-by: Kudu Jenkins Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/38a48476 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/38a48476 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/38a48476 Branch: refs/heads/master Commit: 38a48476ee98e180d09e71cd0547119ff43423e4 Parents: 953a09b Author: Alexey Serbin Authored: Fri Sep 7 13:50:29 2018 -0700 Committer: Alexey Serbin Committed: Mon Sep 10 21:18:25 2018 + -- src/kudu/gutil/map-util.h | 15 ++ src/kudu/util/map-util-test.cc | 59 + 2 files changed, 74 insertions(+) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/38a48476/src/kudu/gutil/map-util.h -- diff --git a/src/kudu/gutil/map-util.h b/src/kudu/gutil/map-util.h index 82d266f..ef6baae 100644 --- a/src/kudu/gutil/map-util.h +++ b/src/kudu/gutil/map-util.h @@ -495,6 +495,21 @@ LookupOrInsert(Collection* const collection, collection, typename Collection::value_type(key, value)); } +// It's similar to LookupOrInsert() but uses the emplace and r-value mechanics +// to achieve the desired results. The constructor of the new element is called +// with exactly the same arguments as supplied to emplace, forwarded via +// std::forward(args). The element may be constructed even if there +// already is an element with the same key in the container, in which case the +// newly constructed element will be destroyed immediately. +// For details, see +// https://en.cppreference.com/w/cpp/container/map/emplace +// https://en.cppreference.com/w/cpp/container/unordered_map/emplace +template +typename Collection::mapped_type& +LookupOrEmplace(Collection* const collection, Args&&... args) { + return collection->emplace(std::forward(args)...).first->second; +} + // Counts the number of equivalent elements in the given "sequence", and stores // the results in "count_map" with element as the key and count as the value. // http://git-wip-us.apache.org/repos/asf/kudu/blob/38a48476/src/kudu/util/map-util-test.cc -- diff --git a/src/kudu/util/map-util-test.cc b/src/kudu/util/map-util-test.cc index b074007..78a3484 100644 --- a/src/kudu/util/map-util-test.cc +++ b/src/kudu/util/map-util-test.cc @@ -123,4 +123,63 @@ TEST(EmplaceTest, TestEmplace) { ASSERT_EQ("foobar", *FindOrDie(my_map, key2)); } +TEST(LookupOrEmplaceTest, IntMap) { + const string key = "mega"; + map int_map; + + { +const auto& val = LookupOrEmplace(&int_map, key, 0); +ASSERT_EQ(0, val); +auto* val_ptr = FindOrNull(int_map, key); +ASSERT_NE(nullptr, val_ptr); +ASSERT_EQ(0, *val_ptr); + } + + { +auto& val = LookupOrEmplace(&int_map, key, 10); +ASSERT_EQ(0, val); +++val; +auto* val_ptr = FindOrNull(int_map, key); +ASSERT_NE(nullptr, val_ptr); +ASSERT_EQ(1, *val_ptr); + } + + { +LookupOrEmplace(&int_map, key, 100) += 1000; +auto* val_ptr = FindOrNull(int_map, key); +ASSERT_NE(nullptr, val_ptr); +ASSERT_EQ(1001, *val_ptr); + } +} + +TEST(LookupOrEmplaceTest, UniquePtrMap) { + constexpr int key = 0; + const string ref_str = "turbo"; + map> uptr_map; + + { +unique_ptr val(new string(ref_str)); +const auto& lookup_val = LookupOrEmplace(&uptr_map, key, std::move(val)); +ASSERT_EQ(nullptr, val.get()); +ASSERT_NE(nullptr, lookup_val.get()); +ASSERT_EQ(ref_str, *lookup_val); + } + + { +unique_ptr val(new string("giga")); +auto& lookup_val = LookupOrEmplace(&uptr_map, key, std::move(val)); +ASSERT_NE(nullptr, lookup_val.get()); +ASSERT_EQ(ref_str, *lookup_val); +// Update the stored value. +*lookup_val = "giga"; + } + + { +unique_ptr val(new string(ref_str)); +const auto& lookup_val = LookupOrEmplace(&uptr_map, key, std::move(val)); +ASSERT_NE(nullptr, lookup_val.get()); +ASSERT_EQ("giga", *lookup_val); + } +} + } // namespace kudu
kudu git commit: [jepsen] a bunch of minor updates on README.adoc
Repository: kudu Updated Branches: refs/heads/master edde4bebe -> e709db1f3 [jepsen] a bunch of minor updates on README.adoc Added a link to the reference script for building Kudu and running Jepsen tests. Added information on how to quickly locate corresponding sections in the Jepsen output in case if an error happened during Jepsen tests run. Updated the command to build the kudu-jepnsen package. This patch does not contain any functional changes. Change-Id: Ibd4c7a522829e774f786a85d4bdfa01d0be553f5 Reviewed-on: http://gerrit.cloudera.org:8080/11414 Tested-by: Kudu Jenkins Reviewed-by: Hao Hao Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/e709db1f Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/e709db1f Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/e709db1f Branch: refs/heads/master Commit: e709db1f379e2375841a2c6357cd3b6a6ce4618b Parents: edde4be Author: Alexey Serbin Authored: Mon Sep 10 16:27:19 2018 -0700 Committer: Alexey Serbin Committed: Tue Sep 11 05:34:53 2018 + -- java/kudu-jepsen/README.adoc | 24 +++- 1 file changed, 19 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/e709db1f/java/kudu-jepsen/README.adoc -- diff --git a/java/kudu-jepsen/README.adoc b/java/kudu-jepsen/README.adoc index 7dfaaee..fc7663b 100644 --- a/java/kudu-jepsen/README.adoc +++ b/java/kudu-jepsen/README.adoc @@ -47,10 +47,11 @@ To build the library the following components are required: * JDK 8 * Apache Maven version 3.3.6 or higher -To build the project, run in the parent directory (i.e. `$KUDU_HOME/java`) +To build the project and install necessary Java packages into the local mvn +repository, run in the parent directory (i.e. `$KUDU_HOME/java`) [listing] -$ mvn clean compile test-compile -Pjepsen +$ mvn -Pjepsen -DskipTests clean install === Running @@ -101,6 +102,10 @@ In the Jepsen terminology, Kudu master and tserver nodes are playing *Jepsen DB node* roles. The machine where the above mentioned maven command is run plays *Jepsen control node* role. +=== A reference script to build Kudu and run Jepsen tests +The following link:../../src/kudu/scripts/jepsen.sh[Bourne-again shell script] +can be used as a reference to build Kudu from source and run Jepsen tests. + === Troubleshooting When Jepsen's analysis doesn't find inconsistencies in the history of operations it outputs the following in the end of a test: @@ -132,8 +137,16 @@ summary message below reports on 10 errors in 10 tests ran: 21:41:42 0 failures, 10 errors. To get more details, take a closer look at the output of `mvn clojure:run` -or at particular `jepsen.log` files under -$KUDU_HOME/java/kudu-jepsen/store/rw-register/ directories. +or at particular `jepsen.log` files in +`$KUDU_HOME/java/kudu-jepsen/store/rw-register/` directory. A +quick way to locate the corresponding section in the error log is to search for +`^ERROR in \(` regex pattern. An example of error message from Jepsen's output: +[listing] + +ERROR in (register-test-tserver-random-halves) (KuduException.java:110) +expected: (:valid? (:results (jepsen/run! (tcasefun opts + actual: org.apache.kudu.client.NonRecoverableException: can not complete before timeout: KuduRpc(method=IsCreateTableDone, tablet=null, attempt=28, DeadlineTracker(timeout=3, elapsed=28571), ... + The latter class represents more serious issue: a manifestation of non-linearizable history of operations. This is reported as _failure_ in the @@ -154,7 +167,8 @@ Analysis invalid! (ï¾à²¥çಥï¼ï¾ â»ââ» To troubleshoot, first it's necessary to find where the failed test stores the results: it should be one of the timestamp-named sub-directories (e.g. `20170109T071938.000-0800`) under -`$KUDU_HOME/java/kudu-jepsen/store/rw-register`. One of the possible ways +`$KUDU_HOME/java/kudu-jepsen/store/rw-register` in case of a linearizability +failure in one of the `rw-register` test scenarios. One of the possible ways to find the directory: [listing]
[2/2] kudu git commit: Remove unnecessary friendship
Remove unnecessary friendship This commit removes unnecessary friendship and declaration in client.h. Change-Id: Ia7c5f35a8b6a6ec2de1d57bbb4f68b53aae4e700 Reviewed-on: http://gerrit.cloudera.org:8080/11416 Tested-by: Kudu Jenkins Reviewed-by: Adar Dembo Reviewed-by: Alexey Serbin Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/f0716cc5 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/f0716cc5 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/f0716cc5 Branch: refs/heads/master Commit: f0716cc521554bc6c4b785d339ecf17b8ca983e6 Parents: f02a0c0 Author: Hao Hao Authored: Mon Sep 10 17:52:29 2018 -0700 Committer: Alexey Serbin Committed: Tue Sep 11 18:15:39 2018 + -- src/kudu/client/client.h | 5 - 1 file changed, 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/f0716cc5/src/kudu/client/client.h -- diff --git a/src/kudu/client/client.h b/src/kudu/client/client.h index 640afce..bfc2c2b 100644 --- a/src/kudu/client/client.h +++ b/src/kudu/client/client.h @@ -51,8 +51,6 @@ namespace kudu { class ClientStressTest_TestUniqueClientIds_Test; class KuduPartialRow; -class MasterHmsTest_TestAlterTable_Test; -class MasterHmsUpgradeTest_TestRenameExistingTables_Test; class MonoDelta; class PartitionSchema; class SecurityUnknownTskTest; @@ -1229,9 +1227,6 @@ class KUDU_EXPORT KuduTableAlterer { friend class KuduClient; - FRIEND_TEST(kudu::MasterHmsTest, TestAlterTable); - FRIEND_TEST(kudu::MasterHmsUpgradeTest, TestRenameExistingTables); - KuduTableAlterer(KuduClient* client, const std::string& name);
[1/2] kudu git commit: KUDU-2566: Enhance rowset tree pruning and stop copying strings
Repository: kudu Updated Branches: refs/heads/master e709db1f3 -> f0716cc52 KUDU-2566: Enhance rowset tree pruning and stop copying strings Three improvements: 1) Support open-ended intervals: Previously, the rowset tree could only compute overlap with a closed and bounded interval. This changeset adds the ability to find overlap for unbounded intervals as well. As a result, tablet iterators with a single primary key bound are more efficient because they do not iterate over rowsets that don't overlap with the key bound. 2) End up fetching one more rowset for the upper bound which is exclusive: This changeset also adds the ability to query the rowset tree using an exclusive upper bound, whereas previously only inclusive intervals were supported. This makes scans more efficient since primary key upper bounds are passed as exclusive bounds, so now rowsets that overlap only in the upper bound are excluded. 3) Perf improvement: Using raw slices instead of copying to strings while querying. The copying from slices to string is discarded, which could help to increase the query performance. Change-Id: I0e34b4169f93f3519f694c9bf86ca5295c318e79 Reviewed-on: http://gerrit.cloudera.org:8080/11381 Tested-by: Kudu Jenkins Reviewed-by: Will Berkeley Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/f02a0c0f Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/f02a0c0f Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/f02a0c0f Branch: refs/heads/master Commit: f02a0c0f29b6ef5468137869764119b1a8f8b3ae Parents: e709db1 Author: helifu Authored: Mon Sep 3 19:12:16 2018 +0800 Committer: Will Berkeley Committed: Tue Sep 11 16:25:48 2018 + -- src/kudu/tablet/rowset_tree-test.cc | 174 +-- src/kudu/tablet/rowset_tree.cc | 32 -- src/kudu/tablet/rowset_tree.h | 15 ++- src/kudu/tablet/tablet.cc | 22 ++-- src/kudu/util/interval_tree-inl.h | 44 src/kudu/util/interval_tree-test.cc | 132 +-- src/kudu/util/interval_tree.h | 12 ++- 7 files changed, 349 insertions(+), 82 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/f02a0c0f/src/kudu/tablet/rowset_tree-test.cc -- diff --git a/src/kudu/tablet/rowset_tree-test.cc b/src/kudu/tablet/rowset_tree-test.cc index 9cfb7f7..a248a5f 100644 --- a/src/kudu/tablet/rowset_tree-test.cc +++ b/src/kudu/tablet/rowset_tree-test.cc @@ -18,15 +18,19 @@ #include #include #include -#include #include +#include +#include +#include #include +#include #include #include #include "kudu/gutil/map-util.h" #include "kudu/gutil/stringprintf.h" +#include "kudu/gutil/strings/substitute.h" #include "kudu/tablet/mock-rowsets.h" #include "kudu/tablet/rowset.h" #include "kudu/tablet/rowset_tree.h" @@ -39,6 +43,7 @@ using std::shared_ptr; using std::string; using std::unordered_set; using std::vector; +using strings::Substitute; namespace kudu { namespace tablet { @@ -86,30 +91,185 @@ TEST_F(TestRowSetTree, TestTree) { ASSERT_EQ(vec[3].get(), out[0]); // MemRowSet ASSERT_EQ(vec[0].get(), out[1]); ASSERT_EQ(vec[1].get(), out[2]); + + // interval [3,4) overlaps 0-5, 3-5 and the MemRowSet + out.clear(); + tree.FindRowSetsIntersectingInterval(Slice("3"), Slice("4"), &out); + ASSERT_EQ(3, out.size()); + ASSERT_EQ(vec[3].get(), out[0]); + ASSERT_EQ(vec[0].get(), out[1]); + ASSERT_EQ(vec[1].get(), out[2]); + + // interval [0,2) overlaps 0-5 and the MemRowSet + out.clear(); + tree.FindRowSetsIntersectingInterval(Slice("0"), Slice("2"), &out); + ASSERT_EQ(2, out.size()); + ASSERT_EQ(vec[3].get(), out[0]); + ASSERT_EQ(vec[0].get(), out[1]); + + // interval [5,7) overlaps 0-5, 3-5, 5-9 and the MemRowSet + out.clear(); + tree.FindRowSetsIntersectingInterval(Slice("5"), Slice("7"), &out); + ASSERT_EQ(4, out.size()); + ASSERT_EQ(vec[3].get(), out[0]); + ASSERT_EQ(vec[0].get(), out[1]); + ASSERT_EQ(vec[1].get(), out[2]); + ASSERT_EQ(vec[2].get(), out[3]); + + // "3" overlaps 0-5, 3-5, and the MemRowSet. + out.clear(); + tree.FindRowSetsWithKeyInRange("3", &out); + ASSERT_EQ(3, out.size()); + ASSERT_EQ(vec[3].get(), out[0]); // MemRowSet + ASSERT_EQ(vec[0].get(), out[1]); + ASSERT_EQ(vec[1].get(), out[2]); - // interval (2,4) overlaps 0-5, 3-5 and the MemRowSet + // "5" overlaps 0-5, 3-5, 5-9, and the MemRowSet + out.clear(); + tree.FindRowSetsWithKeyInRange("5", &out); + ASSERT_EQ(4, out.size()); + ASSERT_EQ(vec[3].get(), out[0]); // MemRowSet + ASSERT_EQ(vec[0].get(), out[1]); + ASSERT_EQ(vec[1].get(), out[2]); + ASSERT_EQ(vec[2].get(), out[3]); + + // interval [0,5) overlaps 0-5, 3-5, and the MemRowSet +