This is an automated email from the ASF dual-hosted git repository.
tarmstrong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit ba509ad1809138844d52a8b5877880e5fc9cfe9a
Author: Tim Armstrong <tarmstr...@cloudera.com>
AuthorDate: Wed Nov 18 17:18:17 2020 -0800
IMPALA-9121: try to avoid ASAN error in hdfs-util-test
I couldn't discern the likely root cause of the ASAN error,
but have a hunch that it's a background thread accessing
some data structure that is being torn down as the
process exits.
The tests in this file are simple so there shouldn't really
be that much that can go wrong, except for the stuff
started by ExecEnv::Init().
I modified the test to only initialize the necessary configs
in ExecEnv, not start up the whole thing. Hopefully that
make the problem go away.
Testing:
Looped the test locally with ASAN.
Change-Id: Ic7b42be0f8b5d6c6a31095f9d1a278fd82bd500c
Reviewed-on: http://gerrit.cloudera.org:8080/16748
Reviewed-by: Impala Public Jenkins <impala-public-jenk...@cloudera.com>
Tested-by: Impala Public Jenkins <impala-public-jenk...@cloudera.com>
---
be/src/runtime/exec-env.cc | 5 ++++-
be/src/runtime/exec-env.h | 7 +++++++
be/src/util/hdfs-util-test.cc | 21 +++++++--------------
3 files changed, 18 insertions(+), 15 deletions(-)
diff --git a/be/src/runtime/exec-env.cc b/be/src/runtime/exec-env.cc
index 0dd78bc..48dd888 100644
--- a/be/src/runtime/exec-env.cc
+++ b/be/src/runtime/exec-env.cc
@@ -461,7 +461,11 @@ Status ExecEnv::Init() {
}
RETURN_IF_ERROR(admission_controller_->Init());
+ RETURN_IF_ERROR(InitHadoopConfig());
+ return Status::OK();
+}
+Status ExecEnv::InitHadoopConfig() {
// Get the fs.defaultFS value set in core-site.xml and assign it to
configured_defaultFs
TGetHadoopConfigRequest config_request;
config_request.__set_name(DEFAULT_FS);
@@ -472,7 +476,6 @@ Status ExecEnv::Init() {
} else {
default_fs_ = "hdfs://";
}
-
return Status::OK();
}
diff --git a/be/src/runtime/exec-env.h b/be/src/runtime/exec-env.h
index e96a070..c685613 100644
--- a/be/src/runtime/exec-env.h
+++ b/be/src/runtime/exec-env.h
@@ -27,6 +27,7 @@
#include "common/global-types.h"
#include "common/status.h"
#include "runtime/client-cache-types.h"
+#include "testutil/gtest-util.h"
#include "util/hdfs-bulk-ops-defs.h" // For declaration of HdfsOpThreadPool
#include "util/network-util.h"
#include "util/spinlock.h"
@@ -226,6 +227,9 @@ class ExecEnv {
friend class TestEnv;
friend class DataStreamTest;
+ // For access to InitHadoopConfig().
+ FRIEND_TEST(HdfsUtilTest, CheckFilesystemsMatch);
+
static ExecEnv* exec_env_;
bool is_fe_tests_ = false;
@@ -272,6 +276,9 @@ class ExecEnv {
/// this backend. Queries take up multiple slots only when mt_dop > 1.
int64_t admission_slots_;
+ /// Initialize ExecEnv based on Hadoop config from frontend.
+ Status InitHadoopConfig();
+
/// Choose a memory limit (returned in *bytes_limit) based on the
--mem_limit flag and
/// the memory available to the daemon process. Returns an error if the
memory limit is
/// invalid or another error is encountered that should prevent starting up
the daemon.
diff --git a/be/src/util/hdfs-util-test.cc b/be/src/util/hdfs-util-test.cc
index cbeac16..748ab23 100644
--- a/be/src/util/hdfs-util-test.cc
+++ b/be/src/util/hdfs-util-test.cc
@@ -24,18 +24,13 @@
#include "runtime/exec-env.h"
#include "service/fe-support.h"
-using namespace impala;
-
-DECLARE_bool(enable_webserver);
+namespace impala {
TEST(HdfsUtilTest, CheckFilesystemsMatch) {
- // We do not want to start the webserver.
- FLAGS_enable_webserver = false;
- ExecEnv* exec_env = new ExecEnv();
-
- // We do this to retrieve the default FS from the frontend.
- // It doesn't matter if initializing the ExecEnv fails.
- discard_result(exec_env->Init());
+ // We do this to retrieve the default FS from the frontend without starting
the rest
+ // of the ExecEnv services.
+ ExecEnv exec_env;
+ ASSERT_OK(exec_env.InitHadoopConfig());
// Tests with both paths qualified.
EXPECT_TRUE(FilesystemsMatch("s3a://dummybucket/temp_dir/temp_path",
@@ -65,7 +60,7 @@ TEST(HdfsUtilTest, CheckFilesystemsMatch) {
EXPECT_TRUE(FilesystemsMatch("tempdir/temppath", "tempdir2/temppath2"));
// Tests with one path qualified and the other unqualified.
- const char* default_fs = exec_env->default_fs().c_str();
+ const char* default_fs = exec_env.default_fs().c_str();
EXPECT_TRUE(FilesystemsMatch(default_fs, "temp_dir/temp_path"));
EXPECT_TRUE(FilesystemsMatch("temp_dir/temp_path", default_fs));
EXPECT_FALSE(FilesystemsMatch("badscheme://namenode/temp_dir/temp_path",
@@ -75,9 +70,6 @@ TEST(HdfsUtilTest, CheckFilesystemsMatch) {
}
TEST(HdfsUtilTest, CheckGetBaseName) {
- // We do not want to start the webserver.
- FLAGS_enable_webserver = false;
-
EXPECT_EQ(".", GetBaseName("s3a://"));
EXPECT_EQ(".", GetBaseName("s3a://dummybucket"));
EXPECT_EQ(".", GetBaseName("s3a://dummybucket/"));
@@ -111,3 +103,4 @@ TEST(HdfsUtilTest, CheckGetBaseName) {
EXPECT_EQ("c", GetBaseName("/c"));
EXPECT_EQ("c", GetBaseName("/a//b/c/"));
}
+};
