Repository: incubator-hawq Updated Branches: refs/heads/master 1a835a945 -> f04c9673f
HAWQ-1426. Fix bug that hawq extract meets error after the table was reorganized Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/f04c9673 Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/f04c9673 Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/f04c9673 Branch: refs/heads/master Commit: f04c9673ff094a7b794931b008d72ddd5bc4868a Parents: 1a835a9 Author: Chunling Wang <wangchunlin...@126.com> Authored: Fri Apr 7 16:28:22 2017 +0800 Committer: Lili Ma <ictmal...@gmail.com> Committed: Mon Apr 10 10:32:16 2017 +0800 ---------------------------------------------------------------------- .../ManagementTool/test_hawq_extract.cpp | 103 +++++++++++++++++++ .../feature/ManagementTool/test_hawq_extract.h | 46 +++++++++ tools/bin/hawqextract | 27 ++++- 3 files changed, 172 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f04c9673/src/test/feature/ManagementTool/test_hawq_extract.cpp ---------------------------------------------------------------------- diff --git a/src/test/feature/ManagementTool/test_hawq_extract.cpp b/src/test/feature/ManagementTool/test_hawq_extract.cpp new file mode 100644 index 0000000..5b6e5ee --- /dev/null +++ b/src/test/feature/ManagementTool/test_hawq_extract.cpp @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <string> + +#include "lib/command.h" +#include "lib/sql_util.h" +#include "lib/string_util.h" +#include "lib/hdfs_config.h" +#include "lib/file_replace.h" +#include "test_hawq_extract.h" + +#include "gtest/gtest.h" + +using std::string; +using hawq::test::SQLUtility; +using hawq::test::Command; +using hawq::test::HdfsConfig; + +TEST_F(TestHawqExtract, TestExtractAfterReorganize) { + SQLUtility util; + util.execute("drop table if exists table_extract_ao;"); + util.execute("drop table if exists table_extract_parquet;"); + util.execute("drop table if exists table_extract_ao_new;"); + util.execute("drop table if exists table_extract_parquet_new;"); + // create an ao and a parquet table and insert data + util.execute("CREATE TABLE table_extract_ao(id int);"); + util.execute("CREATE TABLE table_extract_parquet(id int) WITH (APPENDONLY=true, ORIENTATION=parquet);"); + util.execute("insert into table_extract_ao values(1),(2),(3);"); + util.execute("insert into table_extract_parquet values(1),(2),(3);"); + + // reorganize table + util.execute("alter table table_extract_ao set with (reorganize=true);"); + util.execute("alter table table_extract_parquet set with (reorganize=true);"); + + // extract table to .yml + EXPECT_EQ(0, Command::getCommandStatus("hawq extract -d " + (string) HAWQ_DB + " -o table_extract_ao.yml testhawqextract_testextractafterreorganize.table_extract_ao")); + EXPECT_EQ(0, Command::getCommandStatus("hawq extract -d " + (string) HAWQ_DB + " -o table_extract_parquet.yml testhawqextract_testextractafterreorganize.table_extract_parquet")); + + // register .yml to new table + EXPECT_EQ(0, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c table_extract_ao.yml testhawqextract_testextractafterreorganize.table_extract_ao_new")); + EXPECT_EQ(0, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c table_extract_parquet.yml testhawqextract_testextractafterreorganize.table_extract_parquet_new")); + util.query("select * from table_extract_ao_new;", 3); + util.query("select * from table_extract_parquet_new;", 3); + + EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("rm -rf table_extract_ao.yml"))); + EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("rm -rf table_extract_parquet.yml"))); + util.execute("drop table table_extract_ao;"); + util.execute("drop table table_extract_parquet;"); + util.execute("drop table table_extract_ao_new;"); + util.execute("drop table table_extract_parquet_new;"); +} + +TEST_F(TestHawqExtract, TestExtractAfterTruncate) { + SQLUtility util; + util.execute("drop table if exists table_extract_ao;"); + util.execute("drop table if exists table_extract_parquet;"); + util.execute("drop table if exists table_extract_ao_new;"); + util.execute("drop table if exists table_extract_parquet_new;"); + // create an ao and a parquet table and insert data + util.execute("CREATE TABLE table_extract_ao(id int);"); + util.execute("CREATE TABLE table_extract_parquet(id int) WITH (APPENDONLY=true, ORIENTATION=parquet);"); + util.execute("insert into table_extract_ao values(1),(2),(3);"); + util.execute("insert into table_extract_parquet values(1),(2),(3);"); + + // truncate table and insert again + util.execute("TRUNCATE table table_extract_ao;"); + util.execute("TRUNCATE table table_extract_parquet;"); + util.execute("insert into table_extract_ao values(1),(2),(3);"); + util.execute("insert into table_extract_parquet values(1),(2),(3);"); + + // extract table + EXPECT_EQ(0, Command::getCommandStatus("hawq extract -d " + (string) HAWQ_DB + " -o table_extract_ao.yml testhawqextract_testextractaftertruncate.table_extract_ao")); + EXPECT_EQ(0, Command::getCommandStatus("hawq extract -d " + (string) HAWQ_DB + " -o table_extract_parquet.yml testhawqextract_testextractaftertruncate.table_extract_parquet")); + + // register .yml to new table + EXPECT_EQ(0, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c table_extract_ao.yml testhawqextract_testextractaftertruncate.table_extract_ao_new")); + EXPECT_EQ(0, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c table_extract_parquet.yml testhawqextract_testextractaftertruncate.table_extract_parquet_new")); + util.query("select * from table_extract_ao_new;", 3); + util.query("select * from table_extract_parquet_new;", 3); + + EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("rm -rf table_extract_ao.yml"))); + EXPECT_EQ(0, Command::getCommandStatus(hawq::test::stringFormat("rm -rf table_extract_parquet.yml"))); + util.execute("drop table table_extract_ao;"); + util.execute("drop table table_extract_parquet;"); + util.execute("drop table table_extract_ao_new;"); + util.execute("drop table table_extract_parquet_new;"); +} http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f04c9673/src/test/feature/ManagementTool/test_hawq_extract.h ---------------------------------------------------------------------- diff --git a/src/test/feature/ManagementTool/test_hawq_extract.h b/src/test/feature/ManagementTool/test_hawq_extract.h new file mode 100644 index 0000000..fc65be1 --- /dev/null +++ b/src/test/feature/ManagementTool/test_hawq_extract.h @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TEST_HAWQ_EXTRACT_H +#define TEST_HAWQ_EXTRACT_H + +#include <string> +#include <pwd.h> +#include <fstream> +#include "lib/hdfs_config.h" +#include "gtest/gtest.h" + +class TestHawqExtract: public ::testing::Test { + public: + TestHawqExtract() { + std::string user = HAWQ_USER; + if(user.empty()) { + struct passwd *pw; + uid_t uid = geteuid(); + pw = getpwuid(uid); + user.assign(pw->pw_name); + } + conn.reset(new hawq::test::PSQL(HAWQ_DB, HAWQ_HOST, HAWQ_PORT, user, HAWQ_PASSWORD)); + } + ~TestHawqExtract() {} + + private: + std::unique_ptr<hawq::test::PSQL> conn; +}; + +#endif http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f04c9673/tools/bin/hawqextract ---------------------------------------------------------------------- diff --git a/tools/bin/hawqextract b/tools/bin/hawqextract index 255db84..5ff2364 100644 --- a/tools/bin/hawqextract +++ b/tools/bin/hawqextract @@ -145,6 +145,23 @@ class GpMetadataAccessor: rows = self.exec_query(qry) return [r['fselocation'] for r in rows] + def get_seg_name(self, oid): + ''' + Return relname of pg_aoseg_`oid` table + + Example: + >>> accessor.get_seg_name(35709) + >>> [{'relname':'pg_aoseg_35709'}] + ''' + qry = """ + SELECT pg_class2.relname + FROM pg_class as pg_class1, pg_appendonly, pg_class as pg_class2 + WHERE pg_class1.oid = %d + AND pg_class1.oid = pg_appendonly.relid + AND pg_appendonly.segrelid = pg_class2.oid; + """ % oid + return self.exec_query(qry) + def get_aoseg_files(self, oid): ''' Return rows in pg_aoseg_`oid` table @@ -155,12 +172,13 @@ class GpMetadataAccessor: ... {'fileno':'2', 'filesize':'880', 'tupcount':'27', 'varblockcount':'3', eofuncompressed:'880'}, ... {'fileno':'3', 'filesize':'160', 'tupcount':'5', 'varblockcount':'2', eofuncompressed:'160'}] ''' + seg_name = self.get_seg_name(oid)[0]['relname'] qry = """ SELECT segno as fileno, eof as filesize, tupcount as tupcount, varblockcount as varblockcount, eofuncompressed as eofuncompressed - FROM pg_aoseg.pg_aoseg_%d + FROM pg_aoseg.%s ORDER by fileno; - """ % oid + """ % seg_name return self.exec_query(qry) def get_paqseg_files(self, oid): @@ -174,11 +192,12 @@ class GpMetadataAccessor: ... {'fileno':'2', 'filesize':'880', 'tupcount':'27', 'eofuncompressed':'880'}, ... {'fileno':'3', 'filesize':'160', 'tupcount':'5', 'eofuncompressed':'160'}] ''' + seg_name = self.get_seg_name(oid)[0]['relname'] qry = """ SELECT segno as fileno, eof as filesize, tupcount, eofuncompressed - FROM pg_aoseg.pg_paqseg_%d + FROM pg_aoseg.%s ORDER by fileno; - """ % oid + """ % seg_name return self.exec_query(qry) def get_pgclass(self, nspname, relname):
