This is an automated email from the ASF dual-hosted git repository.
zabetak pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 6cd53ed2bbb HIVE-27747: Generalize TestSchemaToolForMetastore to run
on every supported DBMS (Stamatis Zampetakis reviewed by Zhihua Deng)
6cd53ed2bbb is described below
commit 6cd53ed2bbbe8ff1f5e52fcfd15e2d55b7ee393f
Author: Stamatis Zampetakis <zabe...@gmail.com>
AuthorDate: Wed Sep 27 22:04:23 2023 +0200
HIVE-27747: Generalize TestSchemaToolForMetastore to run on every supported
DBMS (Stamatis Zampetakis reviewed by Zhihua Deng)
The goal is to make the tests run on every supported metastore DBMS (not
only Derby), to increase code coverage for the SchemaTool. An overview of the
changes can be seen below.
1. Parameterize the test using all available dockerized DBMS (except
Mariadb which currently fails due to HIVE-27749)
2. Refactor hardcoded references to derby with the appropriate database type
3. Add missing semicolons in some statements in the test scripts
4. Use appropriate DDL statements for renaming a table based on the
underlying database
5. Post-process DML statements in test scripts to:
* quote table/column identifiers when necessary (only Postgres at the
moment)
* pick appropriate literal for indicating false in boolean/bit datatype
based on the database
Close apache/hive#4754
---
.../schematool/TestSchemaToolForMetastore.java | 149 +++++++++++++++------
1 file changed, 110 insertions(+), 39 deletions(-)
diff --git
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/tools/schematool/TestSchemaToolForMetastore.java
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/tools/schematool/TestSchemaToolForMetastore.java
index f6a029a4470..e82b943915b 100644
---
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/tools/schematool/TestSchemaToolForMetastore.java
+++
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/tools/schematool/TestSchemaToolForMetastore.java
@@ -29,10 +29,14 @@ import java.net.URI;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.SQLException;
-import java.util.Random;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Pattern;
import org.apache.commons.dbcp2.DelegatingConnection;
-import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.text.StrTokenizer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.metastore.HiveMetaException;
@@ -40,39 +44,64 @@ import
org.apache.hadoop.hive.metastore.IMetaStoreSchemaInfo;
import org.apache.hadoop.hive.metastore.MetaStoreSchemaInfoFactory;
import org.apache.hadoop.hive.metastore.annotation.MetastoreCheckinTest;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
-import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
+import org.apache.hadoop.hive.metastore.dbinstall.rules.DatabaseRule;
+import org.apache.hadoop.hive.metastore.dbinstall.rules.Derby;
+import org.apache.hadoop.hive.metastore.dbinstall.rules.Mssql;
+import org.apache.hadoop.hive.metastore.dbinstall.rules.Mysql;
+import org.apache.hadoop.hive.metastore.dbinstall.rules.Oracle;
+import org.apache.hadoop.hive.metastore.dbinstall.rules.Postgres;
+
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import static java.lang.String.format;
@Category(MetastoreCheckinTest.class)
+@RunWith(Parameterized.class)
public class TestSchemaToolForMetastore {
+ private static final Pattern IDENTIFIER = Pattern.compile("[A-Z_]+");
+
private MetastoreSchemaTool schemaTool;
private Connection conn;
private Configuration conf;
- private String testMetastoreDB;
+ private final DatabaseRule dbms;
private PrintStream errStream;
private PrintStream outStream;
- private String argsBase;
private SchemaToolTaskValidate validator;
+ public TestSchemaToolForMetastore(DatabaseRule dbms){
+ this.dbms = dbms;
+ }
+
+ @Parameterized.Parameters(name = "{0}")
+ public static Collection<Object[]> databases() {
+ List<Object[]> dbs = new ArrayList<>();
+ dbs.add(new Object[] { new Derby(true) });
+ dbs.add(new Object[] { new Mysql() });
+ dbs.add(new Object[] { new Oracle() });
+ dbs.add(new Object[] { new Postgres() });
+// dbs.add(new Object[] { new Mariadb() }); Disabled due to HIVE-27749
+ dbs.add(new Object[] { new Mssql() });
+ return dbs;
+ }
+
@Before
- public void setUp() throws HiveMetaException, IOException {
- testMetastoreDB = System.getProperty("java.io.tmpdir") +
- File.separator + "test_metastore-" + new Random().nextInt();
- System.setProperty(ConfVars.CONNECT_URL_KEY.toString(),
- "jdbc:derby:" + testMetastoreDB + ";create=true");
+ public void setUp() throws Exception {
+ dbms.before();
+ dbms.createUser();
conf = MetastoreConf.newMetastoreConf();
schemaTool = new MetastoreSchemaTool();
schemaTool.init(System.getProperty("test.tmp.dir", "target/tmp"),
- new String[]{"-dbType", "derby", "--info"}, null, conf);
- String userName = MetastoreConf.getVar(schemaTool.getConf(),
ConfVars.CONNECTION_USER_NAME);
- String passWord = MetastoreConf.getPassword(schemaTool.getConf(),
ConfVars.PWD);
- schemaTool.setUserName(userName);
- schemaTool.setPassWord(passWord);
- argsBase = "-dbType derby -userName " + userName + " -passWord " +
passWord + " ";
+ new String[]{"-dbType", dbms.getDbType(), "--info"}, null, conf);
+ schemaTool.setUserName(dbms.getHiveUser());
+ schemaTool.setPassWord(dbms.getHivePassword());
+ schemaTool.setUrl(dbms.getJdbcUrl());
+ schemaTool.setDriver(dbms.getJdbcDriver());
System.setProperty("beeLine.system.exit", "true");
errStream = System.err;
outStream = System.out;
@@ -84,15 +113,12 @@ public class TestSchemaToolForMetastore {
@After
public void tearDown() throws IOException, SQLException {
- File metaStoreDir = new File(testMetastoreDB);
- if (metaStoreDir.exists()) {
- FileUtils.forceDeleteOnExit(metaStoreDir);
- }
System.setOut(outStream);
System.setErr(errStream);
if (conn != null) {
conn.close();
}
+ dbms.after();
}
/*
@@ -147,23 +173,27 @@ public class TestSchemaToolForMetastore {
Assert.assertTrue(isValid);
// Simulate a missing table scenario by renaming a couple of tables
- String[] scripts = new String[] {
- "RENAME TABLE SEQUENCE_TABLE to SEQUENCE_TABLE_RENAMED;",
- "RENAME TABLE NUCLEUS_TABLES to NUCLEUS_TABLES_RENAMED;"
- };
+ Map<String, String> tblRenames = new HashMap<>();
+ tblRenames.put("SEQUENCE_TABLE", "SEQUENCE_TABLE_RENAMED");
+ if (!dbms.getDbType().equals("mssql")) {
+ // HIVE-27748: NUCLEUS_TABLES DDLs are missing from MSSQL metastore
installation scripts
+ tblRenames.put("NUCLEUS_TABLES", "NUCLEUS_TABLES_RENAMED");
+ }
+ String[] deleteScripts = new String[tblRenames.size()];
+ String[] restoreScripts = new String[tblRenames.size()];
+ int i = 0;
+ for (Map.Entry<String, String> namePair : tblRenames.entrySet()) {
+ deleteScripts[i] = renameTableStmt(namePair.getKey(),
namePair.getValue());
+ restoreScripts[i] = renameTableStmt(namePair.getValue(),
namePair.getKey());
+ i++;
+ }
- File scriptFile = generateTestScript(scripts);
+ File scriptFile = generateTestScript(deleteScripts);
schemaTool.execSql(scriptFile.getPath());
isValid = validator.validateSchemaTables(conn);
Assert.assertFalse(isValid);
- // Restored the renamed tables
- scripts = new String[] {
- "RENAME TABLE SEQUENCE_TABLE_RENAMED to SEQUENCE_TABLE;",
- "RENAME TABLE NUCLEUS_TABLES_RENAMED to NUCLEUS_TABLES;"
- };
-
- scriptFile = generateTestScript(scripts);
+ scriptFile = generateTestScript(restoreScripts);
schemaTool.execSql(scriptFile.getPath());
isValid = validator.validateSchemaTables(conn);
Assert.assertTrue(isValid);
@@ -248,7 +278,7 @@ public class TestSchemaToolForMetastore {
@Test
public void testSchemaInit() throws Exception {
IMetaStoreSchemaInfo metastoreSchemaInfo =
MetaStoreSchemaInfoFactory.get(conf,
- System.getProperty("test.tmp.dir", "target/tmp"), "derby");
+ System.getProperty("test.tmp.dir", "target/tmp"), dbms.getDbType());
execute(new SchemaToolTaskInit(), "-initSchemaTo " +
metastoreSchemaInfo.getHiveSchemaVersion());
schemaTool.verifySchemaVersion();
}
@@ -335,15 +365,16 @@ public class TestSchemaToolForMetastore {
"Hive operations shouldn't pass with older version schema");
}
+ String db = dbms.getDbType();
// Generate dummy pre-upgrade script with errors
String invalidPreUpgradeScript = writeDummyPreUpgradeScript(
- 0, "upgrade-2.3.0-to-3.0.0.derby.sql", "foo bar;");
+ 0, "upgrade-2.3.0-to-3.0.0."+db+".sql", "foo bar;");
// Generate dummy pre-upgrade scripts with valid SQL
String validPreUpgradeScript0 = writeDummyPreUpgradeScript(
- 1, "upgrade-2.3.0-to-3.0.0.derby.sql",
+ 1, "upgrade-2.3.0-to-3.0.0."+db+".sql",
"CREATE TABLE schema_test0 (id integer);");
String validPreUpgradeScript1 = writeDummyPreUpgradeScript(
- 2, "upgrade-2.3.0-to-3.0.0.derby.sql",
+ 2, "upgrade-2.3.0-to-3.0.0."+db+".sql",
"CREATE TABLE schema_test1 (id integer);");
// Capture system out and err
@@ -473,6 +504,8 @@ public class TestSchemaToolForMetastore {
FileWriter fstream = new FileWriter(testScriptFile.getPath());
BufferedWriter out = new BufferedWriter(fstream);
for (String line: stmts) {
+ line = quoteIdentifiers(line);
+ line = line.replaceAll("'[Nn]'", booleanFalse());
out.write(line);
out.newLine();
}
@@ -480,13 +513,49 @@ public class TestSchemaToolForMetastore {
return testScriptFile;
}
+ private String quoteIdentifiers(String line) {
+ if (!dbms.getDbType().equalsIgnoreCase("postgres")) {
+ return line;
+ }
+ String idWithQuote = "\"$0\"";
+ String[] part = line.split("values");
+ if (part.length == 2) {
+ return IDENTIFIER.matcher(part[0]).replaceAll(idWithQuote) + " values "
+ part[1];
+ } else {
+ return IDENTIFIER.matcher(line).replaceAll(idWithQuote);
+ }
+ }
+
+ private String booleanFalse() {
+ switch (dbms.getDbType()) {
+ case "derby":
+ return "'N'";
+ case "postgres":
+ return "'0'";
+ default:
+ return "0";
+ }
+ }
+
+ private String renameTableStmt(String oldName, String newName) {
+ switch (dbms.getDbType()) {
+ case "mssql":
+ return format("exec sp_rename '%s', '%s';", oldName, newName);
+ case "postgres":
+ case "oracle":
+ return format("alter table %s rename to %s;", oldName, newName);
+ default:
+ return format("rename table %s to %s;", oldName, newName);
+ }
+ }
+
private void validateMetastoreDbPropertiesTable() throws HiveMetaException,
IOException {
boolean isValid = (boolean) validator.validateSchemaTables(conn);
Assert.assertTrue(isValid);
// adding same property key twice should throw unique key constraint
violation exception
String[] scripts = new String[] {
- "insert into METASTORE_DB_PROPERTIES values ('guid', 'test-uuid-1',
'dummy uuid 1')",
- "insert into METASTORE_DB_PROPERTIES values ('guid', 'test-uuid-2',
'dummy uuid 2')", };
+ "insert into METASTORE_DB_PROPERTIES values ('guid', 'test-uuid-1',
'dummy uuid 1');",
+ "insert into METASTORE_DB_PROPERTIES values ('guid', 'test-uuid-2',
'dummy uuid 2');", };
File scriptFile = generateTestScript(scripts);
Exception ex = null;
try {
@@ -505,7 +574,7 @@ public class TestSchemaToolForMetastore {
String preUpgradeScript = "pre-" + index + "-" + upgradeScriptName;
String dummyPreScriptPath = System.getProperty("test.tmp.dir",
"target/tmp") +
File.separatorChar + "scripts" + File.separatorChar + "metastore" +
- File.separatorChar + "upgrade" + File.separatorChar + "derby" +
+ File.separatorChar + "upgrade" + File.separatorChar + dbms.getDbType()
+
File.separatorChar + preUpgradeScript;
FileWriter fstream = new FileWriter(dummyPreScriptPath);
BufferedWriter out = new BufferedWriter(fstream);
@@ -547,6 +616,8 @@ public class TestSchemaToolForMetastore {
}
private void execute(SchemaToolTask task, String taskArgs) throws
HiveMetaException {
+ String argsBase =
+ format("-dbType %s -userName %s -passWord %s ", dbms.getDbType(),
dbms.getHiveUser(), dbms.getHivePassword());
try {
StrTokenizer tokenizer = new StrTokenizer(argsBase + taskArgs, ' ',
'\"');
SchemaToolCommandLine cl = new
SchemaToolCommandLine(tokenizer.getTokenArray(), null);
