This is an automated email from the ASF dual-hosted git repository. chinmayskulkarni pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/master by this push: new 218a71c PHOENIX-6026: Fix BackwardCompatibilityIT so it can run locally 218a71c is described below commit 218a71c07259a6ac4660335bf3636325e1ee138d Author: Chinmay Kulkarni <chinmayskulka...@gmail.com> AuthorDate: Fri Jul 17 20:20:27 2020 -0700 PHOENIX-6026: Fix BackwardCompatibilityIT so it can run locally --- .../phoenix/end2end/BackwardCompatibilityIT.java | 216 +++++++++++---------- .../it/{ => resources}/scripts/execute_query.sh | 0 2 files changed, 118 insertions(+), 98 deletions(-) diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityIT.java index cf28559..fde2f69 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityIT.java @@ -21,15 +21,18 @@ import static org.apache.phoenix.query.BaseTest.setUpConfigForMiniCluster; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertNotNull; import static org.junit.Assume.assumeFalse; import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.File; +import java.io.FileNotFoundException; import java.io.FileReader; import java.io.FileWriter; +import java.io.InputStream; import java.io.InputStreamReader; +import java.net.URL; import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; @@ -80,10 +83,12 @@ import com.google.common.collect.Lists; @Category(NeedsOwnMiniClusterTest.class) public class BackwardCompatibilityIT { - private static final String SQL_DIR = "src/it/resources/sql_files/"; - private static final String RESULT_DIR = "src/it/resources/gold_files/"; + private static final String SQL_DIR = "sql_files/"; + private static final String RESULT_DIR = "gold_files/"; private static final String COMPATIBLE_CLIENTS_JSON = - "src/it/resources/compatible_client_versions.json"; + "compatible_client_versions.json"; + private static final String BASH = "/bin/bash"; + private static final String EXECUTE_QUERY_SH = "scripts/execute_query.sh"; private static final String RESULT_PREFIX = "result_"; private static final String SQL_EXTENSION = ".sql"; private static final String TEXT_EXTENSION = ".txt"; @@ -93,6 +98,8 @@ public class BackwardCompatibilityIT { private static final String QUERY = "query"; private static final String QUERY_MORE = "query_more"; private static final String QUERY_ADD_DELETE = "query_add_delete"; + private static final String MVN_HOME = "maven.home"; + private static final String JAVA_TMP_DIR = "java.io.tmpdir"; private final String compatibleClientVersion; private static Configuration conf; @@ -119,6 +126,7 @@ public class BackwardCompatibilityIT { zkQuorum = "localhost:" + hbaseTestUtil.getZkCluster().getClientPort(); url = PhoenixRuntime.JDBC_PROTOCOL + PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR + zkQuorum; DriverManager.registerDriver(PhoenixDriver.INSTANCE); + checkForPreConditions(); } @After @@ -141,10 +149,14 @@ public class BackwardCompatibilityIT { List<String> clientVersions = Lists.newArrayList(); ObjectMapper mapper = new ObjectMapper(); mapper.configure(JsonParser.Feature.ALLOW_COMMENTS, true); - JsonNode jsonNode = mapper.readTree(new FileReader(COMPATIBLE_CLIENTS_JSON)); - JsonNode HBaseProfile = jsonNode.get(hbaseProfile); - for (final JsonNode clientVersion : HBaseProfile) { - clientVersions.add(clientVersion.textValue() + "-HBase-" + hbaseProfile); + try (InputStream inputStream = BackwardCompatibilityIT.class + .getClassLoader().getResourceAsStream(COMPATIBLE_CLIENTS_JSON)) { + assertNotNull(inputStream); + JsonNode jsonNode = mapper.readTree(inputStream); + JsonNode HBaseProfile = jsonNode.get(hbaseProfile); + for (final JsonNode clientVersion : HBaseProfile) { + clientVersions.add(clientVersion.textValue() + "-HBase-" + hbaseProfile); + } } return clientVersions; } @@ -155,15 +167,14 @@ public class BackwardCompatibilityIT { * 2. Old Client creates tables and inserts data * 3. New Client reads the data inserted by the old client * - * @throws Exception + * @throws Exception thrown if any errors encountered during query execution or file IO */ @Test public void testUpsertWithOldClient() throws Exception { - checkForPreConditions(); // Insert data with old client and read with new client executeQueryWithClientVersion(compatibleClientVersion, CREATE_ADD); executeQueriesWithCurrentVersion(QUERY); - assertTrue(compareOutput(CREATE_ADD, QUERY)); + assertExpectedOutput(CREATE_ADD, QUERY); } /** @@ -172,15 +183,14 @@ public class BackwardCompatibilityIT { * 2. New Client creates tables and inserts data * 3. Old Client reads the data inserted by the old client * - * @throws Exception + * @throws Exception thrown if any errors encountered during query execution or file IO */ @Test public void testSelectWithOldClient() throws Exception { - checkForPreConditions(); // Insert data with new client and read with old client executeQueriesWithCurrentVersion(CREATE_ADD); executeQueryWithClientVersion(compatibleClientVersion, QUERY); - assertTrue(compareOutput(CREATE_ADD, QUERY)); + assertExpectedOutput(CREATE_ADD, QUERY); } /** @@ -191,20 +201,19 @@ public class BackwardCompatibilityIT { * 4. New Client inserts more data into the tables created by old client * 5. Old Client reads the data inserted by new client * - * @throws Exception + * @throws Exception thrown if any errors encountered during query execution or file IO */ @Test public void testSelectUpsertWithNewClient() throws Exception { - checkForPreConditions(); // Insert data with old client and read with new client executeQueryWithClientVersion(compatibleClientVersion, CREATE_ADD); executeQueriesWithCurrentVersion(QUERY); - assertTrue(compareOutput(CREATE_ADD, QUERY)); + assertExpectedOutput(CREATE_ADD, QUERY); // Insert more data with new client and read with old client executeQueriesWithCurrentVersion(ADD_DATA); executeQueryWithClientVersion(compatibleClientVersion, QUERY_MORE); - assertTrue(compareOutput(ADD_DATA, QUERY_MORE)); + assertExpectedOutput(ADD_DATA, QUERY_MORE); } /** @@ -215,20 +224,19 @@ public class BackwardCompatibilityIT { * 4. Old Client inserts more data into the tables created by old client * 5. New Client reads the data inserted by new client * - * @throws Exception + * @throws Exception thrown if any errors encountered during query execution or file IO */ @Test public void testSelectUpsertWithOldClient() throws Exception { - checkForPreConditions(); // Insert data with new client and read with old client executeQueriesWithCurrentVersion(CREATE_ADD); executeQueryWithClientVersion(compatibleClientVersion, QUERY); - assertTrue(compareOutput(CREATE_ADD, QUERY)); + assertExpectedOutput(CREATE_ADD, QUERY); // Insert more data with old client and read with new client executeQueryWithClientVersion(compatibleClientVersion, ADD_DATA); executeQueriesWithCurrentVersion(QUERY_MORE); - assertTrue(compareOutput(ADD_DATA, QUERY_MORE)); + assertExpectedOutput(ADD_DATA, QUERY_MORE); } /** @@ -238,20 +246,19 @@ public class BackwardCompatibilityIT { * 3. New Client reads the data inserted by the old client * 4. Old Client creates and deletes the data * - * @throws Exception + * @throws Exception thrown if any errors encountered during query execution or file IO */ @Test public void testUpsertDeleteWithOldClient() throws Exception { - checkForPreConditions(); // Insert data with old client and read with new client executeQueryWithClientVersion(compatibleClientVersion, CREATE_ADD); executeQueriesWithCurrentVersion(QUERY); - assertTrue(compareOutput(CREATE_ADD, QUERY)); + assertExpectedOutput(CREATE_ADD, QUERY); // Deletes with the old client executeQueryWithClientVersion(compatibleClientVersion, ADD_DELETE); executeQueryWithClientVersion(compatibleClientVersion, QUERY_ADD_DELETE); - assertTrue(compareOutput(ADD_DELETE, QUERY_ADD_DELETE)); + assertExpectedOutput(ADD_DELETE, QUERY_ADD_DELETE); } /** @@ -261,20 +268,19 @@ public class BackwardCompatibilityIT { * 3. Old Client reads the data inserted by the old client * 4. New Client creates and deletes the data * - * @throws Exception + * @throws Exception thrown if any errors encountered during query execution or file IO */ @Test public void testUpsertDeleteWithNewClient() throws Exception { - checkForPreConditions(); // Insert data with old client and read with new client executeQueriesWithCurrentVersion(CREATE_ADD); executeQueryWithClientVersion(compatibleClientVersion, QUERY); - assertTrue(compareOutput(CREATE_ADD, QUERY)); + assertExpectedOutput(CREATE_ADD, QUERY); // Deletes with the new client executeQueriesWithCurrentVersion(ADD_DELETE); executeQueriesWithCurrentVersion(QUERY_ADD_DELETE); - assertTrue(compareOutput(ADD_DELETE, QUERY_ADD_DELETE)); + assertExpectedOutput(ADD_DELETE, QUERY_ADD_DELETE); } private void checkForPreConditions() throws Exception { @@ -284,32 +290,37 @@ public class BackwardCompatibilityIT { assumeFalse(compatibleClientVersion.contains(MetaDataProtocol.CURRENT_CLIENT_VERSION)); // Make sure that cluster is clean before test execution with no system tables try (org.apache.hadoop.hbase.client.Connection conn = - ConnectionFactory.createConnection(conf)) { - Admin admin = conn.getAdmin(); - assertFalse(admin.tableExists(TableName.valueOf(QueryConstants.SYSTEM_SCHEMA_NAME, + ConnectionFactory.createConnection(conf); + Admin admin = conn.getAdmin()) { + assertFalse(admin.tableExists(TableName.valueOf(QueryConstants.SYSTEM_SCHEMA_NAME, PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE))); - } + } } // Executes the queries listed in the operation file with a given client version private void executeQueryWithClientVersion(String clientVersion, String operation) throws Exception { - String BASH = "/bin/bash"; - String EXECUTE_QUERY_SH = "src/it/scripts/execute_query.sh"; - List<String> cmdParams = Lists.newArrayList(); cmdParams.add(BASH); - cmdParams.add(EXECUTE_QUERY_SH); + // Note that auto-commit is true for queries executed via SQLline + URL fileUrl = BackwardCompatibilityIT.class.getClassLoader().getResource(EXECUTE_QUERY_SH); + assertNotNull(fileUrl); + cmdParams.add(new File(fileUrl.getFile()).getAbsolutePath()); cmdParams.add(zkQuorum); cmdParams.add(clientVersion); - cmdParams.add(new File(SQL_DIR + operation + SQL_EXTENSION).getAbsolutePath()); - cmdParams.add( - new File(RESULT_DIR + RESULT_PREFIX + operation + TEXT_EXTENSION).getAbsolutePath()); - cmdParams.add(System.getProperty("java.io.tmpdir")); - - if (System.getProperty("maven.home") != null) { - cmdParams.add(System.getProperty("maven.home")); + fileUrl = BackwardCompatibilityIT.class.getClassLoader() + .getResource(SQL_DIR + operation + SQL_EXTENSION); + assertNotNull(fileUrl); + cmdParams.add(new File(fileUrl.getFile()).getAbsolutePath()); + fileUrl = BackwardCompatibilityIT.class.getClassLoader().getResource(RESULT_DIR); + assertNotNull(fileUrl); + cmdParams.add(new File(fileUrl.getFile()).getAbsolutePath() + "/" + + RESULT_PREFIX + operation + TEXT_EXTENSION); + cmdParams.add(System.getProperty(JAVA_TMP_DIR)); + + if (System.getProperty(MVN_HOME) != null) { + cmdParams.add(System.getProperty(MVN_HOME)); } ProcessBuilder pb = new ProcessBuilder(cmdParams); @@ -319,72 +330,77 @@ public class BackwardCompatibilityIT { Thread errorStreamThread = new Thread() { @Override public void run() { - try { - BufferedReader reader = new BufferedReader( - new InputStreamReader(p.getErrorStream())); - String line = null; + try (BufferedReader reader = new BufferedReader( + new InputStreamReader(p.getErrorStream()))) { + String line; while ((line = reader.readLine()) != null) { sb.append(line); } - reader.close(); } catch (final Exception e) { - e.printStackTrace(); + sb.append(e.getMessage()); } } }; errorStreamThread.start(); - assertEquals(sb.toString(), 0, p.waitFor()); + p.waitFor(); + assertEquals(sb.toString(), 0, p.exitValue()); } // Executes the SQL commands listed in the given operation file from the sql_files directory - private ResultSet executeQueriesWithCurrentVersion(String operation) throws Exception { + private void executeQueriesWithCurrentVersion(String operation) throws Exception { Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); try (Connection conn = DriverManager.getConnection(url, props)) { StringBuilder sb = new StringBuilder(); - BufferedReader reader = - new BufferedReader(new FileReader(SQL_DIR + operation + SQL_EXTENSION)); - String sqlCommand; - ResultSet rs = null; - while ((sqlCommand = reader.readLine()) != null) { - sqlCommand = sqlCommand.trim(); - if (sqlCommand.length() == 0 || sqlCommand.startsWith("/") || sqlCommand.startsWith("*")) continue; - sb.append(sqlCommand); + try (BufferedReader reader = + getBufferedReaderForResource(SQL_DIR + operation + SQL_EXTENSION)) { + String sqlCommand; + while ((sqlCommand = reader.readLine()) != null) { + sqlCommand = sqlCommand.trim(); + if (sqlCommand.length() == 0 || sqlCommand.startsWith("/") + || sqlCommand.startsWith("*")) { + continue; + } + sb.append(sqlCommand); + } } - reader.close(); - + ResultSet rs; String[] sqlCommands = sb.toString().split(";"); - try (BufferedWriter br = new BufferedWriter( - new FileWriter(RESULT_DIR + RESULT_PREFIX + operation + TEXT_EXTENSION))) { - for (int i = 0; i < sqlCommands.length; i++) { - PreparedStatement stmt = conn.prepareStatement(sqlCommands[i]); - stmt.execute(); - rs = stmt.getResultSet(); - if (rs != null) { - saveResultSet(rs, br); + + URL fileUrl = BackwardCompatibilityIT.class.getClassLoader().getResource(RESULT_DIR); + assertNotNull(fileUrl); + final String resultFile = new File(fileUrl.getFile()).getAbsolutePath() + "/" + + RESULT_PREFIX + operation + TEXT_EXTENSION; + try (BufferedWriter br = new BufferedWriter(new FileWriter(resultFile))) { + for (String command : sqlCommands) { + try (PreparedStatement stmt = conn.prepareStatement(command)) { + stmt.execute(); + rs = stmt.getResultSet(); + if (rs != null) { + saveResultSet(rs, br); + } } conn.commit(); } - return rs; } - } + } } // Saves the result set to a text file to be compared against the gold file for difference private void saveResultSet(ResultSet rs, BufferedWriter br) throws Exception { ResultSetMetaData rsm = rs.getMetaData(); int columnCount = rsm.getColumnCount(); - String row = formatStringWithQuotes(rsm.getColumnName(1)); + StringBuilder row = new StringBuilder(formatStringWithQuotes(rsm.getColumnName(1))); for (int i = 2; i <= columnCount; i++) { - row = row + "," + formatStringWithQuotes(rsm.getColumnName(i)); + row.append(",").append(formatStringWithQuotes(rsm.getColumnName(i))); } - br.write(row); + br.write(row.toString()); br.write("\n"); while (rs.next()) { - row = formatStringWithQuotes(rs.getString(1)); + row = new StringBuilder(formatStringWithQuotes(rs.getString(1))); for (int i = 2; i <= columnCount; i++) { - row = row + "," + formatStringWithQuotes(rs.getString(i)); + row.append(",").append(formatStringWithQuotes(rs.getString(i))); } - br.write(row); + br.write(row.toString()); br.write("\n"); } } @@ -393,35 +409,39 @@ public class BackwardCompatibilityIT { return (str != null) ? String.format("\'%s\'", str) : "\'\'"; } + private BufferedReader getBufferedReaderForResource(String relativePath) + throws FileNotFoundException { + URL fileUrl = getClass().getClassLoader().getResource(relativePath); + assertNotNull(fileUrl); + return new BufferedReader(new FileReader(new File(fileUrl.getFile()))); + } + // Compares the result file against the gold file to match for the expected output // for the given operation - private boolean compareOutput(String gold, String result) throws Exception { - BufferedReader goldFileReader = new BufferedReader(new FileReader( - new File(RESULT_DIR + "gold_query_" + gold + TEXT_EXTENSION))); - BufferedReader resultFileReader = new BufferedReader(new FileReader( - new File(RESULT_DIR + RESULT_PREFIX + result + TEXT_EXTENSION))); - + private void assertExpectedOutput(String gold, String result) throws Exception { List<String> resultFile = Lists.newArrayList(); List<String> goldFile = Lists.newArrayList(); - - String line = null; - while ((line = resultFileReader.readLine()) != null) { - resultFile.add(line.trim()); + String line; + try (BufferedReader resultFileReader = getBufferedReaderForResource( + RESULT_DIR + RESULT_PREFIX + result + TEXT_EXTENSION)) { + while ((line = resultFileReader.readLine()) != null) { + resultFile.add(line.trim()); + } } - resultFileReader.close(); - - while ((line = goldFileReader.readLine()) != null) { - line = line.trim(); - if ( !(line.isEmpty() || line.startsWith("*") || line.startsWith("/"))) { - goldFile.add(line); - } + try (BufferedReader goldFileReader = getBufferedReaderForResource( + RESULT_DIR + "gold_query_" + gold + TEXT_EXTENSION)) { + while ((line = goldFileReader.readLine()) != null) { + line = line.trim(); + if ( !(line.isEmpty() || line.startsWith("*") || line.startsWith("/"))) { + goldFile.add(line); + } + } } - goldFileReader.close(); // We take the first line in gold file and match against the result file to exclude any // other WARNING messages that comes as a result of the query execution int index = resultFile.indexOf(goldFile.get(0)); resultFile = resultFile.subList(index, resultFile.size()); - return resultFile.equals(goldFile); + assertEquals(resultFile, goldFile); } } \ No newline at end of file diff --git a/phoenix-core/src/it/scripts/execute_query.sh b/phoenix-core/src/it/resources/scripts/execute_query.sh similarity index 100% rename from phoenix-core/src/it/scripts/execute_query.sh rename to phoenix-core/src/it/resources/scripts/execute_query.sh