This is an automated email from the ASF dual-hosted git repository. stoty pushed a commit to branch 4.x in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/4.x by this push: new 43b56d4 PHOENIX-3499 Enable null value for quote character for CSVBulkLoad tool 43b56d4 is described below commit 43b56d40e9d38a4120865be4bac098236fb5fe26 Author: Sergey Soldatov <s...@apache.org> AuthorDate: Fri Nov 18 03:24:00 2016 -0800 PHOENIX-3499 Enable null value for quote character for CSVBulkLoad tool --- .../apache/phoenix/end2end/CsvBulkLoadToolIT.java | 37 +++++++++++++++++++++- .../phoenix/mapreduce/CsvBulkImportUtil.java | 10 +++--- .../apache/phoenix/mapreduce/CsvBulkLoadTool.java | 26 +++++++++------ .../phoenix/mapreduce/CsvToKeyValueMapper.java | 2 +- 4 files changed, 60 insertions(+), 15 deletions(-) diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CsvBulkLoadToolIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CsvBulkLoadToolIT.java index 6f10d71..2de2c82 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CsvBulkLoadToolIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CsvBulkLoadToolIT.java @@ -139,7 +139,6 @@ public class CsvBulkLoadToolIT extends BaseOwnClusterIT { stmt.close(); } - @Test public void testImportWithTabs() throws Exception { @@ -173,6 +172,42 @@ public class CsvBulkLoadToolIT extends BaseOwnClusterIT { } @Test + public void testImportWithTabsAndEmptyQuotes() throws Exception { + + Statement stmt = conn.createStatement(); + stmt.execute("CREATE TABLE TABLE8 (ID INTEGER NOT NULL PRIMARY KEY, " + + "NAME1 VARCHAR, NAME2 VARCHAR)"); + + FileSystem fs = FileSystem.get(getUtility().getConfiguration()); + FSDataOutputStream outputStream = fs.create(new Path("/tmp/input8.csv")); + PrintWriter printWriter = new PrintWriter(outputStream); + printWriter.println("1\t\"\\t123\tName 2a"); + printWriter.println("2\tName 2a\tName 2b"); + printWriter.close(); + + CsvBulkLoadTool csvBulkLoadTool = new CsvBulkLoadTool(); + csvBulkLoadTool.setConf(getUtility().getConfiguration()); + int exitCode = csvBulkLoadTool.run(new String[] { + "--input", "/tmp/input8.csv", + "--table", "table8", + "--zookeeper", zkQuorum, + "-q", "\"\"", + "-e", "\"\"", + "--delimiter", "\\t" + }); + assertEquals(0, exitCode); + + ResultSet rs = stmt.executeQuery("SELECT id, name1, name2 FROM table8 ORDER BY id"); + assertTrue(rs.next()); + assertEquals(1, rs.getInt(1)); + assertEquals("\"\\t123", rs.getString(2)); + assertEquals("Name 2a", rs.getString(3)); + + rs.close(); + stmt.close(); + } + + @Test public void testFullOptionImport() throws Exception { Statement stmt = conn.createStatement(); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvBulkImportUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvBulkImportUtil.java index ff9ff72..4ea2d21 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvBulkImportUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvBulkImportUtil.java @@ -41,8 +41,8 @@ public class CsvBulkImportUtil { * @param arrayDelimiter array delimiter character, can be null * @param binaryEncoding */ - public static void initCsvImportJob(Configuration conf, char fieldDelimiter, char quoteChar, - char escapeChar, String arrayDelimiter, String binaryEncoding) { + public static void initCsvImportJob(Configuration conf, char fieldDelimiter, Character quoteChar, + Character escapeChar, String arrayDelimiter, String binaryEncoding) { setChar(conf, CsvToKeyValueMapper.FIELD_DELIMITER_CONFKEY, fieldDelimiter); setChar(conf, CsvToKeyValueMapper.QUOTE_CHAR_CONFKEY, quoteChar); setChar(conf, CsvToKeyValueMapper.ESCAPE_CHAR_CONFKEY, escapeChar); @@ -67,8 +67,10 @@ public class CsvBulkImportUtil { } @VisibleForTesting - static void setChar(Configuration conf, String confKey, char charValue) { - conf.set(confKey, Base64.encodeBytes(Character.toString(charValue).getBytes())); + static void setChar(Configuration conf, String confKey, Character charValue) { + if(charValue!=null) { + conf.set(confKey, Base64.encodeBytes(charValue.toString().getBytes())); + } } @VisibleForTesting diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvBulkLoadTool.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvBulkLoadTool.java index 14b8c34..a00dcb8 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvBulkLoadTool.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvBulkLoadTool.java @@ -55,31 +55,39 @@ public class CsvBulkLoadTool extends AbstractBulkLoadTool { // we don't parse ZK_QUORUM_OPT here because we need it in order to // create the connection we need to build importColumns. - char delimiterChar = ','; + Character delimiterChar = ','; if (cmdLine.hasOption(DELIMITER_OPT.getOpt())) { - String delimString = StringEscapeUtils.unescapeJava(cmdLine.getOptionValue(DELIMITER_OPT.getOpt())); + String delimString = StringEscapeUtils.unescapeJava(cmdLine.getOptionValue + (DELIMITER_OPT.getOpt())); if (delimString.length() != 1) { throw new IllegalArgumentException("Illegal delimiter character: " + delimString); } delimiterChar = delimString.charAt(0); } - char quoteChar = '"'; + Character quoteChar = '"'; if (cmdLine.hasOption(QUOTE_OPT.getOpt())) { - String quoteString = cmdLine.getOptionValue(QUOTE_OPT.getOpt()); - if (quoteString.length() != 1) { + String quoteString = StringEscapeUtils.unescapeJava(cmdLine.getOptionValue(QUOTE_OPT + .getOpt())); + if(quoteString.length() == 0) { + quoteChar = null; + } else if (quoteString.length() != 1) { throw new IllegalArgumentException("Illegal quote character: " + quoteString); + } else { + quoteChar = quoteString.charAt(0); } - quoteChar = quoteString.charAt(0); } - char escapeChar = '\\'; + Character escapeChar = '\\'; if (cmdLine.hasOption(ESCAPE_OPT.getOpt())) { String escapeString = cmdLine.getOptionValue(ESCAPE_OPT.getOpt()); - if (escapeString.length() != 1) { + if(escapeString.length() == 0) { + escapeChar = null; + } else if (escapeString.length() != 1) { throw new IllegalArgumentException("Illegal escape character: " + escapeString); + } else { + escapeChar = escapeString.charAt(0); } - escapeChar = escapeString.charAt(0); } String binaryEncoding = null; diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvToKeyValueMapper.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvToKeyValueMapper.java index e5a6e5b..3b2293e 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvToKeyValueMapper.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/CsvToKeyValueMapper.java @@ -92,7 +92,7 @@ public class CsvToKeyValueMapper extends FormatToBytesWritableMapper<CSVRecord> static class CsvLineParser implements LineParser<CSVRecord> { private final CSVFormat csvFormat; - CsvLineParser(char fieldDelimiter, char quote, char escape) { + CsvLineParser(Character fieldDelimiter, Character quote, Character escape) { this.csvFormat = CSVFormat.DEFAULT .withIgnoreEmptyLines(true) .withDelimiter(fieldDelimiter)