This is an automated email from the ASF dual-hosted git repository. zabetak pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push: new 2bdec6f HIVE-26015: CREATE HBase table fails when SERDEPROPERTIES contain special characters (Steve Carlin, reviewed by Alessandro Solimando, Stamatis Zampetakis) 2bdec6f is described below commit 2bdec6f63d217cb42720fd7fe5cee804a2e5803c Author: Steve Carlin <scar...@cloudera.com> AuthorDate: Tue Mar 8 12:29:04 2022 -0800 HIVE-26015: CREATE HBase table fails when SERDEPROPERTIES contain special characters (Steve Carlin, reviewed by Alessandro Solimando, Stamatis Zampetakis) Fields in the Serde Properties can have a hash tag (#) in it, so the URI needs to be URLEncoded. Closes #3084 --- .../hadoop/hive/hbase/HBaseStorageHandler.java | 31 ++++++--- .../hadoop/hive/hbase/TestHBaseStorageHandler.java | 76 ++++++++++++++++++++++ 2 files changed, 99 insertions(+), 8 deletions(-) diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java index 302c09c..03d455f 100644 --- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java +++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java @@ -19,8 +19,10 @@ package org.apache.hadoop.hive.hbase; import java.io.IOException; +import java.io.UnsupportedEncodingException; import java.net.URI; import java.net.URISyntaxException; +import java.net.URLEncoder; import java.util.ArrayList; import java.util.LinkedHashSet; import java.util.List; @@ -293,14 +295,27 @@ public class HBaseStorageHandler extends DefaultStorageHandler public URI getURIForAuth(Table table) throws URISyntaxException { Map<String, String> tableProperties = HiveCustomStorageHandlerUtils.getTableProperties(table); hbaseConf = getConf(); - String hbase_host = tableProperties.containsKey(HBASE_HOST_NAME)? tableProperties.get(HBASE_HOST_NAME) : hbaseConf.get(HBASE_HOST_NAME); - String hbase_port = tableProperties.containsKey(HBASE_CLIENT_PORT)? tableProperties.get(HBASE_CLIENT_PORT) : hbaseConf.get(HBASE_CLIENT_PORT); - String table_name = tableProperties.getOrDefault(HBaseSerDe.HBASE_TABLE_NAME, null); - String column_family = tableProperties.getOrDefault(HBaseSerDe.HBASE_COLUMNS_MAPPING, null); - if (column_family != null) - return new URI(HBASE_PREFIX+"//"+hbase_host+":"+hbase_port+"/"+table_name+"/"+column_family); - else - return new URI(HBASE_PREFIX+"//"+hbase_host+":"+hbase_port+"/"+table_name); + String hbase_host = tableProperties.getOrDefault(HBASE_HOST_NAME, + hbaseConf.get(HBASE_HOST_NAME)); + String hbase_port = tableProperties.getOrDefault(HBASE_CLIENT_PORT, + hbaseConf.get(HBASE_CLIENT_PORT)); + String table_name = encodeString(tableProperties.getOrDefault(HBaseSerDe.HBASE_TABLE_NAME, + null)); + String column_family = encodeString(tableProperties.getOrDefault( + HBaseSerDe.HBASE_COLUMNS_MAPPING, null)); + String URIString = HBASE_PREFIX + "//" + hbase_host + ":" + hbase_port + "/" + table_name; + if (column_family != null) { + URIString += "/" + column_family; + } + return new URI(URIString); + } + + private static String encodeString(String rawString) throws URISyntaxException { + try { + return rawString != null ? URLEncoder.encode(rawString, "UTF-8"): null; + } catch (UnsupportedEncodingException e) { + throw new URISyntaxException(rawString, "Could not URLEncode string"); + } } /** diff --git a/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestHBaseStorageHandler.java b/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestHBaseStorageHandler.java index 8c8702a..b12df94 100644 --- a/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestHBaseStorageHandler.java +++ b/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestHBaseStorageHandler.java @@ -17,9 +17,16 @@ */ package org.apache.hadoop.hive.hbase; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.HashMap; +import java.util.Map; import java.util.Properties; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.api.SerDeInfo; +import org.apache.hadoop.hive.metastore.api.StorageDescriptor; +import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.mapred.JobConf; import org.junit.Assert; @@ -46,6 +53,58 @@ public class TestHBaseStorageHandler { jobConfToConfigure.get("hbase.some.fake.option.from.xml.file") != null); } + @Test + public void testGetUriForAuthEmptyTableDefaultHostPort() throws URISyntaxException { + Table table = createMockTable(new HashMap<>()); + URI uri = checkURIForAuth(table); + // If there is no tablename provided, the default "null" is still + // written out. At the time this test was written, this was the current + // behavior, so I left this test as/is. Need to research if a null + // table can be provided here. + Assert.assertEquals("hbase://localhost:2181/null", uri.toString()); + } + + @Test + public void testGetUriForAuthEmptyTable() throws URISyntaxException { + Map<String, String> serdeParams = new HashMap<>(); + serdeParams.put("hbase.zookeeper.quorum", "testhost"); + serdeParams.put("hbase.zookeeper.property.clientPort", "8765"); + URI uri = checkURIForAuth(createMockTable(serdeParams)); + Assert.assertEquals("hbase://testhost:8765/null", uri.toString()); + } + + @Test + public void testGetUriForAuthWithTable() throws URISyntaxException { + Map<String, String> serdeParams = new HashMap<>(); + serdeParams.put("hbase.zookeeper.quorum", "testhost"); + serdeParams.put("hbase.zookeeper.property.clientPort", "8765"); + serdeParams.put("hbase.table.name", "mytbl"); + URI uri = checkURIForAuth(createMockTable(serdeParams)); + Assert.assertEquals("hbase://testhost:8765/mytbl", uri.toString()); + } + + @Test + public void testGetUriForAuthWithTableAndColumns() throws URISyntaxException { + Map<String, String> serdeParams = new HashMap<>(); + serdeParams.put("hbase.zookeeper.quorum", "testhost"); + serdeParams.put("hbase.zookeeper.property.clientPort", "8765"); + serdeParams.put("hbase.table.name", "mytbl"); + serdeParams.put("hbase.columns.mapping", "mycolumns"); + URI uri = checkURIForAuth(createMockTable(serdeParams)); + Assert.assertEquals("hbase://testhost:8765/mytbl/mycolumns", uri.toString()); + } + + @Test + public void testGetUriForAuthWithTableAndEncodedColumns() throws URISyntaxException { + Map<String, String> serdeParams = new HashMap<>(); + serdeParams.put("hbase.zookeeper.quorum", "testhost"); + serdeParams.put("hbase.zookeeper.property.clientPort", "8765"); + serdeParams.put("hbase.table.name", "my#tbl"); + serdeParams.put("hbase.columns.mapping", "myco#lumns"); + URI uri = checkURIForAuth(createMockTable(serdeParams)); + Assert.assertEquals("hbase://testhost:8765/my%23tbl/myco%23lumns", uri.toString()); + } + private TableDesc getHBaseTableDesc() { TableDesc tableDesc = Mockito.mock(TableDesc.class); Properties properties = new Properties(); @@ -56,4 +115,21 @@ public class TestHBaseStorageHandler { Mockito.when(tableDesc.getProperties()).thenReturn(properties); return tableDesc; } + + private static URI checkURIForAuth(Table table) throws URISyntaxException { + HBaseStorageHandler hbaseStorageHandler = new HBaseStorageHandler(); + hbaseStorageHandler.setConf(new JobConf(new HiveConf())); + return hbaseStorageHandler.getURIForAuth(table); + } + + private static Table createMockTable(Map<String, String> serdeParams) { + Table table = new Table(); + StorageDescriptor sd = new StorageDescriptor(); + SerDeInfo sdi = new SerDeInfo(); + sdi.setParameters(serdeParams); + sd.setSerdeInfo(sdi); + table.setSd(sd); + table.setParameters(new HashMap<>()); + return table; + } }