DRILL-696: Use standard HBase configuration name/value in HBaseStoragePluginConfig
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/6e6c661f Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/6e6c661f Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/6e6c661f Branch: refs/heads/master Commit: 6e6c661fd8099fb7f385346ba38e2026c84fc67a Parents: 508cb5d Author: Aditya Kishore <[email protected]> Authored: Sun May 11 03:19:43 2014 -0700 Committer: Jacques Nadeau <[email protected]> Committed: Mon May 12 11:46:38 2014 -0700 ---------------------------------------------------------------------- .../store/hbase/HBaseStoragePluginConfig.java | 52 +++++++++++--------- .../org/apache/drill/hbase/BaseHBaseTest.java | 2 +- .../org/apache/drill/hbase/HBaseTestsSuite.java | 13 +++-- .../hbase/hbase_scan_screen_physical.json | 6 ++- ...base_scan_screen_physical_column_select.json | 6 ++- ...base_scan_screen_physical_family_select.json | 6 ++- .../src/test/resources/storage-plugins.json | 39 ++------------- distribution/src/resources/storage-plugins.json | 8 +-- 8 files changed, 58 insertions(+), 74 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6e6c661f/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePluginConfig.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePluginConfig.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePluginConfig.java index 5a434d6..054d65f 100644 --- a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePluginConfig.java +++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePluginConfig.java @@ -17,45 +17,43 @@ */ package org.apache.drill.exec.store.hbase; +import java.util.Map; + import org.apache.drill.common.logical.StoragePluginConfigBase; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.client.HConnectionManager.HConnectionKey; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeName; import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; @JsonTypeName("hbase") public class HBaseStoragePluginConfig extends StoragePluginConfigBase implements DrillHBaseConstants { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HBaseStoragePluginConfig.class); - @JsonProperty - public String zookeeperQuorum; - - @JsonProperty - public int zookeeperPort; + private Map<String, String> config; + @JsonIgnore private Configuration hbaseConf; - private HConnectionKey hbaseConfKey; @JsonCreator - public HBaseStoragePluginConfig(@JsonProperty("zookeeperQuorum") String zookeeperQuorum, - @JsonProperty("zookeeperPort") int zookeeperPort) { - this.zookeeperQuorum = zookeeperQuorum; - this.zookeeperPort = zookeeperPort; - - this.hbaseConf = HBaseConfiguration.create(); - logger.debug("Configuring HBase StoragePlugin with zookeeper quorum '{}', port '{}' node '{}'.", - zookeeperQuorum, zookeeperPort, hbaseConf.get(HConstants.ZOOKEEPER_ZNODE_PARENT)); - if (zookeeperQuorum != null && zookeeperQuorum.length() != 0) { - hbaseConf.set(HConstants.ZOOKEEPER_QUORUM, zookeeperQuorum); - hbaseConf.setInt(HBASE_ZOOKEEPER_PORT, zookeeperPort); + public HBaseStoragePluginConfig(@JsonProperty("config") Map<String, String> props) { + this.config = props; + if (config == null) { + config = Maps.newHashMap(); } - this.hbaseConfKey = new HConnectionKey(hbaseConf); + logger.debug("Configuring HBase StoragePlugin with zookeeper quorum '{}', port '{}'.", + config.get(HConstants.ZOOKEEPER_QUORUM), config.get(HBASE_ZOOKEEPER_PORT)); + } + + @JsonProperty + public Map<String, String> getConfig() { + return ImmutableMap.copyOf(config); } @Override @@ -66,24 +64,32 @@ public class HBaseStoragePluginConfig extends StoragePluginConfigBase implements return false; } HBaseStoragePluginConfig that = (HBaseStoragePluginConfig) o; - return this.hbaseConfKey.equals(that.hbaseConfKey); + return config.equals(that.config); } @Override public int hashCode() { - return this.hbaseConfKey != null ? this.hbaseConfKey.hashCode() : 0; + return this.config != null ? this.config.hashCode() : 0; } @JsonIgnore public Configuration getHBaseConf() { + if (hbaseConf == null) { + hbaseConf = HBaseConfiguration.create(); + if (config != null) { + for (Map.Entry<String, String> entry : config.entrySet()) { + hbaseConf.set(entry.getKey(), entry.getValue()); + } + } + } return hbaseConf; } @JsonIgnore @VisibleForTesting public void setZookeeperPort(int zookeeperPort) { - this.zookeeperPort = zookeeperPort; - hbaseConf.setInt(HBASE_ZOOKEEPER_PORT, zookeeperPort); + this.config.put(HBASE_ZOOKEEPER_PORT, String.valueOf(zookeeperPort)); + getHBaseConf().setInt(HBASE_ZOOKEEPER_PORT, zookeeperPort); } } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6e6c661f/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java index 0753a4d..a68cf70 100644 --- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java +++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java @@ -76,7 +76,7 @@ public class BaseHBaseTest extends BaseTestQuery { protected String getPlanText(String planFile, String tableName) throws IOException { return Files.toString(FileUtils.getResourceAsFile(planFile), Charsets.UTF_8) - .replaceFirst("\"zookeeperPort\".*:.*\\d+", "\"zookeeperPort\" : " + HBaseTestsSuite.getZookeeperPort()) + .replaceFirst("\"hbase\\.zookeeper\\.property\\.clientPort\".*:.*\\d+", "\"hbase.zookeeper.property.clientPort\" : " + HBaseTestsSuite.getZookeeperPort()) .replace("[TABLE_NAME]", tableName); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6e6c661f/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java index 36c31b7..32d7aaa 100644 --- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java +++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/HBaseTestsSuite.java @@ -21,8 +21,6 @@ import java.io.IOException; import java.lang.management.ManagementFactory; import java.util.concurrent.atomic.AtomicInteger; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtility; @@ -40,7 +38,8 @@ import org.junit.runners.Suite.SuiteClasses; TestHBaseFilterPushDown.class, TestHBaseProjectPushDown.class}) public class HBaseTestsSuite { - private static final Log LOG = LogFactory.getLog(HBaseTestsSuite.class); + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HBaseTestsSuite.class); + private static final boolean IS_DEBUG = ManagementFactory.getRuntimeMXBean().getInputArguments().toString().indexOf("-agentlib:jdwp") > 0; protected static final String TEST_TABLE_1 = "TestTable1"; @@ -70,11 +69,11 @@ public class HBaseTestsSuite { } if (manageHBaseCluster) { - LOG.info("Starting HBase mini cluster."); + logger.info("Starting HBase mini cluster."); UTIL = new HBaseTestingUtility(conf); UTIL.startMiniCluster(); hbaseClusterCreated = true; - LOG.info("HBase mini cluster started."); + logger.info("HBase mini cluster started. Zookeeper port: '{}'", getZookeeperPort()); } admin = new HBaseAdmin(conf); @@ -104,9 +103,9 @@ public class HBaseTestsSuite { } if (hbaseClusterCreated) { - LOG.info("Shutting down HBase mini cluster."); + logger.info("Shutting down HBase mini cluster."); UTIL.shutdownMiniCluster(); - LOG.info("HBase mini cluster stopped."); + logger.info("HBase mini cluster stopped."); } } } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6e6c661f/contrib/storage-hbase/src/test/resources/hbase/hbase_scan_screen_physical.json ---------------------------------------------------------------------- diff --git a/contrib/storage-hbase/src/test/resources/hbase/hbase_scan_screen_physical.json b/contrib/storage-hbase/src/test/resources/hbase/hbase_scan_screen_physical.json index 17e17e4..7f9015b 100644 --- a/contrib/storage-hbase/src/test/resources/hbase/hbase_scan_screen_physical.json +++ b/contrib/storage-hbase/src/test/resources/hbase/hbase_scan_screen_physical.json @@ -15,8 +15,10 @@ storage: { "type":"hbase", - "zookeeperQuorum" : "localhost", - "zookeeperPort" : 2181 + config : { + "hbase.zookeeper.quorum" : "localhost", + "hbase.zookeeper.property.clientPort" : 2181 + } } }, { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6e6c661f/contrib/storage-hbase/src/test/resources/hbase/hbase_scan_screen_physical_column_select.json ---------------------------------------------------------------------- diff --git a/contrib/storage-hbase/src/test/resources/hbase/hbase_scan_screen_physical_column_select.json b/contrib/storage-hbase/src/test/resources/hbase/hbase_scan_screen_physical_column_select.json index dc08031..f399f6f 100644 --- a/contrib/storage-hbase/src/test/resources/hbase/hbase_scan_screen_physical_column_select.json +++ b/contrib/storage-hbase/src/test/resources/hbase/hbase_scan_screen_physical_column_select.json @@ -15,8 +15,10 @@ storage: { "type":"hbase", - "zookeeperQuorum" : "localhost", - "zookeeperPort" : 2181 + config : { + "hbase.zookeeper.quorum" : "localhost", + "hbase.zookeeper.property.clientPort" : 2181 + } }, columns: [ "`f2`.c1", "`f2`.c2", "row_key" http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6e6c661f/contrib/storage-hbase/src/test/resources/hbase/hbase_scan_screen_physical_family_select.json ---------------------------------------------------------------------- diff --git a/contrib/storage-hbase/src/test/resources/hbase/hbase_scan_screen_physical_family_select.json b/contrib/storage-hbase/src/test/resources/hbase/hbase_scan_screen_physical_family_select.json index ce027a0..0002164 100644 --- a/contrib/storage-hbase/src/test/resources/hbase/hbase_scan_screen_physical_family_select.json +++ b/contrib/storage-hbase/src/test/resources/hbase/hbase_scan_screen_physical_family_select.json @@ -15,8 +15,10 @@ storage: { "type":"hbase", - "zookeeperQuorum" : "localhost", - "zookeeperPort" : 2181 + config : { + "hbase.zookeeper.quorum" : "localhost", + "hbase.zookeeper.property.clientPort" : 2181 + } }, columns: [ "f2" http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6e6c661f/contrib/storage-hbase/src/test/resources/storage-plugins.json ---------------------------------------------------------------------- diff --git a/contrib/storage-hbase/src/test/resources/storage-plugins.json b/contrib/storage-hbase/src/test/resources/storage-plugins.json index 160583a..0e93f7e 100644 --- a/contrib/storage-hbase/src/test/resources/storage-plugins.json +++ b/contrib/storage-hbase/src/test/resources/storage-plugins.json @@ -1,40 +1,11 @@ { "storage":{ - dfs: { - type: "file", - connection: "file:///", - formats: { - "psv" : { - type: "text", - extensions: [ "tbl" ], - delimiter: "|" - }, - "csv" : { - type: "text", - extensions: [ "csv" ], - delimiter: "," - }, - "tsv" : { - type: "text", - extensions: [ "tsv" ], - delimiter: "\t" - }, - "parquet" : { - type: "parquet" - }, - "json" : { - type: "json" - } - } - }, - cp: { - type: "file", - connection: "classpath:///" - }, hbase : { type:"hbase", - zookeeperQuorum : "localhost", - zookeeperPort : 2181 + config : { + "hbase.zookeeper.quorum" : "localhost", + "hbase.zookeeper.property.clientPort" : 2181 + } } } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6e6c661f/distribution/src/resources/storage-plugins.json ---------------------------------------------------------------------- diff --git a/distribution/src/resources/storage-plugins.json b/distribution/src/resources/storage-plugins.json index c3f4a4b..3b1cbd0 100644 --- a/distribution/src/resources/storage-plugins.json +++ b/distribution/src/resources/storage-plugins.json @@ -59,9 +59,11 @@ /*, hbase : { - type:"hbase", - zookeeperQuorum : "localhost", - zookeeperPort : 2181 + type:"hbase", + config : { + "hbase.zookeeper.quorum" : "localhost", + "hbase.zookeeper.property.clientPort" : 2181 + } } */ }
