DRILL-680: INFORMATION_SCHEMA.COLUMNS does not display HBase column families
+ Enhanced result layout with option to set output width on per column basis. + Pretty print plan fragments. Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/ae570aad Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/ae570aad Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/ae570aad Branch: refs/heads/master Commit: ae570aad8b572a5c1fe612f79fe2524cb5e4e879 Parents: ea1ab62 Author: Aditya Kishore <[email protected]> Authored: Tue May 20 06:34:48 2014 -0700 Committer: Jacques Nadeau <[email protected]> Committed: Wed May 21 08:41:57 2014 -0700 ---------------------------------------------------------------------- .../drill/exec/store/hbase/DrillHBaseTable.java | 62 ++++++++++++++++++++ .../exec/store/hbase/HBaseSchemaFactory.java | 6 +- .../org/apache/drill/hbase/BaseHBaseTest.java | 12 ++-- .../drill/hbase/TestHBaseFilterPushDown.java | 6 ++ .../drill/hbase/TestHBaseProjectPushDown.java | 8 ++- .../planner/fragment/SimpleParallelizer.java | 8 +-- .../org/apache/drill/exec/rpc/RpcException.java | 10 ++-- .../org/apache/drill/exec/util/VectorUtil.java | 52 ++++++++++------ 8 files changed, 126 insertions(+), 38 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ae570aad/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/DrillHBaseTable.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/DrillHBaseTable.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/DrillHBaseTable.java new file mode 100644 index 0000000..ff6991f --- /dev/null +++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/DrillHBaseTable.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.store.hbase; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Set; + +import org.apache.drill.exec.planner.logical.DrillTable; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.util.Bytes; +import org.eigenbase.reltype.RelDataType; +import org.eigenbase.reltype.RelDataTypeFactory; +import org.eigenbase.sql.type.SqlTypeName; + +public class DrillHBaseTable extends DrillTable implements DrillHBaseConstants { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillHBaseTable.class); + + private HTableDescriptor table; + + public DrillHBaseTable(String storageEngineName, HBaseStoragePlugin plugin, HBaseScanSpec scanSpec) { + super(storageEngineName, plugin, scanSpec); + try(HBaseAdmin admin = new HBaseAdmin(plugin.getConfig().getHBaseConf())) { + table = admin.getTableDescriptor(HBaseUtils.getBytes(scanSpec.getTableName())); + } catch (IOException e) { + logger.warn("Failure while loading table names for database '{}'.", storageEngineName, e); + } + } + + @Override + public RelDataType getRowType(RelDataTypeFactory typeFactory) { + ArrayList<RelDataType> typeList = new ArrayList<>(); + ArrayList<String> fieldNameList = new ArrayList<>(); + + fieldNameList.add(ROW_KEY); + typeList.add(typeFactory.createSqlType(SqlTypeName.ANY)); + + Set<byte[]> families = table.getFamiliesKeys(); + for (byte[] family : families) { + fieldNameList.add(Bytes.toString(family)); + typeList.add(typeFactory.createMapType(typeFactory.createSqlType(SqlTypeName.VARCHAR), typeFactory.createSqlType(SqlTypeName.ANY))); + } + return typeFactory.createStructType(typeList, fieldNameList); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ae570aad/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseSchemaFactory.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseSchemaFactory.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseSchemaFactory.java index 84f363b..7b76092 100644 --- a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseSchemaFactory.java +++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseSchemaFactory.java @@ -25,7 +25,6 @@ import net.hydromatic.optiq.Schema; import net.hydromatic.optiq.SchemaPlus; import net.hydromatic.optiq.Table; -import org.apache.drill.exec.planner.logical.DynamicDrillTable; import org.apache.drill.exec.rpc.user.UserSession; import org.apache.drill.exec.store.AbstractSchema; import org.apache.drill.exec.store.SchemaFactory; @@ -74,8 +73,8 @@ public class HBaseSchemaFactory implements SchemaFactory { @Override public Table getTable(String name) { - Object selection = new HBaseScanSpec(name); - return new DynamicDrillTable(plugin, schemaName, selection); + HBaseScanSpec scanSpec = new HBaseScanSpec(name); + return new DrillHBaseTable(schemaName, plugin, scanSpec); } @Override @@ -99,4 +98,5 @@ public class HBaseSchemaFactory implements SchemaFactory { } } + } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ae570aad/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java index 9e07d9f..96f0c4a 100644 --- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java +++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java @@ -45,7 +45,7 @@ public class BaseHBaseTest extends BaseTestQuery { @Rule public TestName TEST_NAME = new TestName(); - private int columnWidth = 8; + private int[] columnWidths = new int[] { 8 }; @Before public void printID() throws Exception { @@ -58,7 +58,7 @@ public class BaseHBaseTest extends BaseTestQuery { * Change the following to HBaseTestsSuite.configure(false, true) * if you want to test against an externally running HBase cluster. */ - HBaseTestsSuite.configure(true, true); + HBaseTestsSuite.configure(false, true); HBaseTestsSuite.initCluster(); HBaseStoragePlugin plugin = (HBaseStoragePlugin) bit.getContext().getStorage().getPlugin("hbase"); @@ -71,9 +71,13 @@ public class BaseHBaseTest extends BaseTestQuery { } protected void setColumnWidth(int columnWidth) { - this.columnWidth = columnWidth; + this.columnWidths = new int[] { columnWidth }; } + protected void setColumnWidths(int[] columnWidths) { + this.columnWidths = columnWidths; + } + protected String getPlanText(String planFile, String tableName) throws IOException { return Files.toString(FileUtils.getResourceAsFile(planFile), Charsets.UTF_8) .replaceFirst("\"hbase\\.zookeeper\\.property\\.clientPort\".*:.*\\d+", "\"hbase.zookeeper.property.clientPort\" : " + HBaseTestsSuite.getZookeeperPort()) @@ -102,7 +106,7 @@ public class BaseHBaseTest extends BaseTestQuery { if (loader.getRecordCount() <= 0) { break; } - VectorUtil.showVectorAccessibleContent(loader, columnWidth); + VectorUtil.showVectorAccessibleContent(loader, columnWidths); loader.clear(); result.release(); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ae570aad/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java index 90404b7..2071e90 100644 --- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java +++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java @@ -23,6 +23,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest { @Test public void testFilterPushDownRowKeyEqual() throws Exception { + setColumnWidths(new int[] {8, 38, 38}); runSQLVerifyCount("SELECT\n" + " *\n" + "FROM\n" @@ -34,6 +35,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest { @Test public void testFilterPushDownRowKeyGreaterThan() throws Exception { + setColumnWidths(new int[] {8, 38, 38}); runSQLVerifyCount("SELECT\n" + " *\n" + "FROM\n" @@ -45,6 +47,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest { @Test public void testFilterPushDownRowKeyBetween() throws Exception { + setColumnWidths(new int[] {8, 74, 38}); runSQLVerifyCount("SELECT\n" + " *\n" + "FROM\n" @@ -56,6 +59,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest { @Test public void testFilterPushDownMultiColumns() throws Exception { + setColumnWidths(new int[] {8, 74, 38}); runSQLVerifyCount("SELECT\n" + " *\n" + "FROM\n" @@ -67,6 +71,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest { @Test public void testFilterPushDownConvertExpression() throws Exception { + setColumnWidths(new int[] {8, 38, 38}); runSQLVerifyCount("SELECT\n" + " *\n" + "FROM\n" @@ -78,6 +83,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest { @Test public void testFilterPushDownRowKeyLessThanOrEqualTo() throws Exception { + setColumnWidths(new int[] {8, 74, 38}); runSQLVerifyCount("SELECT\n" + " *\n" + "FROM\n" http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ae570aad/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseProjectPushDown.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseProjectPushDown.java b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseProjectPushDown.java index b66d2ed..7037b9a 100644 --- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseProjectPushDown.java +++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseProjectPushDown.java @@ -23,6 +23,7 @@ public class TestHBaseProjectPushDown extends BaseHBaseTest { @Test public void testRowKeyPushDown() throws Exception{ + setColumnWidth(8); runSQLVerifyCount("SELECT\n" + "row_key\n" + "FROM\n" @@ -32,6 +33,7 @@ public class TestHBaseProjectPushDown extends BaseHBaseTest { @Test public void testColumnWith1RowPushDown() throws Exception{ + setColumnWidth(6); runSQLVerifyCount("SELECT\n" + "f2['c7'] as `f[c7]`\n" + "FROM\n" @@ -41,7 +43,7 @@ public class TestHBaseProjectPushDown extends BaseHBaseTest { @Test public void testRowKeyAndColumnPushDown() throws Exception{ - setColumnWidth(9); + setColumnWidths(new int[] {8, 9, 6, 2, 6}); runSQLVerifyCount("SELECT\n" + "row_key, f['c1']*31 as `f[c1]*31`, f['c2'] as `f[c2]`, 5 as `5`, 'abc' as `'abc'`\n" + "FROM\n" @@ -51,9 +53,9 @@ public class TestHBaseProjectPushDown extends BaseHBaseTest { @Test public void testColumnFamilyPushDown() throws Exception{ - setColumnWidth(74); + setColumnWidths(new int[] {8, 74, 38}); runSQLVerifyCount("SELECT\n" - + "f, f2\n" + + "row_key, f, f2\n" + "FROM\n" + " hbase.`[TABLE_NAME]` tableName" , 6); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ae570aad/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java index c34869d..313a81d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java @@ -20,6 +20,7 @@ package org.apache.drill.exec.planner.fragment; import java.util.Collection; import java.util.List; +import org.apache.commons.lang3.StringEscapeUtils; import org.apache.drill.common.exceptions.ExecutionSetupException; import org.apache.drill.common.exceptions.PhysicalOperatorSetupException; import org.apache.drill.exec.exception.FragmentSetupException; @@ -38,9 +39,6 @@ import org.apache.drill.exec.work.QueryWorkUnit; import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; -import org.apache.drill.exec.expr.fn.impl.DateUtility; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; /** * The simple parallelizer determines the level of parallelization of a plan based on the cost of the underlying @@ -143,11 +141,11 @@ public class SimpleParallelizer { .build(); if (isRootNode) { - logger.debug("Root fragment:\n {}", fragment); + logger.debug("Root fragment:\n {}", StringEscapeUtils.unescapeJava(fragment.toString())); rootFragment = fragment; rootOperator = root; } else { - logger.debug("Remote fragment:\n {}", fragment); + logger.debug("Remote fragment:\n {}", StringEscapeUtils.unescapeJava(fragment.toString())); fragments.add(fragment); } } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ae570aad/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcException.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcException.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcException.java index dba72a1..9b5eb1d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcException.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcException.java @@ -19,13 +19,14 @@ package org.apache.drill.exec.rpc; import java.util.concurrent.ExecutionException; -import org.apache.commons.lang.StringEscapeUtils; +import org.apache.commons.lang3.StringEscapeUtils; import org.apache.drill.common.exceptions.DrillIOException; /** * Parent class for all rpc exceptions. */ public class RpcException extends DrillIOException{ + private static final long serialVersionUID = -5964230316010502319L; static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RpcException.class); public RpcException() { @@ -47,17 +48,16 @@ public class RpcException extends DrillIOException{ public RpcException(Throwable cause) { super(cause); } - + public static RpcException mapException(Throwable t){ while(t instanceof ExecutionException) t = ((ExecutionException)t).getCause(); if(t instanceof RpcException) return ((RpcException) t); return new RpcException(t); } - + public static RpcException mapException(String message, Throwable t){ while(t instanceof ExecutionException) t = ((ExecutionException)t).getCause(); return new RpcException(message, t); } - - + } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ae570aad/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java index a43225e..bdba1d8 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java @@ -17,14 +17,11 @@ */ package org.apache.drill.exec.util; -import java.util.Arrays; import java.util.List; import org.apache.commons.lang.StringUtils; -import org.apache.drill.exec.exception.SchemaChangeException; import org.apache.drill.exec.record.VectorAccessible; import org.apache.drill.exec.record.VectorWrapper; -import org.apache.drill.exec.rpc.RpcException; import com.beust.jcommander.internal.Lists; @@ -37,7 +34,7 @@ public class VectorUtil { int rows = va.getRecordCount(); List<String> columns = Lists.newArrayList(); for (VectorWrapper<?> vw : va) { - columns.add(vw.getValueVector().getField().getAsSchemaPath().getAsUnescapedPath()); + columns.add(vw.getValueVector().getField().getPath().getAsUnescapedPath()); } int width = columns.size(); @@ -71,55 +68,74 @@ public class VectorUtil { } public static void showVectorAccessibleContent(VectorAccessible va) { - showVectorAccessibleContent(va, DEFAULT_COLUMN_WIDTH); + showVectorAccessibleContent(va, DEFAULT_COLUMN_WIDTH); } + public static void showVectorAccessibleContent(VectorAccessible va, int columnWidth) { + showVectorAccessibleContent(va, new int[]{ columnWidth }); + } - int rows = va.getRecordCount(); + public static void showVectorAccessibleContent(VectorAccessible va, int[] columnWidths) { + int width = 0; + int columnIndex = 0; List<String> columns = Lists.newArrayList(); + List<String> formats = Lists.newArrayList(); for (VectorWrapper<?> vw : va) { - columns.add(vw.getValueVector().getField().getAsSchemaPath().getAsUnescapedPath()); + int columnWidth = getColumnWidth(columnWidths, columnIndex); + width += columnWidth + 2; + formats.add("| %-" + columnWidth + "s"); + columns.add(vw.getValueVector().getField().getPath().getAsUnescapedPath()); + columnIndex++; } - int width = columns.size() * (columnWidth + 2); - - String format = ("| %-" + columnWidth + "s"); - + int rows = va.getRecordCount(); for (int row = 0; row < rows; row++) { + // header, every 50 rows. if (row%50 == 0) { System.out.println(StringUtils.repeat("-", width + 1)); + columnIndex = 0; for (String column : columns) { - System.out.printf(format, column.length() <= columnWidth ? column : column.substring(0, columnWidth - 1)); + int columnWidth = getColumnWidth(columnWidths, columnIndex); + System.out.printf(formats.get(columnIndex), column.length() <= columnWidth ? column : column.substring(0, columnWidth - 1)); + columnIndex++; } System.out.printf("|\n"); System.out.println(StringUtils.repeat("-", width + 1)); } + // column values + columnIndex = 0; for (VectorWrapper<?> vw : va) { + int columnWidth = getColumnWidth(columnWidths, columnIndex); Object o = vw.getValueVector().getAccessor().getObject(row); if (o == null) { //null value - System.out.printf(format, ""); + System.out.printf(formats.get(columnIndex), ""); } else if (o instanceof byte[]) { String value = new String((byte[]) o); - System.out.printf(format, value.length() <= columnWidth ? value : value.substring(0, columnWidth - 1)); + System.out.printf(formats.get(columnIndex), value.length() <= columnWidth ? value : value.substring(0, columnWidth - 1)); } else if (o instanceof List) { System.out.printf("| %s", o); } else { String value = o.toString(); - System.out.printf(format, value.length() <= columnWidth ? value : value.substring(0,columnWidth - 1)); + System.out.printf(formats.get(columnIndex), value.length() <= columnWidth ? value : value.substring(0,columnWidth - 1)); } + columnIndex++; } System.out.printf("|\n"); } + if (rows > 0) { + System.out.println(StringUtils.repeat("-", width + 1)); + } for (VectorWrapper<?> vw : va) { vw.clear(); } - - if (rows > 0 ) - System.out.println(StringUtils.repeat("-", width + 1)); } + private static int getColumnWidth(int[] columnWidths, int columnIndex) { + return (columnWidths == null) ? DEFAULT_COLUMN_WIDTH + : (columnWidths.length > columnIndex) ? columnWidths[columnIndex] : columnWidths[0]; + } }
