This is an automated email from the ASF dual-hosted git repository.
krisztiankasa pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 5dddb6ebf46 HIVE-1996: Beeline performance poor with drivers having
slow DatabaseMetaData.getPrimaryKeys impl (#6023)
5dddb6ebf46 is described below
commit 5dddb6ebf46ac40b91ab3273e3bb8910d6163dc2
Author: InvisibleProgrammer <[email protected]>
AuthorDate: Fri Aug 15 15:18:36 2025 +0200
HIVE-1996: Beeline performance poor with drivers having slow
DatabaseMetaData.getPrimaryKeys impl (#6023)
---
beeline/src/java/org/apache/hive/beeline/Rows.java | 74 ++++---
.../org/apache/hive/beeline/TableOutputFormat.java | 4 +-
.../test/org/apache/hive/beeline/BeelineMock.java | 46 +++++
.../apache/hive/beeline/TestTableOutputFormat.java | 228 ++++++++++++++++++---
4 files changed, 289 insertions(+), 63 deletions(-)
diff --git a/beeline/src/java/org/apache/hive/beeline/Rows.java
b/beeline/src/java/org/apache/hive/beeline/Rows.java
index 7d9ee5c3d95..7926f31cf81 100644
--- a/beeline/src/java/org/apache/hive/beeline/Rows.java
+++ b/beeline/src/java/org/apache/hive/beeline/Rows.java
@@ -29,9 +29,13 @@
import java.sql.SQLException;
import java.text.DecimalFormat;
import java.text.NumberFormat;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64;
+import java.util.HashMap;
import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
import org.apache.hadoop.hive.common.cli.EscapeCRLFHelper;
@@ -42,7 +46,8 @@
abstract class Rows implements Iterator {
protected final BeeLine beeLine;
final ResultSetMetaData rsMeta;
- final Boolean[] primaryKeys;
+ final boolean[] primaryKeyColumns;
+ boolean isPrimaryKeyColumnsInitialized;
final NumberFormat numberFormat;
private boolean convertBinaryArrayToString;
private final String nullStr;
@@ -51,14 +56,15 @@ abstract class Rows implements Iterator {
this.beeLine = beeLine;
nullStr = beeLine.getOpts().getNullString();
rsMeta = rs.getMetaData();
- int count = rsMeta.getColumnCount();
- primaryKeys = new Boolean[count];
if (beeLine.getOpts().getNumberFormat().equals("default")) {
numberFormat = null;
} else {
numberFormat = new DecimalFormat(beeLine.getOpts().getNumberFormat());
}
this.convertBinaryArrayToString =
beeLine.getOpts().getConvertBinaryArrayToString();
+
+ int count = this.rsMeta.getColumnCount();
+ primaryKeyColumns = new boolean[count];
}
@Override
@@ -78,37 +84,47 @@ public void remove() {
* JDBC driver property implements {@link ResultSetMetaData#getTableName}
(many do not), it
* is not reliable for all databases.
*/
- boolean isPrimaryKey(int col) {
- if (primaryKeys[col] == null) {
- try {
- // this doesn't always work, since some JDBC drivers (e.g.,
- // Oracle's) return a blank string from getDbTableName.
- String table = rsMeta.getTableName(col + 1);
- String column = rsMeta.getColumnName(col + 1);
+ boolean isPrimaryKeyCol(int col) {
+ if (!isPrimaryKeyColumnsInitialized) {
+ initializePrimaryKeyMetadata();
+ }
+
+ return primaryKeyColumns[col];
+ }
+
+ private void initializePrimaryKeyMetadata() {
+ Map<String, List<String>> tablePrimaryKeys = new HashMap<>();
+
+ try {
+ for (int i = 0; i < primaryKeyColumns.length; i++) {
+ String table = rsMeta.getTableName(i + 1);
+ String column = rsMeta.getColumnName(i + 1);
if (table == null || table.isEmpty() || column == null ||
column.isEmpty()) {
- primaryKeys[col] = Boolean.FALSE;
- } else {
- ResultSet pks =
beeLine.getDatabaseConnection().getDatabaseMetaData().getPrimaryKeys(
-
beeLine.getDatabaseConnection().getDatabaseMetaData().getConnection().getCatalog(),
null, table);
+ continue;
+ }
+
+ if (!tablePrimaryKeys.containsKey(table)) {
+ try (ResultSet pks =
beeLine.getDatabaseConnection().getDatabaseMetaData().getPrimaryKeys(
+
beeLine.getDatabaseConnection().getDatabaseMetaData().getConnection().getCatalog(),
null, table)) {
+
+ List<String> pkNames = new ArrayList<>();
- primaryKeys[col] = Boolean.FALSE;
- try {
while (pks.next()) {
- if (column.equalsIgnoreCase(pks.getString("COLUMN_NAME"))) {
- primaryKeys[col] = Boolean.TRUE;
- break;
- }
+ pkNames.add(pks.getString("COLUMN_NAME"));
}
- } finally {
- pks.close();
+
+ tablePrimaryKeys.put(table, pkNames);
}
}
- } catch (SQLException sqle) {
- primaryKeys[col] = Boolean.FALSE;
+
+ primaryKeyColumns[i] = tablePrimaryKeys.get(table).contains(column);
}
+ } catch (SQLException e) {
+ // Do nothing. We cannot decide if the given column is a primary key so
we keep it as false
}
- return primaryKeys[col].booleanValue();
+
+ isPrimaryKeyColumnsInitialized = true;
}
class Row {
@@ -134,7 +150,7 @@ class Row {
}
@Override
- public String toString(){
+ public String toString() {
return Arrays.asList(values).toString();
}
@@ -156,7 +172,7 @@ public String toString(){
} catch (Throwable t) {
}
- for (int i = 0; i < size; i++) {
+ for (int i = 0; i < size; i++) {
Object o = rs.getObject(i + 1);
String value = null;
@@ -164,9 +180,9 @@ public String toString(){
value = nullStr;
} else if (o instanceof Number) {
value = numberFormat != null ? numberFormat.format(o) :
- o instanceof BigDecimal ? ((BigDecimal)o).toPlainString() :
o.toString();
+ o instanceof BigDecimal ? ((BigDecimal) o).toPlainString() :
o.toString();
} else if (o instanceof byte[]) {
- value = convertBinaryArrayToString ? new String((byte[])o,
StandardCharsets.UTF_8) :
Base64.getEncoder().withoutPadding().encodeToString((byte[])o);
+ value = convertBinaryArrayToString ? new String((byte[]) o,
StandardCharsets.UTF_8) :
Base64.getEncoder().withoutPadding().encodeToString((byte[]) o);
} else {
value = rs.getString(i + 1);
}
diff --git a/beeline/src/java/org/apache/hive/beeline/TableOutputFormat.java
b/beeline/src/java/org/apache/hive/beeline/TableOutputFormat.java
index 12985c844ee..bd2a408e0d7 100644
--- a/beeline/src/java/org/apache/hive/beeline/TableOutputFormat.java
+++ b/beeline/src/java/org/apache/hive/beeline/TableOutputFormat.java
@@ -122,14 +122,14 @@ ColorBuffer getOutputString(Rows rows, Rows.Row row,
String delim) {
if (row.isMeta) {
v = beeLine.getColorBuffer().center(row.values[i], row.sizes[i]);
- if (beeLine.getOpts().getColor() && rows.isPrimaryKey(i)) {
+ if (beeLine.getOpts().getColor() && rows.isPrimaryKeyCol(i)) {
buf.cyan(v.getMono());
} else {
buf.bold(v.getMono());
}
} else {
v = beeLine.getColorBuffer().pad(row.values[i], row.sizes[i]);
- if (beeLine.getOpts().getColor() && rows.isPrimaryKey(i)) {
+ if (beeLine.getOpts().getColor() && rows.isPrimaryKeyCol(i)) {
buf.cyan(v.getMono());
} else {
buf.append(v.getMono());
diff --git a/beeline/src/test/org/apache/hive/beeline/BeelineMock.java
b/beeline/src/test/org/apache/hive/beeline/BeelineMock.java
new file mode 100644
index 00000000000..47083c48162
--- /dev/null
+++ b/beeline/src/test/org/apache/hive/beeline/BeelineMock.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hive.beeline;
+
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.List;
+
+public class BeelineMock extends BeeLine {
+ public BeelineMock() {
+ this.allPrintedLines = new ArrayList<>();
+ }
+
+ private String lastPrintedLine;
+ private List<String> allPrintedLines;
+
+ @Override
+ final void output(final ColorBuffer msg, boolean newline, PrintStream out) {
+ lastPrintedLine = msg.getMono();
+ allPrintedLines.add(msg.getColor());
+ super.output(msg, newline, out);
+ }
+
+ public String getLastPrintedLine() {
+ return lastPrintedLine;
+ }
+
+ public List<String> getAllPrintedLines() {
+ return allPrintedLines;
+ }
+}
\ No newline at end of file
diff --git
a/beeline/src/test/org/apache/hive/beeline/TestTableOutputFormat.java
b/beeline/src/test/org/apache/hive/beeline/TestTableOutputFormat.java
index f18fd445fff..eb8259f0644 100644
--- a/beeline/src/test/org/apache/hive/beeline/TestTableOutputFormat.java
+++ b/beeline/src/test/org/apache/hive/beeline/TestTableOutputFormat.java
@@ -17,35 +17,33 @@
*/
package org.apache.hive.beeline;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
import static org.mockito.ArgumentMatchers.anyInt;
-import java.io.PrintStream;
+import java.sql.Connection;
+import java.sql.DatabaseMetaData;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
+import java.util.List;
+
import static org.junit.Assert.assertEquals;
import org.junit.Test;
+
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.ArgumentMatchers.isNull;
import static org.mockito.Mockito.when;
+
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
import static org.mockito.Mockito.mock;
public class TestTableOutputFormat {
- public class BeelineMock extends BeeLine {
-
- private String lastPrintedLine;
-
- @Override
- final void output(final ColorBuffer msg, boolean newline, PrintStream out)
{
- lastPrintedLine = msg.getMono();
- super.output(msg, newline, out);
- }
-
- private String getLastPrintedLine() {
- return lastPrintedLine;
- }
- }
+ private static final String CYAN = "\033[1;36m";
+ private static final String GREEN = "\033[1;32m";
private final String[][] mockRowData = {
{"key1", "aaa"},
@@ -53,8 +51,14 @@ private String getLastPrintedLine() {
{"key3", "ccccccccccccccccccccccccccc"},
{"key4", "ddddddddddddddd"}
};
+
private BeelineMock mockBeeline;
+ private DatabaseConnection dbConnection;
+ private Connection mockSqlConnection;
+ private DatabaseMetaData mockDatabaseMetaData;
private ResultSet mockResultSet;
+ private ResultSetMetaData mockResultSetMetaData;
+ private ResultSet mockMetadataResultSet;
private TestBufferedRows.MockRow mockRow;
/**
@@ -63,25 +67,191 @@ private String getLastPrintedLine() {
*/
@Test
public final void testPrint() throws SQLException {
+ String EXP_LAST_LINE = "+-------+------------------------------+";
+
setupMockData();
BufferedRows bfRows = new BufferedRows(mockBeeline, mockResultSet);
TableOutputFormat instance = new TableOutputFormat(mockBeeline);
- String expResult = "+-------+------------------------------+";
+
instance.print(bfRows);
+
String outPutResults = mockBeeline.getLastPrintedLine();
- assertEquals(expResult, outPutResults);
+ assertEquals(EXP_LAST_LINE, outPutResults);
+ }
+
+ /**
+ * If the DatabaseConnection doesn't provide metadata, there is nothing to
color
+ */
+ @Test
+ public void testColoringWithNoTableMetadata() throws SQLException {
+ setupMockData();
+ when(mockResultSetMetaData.getTableName(anyInt())).thenReturn(null);
+
+ BufferedRows bfRows = new BufferedRows(mockBeeline, mockResultSet);
+ TableOutputFormat instance = new TableOutputFormat(mockBeeline);
+
+ instance.print(bfRows);
+
+ List<String> allPrintedLines = mockBeeline.getAllPrintedLines();
+ assertColumnHasColor(allPrintedLines, 1, GREEN);
+ assertColumnHasColor(allPrintedLines, 2, GREEN);
+ }
+
+ /**
+ * The primary key has one column. It should be CYAN. The second column
should be green
+ */
+ @Test
+ public void testSingleColumnPrimaryKey() throws SQLException {
+ setupMockData();
+ mockBeeline.getOpts().setColor(true);
+
+ BufferedRows bfRows = new BufferedRows(mockBeeline, mockResultSet);
+ TableOutputFormat instance = new TableOutputFormat(mockBeeline);
+
+ instance.print(bfRows);
+
+ List<String> allPrintedLines = mockBeeline.getAllPrintedLines();
+
+ // PK column is CYAN
+ assertColumnHasColor(allPrintedLines, 1, CYAN);
+ // non-PK column is green
+ assertColumnHasColor(allPrintedLines, 2, GREEN);
+ }
+
+ /**
+ * Default behavior: coloring is disabled
+ */
+ @Test
+ public void testMonoTableOutputFormat() throws SQLException {
+ setupMockData();
+ mockBeeline.getOpts().setColor(false);
+ BufferedRows bfRows = new BufferedRows(mockBeeline, mockResultSet);
+ TableOutputFormat instance = new TableOutputFormat(mockBeeline);
+
+ instance.print(bfRows);
+
+ List<String> allPrintedLines = mockBeeline.getAllPrintedLines();
+ for (String line : allPrintedLines) {
+ assertFalse(line.contains("\033["));
+ }
+ }
+
+ /**
+ * No primary key column - all columns should be green
+ */
+ @Test
+ public void testColoredWithoutPrimaryKey() throws SQLException {
+ setupMockData();
+ mockBeeline.getOpts().setColor(true);
+ when(mockMetadataResultSet.next()).thenReturn(false);
+
+ BufferedRows bfRows = new BufferedRows(mockBeeline, mockResultSet);
+ TableOutputFormat instance = new TableOutputFormat(mockBeeline);
+
+ instance.print(bfRows);
+
+ List<String> allPrintedLines = mockBeeline.getAllPrintedLines();
+
+ assertColumnHasColor(allPrintedLines, 1, GREEN);
+ assertColumnHasColor(allPrintedLines, 2, GREEN);
+ }
+
+ /**
+ * The output contains one single table. And both columns are part of the PK
+ */
+ @Test
+ public void testColoredWithPrimaryKeyHasMultipleColumns() throws
SQLException {
+ setupMockData();
+ mockBeeline.getOpts().setColor(true);
+
when(mockMetadataResultSet.next()).thenReturn(true).thenReturn(true).thenReturn(false);
+
+ BufferedRows bfRows = new BufferedRows(mockBeeline, mockResultSet);
+ TableOutputFormat instance = new TableOutputFormat(mockBeeline);
+
+ instance.print(bfRows);
+
+ List<String> allPrintedLines = mockBeeline.getAllPrintedLines();
+
+ // PK column are CYAN
+ assertColumnHasColor(allPrintedLines, 1, CYAN);
+ assertColumnHasColor(allPrintedLines, 2, CYAN);
+ }
+
+ /**
+ * The output contains two tables. Both columns are PKs
+ */
+ @Test
+ public void testColoredWithMultipleTablesWithPrimaryKeys() throws
SQLException {
+ setupMockData();
+ mockBeeline.getOpts().setColor(true);
+
+ when(mockResultSetMetaData.getTableName(eq(1))).thenReturn("Table");
+ when(mockResultSetMetaData.getTableName(eq(2))).thenReturn("Table2");
+
+ DatabaseMetaData databaseMetaData2 = mock(DatabaseMetaData.class);
+ ResultSet resultSet2 = mock(ResultSet.class);
+ when(resultSet2.next()).thenReturn(true).thenReturn(false);
+ when(resultSet2.getString(eq("COLUMN_NAME"))).thenReturn("Value");
+ when(databaseMetaData2.getPrimaryKeys(anyString(), isNull(),
eq("Table2"))).thenReturn(resultSet2);
+ when(databaseMetaData2.getConnection()).thenReturn(mockSqlConnection);
+ when(dbConnection.getDatabaseMetaData()).thenReturn(mockDatabaseMetaData,
databaseMetaData2);
+
+ BufferedRows bfRows = new BufferedRows(mockBeeline, mockResultSet);
+ TableOutputFormat instance = new TableOutputFormat(mockBeeline);
+
+ instance.print(bfRows);
+
+ List<String> allPrintedLines = mockBeeline.getAllPrintedLines();
+
+ // PK column are CYAN
+ assertColumnHasColor(allPrintedLines, 1, CYAN);
+ assertColumnHasColor(allPrintedLines, 2, CYAN);
+ }
+
+ private boolean cellHasColor(List<String> table, int row, int col, String
color) {
+ String rowText = table.get(row);
+ String split = rowText.split("\\|")[col];
+ return split.contains(color);
+ }
+
+ private void assertColumnHasColor(List<String> allPrintedLines, int col,
String color) {
+ assertTrue(cellHasColor(allPrintedLines, 1, col, color));
+ assertTrue(cellHasColor(allPrintedLines, 3, col, color));
+ assertTrue(cellHasColor(allPrintedLines, 4, col, color));
+ assertTrue(cellHasColor(allPrintedLines, 5, col, color));
+ assertTrue(cellHasColor(allPrintedLines, 6, col, color));
}
private void setupMockData() throws SQLException {
mockBeeline = new BeelineMock();
+ mockBeeline.getOpts().setColor(true);
mockResultSet = mock(ResultSet.class);
- ResultSetMetaData mockResultSetMetaData = mock(ResultSetMetaData.class);
+ mockSqlConnection = mock(Connection.class);
+ when(mockSqlConnection.getCatalog()).thenReturn("Catalog");
+
+ mockResultSetMetaData = mock(ResultSetMetaData.class);
when(mockResultSetMetaData.getColumnCount()).thenReturn(2);
when(mockResultSetMetaData.getColumnLabel(1)).thenReturn("Key");
when(mockResultSetMetaData.getColumnLabel(2)).thenReturn("Value");
+ when(mockResultSetMetaData.getTableName(anyInt())).thenReturn("Table");
+ when(mockResultSetMetaData.getColumnName(eq(1))).thenReturn("Key");
+ when(mockResultSetMetaData.getColumnName(eq(2))).thenReturn("Value");
when(mockResultSet.getMetaData()).thenReturn(mockResultSetMetaData);
+ mockMetadataResultSet = mock(ResultSet.class);
+ when(mockMetadataResultSet.next()).thenReturn(true).thenReturn(false);
+ when(mockMetadataResultSet.getString(eq("COLUMN_NAME"))).thenReturn("Key",
"Value");
+
+ mockDatabaseMetaData = mock(DatabaseMetaData.class);
+ when(mockDatabaseMetaData.getPrimaryKeys(anyString(), isNull(),
anyString())).thenReturn(mockMetadataResultSet);
+
+ dbConnection = mock(DatabaseConnection.class);
+ when(dbConnection.getDatabaseMetaData()).thenReturn(mockDatabaseMetaData);
+ when(mockDatabaseMetaData.getConnection()).thenReturn(mockSqlConnection);
+
+ mockBeeline.getDatabaseConnections().setConnection(dbConnection);
+
mockRow = new TestBufferedRows.MockRow();
// returns true as long as there is more data in mockResultData array
when(mockResultSet.next()).thenAnswer(new Answer<Boolean>() {
@@ -99,22 +269,16 @@ public Boolean answer(final InvocationOnMock invocation) {
}
});
- when(mockResultSet.getObject(anyInt())).thenAnswer(new Answer<String>() {
- @Override
- public String answer(final InvocationOnMock invocation) {
- Object[] args = invocation.getArguments();
- int index = ((Integer) args[0]);
- return mockRow.getColumn(index);
- }
+ when(mockResultSet.getObject(anyInt())).thenAnswer((Answer<String>)
invocation -> {
+ Object[] args = invocation.getArguments();
+ int index = ((Integer) args[0]);
+ return mockRow.getColumn(index);
});
- when(mockResultSet.getString(anyInt())).thenAnswer(new Answer<String>() {
- @Override
- public String answer(final InvocationOnMock invocation) {
- Object[] args = invocation.getArguments();
- int index = ((Integer) args[0]);
- return mockRow.getColumn(index);
- }
+ when(mockResultSet.getString(anyInt())).thenAnswer((Answer<String>)
invocation -> {
+ Object[] args = invocation.getArguments();
+ int index = ((Integer) args[0]);
+ return mockRow.getColumn(index);
});
}
}
