This is an automated email from the ASF dual-hosted git repository.
dengzh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 4f7200dc14a HIVE-28399: Improve the fetch size in HiveConnection
(#5362) (Zhihua Deng, reviewed by Butao Zhang)
4f7200dc14a is described below
commit 4f7200dc14acf71e189db05376baf4280742ff94
Author: dengzh <[email protected]>
AuthorDate: Wed Aug 21 16:44:19 2024 +0800
HIVE-28399: Improve the fetch size in HiveConnection (#5362) (Zhihua Deng,
reviewed by Butao Zhang)
---
.../java/org/apache/hive/jdbc/HiveConnection.java | 30 ++++++++++------------
.../java/org/apache/hive/jdbc/HiveStatement.java | 14 +++-------
.../hive/jdbc/TestHivePreparedStatement.java | 4 ++-
.../org/apache/hive/jdbc/TestHiveStatement.java | 19 +++++++-------
4 files changed, 29 insertions(+), 38 deletions(-)
diff --git a/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
b/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
index 197640cd388..2850ccdddde 100644
--- a/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
+++ b/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
@@ -175,8 +175,7 @@ public class HiveConnection implements java.sql.Connection {
private final List<TProtocolVersion> supportedProtocols = new
LinkedList<TProtocolVersion>();
private int loginTimeout = 0;
private TProtocolVersion protocol;
- private final int initFetchSize;
- private int defaultFetchSize;
+ int fetchSize;
private String initFile = null;
private String wmPool = null, wmApp = null;
private Properties clientInfo;
@@ -278,7 +277,7 @@ public class HiveConnection implements java.sql.Connection {
public HiveConnection() {
sessConfMap = null;
isEmbeddedMode = true;
- initFetchSize = 0;
+ fetchSize = 50;
}
public HiveConnection(String uri, Properties info) throws SQLException {
@@ -320,8 +319,6 @@ public class HiveConnection implements java.sql.Connection {
port = connParams.getPort();
isEmbeddedMode = connParams.isEmbeddedMode();
- initFetchSize =
Integer.parseInt(sessConfMap.getOrDefault(JdbcConnectionParams.FETCH_SIZE,
"0"));
-
if (sessConfMap.containsKey(JdbcConnectionParams.INIT_FILE)) {
initFile = sessConfMap.get(JdbcConnectionParams.INIT_FILE);
}
@@ -1250,17 +1247,17 @@ public class HiveConnection implements
java.sql.Connection {
protocol = openResp.getServerProtocolVersion();
sessHandle = openResp.getSessionHandle();
- final String serverFetchSizeString =
-
openResp.getConfiguration().get(ConfVars.HIVE_SERVER2_THRIFT_RESULTSET_DEFAULT_FETCH_SIZE.varname);
- if (serverFetchSizeString == null) {
- throw new IllegalStateException("Server returned a null default fetch
size. Check that "
- + ConfVars.HIVE_SERVER2_THRIFT_RESULTSET_DEFAULT_FETCH_SIZE.varname
+ " is configured correctly.");
- }
-
- this.defaultFetchSize = Integer.parseInt(serverFetchSizeString);
- if (this.defaultFetchSize <= 0) {
+ ConfVars confVars =
ConfVars.HIVE_SERVER2_THRIFT_RESULTSET_DEFAULT_FETCH_SIZE;
+ int serverFetchSize =
Optional.ofNullable(openResp.getConfiguration().get(confVars.varname))
+ .map(size -> Integer.parseInt(size))
+ .orElse(confVars.defaultIntVal);
+ if (serverFetchSize <= 0) {
throw new IllegalStateException("Default fetch size must be greater than
0");
}
+ this.fetchSize =
Optional.ofNullable(sessConfMap.get(JdbcConnectionParams.FETCH_SIZE))
+ .map(size -> Integer.parseInt(size))
+ .filter(v -> v > 0)
+ .orElse(serverFetchSize);
}
/**
@@ -1577,7 +1574,7 @@ public class HiveConnection implements
java.sql.Connection {
if (isClosed) {
throw new SQLException("Can't create Statement, connection is closed");
}
- return new HiveStatement(this, client, sessHandle, false, initFetchSize,
defaultFetchSize);
+ return new HiveStatement(this, client, sessHandle, false, fetchSize);
}
/*
@@ -1600,8 +1597,7 @@ public class HiveConnection implements
java.sql.Connection {
if (isClosed) {
throw new SQLException("Connection is closed");
}
- return new HiveStatement(this, client, sessHandle, resultSetType ==
ResultSet.TYPE_SCROLL_INSENSITIVE,
- initFetchSize, defaultFetchSize);
+ return new HiveStatement(this, client, sessHandle, resultSetType ==
ResultSet.TYPE_SCROLL_INSENSITIVE, fetchSize);
}
/*
diff --git a/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java
b/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java
index ce8197ba0a5..506cfb30449 100644
--- a/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java
+++ b/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java
@@ -20,7 +20,6 @@ package org.apache.hive.jdbc;
import org.apache.commons.lang3.StringUtils;
import
org.apache.hadoop.hive.common.classification.InterfaceAudience.LimitedPrivate;
-import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hive.jdbc.logs.InPlaceUpdateStream;
import org.apache.hive.service.cli.RowSet;
import org.apache.hive.service.cli.RowSetFactory;
@@ -70,8 +69,6 @@ public class HiveStatement implements java.sql.Statement {
private static final Logger LOG =
LoggerFactory.getLogger(HiveStatement.class);
public static final String QUERY_CANCELLED_MESSAGE = "Query was cancelled.";
- private static final int DEFAULT_FETCH_SIZE =
-
HiveConf.ConfVars.HIVE_SERVER2_THRIFT_RESULTSET_DEFAULT_FETCH_SIZE.defaultIntVal;
private final HiveConnection connection;
private TCLIService.Iface client;
@@ -79,7 +76,6 @@ public class HiveStatement implements java.sql.Statement {
private final TSessionHandle sessHandle;
Map<String, String> sessConf = new HashMap<>();
private int fetchSize;
- private final int defaultFetchSize;
private final boolean isScrollableResultset;
private boolean isOperationComplete = false;
private boolean closeOnResultSetCompletion = false;
@@ -130,22 +126,20 @@ public class HiveStatement implements java.sql.Statement {
public HiveStatement(HiveConnection connection, TCLIService.Iface client,
TSessionHandle sessHandle) {
- this(connection, client, sessHandle, false, 0, DEFAULT_FETCH_SIZE);
+ this(connection, client, sessHandle, false, connection.fetchSize);
}
public HiveStatement(HiveConnection connection, TCLIService.Iface client,
TSessionHandle sessHandle,
- boolean isScrollableResultset, int initFetchSize, int defaultFetchSize) {
+ boolean isScrollableResultset, int fetchSize) {
this.connection = Objects.requireNonNull(connection);
this.client = Objects.requireNonNull(client);
this.sessHandle = Objects.requireNonNull(sessHandle);
- if (initFetchSize < 0 || defaultFetchSize <= 0) {
+ if ((this.fetchSize = fetchSize) <= 0) {
throw new IllegalArgumentException();
}
this.isScrollableResultset = isScrollableResultset;
- this.defaultFetchSize = defaultFetchSize;
- this.fetchSize = (initFetchSize == 0) ? defaultFetchSize : initFetchSize;
this.inPlaceUpdateStream = Optional.empty();
this.stmtHandle = Optional.empty();
}
@@ -676,7 +670,7 @@ public class HiveStatement implements java.sql.Statement {
if (rows > 0) {
this.fetchSize = rows;
} else if (rows == 0) {
- this.fetchSize = this.defaultFetchSize;
+ this.fetchSize = connection.fetchSize;
} else {
throw new SQLException("Fetch size must be greater or equal to 0");
}
diff --git a/jdbc/src/test/org/apache/hive/jdbc/TestHivePreparedStatement.java
b/jdbc/src/test/org/apache/hive/jdbc/TestHivePreparedStatement.java
index 394c89cd066..456669f9d7a 100644
--- a/jdbc/src/test/org/apache/hive/jdbc/TestHivePreparedStatement.java
+++ b/jdbc/src/test/org/apache/hive/jdbc/TestHivePreparedStatement.java
@@ -19,9 +19,11 @@ package org.apache.hive.jdbc;
import static org.junit.Assert.assertEquals;
import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.withSettings;
import java.sql.SQLException;
@@ -47,7 +49,6 @@ import java.io.ByteArrayInputStream;
public class TestHivePreparedStatement {
- @Mock
private HiveConnection connection;
@Mock
private Iface client;
@@ -65,6 +66,7 @@ public class TestHivePreparedStatement {
@Before
public void before() throws Exception {
+ connection = mock(HiveConnection.class, withSettings().useConstructor());
MockitoAnnotations.initMocks(this);
when(tExecStatementResp.getStatus()).thenReturn(tStatusSuccess);
when(tExecStatementResp.getOperationHandle()).thenReturn(tOperationHandle);
diff --git a/jdbc/src/test/org/apache/hive/jdbc/TestHiveStatement.java
b/jdbc/src/test/org/apache/hive/jdbc/TestHiveStatement.java
index 62a1fbdebf2..ee88d1c492c 100644
--- a/jdbc/src/test/org/apache/hive/jdbc/TestHiveStatement.java
+++ b/jdbc/src/test/org/apache/hive/jdbc/TestHiveStatement.java
@@ -19,6 +19,7 @@ package org.apache.hive.jdbc;
import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.withSettings;
import java.sql.SQLException;
import java.sql.SQLFeatureNotSupportedException;
@@ -39,7 +40,7 @@ public class TestHiveStatement {
*/
@Test
public void testSetFetchSize() throws SQLException {
- final HiveConnection connection = mock(HiveConnection.class);
+ final HiveConnection connection = mock(HiveConnection.class,
withSettings().useConstructor());
final Iface iface = mock(Iface.class);
final TSessionHandle handle = mock(TSessionHandle.class);
@@ -63,10 +64,11 @@ public class TestHiveStatement {
// No hint specified and no default value passed in through the
constructor,
// so it falls-back to the configuration default value
+ int fetchSize =
HiveConf.ConfVars.HIVE_SERVER2_THRIFT_RESULTSET_DEFAULT_FETCH_SIZE.defaultIntVal;
+ connection.fetchSize = fetchSize;
try (HiveStatement stmt = new HiveStatement(connection, iface, handle)) {
stmt.setFetchSize(0);
-
assertEquals(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_RESULTSET_DEFAULT_FETCH_SIZE.defaultIntVal,
- stmt.getFetchSize());
+ assertEquals(fetchSize, stmt.getFetchSize());
}
}
@@ -83,10 +85,7 @@ public class TestHiveStatement {
final Iface iface = mock(Iface.class);
final TSessionHandle handle = mock(TSessionHandle.class);
- // No hint specified and no default value passed in through the
constructor,
- // so it falls-back to a value 1000
- try (HiveStatement stmt = new HiveStatement(connection, iface, handle,
false, 0, 10)) {
- stmt.setFetchSize(0);
+ try (HiveStatement stmt = new HiveStatement(connection, iface, handle,
false, 10)) {
assertEquals(10, stmt.getFetchSize());
}
}
@@ -106,7 +105,7 @@ public class TestHiveStatement {
final Iface iface = mock(Iface.class);
final TSessionHandle handle = mock(TSessionHandle.class);
- try (HiveStatement stmt = new HiveStatement(connection, iface, handle,
false, 4, 1000)) {
+ try (HiveStatement stmt = new HiveStatement(connection, iface, handle,
false, 4)) {
assertEquals(4, stmt.getFetchSize());
}
}
@@ -121,7 +120,7 @@ public class TestHiveStatement {
*/
@Test(expected = SQLException.class)
public void testSetFetchSizeNegativeValue() throws SQLException {
- final HiveConnection connection = mock(HiveConnection.class);
+ final HiveConnection connection = mock(HiveConnection.class,
withSettings().useConstructor());
final Iface iface = mock(Iface.class);
final TSessionHandle handle = mock(TSessionHandle.class);
@@ -132,7 +131,7 @@ public class TestHiveStatement {
@Test(expected = SQLFeatureNotSupportedException.class)
public void testaddBatch() throws SQLException {
- final HiveConnection connection = mock(HiveConnection.class);
+ final HiveConnection connection = mock(HiveConnection.class,
withSettings().useConstructor());
final Iface iface = mock(Iface.class);
final TSessionHandle handle = mock(TSessionHandle.class);