[jira] [Work logged] (HIVE-23873) Querying Hive JDBCStorageHandler table fails with NPE when CBO is off
[
https://issues.apache.org/jira/browse/HIVE-23873?focusedWorklogId=463720&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-463720
]
ASF GitHub Bot logged work on HIVE-23873:
-
Author: ASF GitHub Bot
Created on: 29/Jul/20 07:25
Start Date: 29/Jul/20 07:25
Worklog Time Spent: 10m
Work Description: cravani commented on a change in pull request #1328:
URL: https://github.com/apache/hive/pull/1328#discussion_r461806183
##
File path:
jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/JdbcRecordIterator.java
##
@@ -59,9 +63,26 @@ public JdbcRecordIterator(Connection conn, PreparedStatement
ps, ResultSet rs, C
fieldNamesProperty =
Preconditions.checkNotNull(conf.get(Constants.JDBC_QUERY_FIELD_NAMES));
fieldTypesProperty =
Preconditions.checkNotNull(conf.get(Constants.JDBC_QUERY_FIELD_TYPES));
} else {
- fieldNamesProperty =
Preconditions.checkNotNull(conf.get(serdeConstants.LIST_COLUMNS));
- fieldTypesProperty =
Preconditions.checkNotNull(conf.get(serdeConstants.LIST_COLUMN_TYPES));
+try {
Review comment:
fixed :)
##
File path:
jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/JdbcRecordIterator.java
##
@@ -59,9 +63,26 @@ public JdbcRecordIterator(Connection conn, PreparedStatement
ps, ResultSet rs, C
fieldNamesProperty =
Preconditions.checkNotNull(conf.get(Constants.JDBC_QUERY_FIELD_NAMES));
fieldTypesProperty =
Preconditions.checkNotNull(conf.get(Constants.JDBC_QUERY_FIELD_TYPES));
} else {
- fieldNamesProperty =
Preconditions.checkNotNull(conf.get(serdeConstants.LIST_COLUMNS));
- fieldTypesProperty =
Preconditions.checkNotNull(conf.get(serdeConstants.LIST_COLUMN_TYPES));
+try {
+ if(conf.get(Constants.JDBC_QUERY) == null){
+ResultSetMetaData metadata = rs.getMetaData();
+int numColumns = metadata.getColumnCount();
+List columnNames = new ArrayList(numColumns);
+for (int i = 0; i < numColumns; i++) {
+ columnNames.add(metadata.getColumnName(i + 1));
+}
+fieldNamesProperty = String.join(",",columnNames);
+ }
+ else
Review comment:
{ & }. are required? I was assuming since we have single statement
within conditional block, it should be fine.
##
File path:
jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/JdbcRecordIterator.java
##
@@ -59,9 +63,26 @@ public JdbcRecordIterator(Connection conn, PreparedStatement
ps, ResultSet rs, C
fieldNamesProperty =
Preconditions.checkNotNull(conf.get(Constants.JDBC_QUERY_FIELD_NAMES));
fieldTypesProperty =
Preconditions.checkNotNull(conf.get(Constants.JDBC_QUERY_FIELD_TYPES));
} else {
- fieldNamesProperty =
Preconditions.checkNotNull(conf.get(serdeConstants.LIST_COLUMNS));
- fieldTypesProperty =
Preconditions.checkNotNull(conf.get(serdeConstants.LIST_COLUMN_TYPES));
+try {
+ if(conf.get(Constants.JDBC_QUERY) == null){
Review comment:
Fixed.
##
File path:
jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/JdbcRecordIterator.java
##
@@ -59,9 +63,26 @@ public JdbcRecordIterator(Connection conn, PreparedStatement
ps, ResultSet rs, C
fieldNamesProperty =
Preconditions.checkNotNull(conf.get(Constants.JDBC_QUERY_FIELD_NAMES));
fieldTypesProperty =
Preconditions.checkNotNull(conf.get(Constants.JDBC_QUERY_FIELD_TYPES));
} else {
- fieldNamesProperty =
Preconditions.checkNotNull(conf.get(serdeConstants.LIST_COLUMNS));
- fieldTypesProperty =
Preconditions.checkNotNull(conf.get(serdeConstants.LIST_COLUMN_TYPES));
+try {
+ if(conf.get(Constants.JDBC_QUERY) == null){
+ResultSetMetaData metadata = rs.getMetaData();
+int numColumns = metadata.getColumnCount();
+List columnNames = new ArrayList(numColumns);
+for (int i = 0; i < numColumns; i++) {
+ columnNames.add(metadata.getColumnName(i + 1));
+}
+fieldNamesProperty = String.join(",",columnNames);
+ }
+ else
+fieldNamesProperty =
Preconditions.checkNotNull(conf.get(serdeConstants.LIST_COLUMNS));
+}
+catch (Exception e) {
+ LOGGER.error("Error while trying to get column names.", e);
+ throw new HiveJdbcDatabaseAccessException("Error while trying to get
column names: " + e.getMessage(), e);
+}
+fieldTypesProperty =
Preconditions.checkNotNull(conf.get(serdeConstants.LIST_COLUMN_TYPES));
}
+LOGGER.debug("Iterator ColumnNames = "+fieldNamesProperty);
Review comment:
fixed.
##
File path:
jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/JdbcRecordIterator.java
##
@@ -59,9
[jira] [Work logged] (HIVE-23873) Querying Hive JDBCStorageHandler table fails with NPE when CBO is off
[
https://issues.apache.org/jira/browse/HIVE-23873?focusedWorklogId=463991&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-463991
]
ASF GitHub Bot logged work on HIVE-23873:
-
Author: ASF GitHub Bot
Created on: 29/Jul/20 07:51
Start Date: 29/Jul/20 07:51
Worklog Time Spent: 10m
Work Description: jcamachor commented on a change in pull request #1328:
URL: https://github.com/apache/hive/pull/1328#discussion_r461672364
##
File path:
jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/JdbcRecordIterator.java
##
@@ -59,9 +63,26 @@ public JdbcRecordIterator(Connection conn, PreparedStatement
ps, ResultSet rs, C
fieldNamesProperty =
Preconditions.checkNotNull(conf.get(Constants.JDBC_QUERY_FIELD_NAMES));
fieldTypesProperty =
Preconditions.checkNotNull(conf.get(Constants.JDBC_QUERY_FIELD_TYPES));
} else {
- fieldNamesProperty =
Preconditions.checkNotNull(conf.get(serdeConstants.LIST_COLUMNS));
- fieldTypesProperty =
Preconditions.checkNotNull(conf.get(serdeConstants.LIST_COLUMN_TYPES));
+try {
+ if(conf.get(Constants.JDBC_QUERY) == null){
+ResultSetMetaData metadata = rs.getMetaData();
+int numColumns = metadata.getColumnCount();
+List columnNames = new ArrayList(numColumns);
+for (int i = 0; i < numColumns; i++) {
+ columnNames.add(metadata.getColumnName(i + 1));
+}
+fieldNamesProperty = String.join(",",columnNames);
+ }
+ else
+fieldNamesProperty =
Preconditions.checkNotNull(conf.get(serdeConstants.LIST_COLUMNS));
+}
+catch (Exception e) {
+ LOGGER.error("Error while trying to get column names.", e);
+ throw new HiveJdbcDatabaseAccessException("Error while trying to get
column names: " + e.getMessage(), e);
+}
+fieldTypesProperty =
Preconditions.checkNotNull(conf.get(serdeConstants.LIST_COLUMN_TYPES));
}
+LOGGER.debug("Iterator ColumnNames = "+fieldNamesProperty);
Review comment:
`LOGGER.debug("Iterator ColumnNames = {}", fieldNamesProperty);`
##
File path:
jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/JdbcRecordIterator.java
##
@@ -59,9 +63,26 @@ public JdbcRecordIterator(Connection conn, PreparedStatement
ps, ResultSet rs, C
fieldNamesProperty =
Preconditions.checkNotNull(conf.get(Constants.JDBC_QUERY_FIELD_NAMES));
fieldTypesProperty =
Preconditions.checkNotNull(conf.get(Constants.JDBC_QUERY_FIELD_TYPES));
} else {
- fieldNamesProperty =
Preconditions.checkNotNull(conf.get(serdeConstants.LIST_COLUMNS));
- fieldTypesProperty =
Preconditions.checkNotNull(conf.get(serdeConstants.LIST_COLUMN_TYPES));
+try {
Review comment:
nit. Indentation seems off.
##
File path:
jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/JdbcRecordIterator.java
##
@@ -59,9 +63,26 @@ public JdbcRecordIterator(Connection conn, PreparedStatement
ps, ResultSet rs, C
fieldNamesProperty =
Preconditions.checkNotNull(conf.get(Constants.JDBC_QUERY_FIELD_NAMES));
fieldTypesProperty =
Preconditions.checkNotNull(conf.get(Constants.JDBC_QUERY_FIELD_TYPES));
} else {
- fieldNamesProperty =
Preconditions.checkNotNull(conf.get(serdeConstants.LIST_COLUMNS));
- fieldTypesProperty =
Preconditions.checkNotNull(conf.get(serdeConstants.LIST_COLUMN_TYPES));
+try {
+ if(conf.get(Constants.JDBC_QUERY) == null){
+ResultSetMetaData metadata = rs.getMetaData();
+int numColumns = metadata.getColumnCount();
+List columnNames = new ArrayList(numColumns);
+for (int i = 0; i < numColumns; i++) {
+ columnNames.add(metadata.getColumnName(i + 1));
+}
+fieldNamesProperty = String.join(",",columnNames);
+ }
+ else
Review comment:
This is by convention, we use { & } even for single line statements.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 463991)
Time Spent: 0.5h (was: 20m)
> Querying Hive JDBCStorageHandler table fails with NPE when CBO is off
> -
>
> Key: HIVE-23873
> URL: https://issues.apache.org/jira/browse/HIVE-23873
> Project: Hive
> Issue Type: Bug
> Components: HiveServer2, JDBC
>Affects Versions: 3.1.0, 3.1.1, 3.1.2
>
[jira] [Work logged] (HIVE-23873) Querying Hive JDBCStorageHandler table fails with NPE when CBO is off
[
https://issues.apache.org/jira/browse/HIVE-23873?focusedWorklogId=464068&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-464068
]
ASF GitHub Bot logged work on HIVE-23873:
-
Author: ASF GitHub Bot
Created on: 29/Jul/20 07:58
Start Date: 29/Jul/20 07:58
Worklog Time Spent: 10m
Work Description: cravani opened a new pull request #1328:
URL: https://github.com/apache/hive/pull/1328
…n CBO is off
## NOTICE
Please create an issue in ASF JIRA before opening a pull request,
and you need to set the title of the pull request which starts with
the corresponding JIRA issue number. (e.g. HIVE-X: Fix a typo in YYY)
For more details, please see
https://cwiki.apache.org/confluence/display/Hive/HowToContribute
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 464068)
Time Spent: 40m (was: 0.5h)
> Querying Hive JDBCStorageHandler table fails with NPE when CBO is off
> -
>
> Key: HIVE-23873
> URL: https://issues.apache.org/jira/browse/HIVE-23873
> Project: Hive
> Issue Type: Bug
> Components: HiveServer2, JDBC
>Affects Versions: 3.1.0, 3.1.1, 3.1.2
>Reporter: Chiran Ravani
>Assignee: Chiran Ravani
>Priority: Critical
> Labels: pull-request-available
> Attachments: HIVE-23873.01.patch, HIVE-23873.02.patch,
> HIVE-23873.3.patch, HIVE-23873.4.patch
>
> Time Spent: 40m
> Remaining Estimate: 0h
>
> Scenario is Hive table having same schema as table in Oracle, however when we
> query the table with data it fails with NPE, below is the trace.
> {code}
> Caused by: java.io.IOException: java.lang.NullPointerException
> at
> org.apache.hadoop.hive.ql.exec.FetchOperator.getNextRow(FetchOperator.java:617)
> ~[hive-exec-3.1.0.3.1.5.0-152.jar:3.1.0.3.1.5.0-152]
> at
> org.apache.hadoop.hive.ql.exec.FetchOperator.pushRow(FetchOperator.java:524)
> ~[hive-exec-3.1.0.3.1.5.0-152.jar:3.1.0.3.1.5.0-152]
> at org.apache.hadoop.hive.ql.exec.FetchTask.fetch(FetchTask.java:146)
> ~[hive-exec-3.1.0.3.1.5.0-152.jar:3.1.0.3.1.5.0-152]
> at org.apache.hadoop.hive.ql.Driver.getResults(Driver.java:2739)
> ~[hive-exec-3.1.0.3.1.5.0-152.jar:3.1.0.3.1.5.0-152]
> at
> org.apache.hadoop.hive.ql.reexec.ReExecDriver.getResults(ReExecDriver.java:229)
> ~[hive-exec-3.1.0.3.1.5.0-152.jar:3.1.0.3.1.5.0-152]
> at
> org.apache.hive.service.cli.operation.SQLOperation.getNextRowSet(SQLOperation.java:473)
> ~[hive-service-3.1.0.3.1.5.0-152.jar:3.1.0.3.1.5.0-152]
> ... 34 more
> Caused by: java.lang.NullPointerException
> at
> org.apache.hive.storage.jdbc.JdbcSerDe.deserialize(JdbcSerDe.java:164)
> ~[hive-jdbc-handler-3.1.0.3.1.5.0-152.jar:3.1.0.3.1.5.0-152]
> at
> org.apache.hadoop.hive.ql.exec.FetchOperator.getNextRow(FetchOperator.java:598)
> ~[hive-exec-3.1.0.3.1.5.0-152.jar:3.1.0.3.1.5.0-152]
> at
> org.apache.hadoop.hive.ql.exec.FetchOperator.pushRow(FetchOperator.java:524)
> ~[hive-exec-3.1.0.3.1.5.0-152.jar:3.1.0.3.1.5.0-152]
> at org.apache.hadoop.hive.ql.exec.FetchTask.fetch(FetchTask.java:146)
> ~[hive-exec-3.1.0.3.1.5.0-152.jar:3.1.0.3.1.5.0-152]
> at org.apache.hadoop.hive.ql.Driver.getResults(Driver.java:2739)
> ~[hive-exec-3.1.0.3.1.5.0-152.jar:3.1.0.3.1.5.0-152]
> at
> org.apache.hadoop.hive.ql.reexec.ReExecDriver.getResults(ReExecDriver.java:229)
> ~[hive-exec-3.1.0.3.1.5.0-152.jar:3.1.0.3.1.5.0-152]
> at
> org.apache.hive.service.cli.operation.SQLOperation.getNextRowSet(SQLOperation.java:473)
> ~[hive-service-3.1.0.3.1.5.0-152.jar:3.1.0.3.1.5.0-152]
> ... 34 more
> {code}
> Problem appears when column names in Oracle are in Upper case and since in
> Hive, table and column names are forced to store in lowercase during
> creation. User runs into NPE error while fetching data.
> While deserializing data, input consists of column names in lower case which
> fails to get the value
> https://github.com/apache/hive/blob/rel/release-3.1.2/jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/JdbcSerDe.java#L136
> {code}
> rowVal = ((ObjectWritable)value).get();
> {code}
> Log Snio:
> =
> {code}
> 2020-07-17T16:49:09,598 INFO [04ed42ec-91d2-4662-aee7-37e840a06036
> HiveServer2-Handler-Pool: Thread-104]: dao.GenericJdbcDatabaseAccessor (:())
> - Query to execute is [select * from TESTHIVEJDBCSTORAGE
[jira] [Work logged] (HIVE-23873) Querying Hive JDBCStorageHandler table fails with NPE when CBO is off
[
https://issues.apache.org/jira/browse/HIVE-23873?focusedWorklogId=463990&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-463990
]
ASF GitHub Bot logged work on HIVE-23873:
-
Author: ASF GitHub Bot
Created on: 29/Jul/20 07:51
Start Date: 29/Jul/20 07:51
Worklog Time Spent: 10m
Work Description: HunterL commented on a change in pull request #1328:
URL: https://github.com/apache/hive/pull/1328#discussion_r461646240
##
File path:
jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/JdbcRecordIterator.java
##
@@ -59,9 +63,26 @@ public JdbcRecordIterator(Connection conn, PreparedStatement
ps, ResultSet rs, C
fieldNamesProperty =
Preconditions.checkNotNull(conf.get(Constants.JDBC_QUERY_FIELD_NAMES));
fieldTypesProperty =
Preconditions.checkNotNull(conf.get(Constants.JDBC_QUERY_FIELD_TYPES));
} else {
- fieldNamesProperty =
Preconditions.checkNotNull(conf.get(serdeConstants.LIST_COLUMNS));
- fieldTypesProperty =
Preconditions.checkNotNull(conf.get(serdeConstants.LIST_COLUMN_TYPES));
+try {
+ if(conf.get(Constants.JDBC_QUERY) == null){
+ResultSetMetaData metadata = rs.getMetaData();
+int numColumns = metadata.getColumnCount();
+List columnNames = new ArrayList(numColumns);
+for (int i = 0; i < numColumns; i++) {
+ columnNames.add(metadata.getColumnName(i + 1));
+}
+fieldNamesProperty = String.join(",",columnNames);
+ }
+ else
Review comment:
Missing `{` & ` }` for this else
##
File path:
jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/JdbcRecordIterator.java
##
@@ -59,9 +63,26 @@ public JdbcRecordIterator(Connection conn, PreparedStatement
ps, ResultSet rs, C
fieldNamesProperty =
Preconditions.checkNotNull(conf.get(Constants.JDBC_QUERY_FIELD_NAMES));
fieldTypesProperty =
Preconditions.checkNotNull(conf.get(Constants.JDBC_QUERY_FIELD_TYPES));
} else {
- fieldNamesProperty =
Preconditions.checkNotNull(conf.get(serdeConstants.LIST_COLUMNS));
- fieldTypesProperty =
Preconditions.checkNotNull(conf.get(serdeConstants.LIST_COLUMN_TYPES));
+try {
+ if(conf.get(Constants.JDBC_QUERY) == null){
Review comment:
Small white space nitpick here, should be `if
(conf.get(Constants.JDBC_QUERY) == null) {`
##
File path:
jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/JdbcRecordIterator.java
##
@@ -59,9 +63,26 @@ public JdbcRecordIterator(Connection conn, PreparedStatement
ps, ResultSet rs, C
fieldNamesProperty =
Preconditions.checkNotNull(conf.get(Constants.JDBC_QUERY_FIELD_NAMES));
fieldTypesProperty =
Preconditions.checkNotNull(conf.get(Constants.JDBC_QUERY_FIELD_TYPES));
} else {
- fieldNamesProperty =
Preconditions.checkNotNull(conf.get(serdeConstants.LIST_COLUMNS));
- fieldTypesProperty =
Preconditions.checkNotNull(conf.get(serdeConstants.LIST_COLUMN_TYPES));
+try {
+ if(conf.get(Constants.JDBC_QUERY) == null){
+ResultSetMetaData metadata = rs.getMetaData();
+int numColumns = metadata.getColumnCount();
+List columnNames = new ArrayList(numColumns);
+for (int i = 0; i < numColumns; i++) {
+ columnNames.add(metadata.getColumnName(i + 1));
+}
+fieldNamesProperty = String.join(",",columnNames);
+ }
+ else
+fieldNamesProperty =
Preconditions.checkNotNull(conf.get(serdeConstants.LIST_COLUMNS));
+}
+catch (Exception e) {
+ LOGGER.error("Error while trying to get column names.", e);
+ throw new HiveJdbcDatabaseAccessException("Error while trying to get
column names: " + e.getMessage(), e);
+}
+fieldTypesProperty =
Preconditions.checkNotNull(conf.get(serdeConstants.LIST_COLUMN_TYPES));
}
+LOGGER.debug("Iterator ColumnNames = "+fieldNamesProperty);
Review comment:
Small whitespace nitpick here `LOGGER.debug("Iterator ColumnNames = " +
fieldNamesProperty);`
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 463990)
Time Spent: 20m (was: 10m)
> Querying Hive JDBCStorageHandler table fails with NPE when CBO is off
> -
>
> Key: HIVE-23873
> URL: https://issues.apache.org/jira/browse/HIVE-23873
> Project: Hive
>
[jira] [Work logged] (HIVE-23873) Querying Hive JDBCStorageHandler table fails with NPE when CBO is off
[
https://issues.apache.org/jira/browse/HIVE-23873?focusedWorklogId=464245&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-464245
]
ASF GitHub Bot logged work on HIVE-23873:
-
Author: ASF GitHub Bot
Created on: 29/Jul/20 17:52
Start Date: 29/Jul/20 17:52
Worklog Time Spent: 10m
Work Description: HunterL commented on a change in pull request #1328:
URL: https://github.com/apache/hive/pull/1328#discussion_r462373165
##
File path:
jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/JdbcRecordIterator.java
##
@@ -22,15 +22,19 @@
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.apache.hive.storage.jdbc.exception.HiveJdbcDatabaseAccessException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
import java.sql.SQLDataException;
import java.sql.SQLException;
import java.sql.Types;
+import java.util.Arrays;
Review comment:
I don't believe this import is used
##
File path:
jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/JdbcRecordIterator.java
##
@@ -58,10 +62,29 @@ public JdbcRecordIterator(Connection conn,
PreparedStatement ps, ResultSet rs, C
if (conf.get(Constants.JDBC_TABLE) != null &&
conf.get(Constants.JDBC_QUERY) != null) {
fieldNamesProperty =
Preconditions.checkNotNull(conf.get(Constants.JDBC_QUERY_FIELD_NAMES));
fieldTypesProperty =
Preconditions.checkNotNull(conf.get(Constants.JDBC_QUERY_FIELD_TYPES));
-} else {
- fieldNamesProperty =
Preconditions.checkNotNull(conf.get(serdeConstants.LIST_COLUMNS));
+}
+else {
+ try {
+if (conf.get(Constants.JDBC_QUERY) == null){
Review comment:
Whitespace at the end of this if is still off a bit, should be just one
space after `)`
##
File path:
jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/JdbcRecordIterator.java
##
@@ -58,10 +62,29 @@ public JdbcRecordIterator(Connection conn,
PreparedStatement ps, ResultSet rs, C
if (conf.get(Constants.JDBC_TABLE) != null &&
conf.get(Constants.JDBC_QUERY) != null) {
fieldNamesProperty =
Preconditions.checkNotNull(conf.get(Constants.JDBC_QUERY_FIELD_NAMES));
fieldTypesProperty =
Preconditions.checkNotNull(conf.get(Constants.JDBC_QUERY_FIELD_TYPES));
-} else {
- fieldNamesProperty =
Preconditions.checkNotNull(conf.get(serdeConstants.LIST_COLUMNS));
+}
+else {
Review comment:
Nitpicking here again sorry, lets move the else a line up `} else {`
Also regarding else without the `{ ... }` being valid, you are correct it is
valid java. Stylistically however using the brackets tends to be more readable
and seems to be the dominant style in this project. :+1:
##
File path:
jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/JdbcRecordIterator.java
##
@@ -58,10 +62,29 @@ public JdbcRecordIterator(Connection conn,
PreparedStatement ps, ResultSet rs, C
if (conf.get(Constants.JDBC_TABLE) != null &&
conf.get(Constants.JDBC_QUERY) != null) {
fieldNamesProperty =
Preconditions.checkNotNull(conf.get(Constants.JDBC_QUERY_FIELD_NAMES));
fieldTypesProperty =
Preconditions.checkNotNull(conf.get(Constants.JDBC_QUERY_FIELD_TYPES));
-} else {
- fieldNamesProperty =
Preconditions.checkNotNull(conf.get(serdeConstants.LIST_COLUMNS));
+}
+else {
+ try {
+if (conf.get(Constants.JDBC_QUERY) == null){
+ ResultSetMetaData metadata = rs.getMetaData();
+ int numColumns = metadata.getColumnCount();
+ List columnNames = new ArrayList(numColumns);
+ for (int i = 0; i < numColumns; i++) {
+columnNames.add(metadata.getColumnName(i + 1));
+ }
+ fieldNamesProperty = String.join(",",columnNames);
+}
+else {
Review comment:
Here as well lets move the else up to line 76, `} else {`
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 464245)
Time Spent: 1h (was: 50m)
> Querying Hive JDBCStorageHandler table fails with NPE when CBO is off
> -
>
> Key: HIVE-23873
> URL: https://issues.apache.org/jira/browse/HIVE-23873
> Project:
[jira] [Work logged] (HIVE-23873) Querying Hive JDBCStorageHandler table fails with NPE when CBO is off
[
https://issues.apache.org/jira/browse/HIVE-23873?focusedWorklogId=464230&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-464230
]
ASF GitHub Bot logged work on HIVE-23873:
-
Author: ASF GitHub Bot
Created on: 29/Jul/20 17:51
Start Date: 29/Jul/20 17:51
Worklog Time Spent: 10m
Work Description: cravani commented on a change in pull request #1328:
URL: https://github.com/apache/hive/pull/1328#discussion_r462403223
##
File path:
jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/JdbcRecordIterator.java
##
@@ -22,15 +22,19 @@
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.apache.hive.storage.jdbc.exception.HiveJdbcDatabaseAccessException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
import java.sql.SQLDataException;
import java.sql.SQLException;
import java.sql.Types;
+import java.util.Arrays;
Review comment:
fixed.
##
File path:
jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/JdbcRecordIterator.java
##
@@ -58,10 +62,29 @@ public JdbcRecordIterator(Connection conn,
PreparedStatement ps, ResultSet rs, C
if (conf.get(Constants.JDBC_TABLE) != null &&
conf.get(Constants.JDBC_QUERY) != null) {
fieldNamesProperty =
Preconditions.checkNotNull(conf.get(Constants.JDBC_QUERY_FIELD_NAMES));
fieldTypesProperty =
Preconditions.checkNotNull(conf.get(Constants.JDBC_QUERY_FIELD_TYPES));
-} else {
- fieldNamesProperty =
Preconditions.checkNotNull(conf.get(serdeConstants.LIST_COLUMNS));
+}
+else {
Review comment:
fixed.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 464230)
Time Spent: 50m (was: 40m)
> Querying Hive JDBCStorageHandler table fails with NPE when CBO is off
> -
>
> Key: HIVE-23873
> URL: https://issues.apache.org/jira/browse/HIVE-23873
> Project: Hive
> Issue Type: Bug
> Components: HiveServer2, JDBC
>Affects Versions: 3.1.0, 3.1.1, 3.1.2
>Reporter: Chiran Ravani
>Assignee: Chiran Ravani
>Priority: Critical
> Labels: pull-request-available
> Attachments: HIVE-23873.01.patch, HIVE-23873.02.patch,
> HIVE-23873.3.patch, HIVE-23873.4.patch
>
> Time Spent: 50m
> Remaining Estimate: 0h
>
> Scenario is Hive table having same schema as table in Oracle, however when we
> query the table with data it fails with NPE, below is the trace.
> {code}
> Caused by: java.io.IOException: java.lang.NullPointerException
> at
> org.apache.hadoop.hive.ql.exec.FetchOperator.getNextRow(FetchOperator.java:617)
> ~[hive-exec-3.1.0.3.1.5.0-152.jar:3.1.0.3.1.5.0-152]
> at
> org.apache.hadoop.hive.ql.exec.FetchOperator.pushRow(FetchOperator.java:524)
> ~[hive-exec-3.1.0.3.1.5.0-152.jar:3.1.0.3.1.5.0-152]
> at org.apache.hadoop.hive.ql.exec.FetchTask.fetch(FetchTask.java:146)
> ~[hive-exec-3.1.0.3.1.5.0-152.jar:3.1.0.3.1.5.0-152]
> at org.apache.hadoop.hive.ql.Driver.getResults(Driver.java:2739)
> ~[hive-exec-3.1.0.3.1.5.0-152.jar:3.1.0.3.1.5.0-152]
> at
> org.apache.hadoop.hive.ql.reexec.ReExecDriver.getResults(ReExecDriver.java:229)
> ~[hive-exec-3.1.0.3.1.5.0-152.jar:3.1.0.3.1.5.0-152]
> at
> org.apache.hive.service.cli.operation.SQLOperation.getNextRowSet(SQLOperation.java:473)
> ~[hive-service-3.1.0.3.1.5.0-152.jar:3.1.0.3.1.5.0-152]
> ... 34 more
> Caused by: java.lang.NullPointerException
> at
> org.apache.hive.storage.jdbc.JdbcSerDe.deserialize(JdbcSerDe.java:164)
> ~[hive-jdbc-handler-3.1.0.3.1.5.0-152.jar:3.1.0.3.1.5.0-152]
> at
> org.apache.hadoop.hive.ql.exec.FetchOperator.getNextRow(FetchOperator.java:598)
> ~[hive-exec-3.1.0.3.1.5.0-152.jar:3.1.0.3.1.5.0-152]
> at
> org.apache.hadoop.hive.ql.exec.FetchOperator.pushRow(FetchOperator.java:524)
> ~[hive-exec-3.1.0.3.1.5.0-152.jar:3.1.0.3.1.5.0-152]
> at org.apache.hadoop.hive.ql.exec.FetchTask.fetch(FetchTask.java:146)
> ~[hive-exec-3.1.0.3.1.5.0-152.jar:3.1.0.3.1.5.0-152]
> at org.apache.hadoop.hive.ql.Driver.getResults(Driver.java:2739)
> ~[hive-exec-3.1.0.3.1.5.0-152.jar:3.1.0.3.1.5.0-152]
>
[jira] [Work logged] (HIVE-23873) Querying Hive JDBCStorageHandler table fails with NPE when CBO is off
[
https://issues.apache.org/jira/browse/HIVE-23873?focusedWorklogId=465559&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-465559
]
ASF GitHub Bot logged work on HIVE-23873:
-
Author: ASF GitHub Bot
Created on: 03/Aug/20 06:24
Start Date: 03/Aug/20 06:24
Worklog Time Spent: 10m
Work Description: shameersss1 commented on a change in pull request #1328:
URL: https://github.com/apache/hive/pull/1328#discussion_r464215422
##
File path:
jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/JdbcRecordIterator.java
##
@@ -59,9 +62,26 @@ public JdbcRecordIterator(Connection conn, PreparedStatement
ps, ResultSet rs, C
fieldNamesProperty =
Preconditions.checkNotNull(conf.get(Constants.JDBC_QUERY_FIELD_NAMES));
fieldTypesProperty =
Preconditions.checkNotNull(conf.get(Constants.JDBC_QUERY_FIELD_TYPES));
} else {
- fieldNamesProperty =
Preconditions.checkNotNull(conf.get(serdeConstants.LIST_COLUMNS));
+ try {
+if (conf.get(Constants.JDBC_QUERY) == null) {
+ ResultSetMetaData metadata = rs.getMetaData();
+ int numColumns = metadata.getColumnCount();
+ List columnNames = new ArrayList(numColumns);
+ for (int i = 0; i < numColumns; i++) {
+columnNames.add(metadata.getColumnName(i + 1));
+ }
+ fieldNamesProperty = String.join(",",columnNames);
Review comment:
nit: space missing: **String.join(",", columnNames);**
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 465559)
Time Spent: 1h 10m (was: 1h)
> Querying Hive JDBCStorageHandler table fails with NPE when CBO is off
> -
>
> Key: HIVE-23873
> URL: https://issues.apache.org/jira/browse/HIVE-23873
> Project: Hive
> Issue Type: Bug
> Components: HiveServer2, JDBC
>Affects Versions: 3.1.0, 3.1.1, 3.1.2
>Reporter: Chiran Ravani
>Assignee: Chiran Ravani
>Priority: Critical
> Labels: pull-request-available
> Attachments: HIVE-23873.01.patch, HIVE-23873.02.patch,
> HIVE-23873.3.patch, HIVE-23873.4.patch
>
> Time Spent: 1h 10m
> Remaining Estimate: 0h
>
> Scenario is Hive table having same schema as table in Oracle, however when we
> query the table with data it fails with NPE, below is the trace.
> {code}
> Caused by: java.io.IOException: java.lang.NullPointerException
> at
> org.apache.hadoop.hive.ql.exec.FetchOperator.getNextRow(FetchOperator.java:617)
> ~[hive-exec-3.1.0.3.1.5.0-152.jar:3.1.0.3.1.5.0-152]
> at
> org.apache.hadoop.hive.ql.exec.FetchOperator.pushRow(FetchOperator.java:524)
> ~[hive-exec-3.1.0.3.1.5.0-152.jar:3.1.0.3.1.5.0-152]
> at org.apache.hadoop.hive.ql.exec.FetchTask.fetch(FetchTask.java:146)
> ~[hive-exec-3.1.0.3.1.5.0-152.jar:3.1.0.3.1.5.0-152]
> at org.apache.hadoop.hive.ql.Driver.getResults(Driver.java:2739)
> ~[hive-exec-3.1.0.3.1.5.0-152.jar:3.1.0.3.1.5.0-152]
> at
> org.apache.hadoop.hive.ql.reexec.ReExecDriver.getResults(ReExecDriver.java:229)
> ~[hive-exec-3.1.0.3.1.5.0-152.jar:3.1.0.3.1.5.0-152]
> at
> org.apache.hive.service.cli.operation.SQLOperation.getNextRowSet(SQLOperation.java:473)
> ~[hive-service-3.1.0.3.1.5.0-152.jar:3.1.0.3.1.5.0-152]
> ... 34 more
> Caused by: java.lang.NullPointerException
> at
> org.apache.hive.storage.jdbc.JdbcSerDe.deserialize(JdbcSerDe.java:164)
> ~[hive-jdbc-handler-3.1.0.3.1.5.0-152.jar:3.1.0.3.1.5.0-152]
> at
> org.apache.hadoop.hive.ql.exec.FetchOperator.getNextRow(FetchOperator.java:598)
> ~[hive-exec-3.1.0.3.1.5.0-152.jar:3.1.0.3.1.5.0-152]
> at
> org.apache.hadoop.hive.ql.exec.FetchOperator.pushRow(FetchOperator.java:524)
> ~[hive-exec-3.1.0.3.1.5.0-152.jar:3.1.0.3.1.5.0-152]
> at org.apache.hadoop.hive.ql.exec.FetchTask.fetch(FetchTask.java:146)
> ~[hive-exec-3.1.0.3.1.5.0-152.jar:3.1.0.3.1.5.0-152]
> at org.apache.hadoop.hive.ql.Driver.getResults(Driver.java:2739)
> ~[hive-exec-3.1.0.3.1.5.0-152.jar:3.1.0.3.1.5.0-152]
> at
> org.apache.hadoop.hive.ql.reexec.ReExecDriver.getResults(ReExecDriver.java:229)
> ~[hive-exec-3.1.0.3.1.5.0-152.jar:3.1.0.3.1.5.0-152]
> at
> org.apache.hive.service.cli.operation.SQLOperation.getNextRowSet(SQLOperation.java:473)
> ~[hive-service-3.1.0.3.1.5.0-152.jar:3.1.0.3.1.5.0-152]
> ... 34 more
> {code}
> Problem appears when colum
[jira] [Work logged] (HIVE-23873) Querying Hive JDBCStorageHandler table fails with NPE when CBO is off
[
https://issues.apache.org/jira/browse/HIVE-23873?focusedWorklogId=465729&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-465729
]
ASF GitHub Bot logged work on HIVE-23873:
-
Author: ASF GitHub Bot
Created on: 03/Aug/20 14:40
Start Date: 03/Aug/20 14:40
Worklog Time Spent: 10m
Work Description: jcamachor merged pull request #1328:
URL: https://github.com/apache/hive/pull/1328
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
---
Worklog Id: (was: 465729)
Time Spent: 1h 20m (was: 1h 10m)
> Querying Hive JDBCStorageHandler table fails with NPE when CBO is off
> -
>
> Key: HIVE-23873
> URL: https://issues.apache.org/jira/browse/HIVE-23873
> Project: Hive
> Issue Type: Bug
> Components: HiveServer2, JDBC
>Affects Versions: 3.1.0, 3.1.1, 3.1.2
>Reporter: Chiran Ravani
>Assignee: Chiran Ravani
>Priority: Critical
> Labels: pull-request-available
> Attachments: HIVE-23873.01.patch, HIVE-23873.02.patch,
> HIVE-23873.3.patch, HIVE-23873.4.patch
>
> Time Spent: 1h 20m
> Remaining Estimate: 0h
>
> Scenario is Hive table having same schema as table in Oracle, however when we
> query the table with data it fails with NPE, below is the trace.
> {code}
> Caused by: java.io.IOException: java.lang.NullPointerException
> at
> org.apache.hadoop.hive.ql.exec.FetchOperator.getNextRow(FetchOperator.java:617)
> ~[hive-exec-3.1.0.3.1.5.0-152.jar:3.1.0.3.1.5.0-152]
> at
> org.apache.hadoop.hive.ql.exec.FetchOperator.pushRow(FetchOperator.java:524)
> ~[hive-exec-3.1.0.3.1.5.0-152.jar:3.1.0.3.1.5.0-152]
> at org.apache.hadoop.hive.ql.exec.FetchTask.fetch(FetchTask.java:146)
> ~[hive-exec-3.1.0.3.1.5.0-152.jar:3.1.0.3.1.5.0-152]
> at org.apache.hadoop.hive.ql.Driver.getResults(Driver.java:2739)
> ~[hive-exec-3.1.0.3.1.5.0-152.jar:3.1.0.3.1.5.0-152]
> at
> org.apache.hadoop.hive.ql.reexec.ReExecDriver.getResults(ReExecDriver.java:229)
> ~[hive-exec-3.1.0.3.1.5.0-152.jar:3.1.0.3.1.5.0-152]
> at
> org.apache.hive.service.cli.operation.SQLOperation.getNextRowSet(SQLOperation.java:473)
> ~[hive-service-3.1.0.3.1.5.0-152.jar:3.1.0.3.1.5.0-152]
> ... 34 more
> Caused by: java.lang.NullPointerException
> at
> org.apache.hive.storage.jdbc.JdbcSerDe.deserialize(JdbcSerDe.java:164)
> ~[hive-jdbc-handler-3.1.0.3.1.5.0-152.jar:3.1.0.3.1.5.0-152]
> at
> org.apache.hadoop.hive.ql.exec.FetchOperator.getNextRow(FetchOperator.java:598)
> ~[hive-exec-3.1.0.3.1.5.0-152.jar:3.1.0.3.1.5.0-152]
> at
> org.apache.hadoop.hive.ql.exec.FetchOperator.pushRow(FetchOperator.java:524)
> ~[hive-exec-3.1.0.3.1.5.0-152.jar:3.1.0.3.1.5.0-152]
> at org.apache.hadoop.hive.ql.exec.FetchTask.fetch(FetchTask.java:146)
> ~[hive-exec-3.1.0.3.1.5.0-152.jar:3.1.0.3.1.5.0-152]
> at org.apache.hadoop.hive.ql.Driver.getResults(Driver.java:2739)
> ~[hive-exec-3.1.0.3.1.5.0-152.jar:3.1.0.3.1.5.0-152]
> at
> org.apache.hadoop.hive.ql.reexec.ReExecDriver.getResults(ReExecDriver.java:229)
> ~[hive-exec-3.1.0.3.1.5.0-152.jar:3.1.0.3.1.5.0-152]
> at
> org.apache.hive.service.cli.operation.SQLOperation.getNextRowSet(SQLOperation.java:473)
> ~[hive-service-3.1.0.3.1.5.0-152.jar:3.1.0.3.1.5.0-152]
> ... 34 more
> {code}
> Problem appears when column names in Oracle are in Upper case and since in
> Hive, table and column names are forced to store in lowercase during
> creation. User runs into NPE error while fetching data.
> While deserializing data, input consists of column names in lower case which
> fails to get the value
> https://github.com/apache/hive/blob/rel/release-3.1.2/jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/JdbcSerDe.java#L136
> {code}
> rowVal = ((ObjectWritable)value).get();
> {code}
> Log Snio:
> =
> {code}
> 2020-07-17T16:49:09,598 INFO [04ed42ec-91d2-4662-aee7-37e840a06036
> HiveServer2-Handler-Pool: Thread-104]: dao.GenericJdbcDatabaseAccessor (:())
> - Query to execute is [select * from TESTHIVEJDBCSTORAGE]
> 2020-07-17T16:49:10,642 INFO [04ed42ec-91d2-4662-aee7-37e840a06036
> HiveServer2-Handler-Pool: Thread-104]: jdbc.JdbcSerDe (:()) - *** ColumnKey =
> ID
> 2020-07-17T16:49:10,642 INFO [04ed42ec-91d2-4662-aee7-37e840a06036
> HiveServer2-Handler-Pool: Thread-104]: jdbc.JdbcSerDe (:()) - *** Blob value
> = {fname=OW[class=class java.lang.Str
