This is an automated email from the ASF dual-hosted git repository.
dengzh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 709b9e221df HIVE-29484: HMS.getFields() fails with 'SchemaReader not
supported' for Avro and Hbase tables (#6350)
709b9e221df is described below
commit 709b9e221dfce3b1d5a69be1274f432dd778ee2d
Author: rtrivedi12 <[email protected]>
AuthorDate: Thu Apr 9 20:45:01 2026 -0500
HIVE-29484: HMS.getFields() fails with 'SchemaReader not supported' for
Avro and Hbase tables (#6350)
---
.../hive/metastore/TestStorageSchemaReader.java | 31 +++++-----
.../hadoop/hive/metastore/conf/MetastoreConf.java | 3 +-
.../apache/hadoop/hive/metastore/HMSHandler.java | 52 +++++++++++-----
.../hadoop/hive/metastore/TestHiveMetaStore.java | 70 +++++++++++++++++++++-
4 files changed, 126 insertions(+), 30 deletions(-)
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestStorageSchemaReader.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestStorageSchemaReader.java
index fadd793bc23..5d3dcc30864 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestStorageSchemaReader.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestStorageSchemaReader.java
@@ -25,6 +25,7 @@
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder;
import org.apache.hadoop.hive.metastore.client.builder.TableBuilder;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat;
import org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat;
import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat;
@@ -60,6 +61,8 @@ public class TestStorageSchemaReader {
@Before public void setUp() throws Exception {
dbName = "sampleDb";
hiveConf = new HiveConf(this.getClass());
+ // Unset the deprecated HiveConf value so MetastoreConf uses its own
default
+
hiveConf.unset(HiveConf.ConfVars.SERDES_USING_METASTORE_FOR_SCHEMA.varname);
new DatabaseBuilder().setName(dbName).create(new
HiveMetaStoreClient(hiveConf), hiveConf);
avroTableParams.put("avro.schema.literal",
"{\"name\":\"nullable\", \"type\":\"record\",
\"fields\":[{\"name\":\"id\", \"type\":\"int\"}, {\"name\":\"value\",
\"type\":\"int\"}]}");
@@ -111,15 +114,15 @@ private void checkFields(List<FieldSchema> fieldSchemas,
List<FieldSchema> field
}
@Test public void testAvroTableWithDefaultSSR() throws Exception {
- hiveConf.set("metastore.storage.schema.reader.impl",
"org.apache.hadoop.hive.metastore.DefaultStorageSchemaReader");
+ MetastoreConf.setVar(hiveConf,
MetastoreConf.ConfVars.STORAGE_SCHEMA_READER_IMPL,
"org.apache.hadoop.hive.metastore.DefaultStorageSchemaReader");
String tblName = "avroTable";
- createTable(tblName, AvroSerDe.class.getName(),
AvroContainerInputFormat.class.getName(),
+ Table tbl = createTable(tblName, AvroSerDe.class.getName(),
AvroContainerInputFormat.class.getName(),
AvroContainerOutputFormat.class.getName(), avroTableParams, new
HashMap<>());
- assertThrows("Storage schema reading not supported", MetaException.class,
() -> client.getSchema(dbName, tblName));
+ checkSchema(tblName, tbl);
}
@Test public void testAvroTableWithSerdeSSR() throws Exception {
- hiveConf.set("metastore.storage.schema.reader.impl",
"org.apache.hadoop.hive.metastore.SerDeStorageSchemaReader");
+ MetastoreConf.setVar(hiveConf,
MetastoreConf.ConfVars.STORAGE_SCHEMA_READER_IMPL,
"org.apache.hadoop.hive.metastore.SerDeStorageSchemaReader");
String tblName = "avroTable";
Table tbl = createTable(tblName, AvroSerDe.class.getName(),
AvroContainerInputFormat.class.getName(),
AvroContainerOutputFormat.class.getName(), avroTableParams, new
HashMap<>());
@@ -127,16 +130,16 @@ private void checkFields(List<FieldSchema> fieldSchemas,
List<FieldSchema> field
}
@Test public void testHbaseTableWithDefaultSSR() throws Exception {
- hiveConf.set("metastore.storage.schema.reader.impl",
"org.apache.hadoop.hive.metastore.DefaultStorageSchemaReader");
- String tblName = "jdbcTable";
+ MetastoreConf.setVar(hiveConf,
MetastoreConf.ConfVars.STORAGE_SCHEMA_READER_IMPL,
"org.apache.hadoop.hive.metastore.DefaultStorageSchemaReader");
+ String tblName = "hbaseTable";
- createTable(tblName, HBaseSerDe.class.getName(), null, null,
hbaseTableParams, hbaseSerdeParams);
- assertThrows("Storage schema reading not supported", MetaException.class,
() -> client.getSchema(dbName, tblName));
+ Table table = createTable(tblName, HBaseSerDe.class.getName(), null, null,
hbaseTableParams, hbaseSerdeParams);
+ checkSchema(tblName, table);
}
@Test public void testHbaseTableWithSerdeSSR() throws Exception {
- hiveConf.set("metastore.storage.schema.reader.impl",
"org.apache.hadoop.hive.metastore.SerDeStorageSchemaReader");
- String tblName = "jdbcTable";
+ MetastoreConf.setVar(hiveConf,
MetastoreConf.ConfVars.STORAGE_SCHEMA_READER_IMPL,
"org.apache.hadoop.hive.metastore.SerDeStorageSchemaReader");
+ String tblName = "hbaseTableSerde";
Table table =
createTable(tblName, "org.apache.hadoop.hive.hbase.HBaseSerDe", null,
null, hbaseTableParams, hbaseSerdeParams);
@@ -144,7 +147,7 @@ private void checkFields(List<FieldSchema> fieldSchemas,
List<FieldSchema> field
}
@Test public void testJdbcTableWithDefaultSSR() throws Exception {
- hiveConf.set("metastore.storage.schema.reader.impl",
"org.apache.hadoop.hive.metastore.DefaultStorageSchemaReader");
+ MetastoreConf.setVar(hiveConf,
MetastoreConf.ConfVars.STORAGE_SCHEMA_READER_IMPL,
"org.apache.hadoop.hive.metastore.DefaultStorageSchemaReader");
String tblName = "jdbcTable";
createTable(tblName, JdbcSerDe.class.getName(), null, null,
jdbcTableParams, jdbcSerdeParams);
@@ -152,7 +155,7 @@ private void checkFields(List<FieldSchema> fieldSchemas,
List<FieldSchema> field
}
@Test public void testJdbcTableWithSerdeSSR() throws Exception {
- hiveConf.set("metastore.storage.schema.reader.impl",
"org.apache.hadoop.hive.metastore.SerDeStorageSchemaReader");
+ MetastoreConf.setVar(hiveConf,
MetastoreConf.ConfVars.STORAGE_SCHEMA_READER_IMPL,
"org.apache.hadoop.hive.metastore.SerDeStorageSchemaReader");
String tblName = "jdbcTable";
Table table = createTable(tblName, JdbcSerDe.class.getName(), null, null,
jdbcTableParams, jdbcSerdeParams);
@@ -160,7 +163,7 @@ private void checkFields(List<FieldSchema> fieldSchemas,
List<FieldSchema> field
}
@Test public void testOrcTableWithDefaultSSR() throws Exception {
- hiveConf.set("metastore.storage.schema.reader.impl",
"org.apache.hadoop.hive.metastore.DefaultStorageSchemaReader");
+ MetastoreConf.setVar(hiveConf,
MetastoreConf.ConfVars.STORAGE_SCHEMA_READER_IMPL,
"org.apache.hadoop.hive.metastore.DefaultStorageSchemaReader");
String tblName = "orcTable2";
Table tbl =
createTable(tblName, OrcSerde.class.getName(),
OrcInputFormat.class.getName(), OrcOutputFormat.class.getName(),
@@ -169,7 +172,7 @@ private void checkFields(List<FieldSchema> fieldSchemas,
List<FieldSchema> field
}
@Test public void testOrcTableWithSerdeSSR() throws Exception {
- hiveConf.set("metastore.storage.schema.reader.impl",
"org.apache.hadoop.hive.metastore.SerDeStorageSchemaReader");
+ MetastoreConf.setVar(hiveConf,
MetastoreConf.ConfVars.STORAGE_SCHEMA_READER_IMPL,
"org.apache.hadoop.hive.metastore.SerDeStorageSchemaReader");
String tblName = "orcTable";
Table tbl =
createTable(tblName, OrcSerde.class.getName(),
OrcInputFormat.class.getName(), OrcOutputFormat.class.getName(),
diff --git
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
index 16ccf508a43..10015f74837 100644
---
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
+++
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
@@ -1441,7 +1441,8 @@ public enum ConfVars {
"org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe," +
"org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe," +
"org.apache.hadoop.hive.serde2.OpenCSVSerde," +
- "org.apache.iceberg.mr.hive.HiveIcebergSerDe",
+ "org.apache.iceberg.mr.hive.HiveIcebergSerDe," +
+ "org.apache.hadoop.hive.hbase.HBaseSerDe",
"SerDes retrieving schema from metastore. This is an internal
parameter."),
SERDES_WITHOUT_FROM_DESERIALIZER("metastore.serdes.without.from.deserializer",
"hive.metastore.serdes.without.from.deserializer",
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
index 9838d9d2573..522a62f90ef 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
@@ -126,6 +126,8 @@ public class HMSHandler extends FacebookBase implements
IHMSHandler {
static final int LOG_SAMPLE_PARTITIONS_HALF_SIZE = 2;
static final String LOG_SAMPLE_PARTITIONS_SEPARATOR = ",";
+
+ private static final String AVRO_SERDE_CLASS =
"org.apache.hadoop.hive.serde2.avro.AvroSerDe";
private Warehouse wh; // hdfs warehouse
private static Striped<Lock> tablelocks;
@@ -3969,26 +3971,17 @@ private List<FieldSchema>
get_fields_with_environment_context_core(String db, St
String base_table_name = names[0];
String[] parsedDbName = parseDbName(db, conf);
- Table tbl;
List<FieldSchema> ret = null;
Exception ex = null;
try {
- try {
- GetTableRequest getTableRequest = new
GetTableRequest(parsedDbName[DB_NAME], base_table_name);
- getTableRequest.setCatName(parsedDbName[CAT_NAME]);
- tbl = get_table_core(getTableRequest);
- firePreEvent(new PreReadTableEvent(tbl, this));
- } catch (NoSuchObjectException e) {
- throw new UnknownTableException(e.getMessage());
- }
- if (null == tbl.getSd().getSerdeInfo().getSerializationLib() ||
+ Table tbl = getTableForGetFields(parsedDbName, base_table_name);
+ String serdeLib = tbl.getSd().getSerdeInfo().getSerializationLib();
+ if (serdeLib == null ||
MetastoreConf.getStringCollection(conf,
- ConfVars.SERDES_USING_METASTORE_FOR_SCHEMA).contains(
- tbl.getSd().getSerdeInfo().getSerializationLib())) {
+
ConfVars.SERDES_USING_METASTORE_FOR_SCHEMA).contains(serdeLib)) {
ret = tbl.getSd().getCols();
} else {
- StorageSchemaReader schemaReader = getStorageSchemaReader();
- ret = schemaReader.readSchema(tbl, envContext, getConf());
+ ret = getFieldsUsingSchemaReader(tbl, db, tableName, serdeLib,
envContext);
}
} catch (Exception e) {
ex = e;
@@ -3999,6 +3992,37 @@ private List<FieldSchema>
get_fields_with_environment_context_core(String db, St
return ret;
}
+ private Table getTableForGetFields(String[] parsedDbName, String
baseTableName)
+ throws UnknownTableException, MetaException {
+ try {
+ GetTableRequest getTableRequest = new
GetTableRequest(parsedDbName[DB_NAME], baseTableName);
+ getTableRequest.setCatName(parsedDbName[CAT_NAME]);
+ Table tbl = get_table_core(getTableRequest);
+ firePreEvent(new PreReadTableEvent(tbl, this));
+ return tbl;
+ } catch (NoSuchObjectException e) {
+ throw new UnknownTableException(e.getMessage());
+ }
+ }
+
+ private List<FieldSchema> getFieldsUsingSchemaReader(Table tbl, String db,
String tableName, String serdeLib,
+ EnvironmentContext envContext) throws MetaException {
+ try {
+ StorageSchemaReader schemaReader = getStorageSchemaReader();
+ return schemaReader.readSchema(tbl, envContext, getConf());
+ } catch (UnsupportedOperationException e) {
+ // Fallback to metastore for Avro Schema
+ if (AVRO_SERDE_CLASS.equals(serdeLib)) {
+ LOG.warn(
+ "Unable to read schema from storage for AvroSerDe table '{}.{}'.
Returning metastore SD columns " +
+ "as fallback; schema may be stale.",
+ db, tableName, e);
+ return tbl.getSd().getCols();
+ }
+ throw e;
+ }
+ }
+
@Override
public GetFieldsResponse get_fields_req(GetFieldsRequest req)
throws MetaException, UnknownTableException, UnknownDBException {
diff --git
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
index 8f1ba073f1b..8ebc1a7ccf9 100644
---
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
+++
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
@@ -154,7 +154,7 @@ public void setUp() throws Exception {
MetaStoreTestUtils.setConfForStandloneMode(conf);
MetastoreConf.setLongVar(conf, ConfVars.BATCH_RETRIEVE_MAX, 2);
MetastoreConf.setLongVar(conf, ConfVars.LIMIT_PARTITION_REQUEST,
DEFAULT_LIMIT_PARTITION_REQUEST);
- MetastoreConf.setVar(conf, ConfVars.STORAGE_SCHEMA_READER_IMPL,
"no.such.class");
+ MetastoreConf.setVar(conf, ConfVars.STORAGE_SCHEMA_READER_IMPL,
"org.apache.hadoop.hive.metastore.DefaultStorageSchemaReader");
MetastoreConf.setBoolVar(conf, ConfVars.INTEGER_JDO_PUSHDOWN, true);
}
@@ -3973,4 +3973,72 @@ public void testDropDataConnectorIfNotExistsTrue()
throws Exception {
// No such data connector, ignore NoSuchObjectException
client.dropDataConnector("no_such_data_connector", true, false);
}
+
+ @Test
+ public void testGetFieldsForAvroSerDe() throws Exception {
+ String dbName = "test_avro_serde_db";
+ String avroTbl = "avro_tbl";
+
+ client.dropTable(dbName, avroTbl, true, true);
+ silentDropDatabase(dbName);
+
+ new DatabaseBuilder()
+ .setName(dbName)
+ .create(client, conf);
+
+ new TableBuilder()
+ .setDbName(dbName)
+ .setTableName(avroTbl)
+ .setSerdeLib("org.apache.hadoop.hive.serde2.avro.AvroSerDe")
+
.setInputFormat("org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat")
+
.setOutputFormat("org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat")
+ .addCol("foo", "int", "")
+ .addCol("bar", "string", "")
+ .addCol("baz", "bigint", "")
+ .create(client, conf);
+
+ List<FieldSchema> fields = client.getFields(dbName, avroTbl);
+ assertEquals("AvroSerDe table should return 3 fields from metastore", 3,
fields.size());
+ assertEquals("foo", fields.get(0).getName());
+ assertEquals("bar", fields.get(1).getName());
+ assertEquals("baz", fields.get(2).getName());
+
+ List<FieldSchema> schema = client.getSchema(dbName, avroTbl);
+ assertEquals("AvroSerDe getSchema should return 3 columns", 3,
schema.size());
+
+ client.dropTable(dbName, avroTbl, true, true);
+ client.dropDatabase(dbName);
+ }
+
+ @Test
+ public void testGetFieldsForHBaseSerDe() throws Exception {
+ String dbName = "test_hbase_serde_db";
+ String hbaseTbl = "hbase_tbl";
+
+ client.dropTable(dbName, hbaseTbl, true, true);
+ silentDropDatabase(dbName);
+
+ new DatabaseBuilder()
+ .setName(dbName)
+ .create(client, conf);
+
+ new TableBuilder()
+ .setDbName(dbName)
+ .setTableName(hbaseTbl)
+ .setSerdeLib("org.apache.hadoop.hive.hbase.HBaseSerDe")
+ .addCol("key", "string", "")
+ .addCol("value", "string", "")
+ .create(client, conf);
+
+ List<FieldSchema> fields = client.getFields(dbName, hbaseTbl);
+ assertEquals("HBaseSerDe table should return 2 fields from metastore", 2,
fields.size());
+ assertEquals("key", fields.get(0).getName());
+ assertEquals("value", fields.get(1).getName());
+
+ List<FieldSchema> schema = client.getSchema(dbName, hbaseTbl);
+ assertEquals("HBaseSerDe getSchema should return 2 columns", 2,
schema.size());
+
+ client.dropTable(dbName, hbaseTbl, true, true);
+ client.dropDatabase(dbName);
+ }
}
