[jira] [Updated] (HUDI-5088) Failed to synchronize the hive metadata of the Flink table
[
https://issues.apache.org/jira/browse/HUDI-5088?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
waywtdcc updated HUDI-5088:
---
Description:
The flash table failed to synchronize the hive metadata. After using the flash
catalog and specifying changlog.enabled=true, the real-time writing to the
table fails to synchronize the hive metadata.
* sql
*
{code:java}
CREATE TABLE datagen (
id INT,
name string,
ts3 timestamp(3)
) WITH (
'connector' = 'datagen',
'rows-per-second'='10',
'fields.id.kind'='sequence',
'fields.id.start'='1',
'fields.id.end'='1000'
);
CREATE CATALOG myhudi WITH(
'type' = 'hudi',
'default-database' = 'default',
'catalog.path' = '/user/hdpu/warehouse',
'mode' = 'hms',
'hive.conf.dir' = 'hdfs:///user/hdpu/streamx/conf_data/hive_conf',
-- table-prop. properties.It is used to add attributes by default when creating
tables. 'table-prop.connector' = 'hudi',
'table-prop.table.type' = 'MERGE_ON_READ',
'table-prop.compaction.tasks' = '1',
'table-prop.write.tasks' = '1',
'table-prop.index.bootstrap.enabled' = 'true',
'table-prop.read.utc-timezone' = 'false',
'table-prop.changelog.enabled' = 'true',
'table-prop.index.global.enabled' = 'true'
);
CREATE CATALOG myhive WITH (
'type' = 'hive',
'default-database' = 'default',
'hive-conf-dir' = 'hdfs:///user/hdpu/streamx/conf_data/hive_conf'
);
drop table if exists myhive.test_hudi3.hudi_datagen_incre;
drop table if exists myhudi.test_hudi3.hudi_datagen_incre;
create table if not exists myhudi.test_hudi3.hudi_datagen_incre
(id bigint not null, name string,ts3 timestamp(3)
,PRIMARY KEY (`id`) NOT ENFORCED
)
;
show create table myhudi.test_hudi3.hudi_datagen_incre;
insert into myhudi.test_hudi3.hudi_datagen_incre
select id,name,ts3
from datagen;
{code}
* error
{code:java}
22/10/25 13:55:01 ERROR HMSDDLExecutor: Failed to update table for
hudi_datagen_incre_ro
InvalidOperationException(message:The following columns have types incompatible
with the existing columns in their respective positions :
id)
at
org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_with_environment_context_result$alter_table_with_environment_context_resultStandardScheme.read(ThriftHiveMetastore.java:59744)
at
org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_with_environment_context_result$alter_table_with_environment_context_resultStandardScheme.read(ThriftHiveMetastore.java:59730)
at
org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_with_environment_context_result.read(ThriftHiveMetastore.java:59672)
at org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:86)
at
org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.recv_alter_table_with_environment_context(ThriftHiveMetastore.java:1693)
at
org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.alter_table_with_environment_context(ThriftHiveMetastore.java:1677)
at
org.apache.hadoop.hive.metastore.HiveMetaStoreClient.alter_table_with_environmentContext(HiveMetaStoreClient.java:375)
at
org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.alter_table_with_environmentContext(SessionHiveMetaStoreClient.java:322)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.invoke(RetryingMetaStoreClient.java:173)
at com.sun.proxy.$Proxy32.alter_table_with_environmentContext(Unknown
Source)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.apache.hadoop.hive.metastore.HiveMetaStoreClient$SynchronizedHandler.invoke(HiveMetaStoreClient.java:2336)
at com.sun.proxy.$Proxy32.alter_table_with_environmentContext(Unknown
Source)
at
org.apache.hudi.hive.ddl.HMSDDLExecutor.updateTableDefinition(HMSDDLExecutor.java:158)
at
org.apache.hudi.hive.HoodieHiveSyncClient.updateTableSchema(HoodieHiveSyncClient.java:169)
at org.apache.hudi.hive.HiveSyncTool.syncSchema(HiveSyncTool.java:289)
at org.apache.hudi.hive.HiveSyncTool.syncHoodieTable(HiveSyncTool.java:217)
at org.apache.hudi.hive.HiveSyncTool.doSync(HiveSyncTool.java:158)
at org.apache.hudi.hive.HiveSyncTool.syncHoodieTable(HiveSyncTool.java:142)
at
org.apache.hudi.sink.StreamWriteOperatorCoordinator.doSyncHiv
[jira] [Updated] (HUDI-5088) Failed to synchronize the hive metadata of the Flink table
[
https://issues.apache.org/jira/browse/HUDI-5088?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
waywtdcc updated HUDI-5088:
---
Description:
The flash table failed to synchronize the hive metadata. After using the flash
catalog and specifying changlog.enabled=true, the real-time writing to the
table fails to synchronize the hive metadata.
I guess the reason is:
When using the Flink catalog to create the Flink table, the hive table will be
created first. The default value for creating a hive table is none_ hoodie_ The
operation field, but the Flink table has this field. The inconsistent fields
cause synchronization failure.
* sql
*
{code:java}
CREATE TABLE datagen (
id INT,
name string,
ts3 timestamp(3)
) WITH (
'connector' = 'datagen',
'rows-per-second'='10',
'fields.id.kind'='sequence',
'fields.id.start'='1',
'fields.id.end'='1000'
);
CREATE CATALOG myhudi WITH(
'type' = 'hudi',
'default-database' = 'default',
'catalog.path' = '/user/hdpu/warehouse',
'mode' = 'hms',
'hive.conf.dir' = 'hdfs:///user/hdpu/streamx/conf_data/hive_conf',
-- table-prop. properties.It is used to add attributes by default when creating
tables. 'table-prop.connector' = 'hudi',
'table-prop.table.type' = 'MERGE_ON_READ',
'table-prop.compaction.tasks' = '1',
'table-prop.write.tasks' = '1',
'table-prop.index.bootstrap.enabled' = 'true',
'table-prop.read.utc-timezone' = 'false',
'table-prop.changelog.enabled' = 'true',
'table-prop.index.global.enabled' = 'true'
);
CREATE CATALOG myhive WITH (
'type' = 'hive',
'default-database' = 'default',
'hive-conf-dir' = 'hdfs:///user/hdpu/streamx/conf_data/hive_conf'
);
drop table if exists myhive.test_hudi3.hudi_datagen_incre;
drop table if exists myhudi.test_hudi3.hudi_datagen_incre;
create table if not exists myhudi.test_hudi3.hudi_datagen_incre
(id bigint not null, name string,ts3 timestamp(3)
,PRIMARY KEY (`id`) NOT ENFORCED
)
;
show create table myhudi.test_hudi3.hudi_datagen_incre;
insert into myhudi.test_hudi3.hudi_datagen_incre
select id,name,ts3
from datagen;
{code}
* error
{code:java}
22/10/25 13:55:01 ERROR HMSDDLExecutor: Failed to update table for
hudi_datagen_incre_ro
InvalidOperationException(message:The following columns have types incompatible
with the existing columns in their respective positions :
id)
at
org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_with_environment_context_result$alter_table_with_environment_context_resultStandardScheme.read(ThriftHiveMetastore.java:59744)
at
org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_with_environment_context_result$alter_table_with_environment_context_resultStandardScheme.read(ThriftHiveMetastore.java:59730)
at
org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_with_environment_context_result.read(ThriftHiveMetastore.java:59672)
at org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:86)
at
org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.recv_alter_table_with_environment_context(ThriftHiveMetastore.java:1693)
at
org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.alter_table_with_environment_context(ThriftHiveMetastore.java:1677)
at
org.apache.hadoop.hive.metastore.HiveMetaStoreClient.alter_table_with_environmentContext(HiveMetaStoreClient.java:375)
at
org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.alter_table_with_environmentContext(SessionHiveMetaStoreClient.java:322)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.invoke(RetryingMetaStoreClient.java:173)
at com.sun.proxy.$Proxy32.alter_table_with_environmentContext(Unknown
Source)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.apache.hadoop.hive.metastore.HiveMetaStoreClient$SynchronizedHandler.invoke(HiveMetaStoreClient.java:2336)
at com.sun.proxy.$Proxy32.alter_table_with_environmentContext(Unknown
Source)
at
org.apache.hudi.hive.ddl.HMSDDLExecutor.updateTableDefinition(HMSDDLExecutor.java:158)
at
org.apache.hudi.hive.HoodieHiveSyncClient.updateTableSchema(HoodieHiveSyncClient.java:169)
at org.apache.hudi.hive.HiveSyncTool.syncSchema(HiveSyncTool.java:289)
a
[jira] [Updated] (HUDI-5088) Failed to synchronize the hive metadata of the Flink table
[
https://issues.apache.org/jira/browse/HUDI-5088?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
waywtdcc updated HUDI-5088:
---
Description:
The flash table failed to synchronize the hive metadata. After using the flash
catalog and specifying changlog.enabled=true and hive_sync.skip_ro_suffix =
true, the real-time writing to the table fails to synchronize the hive metadata.
I guess the reason is:
When using the Flink catalog to create the Flink table, the hive table will be
created first. The default value for creating a hive table is none_ hoodie_ The
operation field, but the Flink table has this field. The inconsistent fields
cause synchronization failure.
* sql
*
{code:java}
CREATE TABLE datagen (
id INT,
name string,
ts3 timestamp(3)
) WITH (
'connector' = 'datagen',
'rows-per-second'='10',
'fields.id.kind'='sequence',
'fields.id.start'='1',
'fields.id.end'='1000'
);
CREATE CATALOG myhudi WITH(
'type' = 'hudi',
'default-database' = 'default',
'catalog.path' = '/user/hdpu/warehouse',
'mode' = 'hms',
'hive.conf.dir' = 'hdfs:///user/hdpu/streamx/conf_data/hive_conf',
-- table-prop. properties.It is used to add attributes by default when creating
tables.
'table-prop.connector' = 'hudi',
'table-prop.table.type' = 'MERGE_ON_READ',
'table-prop.compaction.tasks' = '4',
'table-prop.write.tasks' = '4',
'table-prop.index.bootstrap.enabled' = 'true',
'table-prop.hive_sync.skip_ro_suffix' = 'true',
'table-prop.compaction.delta_commits' = '1',
'table-prop.compaction.async.enabled' = 'true',
'table-prop.changelog.enabled' = 'true',
'table-prop.index.type' = 'BUCKET',
'table-prop.index.global.enabled' = 'true',
'table-prop.read.utc-timezone' = 'false'
);
CREATE CATALOG myhive WITH (
'type' = 'hive',
'default-database' = 'default',
'hive-conf-dir' = 'hdfs:///user/hdpu/streamx/conf_data/hive_conf'
);
drop table if exists myhive.test_hudi3.hudi_datagen_incre;
drop table if exists myhudi.test_hudi3.hudi_datagen_incre;
create table if not exists myhudi.test_hudi3.hudi_datagen_incre
(id bigint not null, name string,ts3 timestamp(3)
,PRIMARY KEY (`id`) NOT ENFORCED
)
;
show create table myhudi.test_hudi3.hudi_datagen_incre;
insert into myhudi.test_hudi3.hudi_datagen_incre
select id,name,ts3
from datagen;
{code}
* error
{code:java}
22/10/25 13:55:01 ERROR HMSDDLExecutor: Failed to update table for
hudi_datagen_incre_ro
InvalidOperationException(message:The following columns have types incompatible
with the existing columns in their respective positions :
id)
at
org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_with_environment_context_result$alter_table_with_environment_context_resultStandardScheme.read(ThriftHiveMetastore.java:59744)
at
org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_with_environment_context_result$alter_table_with_environment_context_resultStandardScheme.read(ThriftHiveMetastore.java:59730)
at
org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_with_environment_context_result.read(ThriftHiveMetastore.java:59672)
at org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:86)
at
org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.recv_alter_table_with_environment_context(ThriftHiveMetastore.java:1693)
at
org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.alter_table_with_environment_context(ThriftHiveMetastore.java:1677)
at
org.apache.hadoop.hive.metastore.HiveMetaStoreClient.alter_table_with_environmentContext(HiveMetaStoreClient.java:375)
at
org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.alter_table_with_environmentContext(SessionHiveMetaStoreClient.java:322)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.invoke(RetryingMetaStoreClient.java:173)
at com.sun.proxy.$Proxy32.alter_table_with_environmentContext(Unknown
Source)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.apache.hadoop.hive.metastore.HiveMetaStoreClient$SynchronizedHandler.invoke(HiveMetaStoreClient.java:2336)
at com.sun.proxy.$Proxy32.alter_table_with_environmentContext(Unknown
Source)
at
org.apache.hudi.hive.ddl.HMSDDLExecutor.updateTableDefinition(HMSDDLExecutor
[jira] [Updated] (HUDI-5088) Failed to synchronize the hive metadata of the Flink table
[
https://issues.apache.org/jira/browse/HUDI-5088?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
ASF GitHub Bot updated HUDI-5088:
-
Labels: pull-request-available (was: )
> Failed to synchronize the hive metadata of the Flink table
> --
>
> Key: HUDI-5088
> URL: https://issues.apache.org/jira/browse/HUDI-5088
> Project: Apache Hudi
> Issue Type: Bug
> Components: flink-sql
>Reporter: waywtdcc
>Priority: Major
> Labels: pull-request-available
> Fix For: 0.12.1
>
>
> The flash table failed to synchronize the hive metadata. After using the
> flash catalog and specifying changlog.enabled=true and
> hive_sync.skip_ro_suffix = true, the real-time writing to the table fails to
> synchronize the hive metadata.
> I guess the reason is:
> When using the Flink catalog to create the Flink table, the hive table will
> be created first. The default value for creating a hive table is none_
> hoodie_ The operation field, but the Flink table has this field. The
> inconsistent fields cause synchronization failure.
> * sql
>
> *
> {code:java}
> CREATE TABLE datagen (
> id INT,
> name string,
> ts3 timestamp(3)
> ) WITH (
> 'connector' = 'datagen',
> 'rows-per-second'='10',
> 'fields.id.kind'='sequence',
> 'fields.id.start'='1',
> 'fields.id.end'='1000'
> );
> CREATE CATALOG myhudi WITH(
> 'type' = 'hudi',
> 'default-database' = 'default',
> 'catalog.path' = '/user/hdpu/warehouse',
> 'mode' = 'hms',
> 'hive.conf.dir' = 'hdfs:///user/hdpu/streamx/conf_data/hive_conf',
> -- table-prop. properties.It is used to add attributes by default when
> creating tables.
> 'table-prop.connector' = 'hudi',
> 'table-prop.table.type' = 'MERGE_ON_READ',
> 'table-prop.compaction.tasks' = '4',
> 'table-prop.write.tasks' = '4',
> 'table-prop.index.bootstrap.enabled' = 'true',
> 'table-prop.hive_sync.skip_ro_suffix' = 'true',
> 'table-prop.compaction.delta_commits' = '1',
> 'table-prop.compaction.async.enabled' = 'true',
> 'table-prop.changelog.enabled' = 'true',
> 'table-prop.index.type' = 'BUCKET',
> 'table-prop.index.global.enabled' = 'true',
> 'table-prop.read.utc-timezone' = 'false'
> );
> CREATE CATALOG myhive WITH (
> 'type' = 'hive',
> 'default-database' = 'default',
> 'hive-conf-dir' = 'hdfs:///user/hdpu/streamx/conf_data/hive_conf'
> );
> drop table if exists myhive.test_hudi3.hudi_datagen_incre;
> drop table if exists myhudi.test_hudi3.hudi_datagen_incre;
> create table if not exists myhudi.test_hudi3.hudi_datagen_incre
> (id bigint not null, name string,ts3 timestamp(3)
> ,PRIMARY KEY (`id`) NOT ENFORCED
> )
> ;
> show create table myhudi.test_hudi3.hudi_datagen_incre;
> insert into myhudi.test_hudi3.hudi_datagen_incre
> select id,name,ts3
> from datagen;
> {code}
> * error
> {code:java}
> 22/10/25 13:55:01 ERROR HMSDDLExecutor: Failed to update table for
> hudi_datagen_incre_ro
> InvalidOperationException(message:The following columns have types
> incompatible with the existing columns in their respective positions :
> id)
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_with_environment_context_result$alter_table_with_environment_context_resultStandardScheme.read(ThriftHiveMetastore.java:59744)
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_with_environment_context_result$alter_table_with_environment_context_resultStandardScheme.read(ThriftHiveMetastore.java:59730)
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_with_environment_context_result.read(ThriftHiveMetastore.java:59672)
> at org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:86)
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.recv_alter_table_with_environment_context(ThriftHiveMetastore.java:1693)
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.alter_table_with_environment_context(ThriftHiveMetastore.java:1677)
> at
> org.apache.hadoop.hive.metastore.HiveMetaStoreClient.alter_table_with_environmentContext(HiveMetaStoreClient.java:375)
> at
> org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.alter_table_with_environmentContext(SessionHiveMetaStoreClient.java:322)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at
> org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.invoke(RetryingMetaStoreClient.java:173)
>
[jira] [Updated] (HUDI-5088) Failed to synchronize the hive metadata of the Flink table
[
https://issues.apache.org/jira/browse/HUDI-5088?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
waywtdcc updated HUDI-5088:
---
Fix Version/s: 0.12.2
(was: 0.12.1)
> Failed to synchronize the hive metadata of the Flink table
> --
>
> Key: HUDI-5088
> URL: https://issues.apache.org/jira/browse/HUDI-5088
> Project: Apache Hudi
> Issue Type: Bug
> Components: flink-sql
>Affects Versions: 0.12.1
>Reporter: waywtdcc
>Assignee: waywtdcc
>Priority: Major
> Labels: pull-request-available
> Fix For: 0.12.2
>
>
> The flash table failed to synchronize the hive metadata. After using the
> flash catalog and specifying changlog.enabled=true and
> hive_sync.skip_ro_suffix = true, the real-time writing to the table fails to
> synchronize the hive metadata.
> I guess the reason is:
> When using the Flink catalog to create the Flink table, the hive table will
> be created first. The default value for creating a hive table is none_
> hoodie_ The operation field, but the Flink table has this field. The
> inconsistent fields cause synchronization failure.
> * sql
>
> *
> {code:java}
> CREATE TABLE datagen (
> id INT,
> name string,
> ts3 timestamp(3)
> ) WITH (
> 'connector' = 'datagen',
> 'rows-per-second'='10',
> 'fields.id.kind'='sequence',
> 'fields.id.start'='1',
> 'fields.id.end'='1000'
> );
> CREATE CATALOG myhudi WITH(
> 'type' = 'hudi',
> 'default-database' = 'default',
> 'catalog.path' = '/user/hdpu/warehouse',
> 'mode' = 'hms',
> 'hive.conf.dir' = 'hdfs:///user/hdpu/streamx/conf_data/hive_conf',
> -- table-prop. properties.It is used to add attributes by default when
> creating tables.
> 'table-prop.connector' = 'hudi',
> 'table-prop.table.type' = 'MERGE_ON_READ',
> 'table-prop.compaction.tasks' = '4',
> 'table-prop.write.tasks' = '4',
> 'table-prop.index.bootstrap.enabled' = 'true',
> 'table-prop.hive_sync.skip_ro_suffix' = 'true',
> 'table-prop.compaction.delta_commits' = '1',
> 'table-prop.compaction.async.enabled' = 'true',
> 'table-prop.changelog.enabled' = 'true',
> 'table-prop.index.type' = 'BUCKET',
> 'table-prop.index.global.enabled' = 'true',
> 'table-prop.read.utc-timezone' = 'false'
> );
> CREATE CATALOG myhive WITH (
> 'type' = 'hive',
> 'default-database' = 'default',
> 'hive-conf-dir' = 'hdfs:///user/hdpu/streamx/conf_data/hive_conf'
> );
> drop table if exists myhive.test_hudi3.hudi_datagen_incre;
> drop table if exists myhudi.test_hudi3.hudi_datagen_incre;
> create table if not exists myhudi.test_hudi3.hudi_datagen_incre
> (id bigint not null, name string,ts3 timestamp(3)
> ,PRIMARY KEY (`id`) NOT ENFORCED
> )
> ;
> show create table myhudi.test_hudi3.hudi_datagen_incre;
> insert into myhudi.test_hudi3.hudi_datagen_incre
> select id,name,ts3
> from datagen;
> {code}
> * error
> {code:java}
> 22/10/25 13:55:01 ERROR HMSDDLExecutor: Failed to update table for
> hudi_datagen_incre_ro
> InvalidOperationException(message:The following columns have types
> incompatible with the existing columns in their respective positions :
> id)
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_with_environment_context_result$alter_table_with_environment_context_resultStandardScheme.read(ThriftHiveMetastore.java:59744)
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_with_environment_context_result$alter_table_with_environment_context_resultStandardScheme.read(ThriftHiveMetastore.java:59730)
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_with_environment_context_result.read(ThriftHiveMetastore.java:59672)
> at org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:86)
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.recv_alter_table_with_environment_context(ThriftHiveMetastore.java:1693)
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.alter_table_with_environment_context(ThriftHiveMetastore.java:1677)
> at
> org.apache.hadoop.hive.metastore.HiveMetaStoreClient.alter_table_with_environmentContext(HiveMetaStoreClient.java:375)
> at
> org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.alter_table_with_environmentContext(SessionHiveMetaStoreClient.java:322)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at
> org.apache.hadoop.hive.metasto
[jira] [Updated] (HUDI-5088) Failed to synchronize the hive metadata of the Flink table
[
https://issues.apache.org/jira/browse/HUDI-5088?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
waywtdcc updated HUDI-5088:
---
Affects Version/s: 0.12.1
> Failed to synchronize the hive metadata of the Flink table
> --
>
> Key: HUDI-5088
> URL: https://issues.apache.org/jira/browse/HUDI-5088
> Project: Apache Hudi
> Issue Type: Bug
> Components: flink-sql
>Affects Versions: 0.12.1
>Reporter: waywtdcc
>Assignee: waywtdcc
>Priority: Major
> Labels: pull-request-available
> Fix For: 0.12.1
>
>
> The flash table failed to synchronize the hive metadata. After using the
> flash catalog and specifying changlog.enabled=true and
> hive_sync.skip_ro_suffix = true, the real-time writing to the table fails to
> synchronize the hive metadata.
> I guess the reason is:
> When using the Flink catalog to create the Flink table, the hive table will
> be created first. The default value for creating a hive table is none_
> hoodie_ The operation field, but the Flink table has this field. The
> inconsistent fields cause synchronization failure.
> * sql
>
> *
> {code:java}
> CREATE TABLE datagen (
> id INT,
> name string,
> ts3 timestamp(3)
> ) WITH (
> 'connector' = 'datagen',
> 'rows-per-second'='10',
> 'fields.id.kind'='sequence',
> 'fields.id.start'='1',
> 'fields.id.end'='1000'
> );
> CREATE CATALOG myhudi WITH(
> 'type' = 'hudi',
> 'default-database' = 'default',
> 'catalog.path' = '/user/hdpu/warehouse',
> 'mode' = 'hms',
> 'hive.conf.dir' = 'hdfs:///user/hdpu/streamx/conf_data/hive_conf',
> -- table-prop. properties.It is used to add attributes by default when
> creating tables.
> 'table-prop.connector' = 'hudi',
> 'table-prop.table.type' = 'MERGE_ON_READ',
> 'table-prop.compaction.tasks' = '4',
> 'table-prop.write.tasks' = '4',
> 'table-prop.index.bootstrap.enabled' = 'true',
> 'table-prop.hive_sync.skip_ro_suffix' = 'true',
> 'table-prop.compaction.delta_commits' = '1',
> 'table-prop.compaction.async.enabled' = 'true',
> 'table-prop.changelog.enabled' = 'true',
> 'table-prop.index.type' = 'BUCKET',
> 'table-prop.index.global.enabled' = 'true',
> 'table-prop.read.utc-timezone' = 'false'
> );
> CREATE CATALOG myhive WITH (
> 'type' = 'hive',
> 'default-database' = 'default',
> 'hive-conf-dir' = 'hdfs:///user/hdpu/streamx/conf_data/hive_conf'
> );
> drop table if exists myhive.test_hudi3.hudi_datagen_incre;
> drop table if exists myhudi.test_hudi3.hudi_datagen_incre;
> create table if not exists myhudi.test_hudi3.hudi_datagen_incre
> (id bigint not null, name string,ts3 timestamp(3)
> ,PRIMARY KEY (`id`) NOT ENFORCED
> )
> ;
> show create table myhudi.test_hudi3.hudi_datagen_incre;
> insert into myhudi.test_hudi3.hudi_datagen_incre
> select id,name,ts3
> from datagen;
> {code}
> * error
> {code:java}
> 22/10/25 13:55:01 ERROR HMSDDLExecutor: Failed to update table for
> hudi_datagen_incre_ro
> InvalidOperationException(message:The following columns have types
> incompatible with the existing columns in their respective positions :
> id)
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_with_environment_context_result$alter_table_with_environment_context_resultStandardScheme.read(ThriftHiveMetastore.java:59744)
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_with_environment_context_result$alter_table_with_environment_context_resultStandardScheme.read(ThriftHiveMetastore.java:59730)
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_with_environment_context_result.read(ThriftHiveMetastore.java:59672)
> at org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:86)
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.recv_alter_table_with_environment_context(ThriftHiveMetastore.java:1693)
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.alter_table_with_environment_context(ThriftHiveMetastore.java:1677)
> at
> org.apache.hadoop.hive.metastore.HiveMetaStoreClient.alter_table_with_environmentContext(HiveMetaStoreClient.java:375)
> at
> org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.alter_table_with_environmentContext(SessionHiveMetaStoreClient.java:322)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at
> org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.invoke
[jira] [Updated] (HUDI-5088) Failed to synchronize the hive metadata of the Flink table
[
https://issues.apache.org/jira/browse/HUDI-5088?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
waywtdcc updated HUDI-5088:
---
Description:
The flash table failed to synchronize the hive metadata. After using the flash
catalog and specifying changlog.enabled=true and hive_sync.skip_ro_suffix =
true, the real-time writing to the table fails to synchronize the hive metadata.
I guess the reason is:
When using the Flink catalog to create the Flink table, the hive table will be
created first. The default value for creating a hive table is none_ hoodie_ The
operation field, but the Flink table has this field. The inconsistent fields
cause synchronization failure.
* sql
*
{code:java}
CREATE TABLE datagen (
id INT,
name string,
ts3 timestamp(3)
) WITH (
'connector' = 'datagen',
'rows-per-second'='10',
'fields.id.kind'='sequence',
'fields.id.start'='1',
'fields.id.end'='1000'
);
CREATE CATALOG myhudi WITH(
'type' = 'hudi',
'default-database' = 'default',
'catalog.path' = '/user/hdpu/warehouse',
'mode' = 'hms',
'hive.conf.dir' = 'hdfs:///user/hdpu/streamx/conf_data/hive_conf',
-- table-prop. properties.It is used to add attributes by default when creating
tables.
'table-prop.connector' = 'hudi',
'table-prop.table.type' = 'MERGE_ON_READ',
'table-prop.compaction.tasks' = '4',
'table-prop.write.tasks' = '4',
'table-prop.index.bootstrap.enabled' = 'true',
'table-prop.hive_sync.skip_ro_suffix' = 'true',
'table-prop.compaction.delta_commits' = '1',
'table-prop.compaction.async.enabled' = 'true',
'table-prop.changelog.enabled' = 'true',
'table-prop.index.type' = 'BUCKET',
'table-prop.index.global.enabled' = 'true',
'table-prop.read.utc-timezone' = 'false'
);
CREATE CATALOG myhive WITH (
'type' = 'hive',
'default-database' = 'default',
'hive-conf-dir' = 'hdfs:///user/hdpu/streamx/conf_data/hive_conf'
);
drop table if exists myhive.test_hudi3.hudi_datagen_incre;
drop table if exists myhudi.test_hudi3.hudi_datagen_incre;
create table if not exists myhudi.test_hudi3.hudi_datagen_incre
(id bigint not null, name string,ts3 timestamp(3)
,PRIMARY KEY (`id`) NOT ENFORCED
)
;
show create table myhudi.test_hudi3.hudi_datagen_incre;
insert into myhudi.test_hudi3.hudi_datagen_incre
select id,name,ts3
from datagen;
{code}
* error
{code:java}
22/10/25 13:55:01 ERROR HMSDDLExecutor: Failed to update table for
hudi_datagen_incre_ro
InvalidOperationException(message:The following columns have types incompatible
with the existing columns in their respective positions :
id)
at
org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_with_environment_context_result$alter_table_with_environment_context_resultStandardScheme.read(ThriftHiveMetastore.java:59744)
at
org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_with_environment_context_result$alter_table_with_environment_context_resultStandardScheme.read(ThriftHiveMetastore.java:59730)
at
org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_with_environment_context_result.read(ThriftHiveMetastore.java:59672)
at org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:86)
at
org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.recv_alter_table_with_environment_context(ThriftHiveMetastore.java:1693)
at
org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.alter_table_with_environment_context(ThriftHiveMetastore.java:1677)
at
org.apache.hadoop.hive.metastore.HiveMetaStoreClient.alter_table_with_environmentContext(HiveMetaStoreClient.java:375)
at
org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.alter_table_with_environmentContext(SessionHiveMetaStoreClient.java:322)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.invoke(RetryingMetaStoreClient.java:173)
at com.sun.proxy.$Proxy32.alter_table_with_environmentContext(Unknown
Source)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.apache.hadoop.hive.metastore.HiveMetaStoreClient$SynchronizedHandler.invoke(HiveMetaStoreClient.java:2336)
at com.sun.proxy.$Proxy32.alter_table_with_environmentContext(Unknown
Source)
at
org.apache.hudi.hive.ddl.HMSDDLExecutor.updateTableDefinition(HMSDDLExecutor
[jira] [Updated] (HUDI-5088) Failed to synchronize the hive metadata of the Flink table
[
https://issues.apache.org/jira/browse/HUDI-5088?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
waywtdcc updated HUDI-5088:
---
Attachment: image-2022-10-25-15-20-50-321.png
> Failed to synchronize the hive metadata of the Flink table
> --
>
> Key: HUDI-5088
> URL: https://issues.apache.org/jira/browse/HUDI-5088
> Project: Apache Hudi
> Issue Type: Bug
> Components: flink-sql
>Affects Versions: 0.12.1
>Reporter: waywtdcc
>Assignee: waywtdcc
>Priority: Major
> Labels: pull-request-available
> Fix For: 0.12.2
>
> Attachments: image-2022-10-25-15-20-50-321.png
>
>
> The flash table failed to synchronize the hive metadata. After using the
> flash catalog and specifying changlog.enabled=true and
> hive_sync.skip_ro_suffix = true, the real-time writing to the table fails to
> synchronize the hive metadata.
> I guess the reason is:
> When using the Flink catalog to create the Flink table, the hive table will
> be created first. The default value for creating a hive table is none_
> hoodie_ The operation field, but the Flink table has this field. The
> inconsistent fields cause synchronization failure.
> * sql
>
> *
> {code:java}
> CREATE TABLE datagen (
> id INT,
> name string,
> ts3 timestamp(3)
> ) WITH (
> 'connector' = 'datagen',
> 'rows-per-second'='10',
> 'fields.id.kind'='sequence',
> 'fields.id.start'='1',
> 'fields.id.end'='1000'
> );
> CREATE CATALOG myhudi WITH(
> 'type' = 'hudi',
> 'default-database' = 'default',
> 'catalog.path' = '/user/hdpu/warehouse',
> 'mode' = 'hms',
> 'hive.conf.dir' = 'hdfs:///user/hdpu/streamx/conf_data/hive_conf',
> -- table-prop. properties.It is used to add attributes by default when
> creating tables.
> 'table-prop.connector' = 'hudi',
> 'table-prop.table.type' = 'MERGE_ON_READ',
> 'table-prop.compaction.tasks' = '4',
> 'table-prop.write.tasks' = '4',
> 'table-prop.index.bootstrap.enabled' = 'true',
> 'table-prop.hive_sync.skip_ro_suffix' = 'true',
> 'table-prop.compaction.delta_commits' = '1',
> 'table-prop.compaction.async.enabled' = 'true',
> 'table-prop.changelog.enabled' = 'true',
> 'table-prop.index.type' = 'BUCKET',
> 'table-prop.index.global.enabled' = 'true',
> 'table-prop.read.utc-timezone' = 'false'
> );
> CREATE CATALOG myhive WITH (
> 'type' = 'hive',
> 'default-database' = 'default',
> 'hive-conf-dir' = 'hdfs:///user/hdpu/streamx/conf_data/hive_conf'
> );
> drop table if exists myhive.test_hudi3.hudi_datagen_incre;
> drop table if exists myhudi.test_hudi3.hudi_datagen_incre;
> create table if not exists myhudi.test_hudi3.hudi_datagen_incre
> (id bigint not null, name string,ts3 timestamp(3)
> ,PRIMARY KEY (`id`) NOT ENFORCED
> )
> ;
> show create table myhudi.test_hudi3.hudi_datagen_incre;
> insert into myhudi.test_hudi3.hudi_datagen_incre
> select id,name,ts3
> from datagen;
> {code}
> * error
> {code:java}
> 22/10/25 13:55:01 ERROR HMSDDLExecutor: Failed to update table for
> hudi_datagen_incre_ro
> InvalidOperationException(message:The following columns have types
> incompatible with the existing columns in their respective positions :
> id)
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_with_environment_context_result$alter_table_with_environment_context_resultStandardScheme.read(ThriftHiveMetastore.java:59744)
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_with_environment_context_result$alter_table_with_environment_context_resultStandardScheme.read(ThriftHiveMetastore.java:59730)
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_with_environment_context_result.read(ThriftHiveMetastore.java:59672)
> at org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:86)
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.recv_alter_table_with_environment_context(ThriftHiveMetastore.java:1693)
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.alter_table_with_environment_context(ThriftHiveMetastore.java:1677)
> at
> org.apache.hadoop.hive.metastore.HiveMetaStoreClient.alter_table_with_environmentContext(HiveMetaStoreClient.java:375)
> at
> org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.alter_table_with_environmentContext(SessionHiveMetaStoreClient.java:322)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:4
[jira] [Updated] (HUDI-5088) Failed to synchronize the hive metadata of the Flink table
[
https://issues.apache.org/jira/browse/HUDI-5088?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
waywtdcc updated HUDI-5088:
---
Description:
The flash table failed to synchronize the hive metadata. After using the flash
catalog and specifying changlog.enabled=true and hive_sync.skip_ro_suffix =
true, the real-time writing to the table fails to synchronize the hive metadata.
I guess the reason is:
When using the Flink catalog to create the Flink table, the hive table will be
created first. The default value for creating a hive table is none_ hoodie_ The
operation field, but the Flink table has this field. The inconsistent fields
cause synchronization failure.
* sql
*
{code:java}
CREATE TABLE datagen (
id INT,
name string,
ts3 timestamp(3)
) WITH (
'connector' = 'datagen',
'rows-per-second'='10',
'fields.id.kind'='sequence',
'fields.id.start'='1',
'fields.id.end'='1000'
);
CREATE CATALOG myhudi WITH(
'type' = 'hudi',
'default-database' = 'default',
'catalog.path' = '/user/hdpu/warehouse',
'mode' = 'hms',
'hive.conf.dir' = 'hdfs:///user/hdpu/streamx/conf_data/hive_conf',
-- table-prop. properties.It is used to add attributes by default when creating
tables.
'table-prop.connector' = 'hudi',
'table-prop.table.type' = 'MERGE_ON_READ',
'table-prop.compaction.tasks' = '4',
'table-prop.write.tasks' = '4',
'table-prop.index.bootstrap.enabled' = 'true',
'table-prop.hive_sync.skip_ro_suffix' = 'true',
'table-prop.compaction.delta_commits' = '1',
'table-prop.compaction.async.enabled' = 'true',
'table-prop.changelog.enabled' = 'true',
'table-prop.index.type' = 'BUCKET',
'table-prop.index.global.enabled' = 'true',
'table-prop.read.utc-timezone' = 'false'
);
CREATE CATALOG myhive WITH (
'type' = 'hive',
'default-database' = 'default',
'hive-conf-dir' = 'hdfs:///user/hdpu/streamx/conf_data/hive_conf'
);
drop table if exists myhive.test_hudi3.hudi_datagen_incre;
drop table if exists myhudi.test_hudi3.hudi_datagen_incre;
create table if not exists myhudi.test_hudi3.hudi_datagen_incre
(id bigint not null, name string,ts3 timestamp(3)
,PRIMARY KEY (`id`) NOT ENFORCED
)
;
show create table myhudi.test_hudi3.hudi_datagen_incre;
insert into myhudi.test_hudi3.hudi_datagen_incre
select id,name,ts3
from datagen;
{code}
* error
{code:java}
22/10/25 13:55:01 ERROR HMSDDLExecutor: Failed to update table for
hudi_datagen_incre_ro
InvalidOperationException(message:The following columns have types incompatible
with the existing columns in their respective positions :
id)
at
org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_with_environment_context_result$alter_table_with_environment_context_resultStandardScheme.read(ThriftHiveMetastore.java:59744)
at
org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_with_environment_context_result$alter_table_with_environment_context_resultStandardScheme.read(ThriftHiveMetastore.java:59730)
at
org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_with_environment_context_result.read(ThriftHiveMetastore.java:59672)
at org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:86)
at
org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.recv_alter_table_with_environment_context(ThriftHiveMetastore.java:1693)
at
org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.alter_table_with_environment_context(ThriftHiveMetastore.java:1677)
at
org.apache.hadoop.hive.metastore.HiveMetaStoreClient.alter_table_with_environmentContext(HiveMetaStoreClient.java:375)
at
org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.alter_table_with_environmentContext(SessionHiveMetaStoreClient.java:322)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.invoke(RetryingMetaStoreClient.java:173)
at com.sun.proxy.$Proxy32.alter_table_with_environmentContext(Unknown
Source)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.apache.hadoop.hive.metastore.HiveMetaStoreClient$SynchronizedHandler.invoke(HiveMetaStoreClient.java:2336)
at com.sun.proxy.$Proxy32.alter_table_with_environmentContext(Unknown
Source)
at
org.apache.hudi.hive.ddl.HMSDDLExecutor.updateTableDefinition(HMSDDLExecutor
[jira] [Updated] (HUDI-5088) Failed to synchronize the hive metadata of the Flink table
[
https://issues.apache.org/jira/browse/HUDI-5088?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
waywtdcc updated HUDI-5088:
---
Attachment: (was: image-2022-10-25-15-20-50-321.png)
> Failed to synchronize the hive metadata of the Flink table
> --
>
> Key: HUDI-5088
> URL: https://issues.apache.org/jira/browse/HUDI-5088
> Project: Apache Hudi
> Issue Type: Bug
> Components: flink-sql
>Affects Versions: 0.12.1
>Reporter: waywtdcc
>Assignee: waywtdcc
>Priority: Major
> Labels: pull-request-available
> Fix For: 0.12.2
>
>
> The flash table failed to synchronize the hive metadata. After using the
> flash catalog and specifying changlog.enabled=true and
> hive_sync.skip_ro_suffix = true, the real-time writing to the table fails to
> synchronize the hive metadata.
> I guess the reason is:
> When using the Flink catalog to create the Flink table, the hive table will
> be created first. The default value for creating a hive table is none_
> hoodie_ The operation field, but the Flink table has this field. The
> inconsistent fields cause synchronization failure.
> * sql
>
> *
> {code:java}
> CREATE TABLE datagen (
> id INT,
> name string,
> ts3 timestamp(3)
> ) WITH (
> 'connector' = 'datagen',
> 'rows-per-second'='10',
> 'fields.id.kind'='sequence',
> 'fields.id.start'='1',
> 'fields.id.end'='1000'
> );
> CREATE CATALOG myhudi WITH(
> 'type' = 'hudi',
> 'default-database' = 'default',
> 'catalog.path' = '/user/hdpu/warehouse',
> 'mode' = 'hms',
> 'hive.conf.dir' = 'hdfs:///user/hdpu/streamx/conf_data/hive_conf',
> -- table-prop. properties.It is used to add attributes by default when
> creating tables.
> 'table-prop.connector' = 'hudi',
> 'table-prop.table.type' = 'MERGE_ON_READ',
> 'table-prop.compaction.tasks' = '4',
> 'table-prop.write.tasks' = '4',
> 'table-prop.index.bootstrap.enabled' = 'true',
> 'table-prop.hive_sync.skip_ro_suffix' = 'true',
> 'table-prop.compaction.delta_commits' = '1',
> 'table-prop.compaction.async.enabled' = 'true',
> 'table-prop.changelog.enabled' = 'true',
> 'table-prop.index.type' = 'BUCKET',
> 'table-prop.index.global.enabled' = 'true',
> 'table-prop.read.utc-timezone' = 'false'
> );
> CREATE CATALOG myhive WITH (
> 'type' = 'hive',
> 'default-database' = 'default',
> 'hive-conf-dir' = 'hdfs:///user/hdpu/streamx/conf_data/hive_conf'
> );
> drop table if exists myhive.test_hudi3.hudi_datagen_incre;
> drop table if exists myhudi.test_hudi3.hudi_datagen_incre;
> create table if not exists myhudi.test_hudi3.hudi_datagen_incre
> (id bigint not null, name string,ts3 timestamp(3)
> ,PRIMARY KEY (`id`) NOT ENFORCED
> )
> ;
> show create table myhudi.test_hudi3.hudi_datagen_incre;
> insert into myhudi.test_hudi3.hudi_datagen_incre
> select id,name,ts3
> from datagen;
> {code}
> * error
> {code:java}
> 22/10/25 13:55:01 ERROR HMSDDLExecutor: Failed to update table for
> hudi_datagen_incre_ro
> InvalidOperationException(message:The following columns have types
> incompatible with the existing columns in their respective positions :
> id)
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_with_environment_context_result$alter_table_with_environment_context_resultStandardScheme.read(ThriftHiveMetastore.java:59744)
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_with_environment_context_result$alter_table_with_environment_context_resultStandardScheme.read(ThriftHiveMetastore.java:59730)
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_with_environment_context_result.read(ThriftHiveMetastore.java:59672)
> at org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:86)
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.recv_alter_table_with_environment_context(ThriftHiveMetastore.java:1693)
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.alter_table_with_environment_context(ThriftHiveMetastore.java:1677)
> at
> org.apache.hadoop.hive.metastore.HiveMetaStoreClient.alter_table_with_environmentContext(HiveMetaStoreClient.java:375)
> at
> org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.alter_table_with_environmentContext(SessionHiveMetaStoreClient.java:322)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at
> org.apache.hadoop.hive.metastore
[jira] [Updated] (HUDI-5088) Failed to synchronize the hive metadata of the Flink table
[
https://issues.apache.org/jira/browse/HUDI-5088?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
waywtdcc updated HUDI-5088:
---
Fix Version/s: 0.13.0
> Failed to synchronize the hive metadata of the Flink table
> --
>
> Key: HUDI-5088
> URL: https://issues.apache.org/jira/browse/HUDI-5088
> Project: Apache Hudi
> Issue Type: Bug
> Components: flink-sql
>Affects Versions: 0.12.1
>Reporter: waywtdcc
>Assignee: waywtdcc
>Priority: Major
> Labels: pull-request-available
> Fix For: 0.12.2, 0.13.0
>
>
> The flash table failed to synchronize the hive metadata. After using the
> flash catalog and specifying changlog.enabled=true and
> hive_sync.skip_ro_suffix = true, the real-time writing to the table fails to
> synchronize the hive metadata.
> I guess the reason is:
> When using the Flink catalog to create the Flink table, the hive table will
> be created first. The default value for creating a hive table is none_
> hoodie_ The operation field, but the Flink table has this field. The
> inconsistent fields cause synchronization failure.
> * sql
>
> *
> {code:java}
> CREATE TABLE datagen (
> id INT,
> name string,
> ts3 timestamp(3)
> ) WITH (
> 'connector' = 'datagen',
> 'rows-per-second'='10',
> 'fields.id.kind'='sequence',
> 'fields.id.start'='1',
> 'fields.id.end'='1000'
> );
> CREATE CATALOG myhudi WITH(
> 'type' = 'hudi',
> 'default-database' = 'default',
> 'catalog.path' = '/user/hdpu/warehouse',
> 'mode' = 'hms',
> 'hive.conf.dir' = 'hdfs:///user/hdpu/streamx/conf_data/hive_conf',
> -- table-prop. properties.It is used to add attributes by default when
> creating tables.
> 'table-prop.connector' = 'hudi',
> 'table-prop.table.type' = 'MERGE_ON_READ',
> 'table-prop.compaction.tasks' = '4',
> 'table-prop.write.tasks' = '4',
> 'table-prop.index.bootstrap.enabled' = 'true',
> 'table-prop.hive_sync.skip_ro_suffix' = 'true',
> 'table-prop.compaction.delta_commits' = '1',
> 'table-prop.compaction.async.enabled' = 'true',
> 'table-prop.changelog.enabled' = 'true',
> 'table-prop.index.type' = 'BUCKET',
> 'table-prop.index.global.enabled' = 'true',
> 'table-prop.read.utc-timezone' = 'false'
> );
> CREATE CATALOG myhive WITH (
> 'type' = 'hive',
> 'default-database' = 'default',
> 'hive-conf-dir' = 'hdfs:///user/hdpu/streamx/conf_data/hive_conf'
> );
> drop table if exists myhive.test_hudi3.hudi_datagen_incre;
> drop table if exists myhudi.test_hudi3.hudi_datagen_incre;
> create table if not exists myhudi.test_hudi3.hudi_datagen_incre
> (id bigint not null, name string,ts3 timestamp(3)
> ,PRIMARY KEY (`id`) NOT ENFORCED
> )
> ;
> show create table myhudi.test_hudi3.hudi_datagen_incre;
> insert into myhudi.test_hudi3.hudi_datagen_incre
> select id,name,ts3
> from datagen;
> {code}
> * error
> {code:java}
> 22/10/25 13:55:01 ERROR HMSDDLExecutor: Failed to update table for
> hudi_datagen_incre_ro
> InvalidOperationException(message:The following columns have types
> incompatible with the existing columns in their respective positions :
> id)
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_with_environment_context_result$alter_table_with_environment_context_resultStandardScheme.read(ThriftHiveMetastore.java:59744)
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_with_environment_context_result$alter_table_with_environment_context_resultStandardScheme.read(ThriftHiveMetastore.java:59730)
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_with_environment_context_result.read(ThriftHiveMetastore.java:59672)
> at org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:86)
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.recv_alter_table_with_environment_context(ThriftHiveMetastore.java:1693)
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.alter_table_with_environment_context(ThriftHiveMetastore.java:1677)
> at
> org.apache.hadoop.hive.metastore.HiveMetaStoreClient.alter_table_with_environmentContext(HiveMetaStoreClient.java:375)
> at
> org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.alter_table_with_environmentContext(SessionHiveMetaStoreClient.java:322)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at
> org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.in
[jira] [Updated] (HUDI-5088) Failed to synchronize the hive metadata of the Flink table
[
https://issues.apache.org/jira/browse/HUDI-5088?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
waywtdcc updated HUDI-5088:
---
Description:
The flash table failed to synchronize the hive metadata. After using the flash
catalog and specifying changlog.enabled=true and hive_sync.skip_ro_suffix =
true, the real-time writing to the table fails to synchronize the hive metadata.
I guess the reason is:
When using the Flink catalog to create the Flink table, the hive table will be
created first. The default value for creating a hive table is none_ hoodie_ The
operation field, but the Flink table has this field. The inconsistent fields
cause synchronization failure.
* sql
*
{code:java}
CREATE TABLE datagen (
id INT,
name string,
ts3 timestamp(3)
) WITH (
'connector' = 'datagen',
'rows-per-second'='10',
'fields.id.kind'='sequence',
'fields.id.start'='1',
'fields.id.end'='1000'
);
CREATE CATALOG myhudi WITH(
'type' = 'hudi',
'default-database' = 'default',
'catalog.path' = '/user/hdpu/warehouse',
'mode' = 'hms',
'hive.conf.dir' = 'hdfs:///user/hdpu/streamx/conf_data/hive_conf',
-- table-prop. properties.It is used to add attributes by default when creating
tables.
'table-prop.connector' = 'hudi',
'table-prop.table.type' = 'MERGE_ON_READ',
'table-prop.compaction.tasks' = '4',
'table-prop.write.tasks' = '4',
'table-prop.index.bootstrap.enabled' = 'true',
'table-prop.hive_sync.skip_ro_suffix' = 'true',
'table-prop.compaction.delta_commits' = '1',
'table-prop.compaction.async.enabled' = 'true',
'table-prop.changelog.enabled' = 'true',
'table-prop.index.type' = 'BUCKET',
'table-prop.index.global.enabled' = 'true',
'table-prop.read.utc-timezone' = 'false'
);
CREATE CATALOG myhive WITH (
'type' = 'hive',
'default-database' = 'default',
'hive-conf-dir' = 'hdfs:///user/hdpu/streamx/conf_data/hive_conf'
);
drop table if exists myhive.test_hudi3.hudi_datagen_incre;
drop table if exists myhudi.test_hudi3.hudi_datagen_incre;
create table if not exists myhudi.test_hudi3.hudi_datagen_incre
(id bigint not null, name string,ts3 timestamp(3)
,PRIMARY KEY (`id`) NOT ENFORCED
)
;
show create table myhudi.test_hudi3.hudi_datagen_incre;
insert into myhudi.test_hudi3.hudi_datagen_incre
select id,name,ts3
from datagen;
{code}
* error
{code:java}
22/10/25 13:55:01 ERROR HMSDDLExecutor: Failed to update table for
hudi_datagen_incre_ro
InvalidOperationException(message:The following columns have types incompatible
with the existing columns in their respective positions :
id)
at
org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_with_environment_context_result$alter_table_with_environment_context_resultStandardScheme.read(ThriftHiveMetastore.java:59744)
at
org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_with_environment_context_result$alter_table_with_environment_context_resultStandardScheme.read(ThriftHiveMetastore.java:59730)
at
org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_with_environment_context_result.read(ThriftHiveMetastore.java:59672)
at org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:86)
at
org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.recv_alter_table_with_environment_context(ThriftHiveMetastore.java:1693)
at
org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.alter_table_with_environment_context(ThriftHiveMetastore.java:1677)
at
org.apache.hadoop.hive.metastore.HiveMetaStoreClient.alter_table_with_environmentContext(HiveMetaStoreClient.java:375)
at
org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.alter_table_with_environmentContext(SessionHiveMetaStoreClient.java:322)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.invoke(RetryingMetaStoreClient.java:173)
at com.sun.proxy.$Proxy32.alter_table_with_environmentContext(Unknown
Source)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.apache.hadoop.hive.metastore.HiveMetaStoreClient$SynchronizedHandler.invoke(HiveMetaStoreClient.java:2336)
at com.sun.proxy.$Proxy32.alter_table_with_environmentContext(Unknown
Source)
at
org.apache.hudi.hive.ddl.HMSDDLExecutor.updateTableDefinition(HMSDDLExecutor
[jira] [Updated] (HUDI-5088) Failed to synchronize the hive metadata of the Flink table
[
https://issues.apache.org/jira/browse/HUDI-5088?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
waywtdcc updated HUDI-5088:
---
Attachment: image-2022-10-27-13-51-06-216.png
> Failed to synchronize the hive metadata of the Flink table
> --
>
> Key: HUDI-5088
> URL: https://issues.apache.org/jira/browse/HUDI-5088
> Project: Apache Hudi
> Issue Type: Bug
> Components: flink-sql
>Affects Versions: 0.12.1
>Reporter: waywtdcc
>Assignee: waywtdcc
>Priority: Major
> Labels: pull-request-available
> Fix For: 0.12.2, 0.13.0
>
> Attachments: image-2022-10-27-13-51-06-216.png
>
>
> The flash table failed to synchronize the hive metadata. After using the
> flash catalog and specifying changlog.enabled=true and
> hive_sync.skip_ro_suffix = true, the real-time writing to the table fails to
> synchronize the hive metadata.
> I guess the reason is:
> When using the Flink catalog to create the Flink table, the hive table will
> be created first. The default value for creating a hive table is none_
> hoodie_ The operation field, but the Flink table has this field. The
> inconsistent fields cause synchronization failure.
> * sql
>
> *
> {code:java}
> CREATE TABLE datagen (
> id INT,
> name string,
> ts3 timestamp(3)
> ) WITH (
> 'connector' = 'datagen',
> 'rows-per-second'='10',
> 'fields.id.kind'='sequence',
> 'fields.id.start'='1',
> 'fields.id.end'='1000'
> );
> CREATE CATALOG myhudi WITH(
> 'type' = 'hudi',
> 'default-database' = 'default',
> 'catalog.path' = '/user/hdpu/warehouse',
> 'mode' = 'hms',
> 'hive.conf.dir' = 'hdfs:///user/hdpu/streamx/conf_data/hive_conf',
> -- table-prop. properties.It is used to add attributes by default when
> creating tables.
> 'table-prop.connector' = 'hudi',
> 'table-prop.table.type' = 'MERGE_ON_READ',
> 'table-prop.compaction.tasks' = '4',
> 'table-prop.write.tasks' = '4',
> 'table-prop.index.bootstrap.enabled' = 'true',
> 'table-prop.hive_sync.skip_ro_suffix' = 'true',
> 'table-prop.compaction.delta_commits' = '1',
> 'table-prop.compaction.async.enabled' = 'true',
> 'table-prop.changelog.enabled' = 'true',
> 'table-prop.index.type' = 'BUCKET',
> 'table-prop.index.global.enabled' = 'true',
> 'table-prop.read.utc-timezone' = 'false'
> );
> CREATE CATALOG myhive WITH (
> 'type' = 'hive',
> 'default-database' = 'default',
> 'hive-conf-dir' = 'hdfs:///user/hdpu/streamx/conf_data/hive_conf'
> );
> drop table if exists myhive.test_hudi3.hudi_datagen_incre;
> drop table if exists myhudi.test_hudi3.hudi_datagen_incre;
> create table if not exists myhudi.test_hudi3.hudi_datagen_incre
> (id bigint not null, name string,ts3 timestamp(3)
> ,PRIMARY KEY (`id`) NOT ENFORCED
> )
> ;
> show create table myhudi.test_hudi3.hudi_datagen_incre;
> insert into myhudi.test_hudi3.hudi_datagen_incre
> select id,name,ts3
> from datagen;
> {code}
> * error
> {code:java}
> 22/10/25 13:55:01 ERROR HMSDDLExecutor: Failed to update table for
> hudi_datagen_incre_ro
> InvalidOperationException(message:The following columns have types
> incompatible with the existing columns in their respective positions :
> id)
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_with_environment_context_result$alter_table_with_environment_context_resultStandardScheme.read(ThriftHiveMetastore.java:59744)
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_with_environment_context_result$alter_table_with_environment_context_resultStandardScheme.read(ThriftHiveMetastore.java:59730)
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_with_environment_context_result.read(ThriftHiveMetastore.java:59672)
> at org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:86)
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.recv_alter_table_with_environment_context(ThriftHiveMetastore.java:1693)
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.alter_table_with_environment_context(ThriftHiveMetastore.java:1677)
> at
> org.apache.hadoop.hive.metastore.HiveMetaStoreClient.alter_table_with_environmentContext(HiveMetaStoreClient.java:375)
> at
> org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.alter_table_with_environmentContext(SessionHiveMetaStoreClient.java:322)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Metho
[jira] [Updated] (HUDI-5088) Failed to synchronize the hive metadata of the Flink table
[
https://issues.apache.org/jira/browse/HUDI-5088?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Ethan Guo updated HUDI-5088:
Fix Version/s: (was: 0.13.0)
> Failed to synchronize the hive metadata of the Flink table
> --
>
> Key: HUDI-5088
> URL: https://issues.apache.org/jira/browse/HUDI-5088
> Project: Apache Hudi
> Issue Type: Bug
> Components: flink-sql
>Affects Versions: 0.12.1
>Reporter: waywtdcc
>Assignee: waywtdcc
>Priority: Blocker
> Labels: pull-request-available
> Fix For: 0.12.2
>
> Attachments: image-2022-10-27-13-51-06-216.png
>
>
> The flash table failed to synchronize the hive metadata. After using the
> flash catalog and specifying changlog.enabled=true and
> hive_sync.skip_ro_suffix = true, the real-time writing to the table fails to
> synchronize the hive metadata.
> I guess the reason is:
> When using the Flink catalog to create the Flink table, the hive table will
> be created first. The default value for creating a hive table is none_
> hoodie_ The operation field, but the Flink table has this field. The
> inconsistent fields cause synchronization failure.
> * sql
>
> *
> {code:java}
> CREATE TABLE datagen (
> id INT,
> name string,
> ts3 timestamp(3)
> ) WITH (
> 'connector' = 'datagen',
> 'rows-per-second'='10',
> 'fields.id.kind'='sequence',
> 'fields.id.start'='1',
> 'fields.id.end'='1000'
> );
> CREATE CATALOG myhudi WITH(
> 'type' = 'hudi',
> 'default-database' = 'default',
> 'catalog.path' = '/user/hdpu/warehouse',
> 'mode' = 'hms',
> 'hive.conf.dir' = 'hdfs:///user/hdpu/streamx/conf_data/hive_conf',
> -- table-prop. properties.It is used to add attributes by default when
> creating tables.
> 'table-prop.connector' = 'hudi',
> 'table-prop.table.type' = 'MERGE_ON_READ',
> 'table-prop.compaction.tasks' = '4',
> 'table-prop.write.tasks' = '4',
> 'table-prop.index.bootstrap.enabled' = 'true',
> 'table-prop.hive_sync.skip_ro_suffix' = 'true',
> 'table-prop.compaction.delta_commits' = '1',
> 'table-prop.compaction.async.enabled' = 'true',
> 'table-prop.changelog.enabled' = 'true',
> 'table-prop.index.type' = 'BUCKET',
> 'table-prop.index.global.enabled' = 'true',
> 'table-prop.read.utc-timezone' = 'false'
> );
> CREATE CATALOG myhive WITH (
> 'type' = 'hive',
> 'default-database' = 'default',
> 'hive-conf-dir' = 'hdfs:///user/hdpu/streamx/conf_data/hive_conf'
> );
> drop table if exists myhive.test_hudi3.hudi_datagen_incre;
> drop table if exists myhudi.test_hudi3.hudi_datagen_incre;
> create table if not exists myhudi.test_hudi3.hudi_datagen_incre
> (id bigint not null, name string,ts3 timestamp(3)
> ,PRIMARY KEY (`id`) NOT ENFORCED
> )
> ;
> show create table myhudi.test_hudi3.hudi_datagen_incre;
> insert into myhudi.test_hudi3.hudi_datagen_incre
> select id,name,ts3
> from datagen;
> {code}
> * error
> {code:java}
> 22/10/25 13:55:01 ERROR HMSDDLExecutor: Failed to update table for
> hudi_datagen_incre_ro
> InvalidOperationException(message:The following columns have types
> incompatible with the existing columns in their respective positions :
> id)
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_with_environment_context_result$alter_table_with_environment_context_resultStandardScheme.read(ThriftHiveMetastore.java:59744)
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_with_environment_context_result$alter_table_with_environment_context_resultStandardScheme.read(ThriftHiveMetastore.java:59730)
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_with_environment_context_result.read(ThriftHiveMetastore.java:59672)
> at org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:86)
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.recv_alter_table_with_environment_context(ThriftHiveMetastore.java:1693)
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.alter_table_with_environment_context(ThriftHiveMetastore.java:1677)
> at
> org.apache.hadoop.hive.metastore.HiveMetaStoreClient.alter_table_with_environmentContext(HiveMetaStoreClient.java:375)
> at
> org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.alter_table_with_environmentContext(SessionHiveMetaStoreClient.java:322)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
>
[jira] [Updated] (HUDI-5088) Failed to synchronize the hive metadata of the Flink table
[
https://issues.apache.org/jira/browse/HUDI-5088?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Ethan Guo updated HUDI-5088:
Priority: Blocker (was: Major)
> Failed to synchronize the hive metadata of the Flink table
> --
>
> Key: HUDI-5088
> URL: https://issues.apache.org/jira/browse/HUDI-5088
> Project: Apache Hudi
> Issue Type: Bug
> Components: flink-sql
>Affects Versions: 0.12.1
>Reporter: waywtdcc
>Assignee: waywtdcc
>Priority: Blocker
> Labels: pull-request-available
> Fix For: 0.12.2, 0.13.0
>
> Attachments: image-2022-10-27-13-51-06-216.png
>
>
> The flash table failed to synchronize the hive metadata. After using the
> flash catalog and specifying changlog.enabled=true and
> hive_sync.skip_ro_suffix = true, the real-time writing to the table fails to
> synchronize the hive metadata.
> I guess the reason is:
> When using the Flink catalog to create the Flink table, the hive table will
> be created first. The default value for creating a hive table is none_
> hoodie_ The operation field, but the Flink table has this field. The
> inconsistent fields cause synchronization failure.
> * sql
>
> *
> {code:java}
> CREATE TABLE datagen (
> id INT,
> name string,
> ts3 timestamp(3)
> ) WITH (
> 'connector' = 'datagen',
> 'rows-per-second'='10',
> 'fields.id.kind'='sequence',
> 'fields.id.start'='1',
> 'fields.id.end'='1000'
> );
> CREATE CATALOG myhudi WITH(
> 'type' = 'hudi',
> 'default-database' = 'default',
> 'catalog.path' = '/user/hdpu/warehouse',
> 'mode' = 'hms',
> 'hive.conf.dir' = 'hdfs:///user/hdpu/streamx/conf_data/hive_conf',
> -- table-prop. properties.It is used to add attributes by default when
> creating tables.
> 'table-prop.connector' = 'hudi',
> 'table-prop.table.type' = 'MERGE_ON_READ',
> 'table-prop.compaction.tasks' = '4',
> 'table-prop.write.tasks' = '4',
> 'table-prop.index.bootstrap.enabled' = 'true',
> 'table-prop.hive_sync.skip_ro_suffix' = 'true',
> 'table-prop.compaction.delta_commits' = '1',
> 'table-prop.compaction.async.enabled' = 'true',
> 'table-prop.changelog.enabled' = 'true',
> 'table-prop.index.type' = 'BUCKET',
> 'table-prop.index.global.enabled' = 'true',
> 'table-prop.read.utc-timezone' = 'false'
> );
> CREATE CATALOG myhive WITH (
> 'type' = 'hive',
> 'default-database' = 'default',
> 'hive-conf-dir' = 'hdfs:///user/hdpu/streamx/conf_data/hive_conf'
> );
> drop table if exists myhive.test_hudi3.hudi_datagen_incre;
> drop table if exists myhudi.test_hudi3.hudi_datagen_incre;
> create table if not exists myhudi.test_hudi3.hudi_datagen_incre
> (id bigint not null, name string,ts3 timestamp(3)
> ,PRIMARY KEY (`id`) NOT ENFORCED
> )
> ;
> show create table myhudi.test_hudi3.hudi_datagen_incre;
> insert into myhudi.test_hudi3.hudi_datagen_incre
> select id,name,ts3
> from datagen;
> {code}
> * error
> {code:java}
> 22/10/25 13:55:01 ERROR HMSDDLExecutor: Failed to update table for
> hudi_datagen_incre_ro
> InvalidOperationException(message:The following columns have types
> incompatible with the existing columns in their respective positions :
> id)
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_with_environment_context_result$alter_table_with_environment_context_resultStandardScheme.read(ThriftHiveMetastore.java:59744)
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_with_environment_context_result$alter_table_with_environment_context_resultStandardScheme.read(ThriftHiveMetastore.java:59730)
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$alter_table_with_environment_context_result.read(ThriftHiveMetastore.java:59672)
> at org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:86)
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.recv_alter_table_with_environment_context(ThriftHiveMetastore.java:1693)
> at
> org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.alter_table_with_environment_context(ThriftHiveMetastore.java:1677)
> at
> org.apache.hadoop.hive.metastore.HiveMetaStoreClient.alter_table_with_environmentContext(HiveMetaStoreClient.java:375)
> at
> org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.alter_table_with_environmentContext(SessionHiveMetaStoreClient.java:322)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498
