[jira] [Updated] (CARBONDATA-4307) Insert into values fails for hive table on CarbonSession in both Spark 2.4.5 as well as Spark 3.1.1 environments.
[ https://issues.apache.org/jira/browse/CARBONDATA-4307?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] PURUJIT CHAUGULE updated CARBONDATA-4307: - Description: *Insert into values fails for hive table on CarbonSession in both Spark 2.4.5 as well as Spark 3.1.1 environments.* *For Spark 2.4.5:* # drop table if exists dpp_table2; # create table dpp_table2(col1 int, col2 string); # insert into dpp_table2 values(1, 'a'),(2, 'b'),(3, 'c'),(4, 'd'); *Error:* org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: Unable to move source hdfs:// hacluster/user/sparkhive/warehouse/sdv.db/dpp_table2/.hive-staging_hive_2021-10-20_15-36-50_393_362300659594489595-6/-ext-1000 0/part-0-463ae254-c7ab-47ae-8cbd-6bb58d191678-c000 to destination hdfs://hacluster/user/sparkhive/warehouse/sdv.db/dpp_tab le2/part-0-463ae254-c7ab-47ae-8cbd-6bb58d191678-c000; (state=,code=0) *For Spark 3.1.1:* # drop table if exists dpp_table2; # create table dpp_table2(col1 int, col2 string); # insert into dpp_table2 values(1, 'a'),(2, 'b'),(3, 'c'),(4, 'd'); *Error:* org.apache.hive.service.cli.HiveSQLException: Error running query: org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: java.io.IOException: Filesystem closed at org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation.org$apache$spark$sql$hive$thriftserver$SparkExecuteStatementOperation$$execute(SparkExecuteStatementOperation.scala:361) at org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation$$anon$2$$anon$3.$anonfun$run$2(SparkExecuteStatementOperation.scala:263) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) at org.apache.spark.sql.hive.thriftserver.SparkOperation.withLocalProperties(SparkOperation.scala:78) at org.apache.spark.sql.hive.thriftserver.SparkOperation.withLocalProperties$(SparkOperation.scala:62) at org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation.withLocalProperties(SparkExecuteStatementOperation.scala:43) at org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation$$anon$2$$anon$3.run(SparkExecuteStatementOperation.scala:263) at org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation$$anon$2$$anon$3.run(SparkExecuteStatementOperation.scala:258) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1746) at org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation$$anon$2.run(SparkExecuteStatementOperation.scala:272) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: java.io.IOException: Filesystem closed at org.apache.spark.sql.hive.HiveExternalCatalog.withClient(HiveExternalCatalog.scala:112) at org.apache.spark.sql.hive.HiveExternalCatalog.loadTable(HiveExternalCatalog.scala:877) at org.apache.spark.sql.catalyst.catalog.ExternalCatalogWithListener.loadTable(ExternalCatalogWithListener.scala:167) at org.apache.spark.sql.hive.execution.InsertIntoHiveTable.processInsert(InsertIntoHiveTable.scala:343) at org.apache.spark.sql.hive.execution.InsertIntoHiveTable.run(InsertIntoHiveTable.scala:104) at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult$lzycompute(commands.scala:108) at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult(commands.scala:106) at org.apache.spark.sql.execution.command.DataWritingCommandExec.executeCollect(commands.scala:120) at org.apache.spark.sql.Dataset.$anonfun$logicalPlan$1(Dataset.scala:228) at org.apache.spark.sql.Dataset.$anonfun$withAction$1(Dataset.scala:3687) at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:103) at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:163) at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:90) at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:772) at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64) at org.apache.spark.sql.Dataset.withAction(Dataset.scala:3685) at org.apache.spark.sql.Dataset.(Dataset.scala:228) at org.apache.spark.sql.Dataset$.$anonfun$ofRows$2(Dataset.scala:99) at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:772) at
[jira] [Created] (CARBONDATA-4307) Insert into values fails for hive table
PURUJIT CHAUGULE created CARBONDATA-4307: Summary: Insert into values fails for hive table Key: CARBONDATA-4307 URL: https://issues.apache.org/jira/browse/CARBONDATA-4307 Project: CarbonData Issue Type: Bug Components: data-load Affects Versions: 2.3.0 Environment: Spark 2.4.5 Spark 3.1.1 Reporter: PURUJIT CHAUGULE *Insert into values fails for hive table on Both enviroments Spark 2.4.5 and Spark 3.1.1.* *For Spark 2.4.5:* # drop table if exists dpp_table2; # create table dpp_table2(col1 int, col2 string); # insert into dpp_table2 values(1, 'a'),(2, 'b'),(3, 'c'),(4, 'd'); *Error:* org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: Unable to move source hdfs:// hacluster/user/sparkhive/warehouse/sdv.db/dpp_table2/.hive-staging_hive_2021-10-20_15-36-50_393_362300659594489595-6/-ext-1000 0/part-0-463ae254-c7ab-47ae-8cbd-6bb58d191678-c000 to destination hdfs://hacluster/user/sparkhive/warehouse/sdv.db/dpp_tab le2/part-0-463ae254-c7ab-47ae-8cbd-6bb58d191678-c000; (state=,code=0) *For Spark 3.1.1:* # drop table if exists dpp_table2; # create table dpp_table2(col1 int, col2 string); # insert into dpp_table2 values(1, 'a'),(2, 'b'),(3, 'c'),(4, 'd'); *Error:* org.apache.hive.service.cli.HiveSQLException: Error running query: org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: java.io.IOException: Filesystem closed at org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation.org$apache$spark$sql$hive$thriftserver$SparkExecuteStatementOperation$$execute(SparkExecuteStatementOperation.scala:361) at org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation$$anon$2$$anon$3.$anonfun$run$2(SparkExecuteStatementOperation.scala:263) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) at org.apache.spark.sql.hive.thriftserver.SparkOperation.withLocalProperties(SparkOperation.scala:78) at org.apache.spark.sql.hive.thriftserver.SparkOperation.withLocalProperties$(SparkOperation.scala:62) at org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation.withLocalProperties(SparkExecuteStatementOperation.scala:43) at org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation$$anon$2$$anon$3.run(SparkExecuteStatementOperation.scala:263) at org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation$$anon$2$$anon$3.run(SparkExecuteStatementOperation.scala:258) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1746) at org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation$$anon$2.run(SparkExecuteStatementOperation.scala:272) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: java.io.IOException: Filesystem closed at org.apache.spark.sql.hive.HiveExternalCatalog.withClient(HiveExternalCatalog.scala:112) at org.apache.spark.sql.hive.HiveExternalCatalog.loadTable(HiveExternalCatalog.scala:877) at org.apache.spark.sql.catalyst.catalog.ExternalCatalogWithListener.loadTable(ExternalCatalogWithListener.scala:167) at org.apache.spark.sql.hive.execution.InsertIntoHiveTable.processInsert(InsertIntoHiveTable.scala:343) at org.apache.spark.sql.hive.execution.InsertIntoHiveTable.run(InsertIntoHiveTable.scala:104) at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult$lzycompute(commands.scala:108) at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult(commands.scala:106) at org.apache.spark.sql.execution.command.DataWritingCommandExec.executeCollect(commands.scala:120) at org.apache.spark.sql.Dataset.$anonfun$logicalPlan$1(Dataset.scala:228) at org.apache.spark.sql.Dataset.$anonfun$withAction$1(Dataset.scala:3687) at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:103) at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:163) at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:90) at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:772) at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64) at org.apache.spark.sql.Dataset.withAction(Dataset.scala:3685) at
[jira] [Resolved] (CARBONDATA-4298) IS_EMPTY_DATA_BAD_RECORD property not supported for complex types.
[ https://issues.apache.org/jira/browse/CARBONDATA-4298?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Indhumathi resolved CARBONDATA-4298. Fix Version/s: 2.3.0 Resolution: Fixed > IS_EMPTY_DATA_BAD_RECORD property not supported for complex types. > -- > > Key: CARBONDATA-4298 > URL: https://issues.apache.org/jira/browse/CARBONDATA-4298 > Project: CarbonData > Issue Type: Bug >Reporter: SHREELEKHYA GAMPA >Priority: Major > Fix For: 2.3.0 > > Time Spent: 10h 50m > Remaining Estimate: 0h > > {{IS_EMPTY_DATA_BAD_RECORD}} property not supported for complex types. A flag > to determine if empty record is to be considered a bad record or not. -- This message was sent by Atlassian Jira (v8.3.4#803005)
[jira] [Resolved] (CARBONDATA-4281) document update for range column and COLUMN_META_CACHE for complex column
[ https://issues.apache.org/jira/browse/CARBONDATA-4281?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Indhumathi resolved CARBONDATA-4281. Fix Version/s: 2.3.0 Resolution: Fixed > document update for range column and COLUMN_META_CACHE for complex column > -- > > Key: CARBONDATA-4281 > URL: https://issues.apache.org/jira/browse/CARBONDATA-4281 > Project: CarbonData > Issue Type: Bug > Components: docs >Affects Versions: 2.3.0 > Environment: Contents verified on Spark 2.4.5 and Spark 3.1.1 >Reporter: PRIYESH RANJAN >Priority: Minor > Fix For: 2.3.0 > > > +Modification 1 :+ > [https://github.com/apache/carbondata/blob/master/docs/ddl-of-carbondata.md] > Range column and COLUMN_META_CACHE does not support complex columns .This > details need to be updated in doc. > > *+Query:+* > CREATE TABLE alter_com(intField INT,EDUCATED string ,rankk string) STORED AS > carbondata > TBLPROPERTIES('inverted_index'='intField','sort_columns'='intField','COLUMN_META_CACHE'='rankk','range_column'='EDUCATED'); > insert into alter_com values(1,'pti','tanj'); > ALTER TABLE alter_com ADD COLUMNS(arr1 array>, arr2 > array>>) ; > *+For Range column :+* > 0: jdbc:hive2://linux-29:22550/> ALTER TABLE alter_com SET > TBLPROPERTIES('COLUMN_META_CACHE'='arr2'); > Error: org.apache.hive.service.cli.HiveSQLException: Error running query: > java.lang.RuntimeException: Alter table newProperties operation failed: arr2 > is a complex type column and *complex type is not allowed for the option(s): > column_meta_cach*e > at > org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation.org$apache$spark$sql$hive$thriftserver$SparkExecuteStatementOperation$$execute(SparkExecuteStatementOperation.scala:387) > at > org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation$$anon$2$$anon$3.$anonfun$run$3(SparkExecuteStatementOperation.scala:276) > at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) > at > org.apache.spark.sql.hive.thriftserver.SparkOperation.withLocalProperties(SparkOperation.scala:78) > at > org.apache.spark.sql.hive.thriftserver.SparkOperation.withLocalProperties$(SparkOperation.scala:62) > at > org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation.withLocalProperties(SparkExecuteStatementOperation.scala:46) > at > org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation$$anon$2$$anon$3.run(SparkExecuteStatementOperation.scala:276) > at > org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation$$anon$2$$anon$3.run(SparkExecuteStatementOperation.scala:263) > at java.security.AccessController.doPrivileged(Native Method) > at javax.security.auth.Subject.doAs(Subject.java:422) > at > org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1761) > at > org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation$$anon$2.run(SparkExecuteStatementOperation.scala:290) > at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) > at java.util.concurrent.FutureTask.run(FutureTask.java:266) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) > at java.lang.Thread.run(Thread.java:748) > > +*For COLUMN_META_CACHE* :+ > 0: jdbc:hive2://linux-29:22550/> ALTER TABLE alter_com SET > TBLPROPERTIES('range_column'='arr2'); > Error: org.apache.hive.service.cli.HiveSQLException: Error running query: > java.lang.RuntimeException: Alter table newProperties operation failed: > *RANGE_COLUMN doesn't support ARRAY data type: arr2* > at > org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation.org$apache$spark$sql$hive$thriftserver$SparkExecuteStatementOperation$$execute(SparkExecuteStatementOperation.scala:387) > at > org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation$$anon$2$$anon$3.$anonfun$run$3(SparkExecuteStatementOperation.scala:276) > at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) > at > org.apache.spark.sql.hive.thriftserver.SparkOperation.withLocalProperties(SparkOperation.scala:78) > at > org.apache.spark.sql.hive.thriftserver.SparkOperation.withLocalProperties$(SparkOperation.scala:62) > at > org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation.withLocalProperties(SparkExecuteStatementOperation.scala:46) > at > org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation$$anon$2$$anon$3.run(SparkExecuteStatementOperation.scala:276) > at > org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation$$anon$2$$anon$3.run(SparkExecuteStatementOperation.scala:263) > at