[jira] [Updated] (SPARK-33144) Connot insert overwite multiple partition, get exception "get partition: Value for key name is null or empty"
[
https://issues.apache.org/jira/browse/SPARK-33144?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
CHC updated SPARK-33144:
Priority: Major (was: Critical)
> Connot insert overwite multiple partition, get exception "get partition:
> Value for key name is null or empty"
> -
>
> Key: SPARK-33144
> URL: https://issues.apache.org/jira/browse/SPARK-33144
> Project: Spark
> Issue Type: Bug
> Components: SQL
>Affects Versions: 3.0.1, 3.2.1
> Environment: hadoop 2.7.3 + spark 3.0.1
> hadoop 2.7.3 + spark 3.2.1
>Reporter: CHC
>Priority: Major
>
> When:
> {code:sql}
> create table tmp.spark_multi_partition(
> id int
> )
> partitioned by (name string, version string)
> stored as orc
> ;
> set hive.exec.dynamic.partition=true;
> set spark.hadoop.hive.exec.dynamic.partition=true;
>
> set hive.exec.dynamic.partition.mode=nonstrict;
> set spark.hadoop.hive.exec.dynamic.partition.mode=nonstrict;
> insert overwrite table tmp.spark_multi_partition partition (name, version)
> select
> *
> from (
> select
> 1 as id
> , 'hadoop' as name
> , '2.7.3' as version
> union
> select
> 2 as id
> , 'spark' as name
> , '3.0.1' as version
> union
> select
> 3 as id
> , 'hive' as name
> , '2.3.4' as version
> ) as A;
> {code}
> and get exception:
> {code:bash}
> INFO load-dynamic-partitions-0 [hive.ql.metadata.Hive:1919]: New loading path
> =
> hdfs://namespace/apps/hive/warehouse/tmp.db/spark_multi_partition/.hive-staging_hive_2020-10-14_09-15-27_718_4118806337003279343-1/-ext-1/name=spark/version=3.0.1
> with partSpec {name=spark, version=3.0.1}
> 20/10/14 09:15:33 INFO load-dynamic-partitions-1
> [hive.ql.metadata.Hive:1919]: New loading path =
> hdfs://namespace/apps/hive/warehouse/tmp.db/spark_multi_partition/.hive-staging_hive_2020-10-14_09-15-27_718_4118806337003279343-1/-ext-1/name=hadoop/version=2.7.3
> with partSpec {name=hadoop, version=2.7.3}
> 20/10/14 09:15:33 INFO load-dynamic-partitions-2
> [hive.ql.metadata.Hive:1919]: New loading path =
> hdfs://namespace/apps/hive/warehouse/tmp.db/spark_multi_partition/.hive-staging_hive_2020-10-14_09-15-27_718_4118806337003279343-1/-ext-1/name=hive/version=2.3.4
> with partSpec {name=hive, version=2.3.4}
> 20/10/14 09:15:33 INFO load-dynamic-partitions-3
> [hive.ql.metadata.Hive:1919]: New loading path =
> hdfs://namespace/apps/hive/warehouse/tmp.db/spark_multi_partition/.hive-staging_hive_2020-10-14_09-15-27_718_4118806337003279343-1/-ext-1/_temporary/0
> with partSpec {name=, version=}
> 20/10/14 09:15:33 ERROR load-dynamic-partitions-3
> [hive.ql.metadata.Hive:1937]: Exception when loading partition with
> parameters
> partPath=hdfs://namespace/apps/hive/warehouse/tmp.db/spark_multi_partition/.hive-staging_hive_2020-10-14_09-15-27_718_4118806337003279343-1/-ext-1/_temporary/0,
> table=spark_multi_partition, partSpec={name=, version=}, replace=true,
> listBucketingEnabled=false, isAcid=false, hasFollowingStatsTask=false
> org.apache.hadoop.hive.ql.metadata.HiveException: get partition: Value for
> key name is null or empty
> at org.apache.hadoop.hive.ql.metadata.Hive.getPartition(Hive.java:2233)
> at org.apache.hadoop.hive.ql.metadata.Hive.getPartition(Hive.java:2181)
> at org.apache.hadoop.hive.ql.metadata.Hive.loadPartition(Hive.java:1611)
> at org.apache.hadoop.hive.ql.metadata.Hive$3.call(Hive.java:1922)
> at org.apache.hadoop.hive.ql.metadata.Hive$3.call(Hive.java:1913)
> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at java.lang.Thread.run(Thread.java:748)
> 20/10/14 09:15:33 INFO Delete-Thread-0
> [org.apache.hadoop.fs.TrashPolicyDefault:168]: Moved:
> 'hdfs://namespace/apps/hive/warehouse/tmp.db/spark_multi_partition/name=spark/version=3.0.1/part-1-b745147b-600f-4c79-8ba2-12a99283b0a9.c000'
> to trash at:
> hdfs://namespace/user/hive/.Trash/Current/apps/hive/warehouse/tmp.db/spark_multi_partition/name=spark/version=3.0.1/part-1-b745147b-600f-4c79-8ba2-12a99283b0a9.c000
> 20/10/14 09:15:33 INFO load-dynamic-partitions-0
> [org.apache.hadoop.hive.common.FileUtils:520]: Creating directory if it
> doesn't exist:
> hdfs://namespace/apps/hive/warehouse/tmp.db/spark_multi_partition/name=spark/version=3.0.1
> 20/10/14 09:15:33 INFO Delete-Thread-0
> [org.apache.hadoop.fs.TrashPolicyDefault:168]: Moved:
> 'hdfs://namespace/apps/hive/warehouse/tmp.db/spark_multi_partition/name=hive/version=2.3.4/part-2-b745147b-600f-4c79-8ba2-12a99283b0a9.c000'
> to trash at:
>
[jira] [Updated] (SPARK-33144) Connot insert overwite multiple partition, get exception "get partition: Value for key name is null or empty"
[
https://issues.apache.org/jira/browse/SPARK-33144?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
CHC updated SPARK-33144:
Environment:
hadoop 2.7.3 + spark 3.0.1
hadoop 2.7.3 + spark 3.2.1
was:hadoop 2.7.3 + spark 3.0.1
> Connot insert overwite multiple partition, get exception "get partition:
> Value for key name is null or empty"
> -
>
> Key: SPARK-33144
> URL: https://issues.apache.org/jira/browse/SPARK-33144
> Project: Spark
> Issue Type: Bug
> Components: SQL
>Affects Versions: 3.0.1, 3.2.1
> Environment: hadoop 2.7.3 + spark 3.0.1
> hadoop 2.7.3 + spark 3.2.1
>Reporter: CHC
>Priority: Critical
>
> When:
> {code:sql}
> create table tmp.spark_multi_partition(
> id int
> )
> partitioned by (name string, version string)
> stored as orc
> ;
> set hive.exec.dynamic.partition=true;
> set spark.hadoop.hive.exec.dynamic.partition=true;
>
> set hive.exec.dynamic.partition.mode=nonstrict;
> set spark.hadoop.hive.exec.dynamic.partition.mode=nonstrict;
> insert overwrite table tmp.spark_multi_partition partition (name, version)
> select
> *
> from (
> select
> 1 as id
> , 'hadoop' as name
> , '2.7.3' as version
> union
> select
> 2 as id
> , 'spark' as name
> , '3.0.1' as version
> union
> select
> 3 as id
> , 'hive' as name
> , '2.3.4' as version
> ) as A;
> {code}
> and get exception:
> {code:bash}
> INFO load-dynamic-partitions-0 [hive.ql.metadata.Hive:1919]: New loading path
> =
> hdfs://namespace/apps/hive/warehouse/tmp.db/spark_multi_partition/.hive-staging_hive_2020-10-14_09-15-27_718_4118806337003279343-1/-ext-1/name=spark/version=3.0.1
> with partSpec {name=spark, version=3.0.1}
> 20/10/14 09:15:33 INFO load-dynamic-partitions-1
> [hive.ql.metadata.Hive:1919]: New loading path =
> hdfs://namespace/apps/hive/warehouse/tmp.db/spark_multi_partition/.hive-staging_hive_2020-10-14_09-15-27_718_4118806337003279343-1/-ext-1/name=hadoop/version=2.7.3
> with partSpec {name=hadoop, version=2.7.3}
> 20/10/14 09:15:33 INFO load-dynamic-partitions-2
> [hive.ql.metadata.Hive:1919]: New loading path =
> hdfs://namespace/apps/hive/warehouse/tmp.db/spark_multi_partition/.hive-staging_hive_2020-10-14_09-15-27_718_4118806337003279343-1/-ext-1/name=hive/version=2.3.4
> with partSpec {name=hive, version=2.3.4}
> 20/10/14 09:15:33 INFO load-dynamic-partitions-3
> [hive.ql.metadata.Hive:1919]: New loading path =
> hdfs://namespace/apps/hive/warehouse/tmp.db/spark_multi_partition/.hive-staging_hive_2020-10-14_09-15-27_718_4118806337003279343-1/-ext-1/_temporary/0
> with partSpec {name=, version=}
> 20/10/14 09:15:33 ERROR load-dynamic-partitions-3
> [hive.ql.metadata.Hive:1937]: Exception when loading partition with
> parameters
> partPath=hdfs://namespace/apps/hive/warehouse/tmp.db/spark_multi_partition/.hive-staging_hive_2020-10-14_09-15-27_718_4118806337003279343-1/-ext-1/_temporary/0,
> table=spark_multi_partition, partSpec={name=, version=}, replace=true,
> listBucketingEnabled=false, isAcid=false, hasFollowingStatsTask=false
> org.apache.hadoop.hive.ql.metadata.HiveException: get partition: Value for
> key name is null or empty
> at org.apache.hadoop.hive.ql.metadata.Hive.getPartition(Hive.java:2233)
> at org.apache.hadoop.hive.ql.metadata.Hive.getPartition(Hive.java:2181)
> at org.apache.hadoop.hive.ql.metadata.Hive.loadPartition(Hive.java:1611)
> at org.apache.hadoop.hive.ql.metadata.Hive$3.call(Hive.java:1922)
> at org.apache.hadoop.hive.ql.metadata.Hive$3.call(Hive.java:1913)
> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at java.lang.Thread.run(Thread.java:748)
> 20/10/14 09:15:33 INFO Delete-Thread-0
> [org.apache.hadoop.fs.TrashPolicyDefault:168]: Moved:
> 'hdfs://namespace/apps/hive/warehouse/tmp.db/spark_multi_partition/name=spark/version=3.0.1/part-1-b745147b-600f-4c79-8ba2-12a99283b0a9.c000'
> to trash at:
> hdfs://namespace/user/hive/.Trash/Current/apps/hive/warehouse/tmp.db/spark_multi_partition/name=spark/version=3.0.1/part-1-b745147b-600f-4c79-8ba2-12a99283b0a9.c000
> 20/10/14 09:15:33 INFO load-dynamic-partitions-0
> [org.apache.hadoop.hive.common.FileUtils:520]: Creating directory if it
> doesn't exist:
> hdfs://namespace/apps/hive/warehouse/tmp.db/spark_multi_partition/name=spark/version=3.0.1
> 20/10/14 09:15:33 INFO Delete-Thread-0
> [org.apache.hadoop.fs.TrashPolicyDefault:168]: Moved:
>
[jira] [Updated] (SPARK-33144) Connot insert overwite multiple partition, get exception "get partition: Value for key name is null or empty"
[
https://issues.apache.org/jira/browse/SPARK-33144?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
CHC updated SPARK-33144:
Priority: Critical (was: Major)
> Connot insert overwite multiple partition, get exception "get partition:
> Value for key name is null or empty"
> -
>
> Key: SPARK-33144
> URL: https://issues.apache.org/jira/browse/SPARK-33144
> Project: Spark
> Issue Type: Bug
> Components: SQL
>Affects Versions: 3.0.1, 3.2.1
> Environment: hadoop 2.7.3 + spark 3.0.1
>Reporter: CHC
>Priority: Critical
>
> When:
> {code:sql}
> create table tmp.spark_multi_partition(
> id int
> )
> partitioned by (name string, version string)
> stored as orc
> ;
> set hive.exec.dynamic.partition=true;
> set spark.hadoop.hive.exec.dynamic.partition=true;
>
> set hive.exec.dynamic.partition.mode=nonstrict;
> set spark.hadoop.hive.exec.dynamic.partition.mode=nonstrict;
> insert overwrite table tmp.spark_multi_partition partition (name, version)
> select
> *
> from (
> select
> 1 as id
> , 'hadoop' as name
> , '2.7.3' as version
> union
> select
> 2 as id
> , 'spark' as name
> , '3.0.1' as version
> union
> select
> 3 as id
> , 'hive' as name
> , '2.3.4' as version
> ) as A;
> {code}
> and get exception:
> {code:bash}
> INFO load-dynamic-partitions-0 [hive.ql.metadata.Hive:1919]: New loading path
> =
> hdfs://namespace/apps/hive/warehouse/tmp.db/spark_multi_partition/.hive-staging_hive_2020-10-14_09-15-27_718_4118806337003279343-1/-ext-1/name=spark/version=3.0.1
> with partSpec {name=spark, version=3.0.1}
> 20/10/14 09:15:33 INFO load-dynamic-partitions-1
> [hive.ql.metadata.Hive:1919]: New loading path =
> hdfs://namespace/apps/hive/warehouse/tmp.db/spark_multi_partition/.hive-staging_hive_2020-10-14_09-15-27_718_4118806337003279343-1/-ext-1/name=hadoop/version=2.7.3
> with partSpec {name=hadoop, version=2.7.3}
> 20/10/14 09:15:33 INFO load-dynamic-partitions-2
> [hive.ql.metadata.Hive:1919]: New loading path =
> hdfs://namespace/apps/hive/warehouse/tmp.db/spark_multi_partition/.hive-staging_hive_2020-10-14_09-15-27_718_4118806337003279343-1/-ext-1/name=hive/version=2.3.4
> with partSpec {name=hive, version=2.3.4}
> 20/10/14 09:15:33 INFO load-dynamic-partitions-3
> [hive.ql.metadata.Hive:1919]: New loading path =
> hdfs://namespace/apps/hive/warehouse/tmp.db/spark_multi_partition/.hive-staging_hive_2020-10-14_09-15-27_718_4118806337003279343-1/-ext-1/_temporary/0
> with partSpec {name=, version=}
> 20/10/14 09:15:33 ERROR load-dynamic-partitions-3
> [hive.ql.metadata.Hive:1937]: Exception when loading partition with
> parameters
> partPath=hdfs://namespace/apps/hive/warehouse/tmp.db/spark_multi_partition/.hive-staging_hive_2020-10-14_09-15-27_718_4118806337003279343-1/-ext-1/_temporary/0,
> table=spark_multi_partition, partSpec={name=, version=}, replace=true,
> listBucketingEnabled=false, isAcid=false, hasFollowingStatsTask=false
> org.apache.hadoop.hive.ql.metadata.HiveException: get partition: Value for
> key name is null or empty
> at org.apache.hadoop.hive.ql.metadata.Hive.getPartition(Hive.java:2233)
> at org.apache.hadoop.hive.ql.metadata.Hive.getPartition(Hive.java:2181)
> at org.apache.hadoop.hive.ql.metadata.Hive.loadPartition(Hive.java:1611)
> at org.apache.hadoop.hive.ql.metadata.Hive$3.call(Hive.java:1922)
> at org.apache.hadoop.hive.ql.metadata.Hive$3.call(Hive.java:1913)
> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at java.lang.Thread.run(Thread.java:748)
> 20/10/14 09:15:33 INFO Delete-Thread-0
> [org.apache.hadoop.fs.TrashPolicyDefault:168]: Moved:
> 'hdfs://namespace/apps/hive/warehouse/tmp.db/spark_multi_partition/name=spark/version=3.0.1/part-1-b745147b-600f-4c79-8ba2-12a99283b0a9.c000'
> to trash at:
> hdfs://namespace/user/hive/.Trash/Current/apps/hive/warehouse/tmp.db/spark_multi_partition/name=spark/version=3.0.1/part-1-b745147b-600f-4c79-8ba2-12a99283b0a9.c000
> 20/10/14 09:15:33 INFO load-dynamic-partitions-0
> [org.apache.hadoop.hive.common.FileUtils:520]: Creating directory if it
> doesn't exist:
> hdfs://namespace/apps/hive/warehouse/tmp.db/spark_multi_partition/name=spark/version=3.0.1
> 20/10/14 09:15:33 INFO Delete-Thread-0
> [org.apache.hadoop.fs.TrashPolicyDefault:168]: Moved:
> 'hdfs://namespace/apps/hive/warehouse/tmp.db/spark_multi_partition/name=hive/version=2.3.4/part-2-b745147b-600f-4c79-8ba2-12a99283b0a9.c000'
> to trash at:
>
[jira] [Updated] (SPARK-33144) Connot insert overwite multiple partition, get exception "get partition: Value for key name is null or empty"
[
https://issues.apache.org/jira/browse/SPARK-33144?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
CHC updated SPARK-33144:
Affects Version/s: 3.2.1
> Connot insert overwite multiple partition, get exception "get partition:
> Value for key name is null or empty"
> -
>
> Key: SPARK-33144
> URL: https://issues.apache.org/jira/browse/SPARK-33144
> Project: Spark
> Issue Type: Bug
> Components: SQL
>Affects Versions: 3.0.1, 3.2.1
> Environment: hadoop 2.7.3 + spark 3.0.1
>Reporter: CHC
>Priority: Major
>
> When:
> {code:sql}
> create table tmp.spark_multi_partition(
> id int
> )
> partitioned by (name string, version string)
> stored as orc
> ;
> set hive.exec.dynamic.partition=true;
> set spark.hadoop.hive.exec.dynamic.partition=true;
>
> set hive.exec.dynamic.partition.mode=nonstrict;
> set spark.hadoop.hive.exec.dynamic.partition.mode=nonstrict;
> insert overwrite table tmp.spark_multi_partition partition (name, version)
> select
> *
> from (
> select
> 1 as id
> , 'hadoop' as name
> , '2.7.3' as version
> union
> select
> 2 as id
> , 'spark' as name
> , '3.0.1' as version
> union
> select
> 3 as id
> , 'hive' as name
> , '2.3.4' as version
> ) as A;
> {code}
> and get exception:
> {code:bash}
> INFO load-dynamic-partitions-0 [hive.ql.metadata.Hive:1919]: New loading path
> =
> hdfs://namespace/apps/hive/warehouse/tmp.db/spark_multi_partition/.hive-staging_hive_2020-10-14_09-15-27_718_4118806337003279343-1/-ext-1/name=spark/version=3.0.1
> with partSpec {name=spark, version=3.0.1}
> 20/10/14 09:15:33 INFO load-dynamic-partitions-1
> [hive.ql.metadata.Hive:1919]: New loading path =
> hdfs://namespace/apps/hive/warehouse/tmp.db/spark_multi_partition/.hive-staging_hive_2020-10-14_09-15-27_718_4118806337003279343-1/-ext-1/name=hadoop/version=2.7.3
> with partSpec {name=hadoop, version=2.7.3}
> 20/10/14 09:15:33 INFO load-dynamic-partitions-2
> [hive.ql.metadata.Hive:1919]: New loading path =
> hdfs://namespace/apps/hive/warehouse/tmp.db/spark_multi_partition/.hive-staging_hive_2020-10-14_09-15-27_718_4118806337003279343-1/-ext-1/name=hive/version=2.3.4
> with partSpec {name=hive, version=2.3.4}
> 20/10/14 09:15:33 INFO load-dynamic-partitions-3
> [hive.ql.metadata.Hive:1919]: New loading path =
> hdfs://namespace/apps/hive/warehouse/tmp.db/spark_multi_partition/.hive-staging_hive_2020-10-14_09-15-27_718_4118806337003279343-1/-ext-1/_temporary/0
> with partSpec {name=, version=}
> 20/10/14 09:15:33 ERROR load-dynamic-partitions-3
> [hive.ql.metadata.Hive:1937]: Exception when loading partition with
> parameters
> partPath=hdfs://namespace/apps/hive/warehouse/tmp.db/spark_multi_partition/.hive-staging_hive_2020-10-14_09-15-27_718_4118806337003279343-1/-ext-1/_temporary/0,
> table=spark_multi_partition, partSpec={name=, version=}, replace=true,
> listBucketingEnabled=false, isAcid=false, hasFollowingStatsTask=false
> org.apache.hadoop.hive.ql.metadata.HiveException: get partition: Value for
> key name is null or empty
> at org.apache.hadoop.hive.ql.metadata.Hive.getPartition(Hive.java:2233)
> at org.apache.hadoop.hive.ql.metadata.Hive.getPartition(Hive.java:2181)
> at org.apache.hadoop.hive.ql.metadata.Hive.loadPartition(Hive.java:1611)
> at org.apache.hadoop.hive.ql.metadata.Hive$3.call(Hive.java:1922)
> at org.apache.hadoop.hive.ql.metadata.Hive$3.call(Hive.java:1913)
> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at java.lang.Thread.run(Thread.java:748)
> 20/10/14 09:15:33 INFO Delete-Thread-0
> [org.apache.hadoop.fs.TrashPolicyDefault:168]: Moved:
> 'hdfs://namespace/apps/hive/warehouse/tmp.db/spark_multi_partition/name=spark/version=3.0.1/part-1-b745147b-600f-4c79-8ba2-12a99283b0a9.c000'
> to trash at:
> hdfs://namespace/user/hive/.Trash/Current/apps/hive/warehouse/tmp.db/spark_multi_partition/name=spark/version=3.0.1/part-1-b745147b-600f-4c79-8ba2-12a99283b0a9.c000
> 20/10/14 09:15:33 INFO load-dynamic-partitions-0
> [org.apache.hadoop.hive.common.FileUtils:520]: Creating directory if it
> doesn't exist:
> hdfs://namespace/apps/hive/warehouse/tmp.db/spark_multi_partition/name=spark/version=3.0.1
> 20/10/14 09:15:33 INFO Delete-Thread-0
> [org.apache.hadoop.fs.TrashPolicyDefault:168]: Moved:
> 'hdfs://namespace/apps/hive/warehouse/tmp.db/spark_multi_partition/name=hive/version=2.3.4/part-2-b745147b-600f-4c79-8ba2-12a99283b0a9.c000'
> to trash at:
>
