[
https://issues.apache.org/jira/browse/SPARK-25135?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Dongjoon Hyun updated SPARK-25135:
----------------------------------
Description:
This happens on parquet.
How to reproduce in parquet.
{code:scala}
val path = "/tmp/spark/parquet"
val cnt = 30
spark.range(cnt).selectExpr("cast(id as bigint) as col1", "cast(id as bigint)
as col2").write.mode("overwrite").parquet(path)
spark.sql(s"CREATE TABLE table1(col1 bigint, col2 bigint) using parquet
location '$path'")
spark.sql("create view view1 as select col1, col2 from table1 where col1 > -20")
spark.sql("create table table2 (COL1 BIGINT, COL2 BIGINT) using parquet")
spark.sql("insert overwrite table table2 select COL1, COL2 from view1")
spark.table("table2").show
{code}
FYI, the following is orc.
{code}
scala> val path = "/tmp/spark/orc"
scala> val cnt = 30
scala> spark.range(cnt).selectExpr("cast(id as bigint) as col1", "cast(id as
bigint) as col2").write.mode("overwrite").orc(path)
scala> spark.sql(s"CREATE TABLE table1(col1 bigint, col2 bigint) using orc
location '$path'")
scala> spark.sql("create view view1 as select col1, col2 from table1 where col1
> -20")
scala> spark.sql("create table table2 (COL1 BIGINT, COL2 BIGINT) using orc")
scala> spark.sql("insert overwrite table table2 select COL1, COL2 from view1")
scala> spark.table("table2").show
+----+----+
|COL1|COL2|
+----+----+
| 15| 15|
| 16| 16|
| 17| 17|
...
{code}
was:
How to reproduce:
{code:scala}
val path = "/tmp/spark/parquet"
val cnt = 30
spark.range(cnt).selectExpr("cast(id as bigint) as col1", "cast(id as bigint)
as col2").write.mode("overwrite").parquet(path)
spark.sql(s"CREATE TABLE table1(col1 bigint, col2 bigint) using parquet
location '$path'")
spark.sql("create view view1 as select col1, col2 from table1 where col1 > -20")
spark.sql("create table table2 (COL1 BIGINT, COL2 BIGINT) using parquet")
spark.sql("insert overwrite table table2 select COL1, COL2 from view1")
spark.table("table2").show
{code}
This happens on parquet.
{code}
scala> val path = "/tmp/spark/orc"
scala> val cnt = 30
scala> spark.range(cnt).selectExpr("cast(id as bigint) as col1", "cast(id as
bigint) as col2").write.mode("overwrite").orc(path)
scala> spark.sql(s"CREATE TABLE table1(col1 bigint, col2 bigint) using orc
location '$path'")
scala> spark.sql("create view view1 as select col1, col2 from table1 where col1
> -20")
scala> spark.sql("create table table2 (COL1 BIGINT, COL2 BIGINT) using orc")
scala> spark.sql("insert overwrite table table2 select COL1, COL2 from view1")
scala> spark.table("table2").show
+----+----+
|COL1|COL2|
+----+----+
| 15| 15|
| 16| 16|
| 17| 17|
...
{code}
> insert datasource table may all null when select from view on parquet
> ---------------------------------------------------------------------
>
> Key: SPARK-25135
> URL: https://issues.apache.org/jira/browse/SPARK-25135
> Project: Spark
> Issue Type: Bug
> Components: SQL
> Affects Versions: 2.3.0, 2.3.1
> Reporter: Yuming Wang
> Priority: Blocker
> Labels: correctness
>
> This happens on parquet.
> How to reproduce in parquet.
> {code:scala}
> val path = "/tmp/spark/parquet"
> val cnt = 30
> spark.range(cnt).selectExpr("cast(id as bigint) as col1", "cast(id as bigint)
> as col2").write.mode("overwrite").parquet(path)
> spark.sql(s"CREATE TABLE table1(col1 bigint, col2 bigint) using parquet
> location '$path'")
> spark.sql("create view view1 as select col1, col2 from table1 where col1 >
> -20")
> spark.sql("create table table2 (COL1 BIGINT, COL2 BIGINT) using parquet")
> spark.sql("insert overwrite table table2 select COL1, COL2 from view1")
> spark.table("table2").show
> {code}
> FYI, the following is orc.
> {code}
> scala> val path = "/tmp/spark/orc"
> scala> val cnt = 30
> scala> spark.range(cnt).selectExpr("cast(id as bigint) as col1", "cast(id as
> bigint) as col2").write.mode("overwrite").orc(path)
> scala> spark.sql(s"CREATE TABLE table1(col1 bigint, col2 bigint) using orc
> location '$path'")
> scala> spark.sql("create view view1 as select col1, col2 from table1 where
> col1 > -20")
> scala> spark.sql("create table table2 (COL1 BIGINT, COL2 BIGINT) using orc")
> scala> spark.sql("insert overwrite table table2 select COL1, COL2 from view1")
> scala> spark.table("table2").show
> +----+----+
> |COL1|COL2|
> +----+----+
> | 15| 15|
> | 16| 16|
> | 17| 17|
> ...
> {code}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
