Github user jiangxb1987 commented on a diff in the pull request: https://github.com/apache/spark/pull/16674#discussion_r100002647 --- Diff: sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewSuite.scala --- @@ -452,311 +506,96 @@ class SQLViewSuite extends QueryTest with SQLTestUtils with TestHiveSingleton { } } - test("create hive view for joined tables") { - // make sure the new flag can handle some complex cases like join and schema change. - withTable("jt1", "jt2") { - spark.range(1, 10).toDF("id1").write.format("json").saveAsTable("jt1") - spark.range(1, 10).toDF("id2").write.format("json").saveAsTable("jt2") - sql("CREATE VIEW testView AS SELECT * FROM jt1 JOIN jt2 ON id1 == id2") - checkAnswer(sql("SELECT * FROM testView ORDER BY id1"), (1 to 9).map(i => Row(i, i))) - - val df = (1 until 10).map(i => i -> i).toDF("id1", "newCol") - df.write.format("json").mode(SaveMode.Overwrite).saveAsTable("jt1") - checkAnswer(sql("SELECT * FROM testView ORDER BY id1"), (1 to 9).map(i => Row(i, i))) - - sql("DROP VIEW testView") - } - } - - test("SPARK-14933 - create view from hive parquet table") { - withTable("t_part") { - withView("v_part") { - spark.sql("create table t_part stored as parquet as select 1 as a, 2 as b") - spark.sql("create view v_part as select * from t_part") - checkAnswer( - sql("select * from t_part"), - sql("select * from v_part")) - } - } - } - - test("SPARK-14933 - create view from hive orc table") { - withTable("t_orc") { - withView("v_orc") { - spark.sql("create table t_orc stored as orc as select 1 as a, 2 as b") - spark.sql("create view v_orc as select * from t_orc") - checkAnswer( - sql("select * from t_orc"), - sql("select * from v_orc")) - } - } - } - - test("create a permanent/temp view using a hive, built-in, and permanent user function") { - val permanentFuncName = "myUpper" - val permanentFuncClass = - classOf[org.apache.hadoop.hive.ql.udf.generic.GenericUDFUpper].getCanonicalName - val builtInFuncNameInLowerCase = "abs" - val builtInFuncNameInMixedCase = "aBs" - val hiveFuncName = "histogram_numeric" - - withUserDefinedFunction(permanentFuncName -> false) { - sql(s"CREATE FUNCTION $permanentFuncName AS '$permanentFuncClass'") - withTable("tab1") { - (1 to 10).map(i => (s"$i", i)).toDF("str", "id").write.saveAsTable("tab1") - Seq("VIEW", "TEMPORARY VIEW").foreach { viewMode => - withView("view1") { - sql( - s""" - |CREATE $viewMode view1 - |AS SELECT - |$permanentFuncName(str), - |$builtInFuncNameInLowerCase(id), - |$builtInFuncNameInMixedCase(id) as aBs, - |$hiveFuncName(id, 5) over() - |FROM tab1 - """.stripMargin) - checkAnswer(sql("select count(*) FROM view1"), Row(10)) - } - } - } - } - } - - test("create a permanent/temp view using a temporary function") { - val tempFunctionName = "temp" - val functionClass = - classOf[org.apache.hadoop.hive.ql.udf.generic.GenericUDFUpper].getCanonicalName - withUserDefinedFunction(tempFunctionName -> true) { - sql(s"CREATE TEMPORARY FUNCTION $tempFunctionName AS '$functionClass'") - withView("view1", "tempView1") { - withTable("tab1") { - (1 to 10).map(i => s"$i").toDF("id").write.saveAsTable("tab1") - - // temporary view - sql(s"CREATE TEMPORARY VIEW tempView1 AS SELECT $tempFunctionName(id) from tab1") - checkAnswer(sql("select count(*) FROM tempView1"), Row(10)) - - // permanent view - val e = intercept[AnalysisException] { - sql(s"CREATE VIEW view1 AS SELECT $tempFunctionName(id) from tab1") - }.getMessage - assert(e.contains("Not allowed to create a permanent view `view1` by referencing " + - s"a temporary function `$tempFunctionName`")) - } - } - } - } - - test("correctly resolve a nested view") { - withTempDatabase { db => - withView(s"$db.view1", s"$db.view2") { - val view1 = CatalogTable( - identifier = TableIdentifier("view1", Some(db)), - tableType = CatalogTableType.VIEW, - storage = CatalogStorageFormat.empty, - schema = new StructType().add("x", "long").add("y", "long"), - viewText = Some("SELECT * FROM jt"), - properties = Map(CatalogTable.VIEW_DEFAULT_DATABASE -> "default", - CatalogTable.VIEW_QUERY_OUTPUT_NUM_COLUMNS -> "2", - s"${CatalogTable.VIEW_QUERY_OUTPUT_COLUMN_NAME_PREFIX}0" -> "id", - s"${CatalogTable.VIEW_QUERY_OUTPUT_COLUMN_NAME_PREFIX}1" -> "id1")) - val view2 = CatalogTable( - identifier = TableIdentifier("view2", Some(db)), - tableType = CatalogTableType.VIEW, - storage = CatalogStorageFormat.empty, - schema = new StructType().add("id", "long").add("id1", "long"), - viewText = Some("SELECT * FROM view1"), - properties = Map(CatalogTable.VIEW_DEFAULT_DATABASE -> db, - CatalogTable.VIEW_QUERY_OUTPUT_NUM_COLUMNS -> "2", - s"${CatalogTable.VIEW_QUERY_OUTPUT_COLUMN_NAME_PREFIX}0" -> "x", - s"${CatalogTable.VIEW_QUERY_OUTPUT_COLUMN_NAME_PREFIX}1" -> "y")) - activateDatabase(db) { - hiveContext.sessionState.catalog.createTable(view1, ignoreIfExists = false) - hiveContext.sessionState.catalog.createTable(view2, ignoreIfExists = false) - checkAnswer(sql("SELECT * FROM view2 ORDER BY id"), (1 to 9).map(i => Row(i, i))) - } - } - } - } - - test("correctly resolve a view with CTE") { - withView("cte_view") { - val cte_view = CatalogTable( - identifier = TableIdentifier("cte_view"), - tableType = CatalogTableType.VIEW, - storage = CatalogStorageFormat.empty, - schema = new StructType().add("n", "int"), - viewText = Some("WITH w AS (SELECT 1 AS n) SELECT n FROM w"), - properties = Map(CatalogTable.VIEW_DEFAULT_DATABASE -> "default", - CatalogTable.VIEW_QUERY_OUTPUT_NUM_COLUMNS -> "1", - s"${CatalogTable.VIEW_QUERY_OUTPUT_COLUMN_NAME_PREFIX}0" -> "n")) - hiveContext.sessionState.catalog.createTable(cte_view, ignoreIfExists = false) - checkAnswer(sql("SELECT * FROM cte_view"), Row(1)) - } - } - - test("correctly resolve a view in a self join") { - withView("join_view") { - val join_view = CatalogTable( - identifier = TableIdentifier("join_view"), - tableType = CatalogTableType.VIEW, - storage = CatalogStorageFormat.empty, - schema = new StructType().add("id", "long").add("id1", "long"), - viewText = Some("SELECT * FROM jt"), - properties = Map(CatalogTable.VIEW_DEFAULT_DATABASE -> "default", - CatalogTable.VIEW_QUERY_OUTPUT_NUM_COLUMNS -> "2", - s"${CatalogTable.VIEW_QUERY_OUTPUT_COLUMN_NAME_PREFIX}0" -> "id", - s"${CatalogTable.VIEW_QUERY_OUTPUT_COLUMN_NAME_PREFIX}1" -> "id1")) - hiveContext.sessionState.catalog.createTable(join_view, ignoreIfExists = false) - checkAnswer( - sql("SELECT * FROM join_view t1 JOIN join_view t2 ON t1.id = t2.id ORDER BY t1.id"), - (1 to 9).map(i => Row(i, i, i, i))) - } - } - - private def assertInvalidReference(query: String): Unit = { - val e = intercept[AnalysisException] { - sql(query) - }.getMessage - assert(e.contains("Table or view not found")) - } - test("error handling: fail if the referenced table or view is invalid") { withView("view1", "view2", "view3") { // Fail if the referenced table is defined in a invalid database. - val view1 = CatalogTable( - identifier = TableIdentifier("view1"), - tableType = CatalogTableType.VIEW, - storage = CatalogStorageFormat.empty, - schema = new StructType().add("id", "long").add("id1", "long"), - viewText = Some("SELECT * FROM invalid_db.jt"), - properties = Map(CatalogTable.VIEW_DEFAULT_DATABASE -> "default", - CatalogTable.VIEW_QUERY_OUTPUT_NUM_COLUMNS -> "2", - s"${CatalogTable.VIEW_QUERY_OUTPUT_COLUMN_NAME_PREFIX}0" -> "id", - s"${CatalogTable.VIEW_QUERY_OUTPUT_COLUMN_NAME_PREFIX}1" -> "id1")) - hiveContext.sessionState.catalog.createTable(view1, ignoreIfExists = false) + withTempDatabase { db => + withTable(s"$db.table1") { + activateDatabase(db) { + sql("CREATE TABLE table1(a int, b string) USING parquet") + sql("CREATE VIEW default.view1 AS SELECT * FROM table1") + } + } + } assertInvalidReference("SELECT * FROM view1") // Fail if the referenced table is invalid. - val view2 = CatalogTable( - identifier = TableIdentifier("view2"), - tableType = CatalogTableType.VIEW, - storage = CatalogStorageFormat.empty, - schema = new StructType().add("id", "long").add("id1", "long"), - viewText = Some("SELECT * FROM invalid_table"), - properties = Map(CatalogTable.VIEW_DEFAULT_DATABASE -> "default", - CatalogTable.VIEW_QUERY_OUTPUT_NUM_COLUMNS -> "2", - s"${CatalogTable.VIEW_QUERY_OUTPUT_COLUMN_NAME_PREFIX}0" -> "id", - s"${CatalogTable.VIEW_QUERY_OUTPUT_COLUMN_NAME_PREFIX}1" -> "id1")) - hiveContext.sessionState.catalog.createTable(view2, ignoreIfExists = false) + withTable("table2") { + sql("CREATE TABLE table2(a int, b string) USING parquet") + sql("CREATE VIEW view2 AS SELECT * FROM table2") + } assertInvalidReference("SELECT * FROM view2") // Fail if the referenced view is invalid. - val view3 = CatalogTable( - identifier = TableIdentifier("view3"), - tableType = CatalogTableType.VIEW, - storage = CatalogStorageFormat.empty, - schema = new StructType().add("id", "long").add("id1", "long"), - viewText = Some("SELECT * FROM view2"), - properties = Map(CatalogTable.VIEW_DEFAULT_DATABASE -> "default", - CatalogTable.VIEW_QUERY_OUTPUT_NUM_COLUMNS -> "2", - s"${CatalogTable.VIEW_QUERY_OUTPUT_COLUMN_NAME_PREFIX}0" -> "id", - s"${CatalogTable.VIEW_QUERY_OUTPUT_COLUMN_NAME_PREFIX}1" -> "id1")) - hiveContext.sessionState.catalog.createTable(view3, ignoreIfExists = false) + withView("testView") { + sql("CREATE VIEW testView AS SELECT * FROM jt") + sql("CREATE VIEW view3 AS SELECT * FROM testView") + } assertInvalidReference("SELECT * FROM view3") } } - test("make sure we can resolve view created by old version of Spark") { --- End diff -- This test case has been moved to `HiveSQLViewSuite`.
--- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- --------------------------------------------------------------------- To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org