[jira] [Closed] (SPARK-25464) Dropping database can remove the hive warehouse directory contents
[
https://issues.apache.org/jira/browse/SPARK-25464?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Sushanta Sen closed SPARK-25464.
> Dropping database can remove the hive warehouse directory contents
> --
>
> Key: SPARK-25464
> URL: https://issues.apache.org/jira/browse/SPARK-25464
> Project: Spark
> Issue Type: Bug
> Components: SQL
>Affects Versions: 2.3.1
>Reporter: Sushanta Sen
>Priority: Major
>
> Create Database.
> CREATE (DATABASE|SCHEMA) [IF NOT EXISTS] db_name [COMMENT comment_text]
> [*LOCATION*path] [WITH DBPROPERTIES (key1=val1, key2=val2, ...)]
> \{{LOCATION }}If the specified path does not already exist in the underlying
> file system, this command tries to create a directory with the path. *When
> the database is dropped later, this directory is not deleted,
> {color:#d04437}but currently it is deleting the directory as well.{color}
> {color:#33}please refer the below link{color}
> {color:#d04437}[databricks documentation|{color}
>
> [https://docs.databricks.com/spark/latest/spark-sql/language-manual/create-database.html]
> {color:#d04437}]{color}
> if i create the database as below
> create database db1 location '/user/hive/warehouse'; //this is hive warehouse
> directory
> *{color:#33}on dropping this db it will also delete the warehouse
> directory which contains the other db information.{color}*
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
-
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
[jira] [Closed] (SPARK-24278) Create table if not exists is throwing table already exists exception
[ https://issues.apache.org/jira/browse/SPARK-24278?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Sushanta Sen closed SPARK-24278. > Create table if not exists is throwing table already exists exception > - > > Key: SPARK-24278 > URL: https://issues.apache.org/jira/browse/SPARK-24278 > Project: Spark > Issue Type: Bug > Components: SQL >Affects Versions: 2.3.0 > Environment: OS: SUSE11 > Spark Version: 2.3 >Reporter: Sushanta Sen >Priority: Major > > # Launch Spark-sql > # create table check(time timestamp, name string, isright boolean, datetoday > date, num binary, height double, score float, decimaler decimal(10,0), id > tinyint, age int, license bigint, length smallint) row format delimited > fields terminated by ',' stored as textfile; > # create table if not exists check (time timestamp, name string, isright > boolean, datetoday date, num binary, height double, score float, decimaler > decimal(10,0), id tinyint, age int, license bigint, length smallint) row > format delimited fields terminated by ','stored as TEXTFILE; *-FAILED* ** > > Exception as below > spark-sql> create table if not exists check (col1 string); > *2018-05-15 14:29:56 ERROR RetryingHMSHandler:159 -* > *AlreadyExistsException(message:Table check already exists)* > *at > org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.create_table_core(HiveMetaStore.java:1372)* > at > org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.create_table_with_environment_context(HiveMetaStore.java:1449) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:497) > at > org.apache.hadoop.hive.metastore.RetryingHMSHandler.invoke(RetryingHMSHandler.java:107) > at com.sun.proxy.$Proxy8.create_table_with_environment_context(Unknown Source) > at > org.apache.hadoop.hive.metastore.HiveMetaStoreClient.create_table_with_environment_context(HiveMetaStoreClient.java:2050) > at > org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.create_table_with_environment_context(SessionHiveMetaStoreClient.java:97) > at > org.apache.hadoop.hive.metastore.HiveMetaStoreClient.createTable(HiveMetaStoreClient.java:669) > at > org.apache.hadoop.hive.metastore.HiveMetaStoreClient.createTable(HiveMetaStoreClient.java:657) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:497) > at > org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.invoke(RetryingMetaStoreClient.java:156) > at com.sun.proxy.$Proxy9.createTable(Unknown Source) > at org.apache.hadoop.hive.ql.metadata.Hive.createTable(Hive.java:714) > at > org.apache.spark.sql.hive.client.HiveClientImpl$$anonfun$createTable$1.apply$mcV$sp(HiveClientImpl.scala:468) > at > org.apache.spark.sql.hive.client.HiveClientImpl$$anonfun$createTable$1.apply(HiveClientImpl.scala:466) > at > org.apache.spark.sql.hive.client.HiveClientImpl$$anonfun$createTable$1.apply(HiveClientImpl.scala:466) > at > org.apache.spark.sql.hive.client.HiveClientImpl$$anonfun$withHiveState$1.apply(HiveClientImpl.scala:272) > at > org.apache.spark.sql.hive.client.HiveClientImpl.liftedTree1$1(HiveClientImpl.scala:210) > at > org.apache.spark.sql.hive.client.HiveClientImpl.retryLocked(HiveClientImpl.scala:209) > at > org.apache.spark.sql.hive.client.HiveClientImpl.withHiveState(HiveClientImpl.scala:255) > at > org.apache.spark.sql.hive.client.HiveClientImpl.createTable(HiveClientImpl.scala:466) > at > org.apache.spark.sql.hive.HiveExternalCatalog$$anonfun$doCreateTable$1.apply$mcV$sp(HiveExternalCatalog.scala:258) > at > org.apache.spark.sql.hive.HiveExternalCatalog$$anonfun$doCreateTable$1.apply(HiveExternalCatalog.scala:216) > at > org.apache.spark.sql.hive.HiveExternalCatalog$$anonfun$doCreateTable$1.apply(HiveExternalCatalog.scala:216) > at > org.apache.spark.sql.hive.HiveExternalCatalog.withClient(HiveExternalCatalog.scala:97) > at > org.apache.spark.sql.hive.HiveExternalCatalog.doCreateTable(HiveExternalCatalog.scala:216) > at > org.apache.spark.sql.catalyst.catalog.ExternalCatalog.createTable(ExternalCatalog.scala:119) > at > org.apache.spark.sql.catalyst.catalog.SessionCatalog.createTable(SessionCatalog.scala:304) > at > org.apache.spark.sql.execution.command.CreateTableCommand.run(tables.scala:128) > at > org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands
[jira] [Commented] (SPARK-24291) Data source table is not displaying records when files are uploaded to table location
[
https://issues.apache.org/jira/browse/SPARK-24291?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16787786#comment-16787786
]
Sushanta Sen commented on SPARK-24291:
--
Yes,after refresh it fetches the data.But why this is happening when tables
created with 'USING'.
> Data source table is not displaying records when files are uploaded to table
> location
> -
>
> Key: SPARK-24291
> URL: https://issues.apache.org/jira/browse/SPARK-24291
> Project: Spark
> Issue Type: Bug
> Components: SQL
>Affects Versions: 2.3.0
> Environment: OS: SUSE11
> Spark Version: 2.3
>Reporter: Sushanta Sen
>Priority: Major
>
> Precondition:
> 1.Already one .orc file exists in the /tmp/orcdata/ location
> # Launch Spark-sql
> # spark-sql> CREATE TABLE os_orc (name string, version string, other string)
> USING ORC OPTIONS (path '/tmp/orcdata/');
> # spark-sql> select * from os_orc;
> Spark 2.3.0 Apache
> Time taken: 2.538 seconds, Fetched 1 row(s)
> # pc1:/opt/# *./hadoop dfs -ls /tmp/orcdata*
> Found 1 items
> -rw-r--r-- 3 spark hadoop 475 2018-05-09 18:21
> /tmp/orcdata/part-0-d488121b-e9fd-4269-a6ea-842c631722ee-c000.snappy.orc
> pc1:/opt/# *./hadoop fs -copyFromLocal
> /opt/OS/loaddata/orcdata/part-1-d488121b-e9fd-4269-a6ea-842c631722ee-c000.snappy.orc
> /tmp/orcdata/data2.orc*
> pc1:/opt/# *./hadoop dfs -ls /tmp/orcdata*
> Found *2* items
> -rw-r--r-- 3 spark hadoop 475 2018-05-15 14:59 /tmp/orcdata/data2.orc
> -rw-r--r-- 3 spark hadoop 475 2018-05-09 18:21
> /tmp/orcdata/part-0-d488121b-e9fd-4269-a6ea-842c631722ee-c000.snappy.orc
> pc1:/opt/# **
> 5. Again execute the select command on the table os_orc
> spark-sql> select * from os_orc;
> Spark 2.3.0 Apache
> Time taken: 1.528 seconds, Fetched {color:#FF}1 row(s){color}
> Actual Result: On executing select command it does not display the all the
> records exist in the data source table location
> Expected Result: All the records should be fetched and displayed for the data
> source table from the location
> NB:
> 1.On exiting and relaunching the spark-sql session, select command fetches
> the correct # of records.
> 2.This issue is valid for all the data source tables created with 'Using' .
> I came across this use case in Spark 2.2.1 when tried to reproduce a customer
> site observation.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
[jira] [Commented] (SPARK-24602) In Spark SQL, ALTER TABLE--CHANGE column1 column2 datatype is not supported in 2.3.1
[
https://issues.apache.org/jira/browse/SPARK-24602?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16784155#comment-16784155
]
Sushanta Sen commented on SPARK-24602:
--
This issue is logged prior to other JIRAs as mentioned in the Issue Links.
> In Spark SQL, ALTER TABLE--CHANGE column1 column2 datatype is not supported
> in 2.3.1
>
>
> Key: SPARK-24602
> URL: https://issues.apache.org/jira/browse/SPARK-24602
> Project: Spark
> Issue Type: Improvement
> Components: SQL
>Affects Versions: 2.3.0
> Environment: OS: SUSE11
> Spark Version: 2.3
>Reporter: Sushanta Sen
>Priority: Major
>
> Precondition:
> Spark cluster 2.3 is up and running
> Test Steps:
> # Launch Spark-sql
> # spark-sql> CREATE TABLE t1(a int,string)
> 0: jdbc:hive2://ha-cluster/default> *alter
> table t1 change a a1 int;*
> Error: org.apache.spark.sql.AnalysisException: {color:#FF}ALTER TABLE
> CHANGE COLUMN is not supported for changing column 'a' with type
> 'IntegerType' to 'b' with type 'IntegerType'; (state=,code=0){color}
> # Launch hive beeliine
> # repeat step1 & 2
> # 0: jdbc:hive2://10.18.108.126:1/> desc del1;
> +---++--+--+
> | col_name | data_type | comment |
> +---++--+--+
> | *a1* | *int* | |
> | dob | int | |
> +---++--+--+
> 2 rows selected (1.572 seconds)
> 0: jdbc:hive2://10.18.108.126:1/>{color:#205081} alter table del1 change
> a1 a bigint;{color}
> No rows affected (0.425 seconds)
> 0: jdbc:hive2://10.18.108.126:1/> desc del1;
> +---++--+--+
> | col_name | data_type | comment |
> +---++--+--+
> | *a* | *bigint* | |
> | dob | int | |
> +---++--+--+
> 2 rows selected (0.364 seconds)
>
> Actual Result: In spark sql, alter table change is not supported, whereas in
> hive beeline it is working fine.
> Expected Result: ALTER Table CHANGE should be supported in Spark-SQL as well
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
[jira] [Updated] (SPARK-26255) Custom error/exception is not thrown for the SQL tab when UI filters are added in spark-sql launch
[ https://issues.apache.org/jira/browse/SPARK-26255?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Sushanta Sen updated SPARK-26255: - Description: 【Detailed description】：Custom error is not thrown for the SQL tab when UI filters are added in spark-sql launch 【Precondition】： 1.Cluster is up and running【Test step】： 1. Launch spark sql as below: [spark-sql --master yarn --conf spark.ui.filters=org.apache.hadoop.security.authentication.server.AuthenticationFilter --conf spark.org.apache.hadoop.security.authentication.server.AuthenticationFilter.params="type=simple"] 2. Go to Yarn application list UI link 3. Launch the application master for the Spark-SQL app ID 4. It will display an error 5. Append /executors, /stages, /jobs, /environment, /SQL 【Expect Output】：An error should be displayed "An error has occurred. Please check for all the TABS 【Actual Output】：The error message is displayed for all the tabs except SQL tab . was: 【Detailed description】：Custom error is not thrown for the SQL tab when UI filters are added in spark-sql launch 【Precondition】： 1.Cluster is up and running【Test step】： 1. Launch spark sql as below: spark-sql --master yarn --conf spark.ui.filters=org.apache.hadoop.security.authentication.server.AuthenticationFilter --conf spark.org.apache.hadoop.security.authentication.server.AuthenticationFilter.params="type=simple" 2. Go to Yarn application list UI link 3. Launch the application master for the Spark-SQL app ID 4. It will display an error 5. Append /executors, /stages, /jobs, /environment, /SQL 【Expect Output】：An error should be displayed "An error has occurred. Please check for all the TABS 【Actual Output】：The error message is displayed for all the tabs except SQL tab . > Custom error/exception is not thrown for the SQL tab when UI filters are > added in spark-sql launch > -- > > Key: SPARK-26255 > URL: https://issues.apache.org/jira/browse/SPARK-26255 > Project: Spark > Issue Type: Bug > Components: Spark Core >Affects Versions: 2.3.2 > Environment: 【Test Environment】： > Server OS :-SUSE > No. of Cluster Node:-3 > Spark Version:- 2.3.2 > Hadoop Version:-3.1 >Reporter: Sushanta Sen >Priority: Major > > 【Detailed description】：Custom error is not thrown for the SQL tab when UI > filters are added in spark-sql launch > 【Precondition】： > 1.Cluster is up and running【Test step】： > 1. Launch spark sql as below: > [spark-sql --master yarn --conf > spark.ui.filters=org.apache.hadoop.security.authentication.server.AuthenticationFilter > --conf > spark.org.apache.hadoop.security.authentication.server.AuthenticationFilter.params="type=simple"] > 2. Go to Yarn application list UI link > 3. Launch the application master for the Spark-SQL app ID > 4. It will display an error > 5. Append /executors, /stages, /jobs, /environment, /SQL > 【Expect Output】：An error should be displayed "An error has occurred. Please > check for all the TABS > 【Actual Output】：The error message is displayed for all the tabs except SQL > tab . -- This message was sent by Atlassian JIRA (v7.6.3#76005) - To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org
[jira] [Updated] (SPARK-26255) Custom error/exception is not thrown for the SQL tab when UI filters are added in spark-sql launch
[ https://issues.apache.org/jira/browse/SPARK-26255?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Sushanta Sen updated SPARK-26255: - Description: 【Detailed description】：Custom error is not thrown for the SQL tab when UI filters are added in spark-sql launch 【Precondition】： 1.Cluster is up and running【Test step】： 1. Launch spark sql as below: spark-sql --master yarn --conf spark.ui.filters=org.apache.hadoop.security.authentication.server.AuthenticationFilter --conf spark.org.apache.hadoop.security.authentication.server.AuthenticationFilter.params="type=simple" 2. Go to Yarn application list UI link 3. Launch the application master for the Spark-SQL app ID 4. It will display an error 5. Append /executors, /stages, /jobs, /environment, /SQL 【Expect Output】：An error should be displayed "An error has occurred. Please check for all the TABS 【Actual Output】：The error message is displayed for all the tabs except SQL tab . was: 【Detailed description】：Custom error is not thrown for the SQL tab when UI filters are added in spark-sql launch 【Precondition】： 1.Cluster is up and running【Test step】： 1. Launch spark sql as below: spark-sql --master yarn --conf spark.ui.filters=org.apache.hadoop.security.authentication.server.AuthenticationFilter --conf spark.org.apache.hadoop.security.authentication.server.AuthenticationFilter.params="type=simple" 2. Go to Yarn application list UI link 3. Launch the application master for the Spark-SQL app ID 4. It will display an error 5. Append /executors, /stages, /jobs, /environment, /SQL 【Expect Output】：An error should be displayed "An error has occurred. Please check for all the TABS 【Actual Output】：The error message is displayed for all the tabs except SQL tab . > Custom error/exception is not thrown for the SQL tab when UI filters are > added in spark-sql launch > -- > > Key: SPARK-26255 > URL: https://issues.apache.org/jira/browse/SPARK-26255 > Project: Spark > Issue Type: Bug > Components: Spark Core >Affects Versions: 2.3.2 > Environment: 【Test Environment】： > Server OS :-SUSE > No. of Cluster Node:-3 > Spark Version:- 2.3.2 > Hadoop Version:-3.1 >Reporter: Sushanta Sen >Priority: Major > > 【Detailed description】：Custom error is not thrown for the SQL tab when UI > filters are added in spark-sql launch > 【Precondition】： > 1.Cluster is up and running【Test step】： > 1. Launch spark sql as below: > spark-sql --master yarn --conf > spark.ui.filters=org.apache.hadoop.security.authentication.server.AuthenticationFilter > --conf > spark.org.apache.hadoop.security.authentication.server.AuthenticationFilter.params="type=simple" > 2. Go to Yarn application list UI link > 3. Launch the application master for the Spark-SQL app ID > 4. It will display an error > 5. Append /executors, /stages, /jobs, /environment, /SQL > 【Expect Output】：An error should be displayed "An error has occurred. Please > check for all the TABS > 【Actual Output】：The error message is displayed for all the tabs except SQL > tab . -- This message was sent by Atlassian JIRA (v7.6.3#76005) - To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org
[jira] [Created] (SPARK-26255) Custom error/exception is not thrown for the SQL tab when UI filters are added in spark-sql launch
Sushanta Sen created SPARK-26255: Summary: Custom error/exception is not thrown for the SQL tab when UI filters are added in spark-sql launch Key: SPARK-26255 URL: https://issues.apache.org/jira/browse/SPARK-26255 Project: Spark Issue Type: Bug Components: Spark Core Affects Versions: 2.3.2 Environment: 【Test Environment】： Server OS :-SUSE No. of Cluster Node:-3 Spark Version:- 2.3.2 Hadoop Version:-3.1 Reporter: Sushanta Sen 【Detailed description】：Custom error is not thrown for the SQL tab when UI filters are added in spark-sql launch 【Precondition】： 1.Cluster is up and running【Test step】： 1. Launch spark sql as below: spark-sql --master yarn --conf spark.ui.filters=org.apache.hadoop.security.authentication.server.AuthenticationFilter --conf spark.org.apache.hadoop.security.authentication.server.AuthenticationFilter.params="type=simple" 2. Go to Yarn application list UI link 3. Launch the application master for the Spark-SQL app ID 4. It will display an error 5. Append /executors, /stages, /jobs, /environment, /SQL 【Expect Output】：An error should be displayed "An error has occurred. Please check for all the TABS 【Actual Output】：The error message is displayed for all the tabs except SQL tab . -- This message was sent by Atlassian JIRA (v7.6.3#76005) - To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org
***UNCHECKED*** [jira] [Updated] (SPARK-25464) Dropping database can remove the hive warehouse directory contents
[
https://issues.apache.org/jira/browse/SPARK-25464?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Sushanta Sen updated SPARK-25464:
-
Description:
Create Database.
CREATE (DATABASE|SCHEMA) [IF NOT EXISTS] db_name [COMMENT comment_text]
[*LOCATION*path] [WITH DBPROPERTIES (key1=val1, key2=val2, ...)]
\{{LOCATION }}If the specified path does not already exist in the underlying
file system, this command tries to create a directory with the path. *When the
database is dropped later, this directory is not deleted, {color:#d04437}but
currently it is deleting the directory as well.{color}
{color:#33}please refer the below link{color}
{color:#d04437}[databricks documentation|{color}
[https://docs.databricks.com/spark/latest/spark-sql/language-manual/create-database.html]
{color:#d04437}]{color}
if i create the database as below
create database db1 location '/user/hive/warehouse'; //this is hive warehouse
directory
*{color:#33}on dropping this db it will also delete the warehouse directory
which contains the other db information.{color}*
was:
Create Database.
CREATE (DATABASE|SCHEMA) [IF NOT EXISTS] db_name [COMMENT comment_text]
[*LOCATION*path] [WITH DBPROPERTIES (key1=val1, key2=val2, ...)]
{\{LOCATION }}If the specified path does not already exist in the underlying
file system, this command tries to create a directory with the path. *When the
database is dropped later, this directory is not deleted, {color:#d04437}but
currently it is deleting the directory as well.{color}
{color:#d04437}[databricks documentation|{color}
[https://docs.databricks.com/spark/latest/spark-sql/language-manual/create-database.html]
{color:#d04437}]{color}
if i create the database as below
create database db1 location '/user/hive/warehouse'; //this is hive warehouse
directory
*{color:#33}on dropping this db it will also delete the warehouse directory
which contains the other db information.{color}*
> Dropping database can remove the hive warehouse directory contents
> --
>
> Key: SPARK-25464
> URL: https://issues.apache.org/jira/browse/SPARK-25464
> Project: Spark
> Issue Type: Bug
> Components: SQL
>Affects Versions: 2.3.1
>Reporter: Sushanta Sen
>Priority: Major
>
> Create Database.
> CREATE (DATABASE|SCHEMA) [IF NOT EXISTS] db_name [COMMENT comment_text]
> [*LOCATION*path] [WITH DBPROPERTIES (key1=val1, key2=val2, ...)]
> \{{LOCATION }}If the specified path does not already exist in the underlying
> file system, this command tries to create a directory with the path. *When
> the database is dropped later, this directory is not deleted,
> {color:#d04437}but currently it is deleting the directory as well.{color}
> {color:#33}please refer the below link{color}
> {color:#d04437}[databricks documentation|{color}
>
> [https://docs.databricks.com/spark/latest/spark-sql/language-manual/create-database.html]
> {color:#d04437}]{color}
> if i create the database as below
> create database db1 location '/user/hive/warehouse'; //this is hive warehouse
> directory
> *{color:#33}on dropping this db it will also delete the warehouse
> directory which contains the other db information.{color}*
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
[jira] [Updated] (SPARK-25464) Dropping database can remove the hive warehouse directory contents
[
https://issues.apache.org/jira/browse/SPARK-25464?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Sushanta Sen updated SPARK-25464:
-
Description:
Create Database.
CREATE (DATABASE|SCHEMA) [IF NOT EXISTS] db_name [COMMENT comment_text]
[*LOCATION*path] [WITH DBPROPERTIES (key1=val1, key2=val2, ...)]
{\{LOCATION }}If the specified path does not already exist in the underlying
file system, this command tries to create a directory with the path. *When the
database is dropped later, this directory is not deleted, {color:#d04437}but
currently it is deleting the directory as well.{color}
{color:#d04437}[databricks documentation|{color}
[https://docs.databricks.com/spark/latest/spark-sql/language-manual/create-database.html]
{color:#d04437}]{color}
if i create the database as below
create database db1 location '/user/hive/warehouse'; //this is hive warehouse
directory
*{color:#33}on dropping this db it will also delete the warehouse directory
which contains the other db information.{color}*
was:
Create Database. CREATE (DATABASE|SCHEMA) [IF NOT EXISTS] db_name [COMMENT
comment_text] [*LOCATION*path] [WITH DBPROPERTIES (key1=val1, key2=val2,
...)]\{{LOCATION }}If the specified path does not already exist in the
underlying file system, this command tries to create a directory with the path.
*When the database is dropped later, this directory is not deleted,
\{color:#d04437}{color:#d04437}but currently it is deleting the directory as
well.{color}
{color:#d04437}[databricks documentation|{color}
[https://docs.databricks.com/spark/latest/spark-sql/language-manual/create-database.html]
{color:#d04437}]{color}
if i create the database as below\{color}{color}*
*\{color:#d04437}{color:#33}create database db1 location
'/user/hive/warehouse'; //this is hive warehouse directory\{color}{color}*
*\{color:#d04437}{color:#33}on dropping this db it will also delete the
warehouse directory which contains the other db information.\{color}{color}*
> Dropping database can remove the hive warehouse directory contents
> --
>
> Key: SPARK-25464
> URL: https://issues.apache.org/jira/browse/SPARK-25464
> Project: Spark
> Issue Type: Bug
> Components: SQL
>Affects Versions: 2.3.1
>Reporter: Sushanta Sen
>Priority: Major
>
> Create Database.
> CREATE (DATABASE|SCHEMA) [IF NOT EXISTS] db_name [COMMENT comment_text]
> [*LOCATION*path] [WITH DBPROPERTIES (key1=val1, key2=val2, ...)]
> {\{LOCATION }}If the specified path does not already exist in the underlying
> file system, this command tries to create a directory with the path. *When
> the database is dropped later, this directory is not deleted,
> {color:#d04437}but currently it is deleting the directory as well.{color}
> {color:#d04437}[databricks documentation|{color}
>
> [https://docs.databricks.com/spark/latest/spark-sql/language-manual/create-database.html]
> {color:#d04437}]{color}
> if i create the database as below
> create database db1 location '/user/hive/warehouse'; //this is hive warehouse
> directory
> *{color:#33}on dropping this db it will also delete the warehouse
> directory which contains the other db information.{color}*
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
***UNCHECKED*** [jira] [Created] (SPARK-25464) Dropping database can remove the hive warehouse directory contents
Sushanta Sen created SPARK-25464:
Summary: Dropping database can remove the hive warehouse directory
contents
Key: SPARK-25464
URL: https://issues.apache.org/jira/browse/SPARK-25464
Project: Spark
Issue Type: Bug
Components: SQL
Affects Versions: 2.3.1
Reporter: Sushanta Sen
Create Database. CREATE (DATABASE|SCHEMA) [IF NOT EXISTS] db_name [COMMENT
comment_text] [*LOCATION*path] [WITH DBPROPERTIES (key1=val1, key2=val2,
...)]\{{LOCATION }}If the specified path does not already exist in the
underlying file system, this command tries to create a directory with the path.
*When the database is dropped later, this directory is not deleted,
\{color:#d04437}{color:#d04437}but currently it is deleting the directory as
well.{color}
{color:#d04437}[databricks documentation|{color}
[https://docs.databricks.com/spark/latest/spark-sql/language-manual/create-database.html]
{color:#d04437}]{color}
if i create the database as below\{color}{color}*
*\{color:#d04437}{color:#33}create database db1 location
'/user/hive/warehouse'; //this is hive warehouse directory\{color}{color}*
*\{color:#d04437}{color:#33}on dropping this db it will also delete the
warehouse directory which contains the other db information.\{color}{color}*
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
[jira] [Created] (SPARK-24602) In Spark SQL, ALTER TABLE--CHANGE column1 column2 datatype is not supported in 2.3.1
Sushanta Sen created SPARK-24602:
Summary: In Spark SQL, ALTER TABLE--CHANGE column1 column2
datatype is not supported in 2.3.1
Key: SPARK-24602
URL: https://issues.apache.org/jira/browse/SPARK-24602
Project: Spark
Issue Type: Improvement
Components: SQL
Affects Versions: 2.3.0
Environment: OS: SUSE11
Spark Version: 2.3
Reporter: Sushanta Sen
Precondition:
Spark cluster 2.3 is up and running
Test Steps:
# Launch Spark-sql
# spark-sql> CREATE TABLE t1(a int,string)
0: jdbc:hive2://ha-cluster/default> *alter table t1
change a a1 int;*
Error: org.apache.spark.sql.AnalysisException: {color:#FF}ALTER TABLE
CHANGE COLUMN is not supported for changing column 'a' with type 'IntegerType'
to 'b' with type 'IntegerType'; (state=,code=0){color}
# Launch hive beeliine
# repeat step1 & 2
# 0: jdbc:hive2://10.18.108.126:1/> desc del1;
+---++--+--+
| col_name | data_type | comment |
+---++--+--+
| *a1* | *int* | |
| dob | int | |
+---++--+--+
2 rows selected (1.572 seconds)
0: jdbc:hive2://10.18.108.126:1/>{color:#205081} alter table del1 change a1
a bigint;{color}
No rows affected (0.425 seconds)
0: jdbc:hive2://10.18.108.126:1/> desc del1;
+---++--+--+
| col_name | data_type | comment |
+---++--+--+
| *a* | *bigint* | |
| dob | int | |
+---++--+--+
2 rows selected (0.364 seconds)
Actual Result: In spark sql, alter table change is not supported, whereas in
hive beeline it is working fine.
Expected Result: ALTER Table CHANGE should be supported in Spark-SQL as well
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
[jira] [Created] (SPARK-24291) Data source table is not displaying records when files are uploaded to table location
Sushanta Sen created SPARK-24291:
Summary: Data source table is not displaying records when files
are uploaded to table location
Key: SPARK-24291
URL: https://issues.apache.org/jira/browse/SPARK-24291
Project: Spark
Issue Type: Bug
Components: SQL
Affects Versions: 2.3.0
Environment: OS: SUSE11
Spark Version: 2.3
Reporter: Sushanta Sen
Precondition:
1.Already one .orc file exists in the /tmp/orcdata/ location
# Launch Spark-sql
# spark-sql> CREATE TABLE os_orc (name string, version string, other string)
USING ORC OPTIONS (path '/tmp/orcdata/');
# spark-sql> select * from os_orc;
Spark 2.3.0 Apache
Time taken: 2.538 seconds, Fetched 1 row(s)
# pc1:/opt/# *./hadoop dfs -ls /tmp/orcdata*
Found 1 items
-rw-r--r-- 3 spark hadoop 475 2018-05-09 18:21
/tmp/orcdata/part-0-d488121b-e9fd-4269-a6ea-842c631722ee-c000.snappy.orc
pc1:/opt/# *./hadoop fs -copyFromLocal
/opt/OS/loaddata/orcdata/part-1-d488121b-e9fd-4269-a6ea-842c631722ee-c000.snappy.orc
/tmp/orcdata/data2.orc*
pc1:/opt/# *./hadoop dfs -ls /tmp/orcdata*
Found *2* items
-rw-r--r-- 3 spark hadoop 475 2018-05-15 14:59 /tmp/orcdata/data2.orc
-rw-r--r-- 3 spark hadoop 475 2018-05-09 18:21
/tmp/orcdata/part-0-d488121b-e9fd-4269-a6ea-842c631722ee-c000.snappy.orc
pc1:/opt/# **
5. Again execute the select command on the table os_orc
spark-sql> select * from os_orc;
Spark 2.3.0 Apache
Time taken: 1.528 seconds, Fetched {color:#FF}1 row(s){color}
Actual Result: On executing select command it does not display the all the
records exist in the data source table location
Expected Result: All the records should be fetched and displayed for the data
source table from the location
NB:
1.On exiting and relaunching the spark-sql session, select command fetches the
correct # of records.
2.This issue is valid for all the data source tables created with 'Using' .
I came across this use case in Spark 2.2.1 when tried to reproduce a customer
site observation.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
-
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
[jira] [Commented] (SPARK-24278) Create table if not exists is throwing table already exists exception
[ https://issues.apache.org/jira/browse/SPARK-24278?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16476837#comment-16476837 ] Sushanta Sen commented on SPARK-24278: -- But then it need not to print all the error logs in the sql console. > Create table if not exists is throwing table already exists exception > - > > Key: SPARK-24278 > URL: https://issues.apache.org/jira/browse/SPARK-24278 > Project: Spark > Issue Type: Bug > Components: SQL >Affects Versions: 2.3.0 > Environment: OS: SUSE11 > Spark Version: 2.3 >Reporter: Sushanta Sen >Priority: Major > > # Launch Spark-sql > # create table check(time timestamp, name string, isright boolean, datetoday > date, num binary, height double, score float, decimaler decimal(10,0), id > tinyint, age int, license bigint, length smallint) row format delimited > fields terminated by ',' stored as textfile; > # create table if not exists check (time timestamp, name string, isright > boolean, datetoday date, num binary, height double, score float, decimaler > decimal(10,0), id tinyint, age int, license bigint, length smallint) row > format delimited fields terminated by ','stored as TEXTFILE; *-FAILED* ** > > Exception as below > spark-sql> create table if not exists check (col1 string); > *2018-05-15 14:29:56 ERROR RetryingHMSHandler:159 -* > *AlreadyExistsException(message:Table check already exists)* > *at > org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.create_table_core(HiveMetaStore.java:1372)* > at > org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.create_table_with_environment_context(HiveMetaStore.java:1449) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:497) > at > org.apache.hadoop.hive.metastore.RetryingHMSHandler.invoke(RetryingHMSHandler.java:107) > at com.sun.proxy.$Proxy8.create_table_with_environment_context(Unknown Source) > at > org.apache.hadoop.hive.metastore.HiveMetaStoreClient.create_table_with_environment_context(HiveMetaStoreClient.java:2050) > at > org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.create_table_with_environment_context(SessionHiveMetaStoreClient.java:97) > at > org.apache.hadoop.hive.metastore.HiveMetaStoreClient.createTable(HiveMetaStoreClient.java:669) > at > org.apache.hadoop.hive.metastore.HiveMetaStoreClient.createTable(HiveMetaStoreClient.java:657) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:497) > at > org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.invoke(RetryingMetaStoreClient.java:156) > at com.sun.proxy.$Proxy9.createTable(Unknown Source) > at org.apache.hadoop.hive.ql.metadata.Hive.createTable(Hive.java:714) > at > org.apache.spark.sql.hive.client.HiveClientImpl$$anonfun$createTable$1.apply$mcV$sp(HiveClientImpl.scala:468) > at > org.apache.spark.sql.hive.client.HiveClientImpl$$anonfun$createTable$1.apply(HiveClientImpl.scala:466) > at > org.apache.spark.sql.hive.client.HiveClientImpl$$anonfun$createTable$1.apply(HiveClientImpl.scala:466) > at > org.apache.spark.sql.hive.client.HiveClientImpl$$anonfun$withHiveState$1.apply(HiveClientImpl.scala:272) > at > org.apache.spark.sql.hive.client.HiveClientImpl.liftedTree1$1(HiveClientImpl.scala:210) > at > org.apache.spark.sql.hive.client.HiveClientImpl.retryLocked(HiveClientImpl.scala:209) > at > org.apache.spark.sql.hive.client.HiveClientImpl.withHiveState(HiveClientImpl.scala:255) > at > org.apache.spark.sql.hive.client.HiveClientImpl.createTable(HiveClientImpl.scala:466) > at > org.apache.spark.sql.hive.HiveExternalCatalog$$anonfun$doCreateTable$1.apply$mcV$sp(HiveExternalCatalog.scala:258) > at > org.apache.spark.sql.hive.HiveExternalCatalog$$anonfun$doCreateTable$1.apply(HiveExternalCatalog.scala:216) > at > org.apache.spark.sql.hive.HiveExternalCatalog$$anonfun$doCreateTable$1.apply(HiveExternalCatalog.scala:216) > at > org.apache.spark.sql.hive.HiveExternalCatalog.withClient(HiveExternalCatalog.scala:97) > at > org.apache.spark.sql.hive.HiveExternalCatalog.doCreateTable(HiveExternalCatalog.scala:216) > at > org.apache.spark.sql.catalyst.catalog.ExternalCatalog.createTable(ExternalCatalog.scala:119) > at > org.apache.spark.sql.catalyst.catalog.SessionCatalog.createTable(SessionCatalog.scala:304) > at > org.apache.spark.sql.execution.command.CreateTableComma
[jira] [Created] (SPARK-24278) Create table if not exists is throwing table already exists exception
Sushanta Sen created SPARK-24278: Summary: Create table if not exists is throwing table already exists exception Key: SPARK-24278 URL: https://issues.apache.org/jira/browse/SPARK-24278 Project: Spark Issue Type: Bug Components: SQL Affects Versions: 2.3.0 Environment: OS: SUSE11 Spark Version: 2.3 Reporter: Sushanta Sen # Launch Spark-sql # create table check(time timestamp, name string, isright boolean, datetoday date, num binary, height double, score float, decimaler decimal(10,0), id tinyint, age int, license bigint, length smallint) row format delimited fields terminated by ',' stored as textfile; # create table if not exists check (time timestamp, name string, isright boolean, datetoday date, num binary, height double, score float, decimaler decimal(10,0), id tinyint, age int, license bigint, length smallint) row format delimited fields terminated by ','stored as TEXTFILE; *-FAILED* ** Exception as below spark-sql> create table if not exists check (col1 string); *2018-05-15 14:29:56 ERROR RetryingHMSHandler:159 -* *AlreadyExistsException(message:Table check already exists)* *at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.create_table_core(HiveMetaStore.java:1372)* at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.create_table_with_environment_context(HiveMetaStore.java:1449) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:497) at org.apache.hadoop.hive.metastore.RetryingHMSHandler.invoke(RetryingHMSHandler.java:107) at com.sun.proxy.$Proxy8.create_table_with_environment_context(Unknown Source) at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.create_table_with_environment_context(HiveMetaStoreClient.java:2050) at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.create_table_with_environment_context(SessionHiveMetaStoreClient.java:97) at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.createTable(HiveMetaStoreClient.java:669) at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.createTable(HiveMetaStoreClient.java:657) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:497) at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.invoke(RetryingMetaStoreClient.java:156) at com.sun.proxy.$Proxy9.createTable(Unknown Source) at org.apache.hadoop.hive.ql.metadata.Hive.createTable(Hive.java:714) at org.apache.spark.sql.hive.client.HiveClientImpl$$anonfun$createTable$1.apply$mcV$sp(HiveClientImpl.scala:468) at org.apache.spark.sql.hive.client.HiveClientImpl$$anonfun$createTable$1.apply(HiveClientImpl.scala:466) at org.apache.spark.sql.hive.client.HiveClientImpl$$anonfun$createTable$1.apply(HiveClientImpl.scala:466) at org.apache.spark.sql.hive.client.HiveClientImpl$$anonfun$withHiveState$1.apply(HiveClientImpl.scala:272) at org.apache.spark.sql.hive.client.HiveClientImpl.liftedTree1$1(HiveClientImpl.scala:210) at org.apache.spark.sql.hive.client.HiveClientImpl.retryLocked(HiveClientImpl.scala:209) at org.apache.spark.sql.hive.client.HiveClientImpl.withHiveState(HiveClientImpl.scala:255) at org.apache.spark.sql.hive.client.HiveClientImpl.createTable(HiveClientImpl.scala:466) at org.apache.spark.sql.hive.HiveExternalCatalog$$anonfun$doCreateTable$1.apply$mcV$sp(HiveExternalCatalog.scala:258) at org.apache.spark.sql.hive.HiveExternalCatalog$$anonfun$doCreateTable$1.apply(HiveExternalCatalog.scala:216) at org.apache.spark.sql.hive.HiveExternalCatalog$$anonfun$doCreateTable$1.apply(HiveExternalCatalog.scala:216) at org.apache.spark.sql.hive.HiveExternalCatalog.withClient(HiveExternalCatalog.scala:97) at org.apache.spark.sql.hive.HiveExternalCatalog.doCreateTable(HiveExternalCatalog.scala:216) at org.apache.spark.sql.catalyst.catalog.ExternalCatalog.createTable(ExternalCatalog.scala:119) at org.apache.spark.sql.catalyst.catalog.SessionCatalog.createTable(SessionCatalog.scala:304) at org.apache.spark.sql.execution.command.CreateTableCommand.run(tables.scala:128) at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:70) at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:68) at org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:79) at org.apache.spark.sql.Dataset$$anonfun$6.apply(Dataset.scala:190) at org.apache.spark.sql.Dataset$$anonfun$6.apply(Dataset.scala:190) at org.apache.spark.sql.Datase
