[24/51] [abbrv] hive git commit: HIVE-13175: Disallow making external tables transactional (Wei Zheng, reviewed by Eugene Koifman)

2016-03-19 Thread jdere 
HIVE-13175: Disallow making external tables transactional (Wei Zheng, reviewed 
by Eugene Koifman)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ff55d0a6
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ff55d0a6
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ff55d0a6

Branch: refs/heads/llap
Commit: ff55d0a67e59c15b5ccfbdf1317bfd60cf057a30
Parents: 1e8a31e
Author: Wei 
Authored: Thu Mar 10 13:39:13 2016 -0800
Committer: Wei 
Committed: Thu Mar 10 13:39:13 2016 -0800

--
 .../hadoop/hive/metastore/TestHiveMetaStore.java   |  1 +
 .../metastore/TransactionalValidationListener.java | 11 +++
 .../test/queries/clientnegative/alter_external_acid.q  |  9 +
 .../test/queries/clientnegative/create_external_acid.q |  6 ++
 .../results/clientnegative/alter_external_acid.q.out   | 13 +
 .../results/clientnegative/create_external_acid.q.out  |  5 +
 6 files changed, 45 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/ff55d0a6/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
--
diff --git 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
index a55c186..5da4165 100644
--- 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
+++ 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
@@ -2944,6 +2944,7 @@ public abstract class TestHiveMetaStore extends TestCase {
 
 tbl.setSd(sd);
 tbl.setLastAccessTime(lastAccessTime);
+tbl.setTableType(TableType.MANAGED_TABLE.toString());
 
 client.createTable(tbl);
 

http://git-wip-us.apache.org/repos/asf/hive/blob/ff55d0a6/metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
--
diff --git 
a/metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
 
b/metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
index 96158f8..3e74675 100644
--- 
a/metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
+++ 
b/metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
@@ -86,6 +86,12 @@ final class TransactionalValidationListener extends 
MetaStorePreEventListener {
 throw new MetaException("The table must be bucketed and stored using 
an ACID compliant" +
 " format (such as ORC)");
   }
+
+  if (newTable.getTableType().equals(TableType.EXTERNAL_TABLE.toString())) 
{
+throw new MetaException(newTable.getDbName() + "." + 
newTable.getTableName() +
+" cannot be declared transactional because it's an external 
table");
+  }
+
   return;
 }
 Table oldTable = context.getOldTable();
@@ -144,6 +150,11 @@ final class TransactionalValidationListener extends 
MetaStorePreEventListener {
 " format (such as ORC)");
   }
 
+  if (newTable.getTableType().equals(TableType.EXTERNAL_TABLE.toString())) 
{
+throw new MetaException(newTable.getDbName() + "." + 
newTable.getTableName() +
+" cannot be declared transactional because it's an external 
table");
+  }
+
   // normalize prop name
   parameters.put(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL, 
Boolean.TRUE.toString());
   return;

http://git-wip-us.apache.org/repos/asf/hive/blob/ff55d0a6/ql/src/test/queries/clientnegative/alter_external_acid.q
--
diff --git a/ql/src/test/queries/clientnegative/alter_external_acid.q 
b/ql/src/test/queries/clientnegative/alter_external_acid.q
new file mode 100644
index 000..7807278
--- /dev/null
+++ b/ql/src/test/queries/clientnegative/alter_external_acid.q
@@ -0,0 +1,9 @@
+set hive.support.concurrency=true;
+set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
+
+
+create external table acid_external (a int, b varchar(128)) clustered by (b) 
into 2 buckets stored as orc;
+
+alter table acid_external set TBLPROPERTIES ('transactional'='true');
+
+drop table acid_external;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/ff55d0a6/ql/src/test/queries/clientnegative/create_external_acid.q
--
diff --git a/ql/src/test/queries/clientnegative/create_external_acid.q 
b/ql/src/test/queries/clientnegative/create_external_acid.q
new file mode 100644
index 000..d6b2d84
--- /dev/null
+++ b/ql/src/test

[3/3] hive git commit: HIVE-13175 : Disallow making external tables transactional (Wei Zheng, reviewed by Eugene Koifman)

2016-03-10 Thread weiz 
HIVE-13175 : Disallow making external tables transactional (Wei Zheng, reviewed 
by Eugene Koifman)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/24b366f0
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/24b366f0
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/24b366f0

Branch: refs/heads/branch-1
Commit: 24b366f0bbe785b6f478881e8224e43d7aa33094
Parents: f7c8fb5
Author: Wei 
Authored: Thu Mar 10 13:58:42 2016 -0800
Committer: Wei 
Committed: Thu Mar 10 13:58:42 2016 -0800

--
 .../hive/metastore/TestHiveMetaStore.java   |1 +
 .../hive/metastore/TestHiveMetaStore.java.orig  | 3224 ++
 .../TransactionalValidationListener.java|   11 +
 .../clientnegative/alter_external_acid.q|9 +
 .../clientnegative/create_external_acid.q   |6 +
 .../clientnegative/alter_external_acid.q.out|   13 +
 .../clientnegative/create_external_acid.q.out   |5 +
 7 files changed, 3269 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/24b366f0/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
--
diff --git 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
index b005759..605dc9d 100644
--- 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
+++ 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
@@ -2941,6 +2941,7 @@ public abstract class TestHiveMetaStore extends TestCase {
 
 tbl.setSd(sd);
 tbl.setLastAccessTime(lastAccessTime);
+tbl.setTableType(TableType.MANAGED_TABLE.toString());
 
 client.createTable(tbl);

[2/3] hive git commit: HIVE-13175 : Disallow making external tables transactional (Wei Zheng, reviewed by Eugene Koifman)

2016-03-10 Thread weiz 
http://git-wip-us.apache.org/repos/asf/hive/blob/24b366f0/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java.orig
--
diff --git 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java.orig
 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java.orig
new file mode 100644
index 000..b005759
--- /dev/null
+++ 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java.orig
@@ -0,0 +1,3224 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import junit.framework.TestCase;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hive.common.FileUtils;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.metastore.api.AggrStats;
+import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
+import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.apache.hadoop.hive.metastore.api.ConfigValSecurityException;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.Function;
+import org.apache.hadoop.hive.metastore.api.FunctionType;
+import org.apache.hadoop.hive.metastore.api.GetAllFunctionsResponse;
+import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
+import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
+import org.apache.hadoop.hive.metastore.api.Order;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.PrincipalType;
+import org.apache.hadoop.hive.metastore.api.ResourceType;
+import org.apache.hadoop.hive.metastore.api.ResourceUri;
+import org.apache.hadoop.hive.metastore.api.SerDeInfo;
+import org.apache.hadoop.hive.metastore.api.SkewedInfo;
+import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
+import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.api.Type;
+import org.apache.hadoop.hive.metastore.api.UnknownDBException;
+import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
+import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.io.HiveInputFormat;
+import org.apache.hadoop.hive.ql.io.HiveOutputFormat;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.thrift.TException;
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.google.common.collect.Lists;
+
+public abstract class TestHiveMetaStore extends TestCase {
+  private static final Log LOG = LogFactory.getLog(TestHiveMetaStore.class);
+  protected static HiveMetaStoreClient client;
+  protected static HiveConf hiveConf;
+  protected static Warehouse warehouse;
+  protected static boolean isThriftClient = false;
+
+  private static final String TEST_DB1_NAME = "testdb1";
+  private static final String TEST_DB2

[1/3] hive git commit: HIVE-13175 : Disallow making external tables transactional (Wei Zheng, reviewed by Eugene Koifman)

2016-03-10 Thread weiz 
Repository: hive
Updated Branches:
  refs/heads/branch-1 f7c8fb527 -> 24b366f0b


http://git-wip-us.apache.org/repos/asf/hive/blob/24b366f0/metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
--
diff --git 
a/metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
 
b/metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
index 96158f8..3e74675 100644
--- 
a/metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
+++ 
b/metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
@@ -86,6 +86,12 @@ final class TransactionalValidationListener extends 
MetaStorePreEventListener {
 throw new MetaException("The table must be bucketed and stored using 
an ACID compliant" +
 " format (such as ORC)");
   }
+
+  if (newTable.getTableType().equals(TableType.EXTERNAL_TABLE.toString())) 
{
+throw new MetaException(newTable.getDbName() + "." + 
newTable.getTableName() +
+" cannot be declared transactional because it's an external 
table");
+  }
+
   return;
 }
 Table oldTable = context.getOldTable();
@@ -144,6 +150,11 @@ final class TransactionalValidationListener extends 
MetaStorePreEventListener {
 " format (such as ORC)");
   }
 
+  if (newTable.getTableType().equals(TableType.EXTERNAL_TABLE.toString())) 
{
+throw new MetaException(newTable.getDbName() + "." + 
newTable.getTableName() +
+" cannot be declared transactional because it's an external 
table");
+  }
+
   // normalize prop name
   parameters.put(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL, 
Boolean.TRUE.toString());
   return;

http://git-wip-us.apache.org/repos/asf/hive/blob/24b366f0/ql/src/test/queries/clientnegative/alter_external_acid.q
--
diff --git a/ql/src/test/queries/clientnegative/alter_external_acid.q 
b/ql/src/test/queries/clientnegative/alter_external_acid.q
new file mode 100644
index 000..7807278
--- /dev/null
+++ b/ql/src/test/queries/clientnegative/alter_external_acid.q
@@ -0,0 +1,9 @@
+set hive.support.concurrency=true;
+set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
+
+
+create external table acid_external (a int, b varchar(128)) clustered by (b) 
into 2 buckets stored as orc;
+
+alter table acid_external set TBLPROPERTIES ('transactional'='true');
+
+drop table acid_external;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/24b366f0/ql/src/test/queries/clientnegative/create_external_acid.q
--
diff --git a/ql/src/test/queries/clientnegative/create_external_acid.q 
b/ql/src/test/queries/clientnegative/create_external_acid.q
new file mode 100644
index 000..d6b2d84
--- /dev/null
+++ b/ql/src/test/queries/clientnegative/create_external_acid.q
@@ -0,0 +1,6 @@
+set hive.support.concurrency=true;
+set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
+
+
+create external table acid_external (a int, b varchar(128)) clustered by (b) 
into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true');
+

http://git-wip-us.apache.org/repos/asf/hive/blob/24b366f0/ql/src/test/results/clientnegative/alter_external_acid.q.out
--
diff --git a/ql/src/test/results/clientnegative/alter_external_acid.q.out 
b/ql/src/test/results/clientnegative/alter_external_acid.q.out
new file mode 100644
index 000..69bba3b
--- /dev/null
+++ b/ql/src/test/results/clientnegative/alter_external_acid.q.out
@@ -0,0 +1,13 @@
+PREHOOK: query: create external table acid_external (a int, b varchar(128)) 
clustered by (b) into 2 buckets stored as orc
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@acid_external
+POSTHOOK: query: create external table acid_external (a int, b varchar(128)) 
clustered by (b) into 2 buckets stored as orc
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@acid_external
+PREHOOK: query: alter table acid_external set TBLPROPERTIES 
('transactional'='true')
+PREHOOK: type: ALTERTABLE_PROPERTIES
+PREHOOK: Input: default@acid_external
+PREHOOK: Output: default@acid_external
+FAILED: Execution Error, return code 1 from 
org.apache.hadoop.hive.ql.exec.DDLTask. Unable to alter table. 
default.acid_external cannot be declared transactional because it's an external 
table

http://git-wip-us.apache.org/repos/asf/hive/blob/24b366f0/ql/src/test/results/clientnegative/create_external_acid.q.out
--
diff --git a/ql/src/test/results/clientnegative/create_external_acid.q.out 
b/ql/src/test/resul

hive git commit: HIVE-13175: Disallow making external tables transactional (Wei Zheng, reviewed by Eugene Koifman)

2016-03-10 Thread weiz 
Repository: hive
Updated Branches:
  refs/heads/master 1e8a31e8f -> ff55d0a67


HIVE-13175: Disallow making external tables transactional (Wei Zheng, reviewed 
by Eugene Koifman)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ff55d0a6
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ff55d0a6
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ff55d0a6

Branch: refs/heads/master
Commit: ff55d0a67e59c15b5ccfbdf1317bfd60cf057a30
Parents: 1e8a31e
Author: Wei 
Authored: Thu Mar 10 13:39:13 2016 -0800
Committer: Wei 
Committed: Thu Mar 10 13:39:13 2016 -0800

--
 .../hadoop/hive/metastore/TestHiveMetaStore.java   |  1 +
 .../metastore/TransactionalValidationListener.java | 11 +++
 .../test/queries/clientnegative/alter_external_acid.q  |  9 +
 .../test/queries/clientnegative/create_external_acid.q |  6 ++
 .../results/clientnegative/alter_external_acid.q.out   | 13 +
 .../results/clientnegative/create_external_acid.q.out  |  5 +
 6 files changed, 45 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/ff55d0a6/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
--
diff --git 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
index a55c186..5da4165 100644
--- 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
+++ 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
@@ -2944,6 +2944,7 @@ public abstract class TestHiveMetaStore extends TestCase {
 
 tbl.setSd(sd);
 tbl.setLastAccessTime(lastAccessTime);
+tbl.setTableType(TableType.MANAGED_TABLE.toString());
 
 client.createTable(tbl);
 

http://git-wip-us.apache.org/repos/asf/hive/blob/ff55d0a6/metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
--
diff --git 
a/metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
 
b/metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
index 96158f8..3e74675 100644
--- 
a/metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
+++ 
b/metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
@@ -86,6 +86,12 @@ final class TransactionalValidationListener extends 
MetaStorePreEventListener {
 throw new MetaException("The table must be bucketed and stored using 
an ACID compliant" +
 " format (such as ORC)");
   }
+
+  if (newTable.getTableType().equals(TableType.EXTERNAL_TABLE.toString())) 
{
+throw new MetaException(newTable.getDbName() + "." + 
newTable.getTableName() +
+" cannot be declared transactional because it's an external 
table");
+  }
+
   return;
 }
 Table oldTable = context.getOldTable();
@@ -144,6 +150,11 @@ final class TransactionalValidationListener extends 
MetaStorePreEventListener {
 " format (such as ORC)");
   }
 
+  if (newTable.getTableType().equals(TableType.EXTERNAL_TABLE.toString())) 
{
+throw new MetaException(newTable.getDbName() + "." + 
newTable.getTableName() +
+" cannot be declared transactional because it's an external 
table");
+  }
+
   // normalize prop name
   parameters.put(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL, 
Boolean.TRUE.toString());
   return;

http://git-wip-us.apache.org/repos/asf/hive/blob/ff55d0a6/ql/src/test/queries/clientnegative/alter_external_acid.q
--
diff --git a/ql/src/test/queries/clientnegative/alter_external_acid.q 
b/ql/src/test/queries/clientnegative/alter_external_acid.q
new file mode 100644
index 000..7807278
--- /dev/null
+++ b/ql/src/test/queries/clientnegative/alter_external_acid.q
@@ -0,0 +1,9 @@
+set hive.support.concurrency=true;
+set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
+
+
+create external table acid_external (a int, b varchar(128)) clustered by (b) 
into 2 buckets stored as orc;
+
+alter table acid_external set TBLPROPERTIES ('transactional'='true');
+
+drop table acid_external;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/ff55d0a6/ql/src/test/queries/clientnegative/create_external_acid.q
--
diff --git a/ql/src/test/queries/clientnegative/create_external_acid.q 
b/ql/src/test/queries/clientnegative/create_external_