hive git commit: HIVE-20451 Metastore client and server tarball issues (Alexander Kolbasov via Alan Gates)
Repository: hive Updated Branches: refs/heads/master 189d3fec2 -> 57f40f71f HIVE-20451 Metastore client and server tarball issues (Alexander Kolbasov via Alan Gates) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/57f40f71 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/57f40f71 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/57f40f71 Branch: refs/heads/master Commit: 57f40f71ff9275458e743aad0ca460dff2291a4a Parents: 189d3fe Author: Alan Gates Authored: Fri Aug 31 14:03:09 2018 -0700 Committer: Alan Gates Committed: Fri Aug 31 14:03:09 2018 -0700 -- .../tools/TestSchemaToolCatalogOps.java | 7 ++ standalone-metastore/metastore-common/pom.xml | 36 +++ .../metastore-common/src/assembly/bin.xml | 108 --- .../metastore-common/src/assembly/src.xml | 53 - standalone-metastore/metastore-server/pom.xml | 4 +- .../metastore-server/src/assembly/src.xml | 53 - standalone-metastore/metastore-tools/pom.xml| 16 +++ standalone-metastore/pom.xml| 32 +- standalone-metastore/src/assembly/src.xml | 55 ++ 9 files changed, 124 insertions(+), 240 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/57f40f71/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/tools/TestSchemaToolCatalogOps.java -- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/tools/TestSchemaToolCatalogOps.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/tools/TestSchemaToolCatalogOps.java index 42eb979..43c0b96 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/tools/TestSchemaToolCatalogOps.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/tools/TestSchemaToolCatalogOps.java @@ -44,6 +44,7 @@ import org.junit.Test; import java.io.File; import java.io.IOException; +import java.io.PrintStream; import java.util.Collections; import java.util.HashSet; import java.util.Set; @@ -56,6 +57,8 @@ public class TestSchemaToolCatalogOps { private static HiveConf conf; private IMetaStoreClient client; private static String testMetastoreDB; + private static PrintStream errStream; + private static PrintStream outStream; private static String argsBase; @BeforeClass @@ -76,6 +79,8 @@ public class TestSchemaToolCatalogOps { String passWord = MetastoreConf.getPassword(conf, MetastoreConf.ConfVars.PWD); schemaTool.setUserName(userName); schemaTool.setPassWord(passWord); +errStream = System.err; +outStream = System.out; argsBase = "-dbType derby -userName " + userName + " -passWord " + passWord + " "; execute(new SchemaToolTaskInit(), "-initSchema"); // Pre-install the database so all the tables are there. @@ -87,6 +92,8 @@ public class TestSchemaToolCatalogOps { if (metaStoreDir.exists()) { FileUtils.forceDeleteOnExit(metaStoreDir); } +System.setOut(outStream); +System.setErr(errStream); } @Before http://git-wip-us.apache.org/repos/asf/hive/blob/57f40f71/standalone-metastore/metastore-common/pom.xml -- diff --git a/standalone-metastore/metastore-common/pom.xml b/standalone-metastore/metastore-common/pom.xml index 334adf8..b05d434 100644 --- a/standalone-metastore/metastore-common/pom.xml +++ b/standalone-metastore/metastore-common/pom.xml @@ -446,28 +446,6 @@ org.apache.maven.plugins -maven-assembly-plugin -${maven.assembly.plugin.version} - - -assemble -package - - single - - - apache-hive-metastore-${project.version} - -src/assembly/bin.xml -src/assembly/src.xml - - gnu - - - - - -org.apache.maven.plugins maven-enforcer-plugin @@ -618,6 +596,20 @@ ${basedir}/src/main/java + + +org.apache.maven.plugins +maven-assembly-plugin + + +assemble +none + + single + + + + http://git-wip-us.apache.org/repos/asf/hive/blob/57f40f71/standalone-metastore/metastore-common/src/assembly/bin.xml -- diff --git a/standalone-metastore/metastore-common/src/assembly/bin.xml b/standalone-metastore/metastore-common/src/assembly/bin.xml deleted file mode
hive git commit: HIVE-14162: Allow disabling of long running job on Hive On Spark On YARN (Sahil Takiar, reviewed by Adam Szita)
Repository: hive Updated Branches: refs/heads/master 219538701 -> 189d3fec2 HIVE-14162: Allow disabling of long running job on Hive On Spark On YARN (Sahil Takiar, reviewed by Adam Szita) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/189d3fec Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/189d3fec Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/189d3fec Branch: refs/heads/master Commit: 189d3fec25dfb94b209b1a34c1be674ce9d85bc5 Parents: 2195387 Author: Sahil Takiar Authored: Mon Jul 16 10:26:21 2018 -0500 Committer: Sahil Takiar Committed: Fri Aug 31 15:42:57 2018 -0500 -- .../org/apache/hadoop/hive/conf/HiveConf.java | 6 + .../ql/exec/spark/TestSparkSessionTimeout.java | 145 +++ .../java/org/apache/hadoop/hive/ql/Driver.java | 10 + .../ql/exec/spark/session/SparkSession.java | 27 ++ .../ql/exec/spark/session/SparkSessionImpl.java | 246 ++- .../spark/session/SparkSessionManagerImpl.java | 63 +++-- 6 files changed, 423 insertions(+), 74 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/189d3fec/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java -- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 8c39de3..40ea3ac 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -4311,6 +4311,12 @@ public class HiveConf extends Configuration { "specified (default) then the spark-submit shell script is used to launch the Spark " + "app. If " + HIVE_SPARK_LAUNCHER_CLIENT + " is specified then Spark's " + "InProcessLauncher is used to programmatically launch the app."), +SPARK_SESSION_TIMEOUT("hive.spark.session.timeout", "30m", new TimeValidator(TimeUnit.MINUTES, +30L, true, null, true), "Amount of time the Spark Remote Driver should wait for " + +" a Spark job to be submitted before shutting down. Minimum value is 30 minutes"), +SPARK_SESSION_TIMEOUT_PERIOD("hive.spark.session.timeout.period", "60s", +new TimeValidator(TimeUnit.SECONDS, 60L, true, null, true), +"How frequently to check for idle Spark sessions. Minimum value is 60 seconds."), NWAYJOINREORDER("hive.reorder.nway.joins", true, "Runs reordering of tables within single n-way join (i.e.: picks streamtable)"), HIVE_MERGE_NWAY_JOINS("hive.merge.nway.joins", true, http://git-wip-us.apache.org/repos/asf/hive/blob/189d3fec/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkSessionTimeout.java -- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkSessionTimeout.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkSessionTimeout.java new file mode 100644 index 000..c887297 --- /dev/null +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkSessionTimeout.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.exec.spark; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.Driver; +import org.apache.hadoop.hive.ql.QueryState; +import org.apache.hadoop.hive.ql.exec.spark.session.SparkSession; +import org.apache.hadoop.hive.ql.exec.spark.session.SparkSessionManagerImpl; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.session.SessionState; + +import org.junit.Assert; +import org.junit.Test; + +import java.io.File; +import java.net.MalformedURLException; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService;
[1/2] hive git commit: HIVE-20225: SerDe to support Teradata Binary Format (Lu Li via cws)
Repository: hive Updated Branches: refs/heads/branch-2 bd32deb44 -> 8b16ad0f5 http://git-wip-us.apache.org/repos/asf/hive/blob/8b16ad0f/src/test/queries/clientpositive/test_teradatabinaryfile.q -- diff --git a/src/test/queries/clientpositive/test_teradatabinaryfile.q b/src/test/queries/clientpositive/test_teradatabinaryfile.q new file mode 100644 index 000..33ab677 --- /dev/null +++ b/src/test/queries/clientpositive/test_teradatabinaryfile.q @@ -0,0 +1,123 @@ +DROP TABLE if exists teradata_binary_table_64kb; +DROP TABLE if exists teradata_binary_table_1mb; +DROP TABLE if exists teradata_binary_table_64kb_insert; +DROP TABLE if exists teradata_binary_table_1mb_insert; + + +CREATE TABLE `teradata_binary_table_64kb`( + `test_tinyint` tinyint, + `test_smallint` smallint, + `test_int` int, + `test_bigint` bigint, + `test_double` double, + `test_decimal` decimal(15,2), + `test_date` date, + `test_timestamp` timestamp, + `test_char` char(1), + `test_varchar` varchar(40), + `test_binary` binary + ) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.teradata.TeradataBinarySerde' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.TeradataBinaryFileInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.TeradataBinaryFileOutputFormat' +TBLPROPERTIES ( + 'teradata.timestamp.precision'='0', + 'teradata.char.charset'='LATIN', + 'teradata.row.length'='64KB' +); + +CREATE TABLE `teradata_binary_table_1mb`( + `test_tinyint` tinyint, + `test_smallint` smallint, + `test_int` int, + `test_bigint` bigint, + `test_double` double, + `test_decimal` decimal(15,2), + `test_date` date, + `test_timestamp` timestamp, + `test_char` char(1), + `test_varchar` varchar(40), + `test_binary` binary + ) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.teradata.TeradataBinarySerde' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.TeradataBinaryFileInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.TeradataBinaryFileOutputFormat' +TBLPROPERTIES ( + 'teradata.timestamp.precision'='6', + 'teradata.char.charset'='UNICODE', + 'teradata.row.length'='1MB' +); + +CREATE TABLE `teradata_binary_table_64kb_insert`( + `test_tinyint` tinyint, + `test_decimal` decimal(15,2), + `test_date` date, + `test_timestamp` timestamp + ) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.teradata.TeradataBinarySerde' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.TeradataBinaryFileInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.TeradataBinaryFileOutputFormat' +TBLPROPERTIES ( + 'teradata.timestamp.precision'='0', + 'teradata.char.charset'='LATIN', + 'teradata.row.length'='64KB' +); + +CREATE TABLE `teradata_binary_table_1mb_insert`( + `test_tinyint` tinyint, + `test_int` int + ) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.teradata.TeradataBinarySerde' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.TeradataBinaryFileInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.TeradataBinaryFileOutputFormat' +TBLPROPERTIES ( + 'teradata.timestamp.precision'='6', + 'teradata.char.charset'='UNICODE', + 'teradata.row.length'='1MB' +); + +LOAD DATA LOCAL INPATH '../../data/files/teradata_binary_file/teradata_binary_table.deflate' OVERWRITE INTO TABLE teradata_binary_table_64kb; +LOAD DATA LOCAL INPATH '../../data/files/teradata_binary_file/td_data_with_1mb_rowsize.teradata.gz' OVERWRITE INTO TABLE teradata_binary_table_1mb; + +SELECT * from teradata_binary_table_64kb; +SELECT * from teradata_binary_table_1mb; + +SELECT COUNT(*) FROM teradata_binary_table_64kb; +SELECT COUNT(*) FROM teradata_binary_table_1mb; + +SELECT max(date_format(test_timestamp, 'y')) FROM teradata_binary_table_64kb; +SELECT max(date_format(test_date, 'y')) FROM teradata_binary_table_64kb; +SELECT max(Floor(test_decimal)) FROM teradata_binary_table_64kb; + +SELECT max(date_format(test_timestamp, 'y')) FROM teradata_binary_table_1mb; +SELECT max(date_format(test_date, 'y')) FROM teradata_binary_table_1mb; +SELECT max(Floor(test_decimal)) FROM teradata_binary_table_1mb; + +SELECT test_tinyint, MAX(test_decimal) FROM teradata_binary_table_64kb GROUP BY test_tinyint; +SELECT test_tinyint, MAX(test_decimal) FROM teradata_binary_table_1mb GROUP BY test_tinyint; + +INSERT OVERWRITE TABLE teradata_binary_table_64kb_insert +SELECT test_tinyint, test_decimal, test_date, test_timestamp FROM teradata_binary_table_64kb; + +INSERT OVERWRITE TABLE teradata_binary_table_1mb_insert +SELECT 1, 15; + +DESC FORMATTED teradata_binary_table_64kb_insert; +DESC FORMATTED teradata_binary_table_1mb_insert; + +DROP TABLE if exists teradata_binary_table_64kb; +DROP TABLE if exists teradata_binary_table_1mb; +DROP TABLE if exists teradata_binary_table_64kb_insert; +DROP TABLE if exists teradata_binary_table_1mb_insert; http://git-wip-us.apache.org/repos/asf/hive/blob/8b16ad0f/src/test/results/clientpositive/test_teradatabinaryfile.q.out
[2/2] hive git commit: HIVE-20225: SerDe to support Teradata Binary Format (Lu Li via cws)
HIVE-20225: SerDe to support Teradata Binary Format (Lu Li via cws) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8b16ad0f Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8b16ad0f Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8b16ad0f Branch: refs/heads/branch-2 Commit: 8b16ad0f54d3bcf56362118f31e25a792ad7291e Parents: bd32deb Author: Daniel Dai Authored: Fri Aug 31 12:36:23 2018 -0700 Committer: Daniel Dai Committed: Fri Aug 31 12:36:23 2018 -0700 -- .../td_data_with_1mb_rowsize.teradata.gz| Bin 0 -> 616 bytes .../teradata_binary_table.deflate | Bin 0 -> 1329 bytes .../ql/io/TeradataBinaryFileInputFormat.java| 66 ++ .../ql/io/TeradataBinaryFileOutputFormat.java | 112 .../hive/ql/io/TeradataBinaryRecordReader.java | 280 + .../teradata/TeradataBinaryDataInputStream.java | 200 +++ .../TeradataBinaryDataOutputStream.java | 270 + .../serde2/teradata/TeradataBinarySerde.java| 597 +++ .../TestTeradataBinarySerdeForDate.java | 72 +++ .../TestTeradataBinarySerdeForDecimal.java | 106 .../TestTeradataBinarySerdeForTimeStamp.java| 96 +++ .../TestTeradataBinarySerdeGeneral.java | 126 .../clientpositive/test_teradatabinaryfile.q| 123 .../test_teradatabinaryfile.q.out | 537 + 14 files changed, 2585 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hive/blob/8b16ad0f/data/files/teradata_binary_file/td_data_with_1mb_rowsize.teradata.gz -- diff --git a/data/files/teradata_binary_file/td_data_with_1mb_rowsize.teradata.gz b/data/files/teradata_binary_file/td_data_with_1mb_rowsize.teradata.gz new file mode 100644 index 000..7319e3c Binary files /dev/null and b/data/files/teradata_binary_file/td_data_with_1mb_rowsize.teradata.gz differ http://git-wip-us.apache.org/repos/asf/hive/blob/8b16ad0f/data/files/teradata_binary_file/teradata_binary_table.deflate -- diff --git a/data/files/teradata_binary_file/teradata_binary_table.deflate b/data/files/teradata_binary_file/teradata_binary_table.deflate new file mode 100644 index 000..fd53dde Binary files /dev/null and b/data/files/teradata_binary_file/teradata_binary_table.deflate differ http://git-wip-us.apache.org/repos/asf/hive/blob/8b16ad0f/src/java/org/apache/hadoop/hive/ql/io/TeradataBinaryFileInputFormat.java -- diff --git a/src/java/org/apache/hadoop/hive/ql/io/TeradataBinaryFileInputFormat.java b/src/java/org/apache/hadoop/hive/ql/io/TeradataBinaryFileInputFormat.java new file mode 100644 index 000..bed87c5 --- /dev/null +++ b/src/java/org/apache/hadoop/hive/ql/io/TeradataBinaryFileInputFormat.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io; + +import java.io.IOException; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.BytesWritable; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.mapred.FileInputFormat; +import org.apache.hadoop.mapred.FileSplit; +import org.apache.hadoop.mapred.InputSplit; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.RecordReader; +import org.apache.hadoop.mapred.Reporter; + +/** + * https://cwiki.apache.org/confluence/display/Hive/TeradataBinarySerde. + * FileInputFormat for Teradata binary files. + * + * In the Teradata Binary File, each record constructs as below: + * The first 2 bytes represents the length of the bytes next for this record. + * Then the null bitmap whose length is depended on the number of fields is followed. + * Then each field of the record is serialized into bytes - the serialization strategy is decided by the type of field. + * At last, there is one byte (0x0a) in the end of the
hive git commit: HIVE-20225: SerDe to support Teradata Binary Format (Lu Li via cws)
Repository: hive Updated Branches: refs/heads/master a77e0468f -> 219538701 HIVE-20225: SerDe to support Teradata Binary Format (Lu Li via cws) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/21953870 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/21953870 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/21953870 Branch: refs/heads/master Commit: 219538701967fc1f5386c2f107fd207988250f2e Parents: a77e046 Author: Daniel Dai Authored: Fri Aug 31 12:19:29 2018 -0700 Committer: Daniel Dai Committed: Fri Aug 31 12:19:29 2018 -0700 -- .../td_data_with_1mb_rowsize.teradata.gz| Bin 0 -> 616 bytes .../teradata_binary_table.deflate | Bin 0 -> 1329 bytes 2 files changed, 0 insertions(+), 0 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/21953870/data/files/teradata_binary_file/td_data_with_1mb_rowsize.teradata.gz -- diff --git a/data/files/teradata_binary_file/td_data_with_1mb_rowsize.teradata.gz b/data/files/teradata_binary_file/td_data_with_1mb_rowsize.teradata.gz new file mode 100644 index 000..7319e3c Binary files /dev/null and b/data/files/teradata_binary_file/td_data_with_1mb_rowsize.teradata.gz differ http://git-wip-us.apache.org/repos/asf/hive/blob/21953870/data/files/teradata_binary_file/teradata_binary_table.deflate -- diff --git a/data/files/teradata_binary_file/teradata_binary_table.deflate b/data/files/teradata_binary_file/teradata_binary_table.deflate new file mode 100644 index 000..fd53dde Binary files /dev/null and b/data/files/teradata_binary_file/teradata_binary_table.deflate differ