Author: ekoifman
Date: Mon Apr 20 21:24:51 2015
New Revision: 1674988
URL: http://svn.apache.org/r1674988
Log:
HIVE-7948 - Add an E2E test to verify fix for HIVE-7155 (Aswathy Chellammal
Sreekumar via Eugene Koifman)
Added:
hive/trunk/hcatalog/src/test/e2e/templeton/deployers/config/webhcat/webhcat-site.updateConfig.xml
hive/trunk/hcatalog/src/test/e2e/templeton/deployers/modify_webhcat_config.sh
hive/trunk/hcatalog/src/test/e2e/templeton/deployers/restore_webhcat_config.sh
hive/trunk/hcatalog/src/test/e2e/templeton/tests/modifyConfiguration.conf
Modified:
hive/trunk/hcatalog/src/test/e2e/templeton/README.txt
hive/trunk/hcatalog/src/test/e2e/templeton/build.xml
hive/trunk/hcatalog/src/test/e2e/templeton/deployers/deploy_e2e_artifacts.sh
hive/trunk/hcatalog/src/test/e2e/templeton/deployers/env.sh
Modified: hive/trunk/hcatalog/src/test/e2e/templeton/README.txt
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/src/test/e2e/templeton/README.txt?rev=1674988&r1=1674987&r2=1674988&view=diff
==============================================================================
--- hive/trunk/hcatalog/src/test/e2e/templeton/README.txt (original)
+++ hive/trunk/hcatalog/src/test/e2e/templeton/README.txt Mon Apr 20 21:24:51
2015
@@ -205,6 +205,19 @@ Here is the schema of the table writen i
To prevent primary key violation and sqoop import directory conflict, make
sure the "PERSON" table is empty
and the folder hdfs://hostname:8020/sqoopoutputdir doesn't exist before
running the test.
+Running updateConfig tests
+--------------------------
+ant test-updateConfig -Dinpdir.hdfs=<location of inpdir on hdfs>
-Dtest.user.name=<user the tests should run as> \
+ -Dsecure.mode=<yes/no> -Dharness.webhdfs.url=<webhdfs url upto port num>
-Dharness.templeton.url=<templeton url upto port num>
+
+This test suite is trying to verify the use of property
templeton.mapper.memory.mb in webhcat-site.xml.
+For this, an attempt is made to load data of size greater than 100MB, from one
hive table to another hive table,
+with the templeton.mapper.memory.mb set to a very low value. This is a
negative test case that expects the failure of map job
+due to insufficient memory.
+
+For running this test suite templeton.mapper.memory.mb property should be set
to 0.01 in webhcat-site.xml. This could be done by
+running modify_webhcat_config.sh in deployers/. Once the test run finishes,
the change could be reverted by running restore_webhcat_config.sh
+
Notes
-----
It's best to set HADOOP_HOME_WARN_SUPPRESS=true everywhere you can.
Modified: hive/trunk/hcatalog/src/test/e2e/templeton/build.xml
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/src/test/e2e/templeton/build.xml?rev=1674988&r1=1674987&r2=1674988&view=diff
==============================================================================
--- hive/trunk/hcatalog/src/test/e2e/templeton/build.xml (original)
+++ hive/trunk/hcatalog/src/test/e2e/templeton/build.xml Mon Apr 20 21:24:51
2015
@@ -197,6 +197,35 @@
</exec>
</target>
+ <target name="test-updateConfig" depends="property-check, init-test">
+ <!-- fork (parallelization) factors for e2e tests execution.
+ Defaults are 1, which means *no* parellelization: -->
+ <property name="fork.factor.group" value="1"/>
+ <property name="fork.factor.conf.file" value="1"/>
+ <property name="e2e.debug" value="false"/>
+ <property name="tests.to.run" value=""/>
+ <exec executable="perl" dir="${test.location}" failonerror="true">
+ <env key="HARNESS_ROOT" value="."/>
+ <env key="TH_WORKING_DIR" value="${test.location}"/>
+ <env key="TH_INPDIR_LOCAL" value="${inpdir.local}"/>
+ <env key="TH_INPDIR_HDFS" value="${inpdir.hdfs}"/>
+ <env key="TH_OUT" value="."/>
+ <env key="TH_ROOT" value="."/>
+ <env key="FORK_FACTOR_GROUP" value="${fork.factor.group}"/>
+ <env key="FORK_FACTOR_FILE" value="${fork.factor.conf.file}"/>
+ <env key="E2E_DEBUG" value="${e2e.debug}"/>
+ <env key="WEBHDFS_URL" value="${harness.webhdfs.url}"/>
+ <env key="TEMPLETON_URL" value="${harness.templeton.url}"/>
+ <env key="USER_NAME" value="${test.user.name}"/>
+ <env key="DOAS_USER" value="${doas.user}"/>
+ <env key="HARNESS_CONF" value="${basedir}/conf/default.conf"/>
+ <env key="SECURE_MODE" value="${secure.mode}"/>
+ <arg value="./test_harness.pl"/>
+ <arg line="${tests.to.run}"/>
+ <arg value="${basedir}/tests/modifyConfiguration.conf"/>
+ </exec>
+ </target>
+
<target name="clean">
<delete dir="${test.location}"/>
<delete file="${tar.name}"/>
Added:
hive/trunk/hcatalog/src/test/e2e/templeton/deployers/config/webhcat/webhcat-site.updateConfig.xml
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/src/test/e2e/templeton/deployers/config/webhcat/webhcat-site.updateConfig.xml?rev=1674988&view=auto
==============================================================================
---
hive/trunk/hcatalog/src/test/e2e/templeton/deployers/config/webhcat/webhcat-site.updateConfig.xml
(added)
+++
hive/trunk/hcatalog/src/test/e2e/templeton/deployers/config/webhcat/webhcat-site.updateConfig.xml
Mon Apr 20 21:24:51 2015
@@ -0,0 +1,118 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<!-- The default settings for Templeton. -->
+<!-- Edit templeton-site.xml to change settings for your local -->
+<!-- install. -->
+
+<configuration>
+<!--TODO:
+1. make pig/hive versions env variables-->
+
+ <property>
+ <name>templeton.hcat</name>
+ <value>${env.HCAT_PREFIX}/bin/hcat</value>
+ <description>The path to the hcatalog executable.</description>
+ </property>
+
+ <property>
+ <name>templeton.libjars</name>
+ <value>${env.TEMPLETON_HOME}/../lib/zookeeper-3.4.5.jar</value>
+ <description>Jars to add to the classpath.</description>
+ </property>
+
+ <property>
+ <name>templeton.pig.archive</name>
+ <value>hdfs:///apps/templeton/pig-${env.PIG_VERSION}.tar.gz</value>
+ <description>The path to the Pig archive.</description>
+ </property>
+ <property>
+ <name>templeton.pig.path</name>
+
<value>pig-${env.PIG_VERSION}.tar.gz/pig-${env.PIG_VERSION}/bin/pig</value>
+ <description>The path to the Pig executable.</description>
+ </property>
+ <property>
+ <name>templeton.hive.archive</name>
+
<value>hdfs:///apps/templeton/apache-hive-${env.HIVE_VERSION}-bin.tar.gz</value>
+ <description>The path to the Hive archive.</description>
+ </property>
+
+ <property>
+ <name>templeton.hive.path</name>
+
<value>apache-hive-${env.HIVE_VERSION}-bin.tar.gz/apache-hive-${env.HIVE_VERSION}-bin/bin/hive</value>
+ <description>The path to the Hive executable.</description>
+ </property>
+
+ <property>
+ <name>templeton.hive.home</name>
+
<value>apache-hive-${env.HIVE_VERSION}-bin.tar.gz/apache-hive-${env.HIVE_VERSION}-bin</value>
+ <description>The path to the Hive home within the tar. This is needed
if
+ Hive is not installed on all nodes in the cluster and needs to be
+ shipped to the target node in the cluster to execute Pig job which
uses
+ HCat, Hive query, etc.</description>
+ </property>
+ <property>
+ <name>templeton.hcat.home</name>
+
<value>apache-hive-${env.HIVE_VERSION}-bin.tar.gz/apache-hive-${env.HIVE_VERSION}-bin/hcatalog</value>
+ <description>The path to the HCat home within the tar. This is needed
if
+ Hive is not installed on all nodes in the cluster and needs to be
+ shipped to the target node in the cluster to execute Pig job which
uses
+ HCat, Hive query, etc.</description>
+ </property>
+ <property>
+ <name>templeton.sqoop.archive</name>
+
<value>hdfs:///apps/templeton/sqoop-1.4.5.bin__hadoop-2.0.4-alpha.tar.gz</value>
+ <description>This should point to Sqoop tar that will be shipped to
target node executing
+ the actual sqoop command. If not set, Sqoop is expected to be
installed on every node of the
+ cluster.</description>
+ </property>
+ <property>
+ <name>templeton.sqoop.path</name>
+
<value>sqoop-1.4.5.bin__hadoop-2.0.4-alpha.tar.gz/sqoop-1.4.5.bin__hadoop-2.0.4-alpha/bin/sqoop</value>
+ <description>The path to the Sqoop executable.</description>
+ </property>
+ <property>
+ <name>templeton.sqoop.home</name>
+
<value>sqoop-1.4.5.bin__hadoop-2.0.4-alpha.tar.gz/sqoop-1.4.5.bin__hadoop-2.0.4-alpha</value>
+ <description>The path to the Sqoop home in the exploded
archive.</description>
+ </property>
+
+ <property>
+ <name>templeton.controller.mr.child.opts</name>
+ <value> -Xmx64m -Djava.net.preferIPv4Stack=true</value>
+ </property>
+
+ <property>
+ <!--\,thrift://127.0.0.1:9933-->
+ <name>templeton.hive.properties</name>
+
<value>hive.metastore.uris=thrift://localhost:9933,hive.metastore.sasl.enabled=false</value>
+ </property>
+ <property>
+ <name>templeton.mapper.memory.mb</name>
+ <value>0.01</value>
+ <description>Templeton controller job's Launch mapper's memory limit in
+ megabytes. When submitting controller job, Templeton will overwrite
+ mapreduce.map.memory.mb with this value. If empty, Templeton will
+ not set mapreduce.map.memory.mb when submitting the controller job,
+ therefore the configuration in mapred-site.xml will be used.
+ </description>
+ </property>
+</configuration>
Modified:
hive/trunk/hcatalog/src/test/e2e/templeton/deployers/deploy_e2e_artifacts.sh
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/src/test/e2e/templeton/deployers/deploy_e2e_artifacts.sh?rev=1674988&r1=1674987&r2=1674988&view=diff
==============================================================================
---
hive/trunk/hcatalog/src/test/e2e/templeton/deployers/deploy_e2e_artifacts.sh
(original)
+++
hive/trunk/hcatalog/src/test/e2e/templeton/deployers/deploy_e2e_artifacts.sh
Mon Apr 20 21:24:51 2015
@@ -25,6 +25,9 @@ source ./env.sh
echo "Deploying artifacts to HDFS..."
+curl -O http://files.grouplens.org/datasets/movielens/ml-1m.zip
+unzip ml-1m.zip
+mv ml-1m/ratings.dat
${PROJ_HOME}/hcatalog/src/test/e2e/templeton/inpdir/ratings.txt
${HADOOP_HOME}/bin/hdfs dfs -put
${PROJ_HOME}/hcatalog/src/test/e2e/templeton/inpdir/ webhcate2e
#For hadoop1 we copy the same file with 2 names
#$HADOOP_HOME/bin/hadoop fs -put hadoop-examples-1.2.1.jar
webhcate2e/hexamples.jar
Modified: hive/trunk/hcatalog/src/test/e2e/templeton/deployers/env.sh
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/src/test/e2e/templeton/deployers/env.sh?rev=1674988&r1=1674987&r2=1674988&view=diff
==============================================================================
--- hive/trunk/hcatalog/src/test/e2e/templeton/deployers/env.sh (original)
+++ hive/trunk/hcatalog/src/test/e2e/templeton/deployers/env.sh Mon Apr 20
21:24:51 2015
@@ -65,3 +65,4 @@ export WEBHCAT_LOG_DIR=/tmp/webhcat_e2e/
export WEBHCAT_PID_DIR=${WEBHCAT_LOG_DIR}
#config/hive/hive-site.xml should match this path - it doesn't understand env
vars
export METASTORE_DB=${WEBHCAT_LOG_DIR}/wehcat_e2e_metastore_db
+export CONF_BACKUP=/Users/${USER}/tmp
\ No newline at end of file
Added:
hive/trunk/hcatalog/src/test/e2e/templeton/deployers/modify_webhcat_config.sh
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/src/test/e2e/templeton/deployers/modify_webhcat_config.sh?rev=1674988&view=auto
==============================================================================
---
hive/trunk/hcatalog/src/test/e2e/templeton/deployers/modify_webhcat_config.sh
(added)
+++
hive/trunk/hcatalog/src/test/e2e/templeton/deployers/modify_webhcat_config.sh
Mon Apr 20 21:24:51 2015
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+#This script stops webhcat server, copies pre-canned webhcat-site.xml file
required for updateConfig test suite to the webhcat config directory,
+#then starts the webhcat server
+
+
+source ./env.sh
+
+echo "Stopping WebHCat...";
+${HIVE_HOME}/hcatalog/sbin/webhcat_server.sh stop;
+
+if [ ${HIVE_HOME}/hcatalog/etc/webhcat/webhcat-site.xml ]; then
+ mkdir -p ${CONF_BACKUP};
+ cp ${HIVE_HOME}/hcatalog/etc/webhcat/webhcat-site.xml
${CONF_BACKUP}/webhcat-site.xml;
+fi
+
+cp
${PROJ_HOME}/hcatalog/src/test/e2e/templeton/deployers/config/webhcat/webhcat-site.updateConfig.xml
${HIVE_HOME}/hcatalog/etc/webhcat/webhcat-site.xml
+
+echo "Starting WebHCat..."
+${HIVE_HOME}/hcatalog/sbin/webhcat_server.sh start
+
+jps;
Added:
hive/trunk/hcatalog/src/test/e2e/templeton/deployers/restore_webhcat_config.sh
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/src/test/e2e/templeton/deployers/restore_webhcat_config.sh?rev=1674988&view=auto
==============================================================================
---
hive/trunk/hcatalog/src/test/e2e/templeton/deployers/restore_webhcat_config.sh
(added)
+++
hive/trunk/hcatalog/src/test/e2e/templeton/deployers/restore_webhcat_config.sh
Mon Apr 20 21:24:51 2015
@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+#This script stops webhcat server and restores the webhcat server
configurations to the one before test run
+
+
+source ./env.sh
+
+echo "Stopping WebHCat...";
+${HIVE_HOME}/hcatalog/sbin/webhcat_server.sh stop;
+
+rm ${HIVE_HOME}/hcatalog/etc/webhcat/webhcat-site.xml
+
+if [ ${CONF_BACKUP}/webhcat-site.xml ]; then
+ cp ${CONF_BACKUP}/webhcat-site.xml
${HIVE_HOME}/hcatalog/etc/webhcat/webhcat-site.xml
+fi
+
+rm -rf ${CONF_BACKUP};
+
Added: hive/trunk/hcatalog/src/test/e2e/templeton/tests/modifyConfiguration.conf
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/src/test/e2e/templeton/tests/modifyConfiguration.conf?rev=1674988&view=auto
==============================================================================
--- hive/trunk/hcatalog/src/test/e2e/templeton/tests/modifyConfiguration.conf
(added)
+++ hive/trunk/hcatalog/src/test/e2e/templeton/tests/modifyConfiguration.conf
Mon Apr 20 21:24:51 2015
@@ -0,0 +1,67 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+###############################################################################
+# curl command tests for templeton
+#
+#
+
+#use Yahoo::Miners::Test::PigSetup;
+
+#PigSetup::setup();
+
+#my $me = `whoami`;
+#chomp $me;
+$cfg =
+{
+ 'driver' => 'Curl',
+
+ 'groups' =>
+ [
+##=============================================================================================================
+ {
+ 'name' => 'ModifyWebhcatConfig',
+ 'tests' =>
+ [
+ {
+ #test to verify templeton.mapper.memory.mb update
+ 'num' => 1,
+ 'setup' => [
+ {
+ 'method' => 'POST',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/ddl',
+ 'status_code' => 200,
+ 'post_options' => ['user.name=:UNAME:','exec=drop table if
exists ratings;CREATE EXTERNAL TABLE ratings(userid INT, movieid INT, ratings
FLOAT, unixtime STRING)
+ ROW FORMAT DELIMITED FIELDS TERMINATED BY
\':\' location \':INPDIR_HDFS:/ratings/\';drop table if exists testtable;create
table testtable
+ (userid INT, movieid INT, ratings FLOAT,
unixtime STRING) location \':INPDIR_HDFS:/testtable/\';'],
+ 'json_field_substr_match' => {'stderr' => 'OK'}
+ }
+ ],
+ 'method' => 'POST',
+ 'url' => ':TEMPLETON_URL:/templeton/v1/hive',
+ 'post_options' => ['user.name=:UNAME:','execute=INSERT OVERWRITE TABLE
testtable SELECT * from ratings','statusdir=:OUTDIR:/status'],
+ 'json_field_substr_match' => { 'id' => '\d+'},
+ #results
+ 'status_code' => 200,
+ 'check_job_created' => 1,
+ 'check_job_complete' => 'FAILURE',
+ }
+ ]
+ }
+ ]
+},
+ ;
\ No newline at end of file
