OOZIE-2613 Upgrade hive version from 0.13.1 to 1.2.0 (abhishekbafna via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/bd544528 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/bd544528 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/bd544528 Branch: refs/heads/oya Commit: bd54452808335cbadc48d4aacd0bd23374bed928 Parents: e0016c9 Author: Robert Kanter <rkan...@cloudera.com> Authored: Mon Oct 10 23:52:34 2016 -0700 Committer: Robert Kanter <rkan...@cloudera.com> Committed: Mon Oct 10 23:52:34 2016 -0700 ---------------------------------------------------------------------- .../action/hadoop/TestFsActionExecutor.java | 3 ++ .../command/coord/TestCoordRerunXCommand.java | 3 ++ .../oozie/dependency/TestHCatURIHandler.java | 3 ++ .../org/apache/oozie/test/MiniHCatServer.java | 10 ---- docs/src/site/twiki/DG_QuickStart.twiki | 2 +- docs/src/site/twiki/ENG_Building.twiki | 2 +- pom.xml | 14 +++-- release-log.txt | 1 + .../oozie/action/hadoop/TestHiveMain.java | 4 +- sharelib/hive2/pom.xml | 56 ++++++++++++++++++++ .../action/hadoop/TestHive2ActionExecutor.java | 1 + 11 files changed, 76 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/bd544528/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java index 5345ae6..b27ede7 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java @@ -35,6 +35,7 @@ import org.apache.oozie.service.HadoopAccessorService; import org.apache.oozie.service.Services; import org.apache.oozie.service.URIHandlerService; import org.apache.oozie.service.WorkflowAppService; +import org.apache.oozie.test.XTestCase; import org.apache.oozie.util.XConfiguration; import org.apache.oozie.util.XmlUtils; import org.jdom.Element; @@ -303,6 +304,8 @@ public class TestFsActionExecutor extends ActionExecutorTestCase { } public void testDeleteHcat() throws Exception { + //setting current user as test user because directory structure created by HCat have current user permissions (755). + setSystemProperty(XTestCase.TEST_USER1_PROP, System.getProperty("user.name")); createTestTable(); addPartition(db, table, "year=2012;month=12;dt=02;country=us"); URI hcatURI = getHCatURI(db, table, "country=us;year=2012;month=12;dt=02"); http://git-wip-us.apache.org/repos/asf/oozie/blob/bd544528/core/src/test/java/org/apache/oozie/command/coord/TestCoordRerunXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/command/coord/TestCoordRerunXCommand.java b/core/src/test/java/org/apache/oozie/command/coord/TestCoordRerunXCommand.java index 891e6ec..9a05a85 100644 --- a/core/src/test/java/org/apache/oozie/command/coord/TestCoordRerunXCommand.java +++ b/core/src/test/java/org/apache/oozie/command/coord/TestCoordRerunXCommand.java @@ -60,6 +60,7 @@ import org.apache.oozie.service.StatusTransitService; import org.apache.oozie.service.URIHandlerService; import org.apache.oozie.store.StoreException; import org.apache.oozie.test.XDataTestCase; +import org.apache.oozie.test.XTestCase; import org.apache.oozie.util.DateUtils; import org.apache.oozie.util.IOUtils; import org.apache.oozie.util.XLog; @@ -569,6 +570,8 @@ public class TestCoordRerunXCommand extends XDataTestCase { * @throws Exception */ public void testCoordRerunCleanupForHCat() throws Exception { + //setting current user as test user because directory structure created by HCat have current user permissions (755). + setSystemProperty(XTestCase.TEST_USER1_PROP, System.getProperty("user.name")); super.setupHCatalogServer(); services = super.setupServicesForHCatalog(); services.init(); http://git-wip-us.apache.org/repos/asf/oozie/blob/bd544528/core/src/test/java/org/apache/oozie/dependency/TestHCatURIHandler.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/dependency/TestHCatURIHandler.java b/core/src/test/java/org/apache/oozie/dependency/TestHCatURIHandler.java index a49eba5..615f5e1 100644 --- a/core/src/test/java/org/apache/oozie/dependency/TestHCatURIHandler.java +++ b/core/src/test/java/org/apache/oozie/dependency/TestHCatURIHandler.java @@ -25,6 +25,7 @@ import org.apache.oozie.service.HCatAccessorService; import org.apache.oozie.service.Services; import org.apache.oozie.service.URIHandlerService; import org.apache.oozie.test.XHCatTestCase; +import org.apache.oozie.test.XTestCase; import org.junit.Test; public class TestHCatURIHandler extends XHCatTestCase { @@ -67,6 +68,8 @@ public class TestHCatURIHandler extends XHCatTestCase { @Test public void testExists() throws Exception { + //setting current user as test user because directory structure created by HCat have current user permissions (755). + setSystemProperty(XTestCase.TEST_USER1_PROP, System.getProperty("user.name")); createTestTable(); addPartition(db, table, "year=2012;month=12;dt=02;country=us"); http://git-wip-us.apache.org/repos/asf/oozie/blob/bd544528/core/src/test/java/org/apache/oozie/test/MiniHCatServer.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/test/MiniHCatServer.java b/core/src/test/java/org/apache/oozie/test/MiniHCatServer.java index 8699ff8..9b7a004 100644 --- a/core/src/test/java/org/apache/oozie/test/MiniHCatServer.java +++ b/core/src/test/java/org/apache/oozie/test/MiniHCatServer.java @@ -38,7 +38,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaStore; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hive.hcatalog.api.HCatAddPartitionDesc; import org.apache.hive.hcatalog.api.HCatClient; @@ -85,7 +84,6 @@ public class MiniHCatServer { initMetastoreServerConf(); } hcatClient = HCatClient.create(hiveConf); - resetDefaultDBCreation(); } public void shutdown() throws Exception { @@ -147,14 +145,6 @@ public class MiniHCatServer { Thread.sleep(10000L); } - public static void resetDefaultDBCreation() throws Exception { - // Need to do this, else default db will not be created for local metastores. - // TestHiveMain will fail with InvalidObjectException(message:There is no database named default) - Field declaredField = HMSHandler.class.getDeclaredField("createDefaultDB"); - declaredField.setAccessible(true); - declaredField.set(null, false); - } - public static void resetHiveConfStaticVariables() throws Exception { // HiveConf initializes location of hive-site.xml in static block. // So this is needed so that tests like TestHiveMain that create hive-site.xml don't fail. http://git-wip-us.apache.org/repos/asf/oozie/blob/bd544528/docs/src/site/twiki/DG_QuickStart.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/DG_QuickStart.twiki b/docs/src/site/twiki/DG_QuickStart.twiki index 3938ff4..1eb02a9 100644 --- a/docs/src/site/twiki/DG_QuickStart.twiki +++ b/docs/src/site/twiki/DG_QuickStart.twiki @@ -54,7 +54,7 @@ profile depending on the hadoop version used. -Dopenjpa.version=<version> - default 2.2.2 -Dxerces.version=<version> - default 2.10.0 -Dcurator.version=<version> - default 2.5.0 --Dhive.version=<version> - default 0.13.1 +-Dhive.version=<version> - default 0.13.1 for hadoop-1, 1.2.0 for hadoop-2 and hadoop-3 profile -Dhbase.version=<version> - default 0.94.2 </verbatim> http://git-wip-us.apache.org/repos/asf/oozie/blob/bd544528/docs/src/site/twiki/ENG_Building.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/ENG_Building.twiki b/docs/src/site/twiki/ENG_Building.twiki index f6c88d6..3560a9b 100644 --- a/docs/src/site/twiki/ENG_Building.twiki +++ b/docs/src/site/twiki/ENG_Building.twiki @@ -225,7 +225,7 @@ profile depending on the hadoop version used. -Dopenjpa.version=<version> - default 2.2.2 -Dxerces.version=<version> - default 2.10.0 -Dcurator.version=<version> - default 2.5.0 --Dhive.version=<version> - default 0.13.1 +-Dhive.version=<version> - default 0.13.1 for hadoop-1, 1.2.0 for hadoop-2 and hadoop-3 profile -Dhbase.version=<version> - default 0.94.2 </verbatim> http://git-wip-us.apache.org/repos/asf/oozie/blob/bd544528/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index a33d2c5..83a1d42 100644 --- a/pom.xml +++ b/pom.xml @@ -89,8 +89,8 @@ <clover.license>/home/jenkins/tools/clover/latest/lib/clover.license</clover.license> <!-- Sharelib component versions --> - <hive.version>0.13.1</hive.version> - <hive.jline.version>0.9.94</hive.jline.version> + <hive.version>1.2.0</hive.version> + <hive.jline.version>2.12</hive.jline.version> <pig.version>0.16.0</pig.version> <pig.classifier></pig.classifier> <sqoop.version>1.4.3</sqoop.version> @@ -106,7 +106,7 @@ <!-- Tomcat version --> <tomcat.version>6.0.44</tomcat.version> - + <jline.version>0.9.94</jline.version> <openjpa.version>2.4.1</openjpa.version> <xerces.version>2.10.0</xerces.version> <curator.version>2.5.0</curator.version> @@ -878,7 +878,7 @@ <dependency> <groupId>jline</groupId> <artifactId>jline</artifactId> - <version>0.9.94</version> + <version>${jline.version}</version> <exclusions> <exclusion> <groupId>junit</groupId> @@ -987,10 +987,6 @@ <groupId>org.apache.hive</groupId> </exclusion> <exclusion> - <artifactId>hive-common</artifactId> - <groupId>org.apache.hive</groupId> - </exclusion> - <exclusion> <artifactId>hive-serde</artifactId> <groupId>org.apache.hive</groupId> </exclusion> @@ -1860,6 +1856,8 @@ <hadoop.majorversion>1</hadoop.majorversion> <pig.classifier></pig.classifier> <sqoop.classifier>hadoop100</sqoop.classifier> + <hive.version>0.13.1</hive.version> + <hive.jline.version>0.9.94</hive.jline.version> </properties> </profile> <profile> http://git-wip-us.apache.org/repos/asf/oozie/blob/bd544528/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index a34525e..f73fbb8 100644 --- a/release-log.txt +++ b/release-log.txt @@ -5,6 +5,7 @@ OOZIE-2634 Queue dump command message is confusing when the queue is empty (andr -- Oozie 4.3.0 release +OOZIE-2613 Upgrade hive version from 0.13.1 to 1.2.0 (abhishekbafna via rkanter) OOZIE-2658 --driver-class-path can overwrite the classpath in SparkMain (gezapeti via rkanter) OOZIE-1814 Oozie should mask any passwords in logs and REST interfaces (andras.piros via rkanter) OOZIE-2622 ExtJS 2.2 is no longer available (rkanter) http://git-wip-us.apache.org/repos/asf/oozie/blob/bd544528/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java ---------------------------------------------------------------------- diff --git a/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java b/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java index 879ae2d..3026311 100644 --- a/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java +++ b/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java @@ -50,6 +50,7 @@ public class TestHiveMain extends MainTestCase { private String getHiveScript(String inputPath, String outputPath) { StringBuilder buffer = new StringBuilder(NEW_LINE); buffer.append("set -v;").append(NEW_LINE); + buffer.append("CREATE DATABASE IF NOT EXISTS default;").append(NEW_LINE); buffer.append("DROP TABLE IF EXISTS test;").append(NEW_LINE); buffer.append("CREATE EXTERNAL TABLE test (a INT) STORED AS"); buffer.append(NEW_LINE).append("TEXTFILE LOCATION '"); @@ -57,7 +58,6 @@ public class TestHiveMain extends MainTestCase { buffer.append("INSERT OVERWRITE DIRECTORY '"); buffer.append(outputPath).append("'").append(NEW_LINE); buffer.append("SELECT (a-1) FROM test;").append(NEW_LINE); - return buffer.toString(); } @@ -133,7 +133,6 @@ public class TestHiveMain extends MainTestCase { os = new FileOutputStream(hiveSite); jobConf.writeXml(os); os.close(); - MiniHCatServer.resetDefaultDBCreation(); MiniHCatServer.resetHiveConfStaticVariables(); HiveMain.main(null); } @@ -166,5 +165,4 @@ public class TestHiveMain extends MainTestCase { } return null; } - } http://git-wip-us.apache.org/repos/asf/oozie/blob/bd544528/sharelib/hive2/pom.xml ---------------------------------------------------------------------- diff --git a/sharelib/hive2/pom.xml b/sharelib/hive2/pom.xml index bc94fd5..3dd9a4d 100644 --- a/sharelib/hive2/pom.xml +++ b/sharelib/hive2/pom.xml @@ -40,6 +40,62 @@ <dependencies> <dependency> <groupId>org.apache.hive</groupId> + <artifactId>hive-cli</artifactId> + <scope>compile</scope> + <exclusions> + <exclusion> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-core</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.hive</groupId> + <artifactId>hive-hwi</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.hive</groupId> + <artifactId>hive-jdbc</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.hive</groupId> + <artifactId>hive-anttasks</artifactId> + </exclusion> + <exclusion> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + </exclusion> + <exclusion> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + </exclusion> + <exclusion> + <groupId>commons-codec</groupId> + <artifactId>commons-codec</artifactId> + </exclusion> + <exclusion> + <groupId>commons-logging</groupId> + <artifactId>commons-logging</artifactId> + </exclusion> + <exclusion> + <groupId>commons-logging</groupId> + <artifactId>commons-logging-api</artifactId> + </exclusion> + <exclusion> + <groupId>org.mockito</groupId> + <artifactId>mockito-all</artifactId> + </exclusion> + <exclusion> + <groupId>javax.servlet</groupId> + <artifactId>servlet-api</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.hive</groupId> + <artifactId>hive-exec</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> + <groupId>org.apache.hive</groupId> <artifactId>hive-beeline</artifactId> <scope>compile</scope> </dependency> http://git-wip-us.apache.org/repos/asf/oozie/blob/bd544528/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2ActionExecutor.java ---------------------------------------------------------------------- diff --git a/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2ActionExecutor.java b/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2ActionExecutor.java index b023b79..4818bb6 100644 --- a/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2ActionExecutor.java +++ b/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2ActionExecutor.java @@ -156,6 +156,7 @@ public class TestHive2ActionExecutor extends ActionExecutorTestCase { private String getHive2Script(String inputPath, String outputPath) { StringBuilder buffer = new StringBuilder(NEW_LINE); buffer.append("set -v;").append(NEW_LINE); + buffer.append("CREATE DATABASE IF NOT EXISTS default;").append(NEW_LINE); buffer.append("DROP TABLE IF EXISTS test;").append(NEW_LINE); buffer.append("CREATE EXTERNAL TABLE test (a INT) STORED AS"); buffer.append(NEW_LINE).append("TEXTFILE LOCATION '");