Author: daijy Date: Fri Dec 20 21:56:10 2013 New Revision: 1552815 URL: http://svn.apache.org/r1552815 Log: HIVE-5957: Fix HCatalog Unit tests on Windows
Modified: hive/trunk/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatUtil.java hive/trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/HCatBaseTest.java hive/trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatPartitionPublish.java hive/trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestMultiOutputFormat.java hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/HcatTestUtils.java hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestMultiOutputFormat.java hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoader.java hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoaderStorer.java hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorerMulti.java hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorerWrapper.java hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java hive/trunk/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java hive/trunk/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHCatHBaseInputFormat.java hive/trunk/hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/SkeletonHBaseTest.java hive/trunk/hcatalog/webhcat/java-client/src/test/java/org/apache/hcatalog/api/TestHCatClient.java hive/trunk/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java hive/trunk/itests/hcatalog-unit/src/test/java/org/apache/hcatalog/mapreduce/TestSequenceFileReadWrite.java hive/trunk/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/hbase/TestPigHBaseStorageHandler.java hive/trunk/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/mapreduce/TestSequenceFileReadWrite.java Modified: hive/trunk/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatUtil.java URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatUtil.java?rev=1552815&r1=1552814&r2=1552815&view=diff ============================================================================== --- hive/trunk/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatUtil.java (original) +++ hive/trunk/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatUtil.java Fri Dec 20 21:56:10 2013 @@ -21,6 +21,7 @@ package org.apache.hive.hcatalog.common; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.File; import java.io.IOException; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; @@ -645,4 +646,10 @@ public class HCatUtil { return true; return false; } + /** + * Used by various tests to make sure the path is safe for Windows + */ + public static String makePathASafeFileName(String filePath) { + return new File(filePath).getPath().replaceAll("\\\\", "/"); + } } Modified: hive/trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/HCatBaseTest.java URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/HCatBaseTest.java?rev=1552815&r1=1552814&r2=1552815&view=diff ============================================================================== --- hive/trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/HCatBaseTest.java (original) +++ hive/trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/HCatBaseTest.java Fri Dec 20 21:56:10 2013 @@ -41,8 +41,7 @@ import java.io.IOException; */ public class HCatBaseTest { protected static final Logger LOG = LoggerFactory.getLogger(HCatBaseTest.class); - protected static final String TEST_DATA_DIR = System.getProperty("user.dir") + - "/build/test/data/" + HCatBaseTest.class.getCanonicalName(); + protected static final String TEST_DATA_DIR = org.apache.hive.hcatalog.mapreduce.HCatBaseTest.TEST_DATA_DIR; protected static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse"; protected HiveConf hiveConf = null; Modified: hive/trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatPartitionPublish.java URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatPartitionPublish.java?rev=1552815&r1=1552814&r2=1552815&view=diff ============================================================================== --- hive/trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatPartitionPublish.java (original) +++ hive/trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatPartitionPublish.java Fri Dec 20 21:56:10 2013 @@ -79,12 +79,12 @@ public class TestHCatPartitionPublish { @BeforeClass public static void setup() throws Exception { + File workDir = org.apache.hive.hcatalog.mapreduce.TestHCatPartitionPublish.handleWorkDir(); conf.set("yarn.scheduler.capacity.root.queues", "default"); conf.set("yarn.scheduler.capacity.root.default.capacity", "100"); fs = FileSystem.get(conf); - System.setProperty("hadoop.log.dir", new File(fs.getWorkingDirectory() - .toString(), "/logs").getAbsolutePath()); + System.setProperty("hadoop.log.dir", new File(workDir, "/logs").getAbsolutePath()); // LocalJobRunner does not work with mapreduce OutputCommitter. So need // to use MiniMRCluster. MAPREDUCE-2350 mrCluster = new MiniMRCluster(1, fs.getUri().toString(), 1, null, null, @@ -99,6 +99,7 @@ public class TestHCatPartitionPublish { MetaStoreUtils.startMetaStore(msPort, ShimLoader .getHadoopThriftAuthBridge()); + Thread.sleep(10000); isServerRunning = true; securityManager = System.getSecurityManager(); System.setSecurityManager(new NoExitSecurityManager()); Modified: hive/trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestMultiOutputFormat.java URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestMultiOutputFormat.java?rev=1552815&r1=1552814&r2=1552815&view=diff ============================================================================== --- hive/trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestMultiOutputFormat.java (original) +++ hive/trunk/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestMultiOutputFormat.java Fri Dec 20 21:56:10 2013 @@ -67,7 +67,7 @@ public class TestMultiOutputFormat { private static final Logger LOG = LoggerFactory.getLogger(TestMultiOutputFormat.class); private static File workDir; - private static Configuration mrConf = null; + private static JobConf mrConf = null; private static FileSystem fs = null; private static MiniMRCluster mrCluster = null; @@ -82,9 +82,8 @@ public class TestMultiOutputFormat { System.setProperty("hadoop.log.dir", new File(workDir, "/logs").getAbsolutePath()); // LocalJobRunner does not work with mapreduce OutputCommitter. So need // to use MiniMRCluster. MAPREDUCE-2350 - mrCluster = new MiniMRCluster(1, fs.getUri().toString(), 1, null, null, - new JobConf(conf)); - mrConf = mrCluster.createJobConf(); + mrConf = new JobConf(conf); + mrCluster = new MiniMRCluster(1, fs.getUri().toString(), 1, null, null, mrConf); } private static void createWorkDir() throws IOException { Modified: hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/HcatTestUtils.java URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/HcatTestUtils.java?rev=1552815&r1=1552814&r2=1552815&view=diff ============================================================================== --- hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/HcatTestUtils.java (original) +++ hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/HcatTestUtils.java Fri Dec 20 21:56:10 2013 @@ -97,4 +97,10 @@ public class HcatTestUtils { } } + /** + * Used by various tests to make sure the path is safe for Windows + */ + public static String makePathASafeFileName(String filePath) { + return new File(filePath).getPath().replaceAll("\\\\", "/"); + } } Modified: hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java?rev=1552815&r1=1552814&r2=1552815&view=diff ============================================================================== --- hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java (original) +++ hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java Fri Dec 20 21:56:10 2013 @@ -25,6 +25,7 @@ import org.apache.hadoop.hive.conf.HiveC import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hive.hcatalog.common.HCatUtil; import org.apache.pig.PigServer; import org.junit.Assert; import org.junit.Before; @@ -40,8 +41,8 @@ import java.io.IOException; */ public class HCatBaseTest { protected static final Logger LOG = LoggerFactory.getLogger(HCatBaseTest.class); - protected static final String TEST_DATA_DIR = - "/tmp/build/test/data/" + HCatBaseTest.class.getCanonicalName(); + public static final String TEST_DATA_DIR = HCatUtil.makePathASafeFileName(System.getProperty("user.dir") + + "/build/test/data/" + HCatBaseTest.class.getCanonicalName() + "-" + System.currentTimeMillis()); protected static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse"; protected HiveConf hiveConf = null; Modified: hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java?rev=1552815&r1=1552814&r2=1552815&view=diff ============================================================================== --- hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java (original) +++ hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java Fri Dec 20 21:56:10 2013 @@ -76,13 +76,17 @@ public class TestHCatPartitionPublish { private static SecurityManager securityManager; private static Configuration conf = new Configuration(true); - @BeforeClass - public static void setup() throws Exception { - String testDir = System.getProperty("test.tmp.dir", "./"); + public static File handleWorkDir() throws IOException { + String testDir = System.getProperty("test.data.dir", "./"); testDir = testDir + "/test_hcat_partitionpublish_" + Math.abs(new Random().nextLong()) + "/"; File workDir = new File(new File(testDir).getCanonicalPath()); FileUtil.fullyDelete(workDir); workDir.mkdirs(); + return workDir; + } + @BeforeClass + public static void setup() throws Exception { + File workDir = handleWorkDir(); conf.set("yarn.scheduler.capacity.root.queues", "default"); conf.set("yarn.scheduler.capacity.root.default.capacity", "100"); Modified: hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestMultiOutputFormat.java URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestMultiOutputFormat.java?rev=1552815&r1=1552814&r2=1552815&view=diff ============================================================================== --- hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestMultiOutputFormat.java (original) +++ hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestMultiOutputFormat.java Fri Dec 20 21:56:10 2013 @@ -64,7 +64,7 @@ public class TestMultiOutputFormat { private static final Logger LOG = LoggerFactory.getLogger(TestMultiOutputFormat.class); private static File workDir; - private static Configuration mrConf = null; + private static JobConf mrConf = null; private static FileSystem fs = null; private static MiniMRCluster mrCluster = null; @@ -79,9 +79,8 @@ public class TestMultiOutputFormat { System.setProperty("hadoop.log.dir", new File(workDir, "/logs").getAbsolutePath()); // LocalJobRunner does not work with mapreduce OutputCommitter. So need // to use MiniMRCluster. MAPREDUCE-2350 - mrCluster = new MiniMRCluster(1, fs.getUri().toString(), 1, null, null, - new JobConf(conf)); - mrConf = mrCluster.createJobConf(); + mrConf = new JobConf(conf); + mrCluster = new MiniMRCluster(1, fs.getUri().toString(), 1, null, null, mrConf); } private static void createWorkDir() throws IOException { Modified: hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoader.java URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoader.java?rev=1552815&r1=1552814&r2=1552815&view=diff ============================================================================== --- hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoader.java (original) +++ hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoader.java Fri Dec 20 21:56:10 2013 @@ -45,6 +45,7 @@ import org.apache.hadoop.mapreduce.Job; import org.apache.hcatalog.HcatTestUtils; import org.apache.hcatalog.common.HCatConstants; import org.apache.hcatalog.data.Pair; +import org.apache.hive.hcatalog.common.HCatUtil; import org.apache.pig.ExecType; import org.apache.pig.PigServer; import org.apache.pig.ResourceStatistics; @@ -60,8 +61,9 @@ import org.junit.Test; * @deprecated Use/modify {@link org.apache.hive.hcatalog.pig.TestHCatLoader} instead */ public class TestHCatLoader { - private static final String TEST_DATA_DIR = System.getProperty("java.io.tmpdir") + File.separator - + TestHCatLoader.class.getCanonicalName() + "-" + System.currentTimeMillis(); + private static final String TEST_DATA_DIR = HCatUtil.makePathASafeFileName( + System.getProperty("java.io.tmpdir") + File.separator + TestHCatLoader.class.getCanonicalName() + "-" + + System.currentTimeMillis()); private static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse"; private static final String BASIC_FILE_NAME = TEST_DATA_DIR + "/basic.input.data"; private static final String COMPLEX_FILE_NAME = TEST_DATA_DIR + "/complex.input.data"; @@ -409,8 +411,8 @@ public class TestHCatLoader { assertEquals(0, driver.run("drop table if exists " + tbl).getResponseCode()); assertEquals(0, driver.run("create external table " + tbl + " (a string, b boolean) row format delimited fields terminated by '\t'" + - " stored as textfile location 'file://" + - inputDataDir.getAbsolutePath() + "'").getResponseCode()); + " stored as textfile location 'file:///" + + inputDataDir.getPath().replaceAll("\\\\", "/") + "'").getResponseCode()); Properties properties = new Properties(); properties.setProperty(HCatConstants.HCAT_DATA_CONVERT_BOOLEAN_TO_INTEGER, "true"); Modified: hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoaderStorer.java URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoaderStorer.java?rev=1552815&r1=1552814&r2=1552815&view=diff ============================================================================== --- hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoaderStorer.java (original) +++ hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoaderStorer.java Fri Dec 20 21:56:10 2013 @@ -69,7 +69,7 @@ public class TestHCatLoaderStorer extend " (my_small_int smallint, my_tiny_int tinyint)" + " row format delimited fields terminated by '\t' stored as textfile").getResponseCode()); Assert.assertEquals(0, driver.run("load data local inpath '" + - dataDir.getAbsolutePath() + "' into table " + readTblName).getResponseCode()); + dataDir.getPath().replaceAll("\\\\", "/") + "' into table " + readTblName).getResponseCode()); PigServer server = new PigServer(ExecType.LOCAL); server.registerQuery( @@ -104,7 +104,7 @@ public class TestHCatLoaderStorer extend String.format("%d\t%d", Short.MIN_VALUE, Byte.MIN_VALUE), String.format("%d\t%d", Short.MAX_VALUE, Byte.MAX_VALUE) }); - smallTinyIntBoundsCheckHelper(writeDataFile.getAbsolutePath(), ExecJob.JOB_STATUS.COMPLETED); + smallTinyIntBoundsCheckHelper(writeDataFile.getPath().replaceAll("\\\\", "/"), ExecJob.JOB_STATUS.COMPLETED); // Values outside the column type bounds will fail at runtime. HcatTestUtils.createTestDataFile(TEST_DATA_DIR + "/shortTooSmall.tsv", new String[]{ Modified: hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorerMulti.java URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorerMulti.java?rev=1552815&r1=1552814&r2=1552815&view=diff ============================================================================== --- hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorerMulti.java (original) +++ hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorerMulti.java Fri Dec 20 21:56:10 2013 @@ -41,8 +41,7 @@ import org.apache.pig.PigServer; * @deprecated Use/modify {@link org.apache.hive.hcatalog.pig.TestHCatStorerMulti} instead */ public class TestHCatStorerMulti extends TestCase { - private static final String TEST_DATA_DIR = System.getProperty("user.dir") + - "/build/test/data/" + TestHCatStorerMulti.class.getCanonicalName(); + private static final String TEST_DATA_DIR = org.apache.hive.hcatalog.pig.TestHCatStorerMulti.TEST_DATA_DIR; private static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse"; private static final String INPUT_FILE_NAME = TEST_DATA_DIR + "/input.data"; Modified: hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorerWrapper.java URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorerWrapper.java?rev=1552815&r1=1552814&r2=1552815&view=diff ============================================================================== --- hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorerWrapper.java (original) +++ hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorerWrapper.java Fri Dec 20 21:56:10 2013 @@ -48,7 +48,7 @@ public class TestHCatStorerWrapper exten @Test public void testStoreExternalTableWithExternalDir() throws IOException, CommandNeedRetryException{ - File tmpExternalDir = new File(SystemUtils.getJavaIoTmpDir(), UUID.randomUUID().toString()); + File tmpExternalDir = new File(TEST_DATA_DIR, UUID.randomUUID().toString()); tmpExternalDir.deleteOnExit(); String part_val = "100"; @@ -71,11 +71,11 @@ public class TestHCatStorerWrapper exten server.setBatchOn(); logAndRegister(server, "A = load '"+INPUT_FILE_NAME+"' as (a:int, b:chararray);"); logAndRegister(server, "store A into 'default.junit_external' using " + HCatStorerWrapper.class.getName() - + "('c=" + part_val + "','" + tmpExternalDir.getAbsolutePath() + "');"); + + "('c=" + part_val + "','" + tmpExternalDir.getPath().replaceAll("\\\\", "/") + "');"); server.executeBatch(); Assert.assertTrue(tmpExternalDir.exists()); - Assert.assertTrue(new File(tmpExternalDir.getAbsoluteFile() + "/" + "part-m-00000").exists()); + Assert.assertTrue(new File(tmpExternalDir.getPath().replaceAll("\\\\", "/") + "/" + "part-m-00000").exists()); driver.run("select * from junit_external"); ArrayList<String> res = new ArrayList<String>(); Modified: hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java?rev=1552815&r1=1552814&r2=1552815&view=diff ============================================================================== --- hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java (original) +++ hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java Fri Dec 20 21:56:10 2013 @@ -43,6 +43,7 @@ import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.mapreduce.Job; import org.apache.hive.hcatalog.HcatTestUtils; +import org.apache.hive.hcatalog.common.HCatUtil; import org.apache.hive.hcatalog.common.HCatConstants; import org.apache.hive.hcatalog.data.Pair; import org.apache.pig.ExecType; @@ -57,8 +58,8 @@ import org.junit.Before; import org.junit.Test; public class TestHCatLoader { - private static final String TEST_DATA_DIR = System.getProperty("java.io.tmpdir") + File.separator - + TestHCatLoader.class.getCanonicalName() + "-" + System.currentTimeMillis(); + private static final String TEST_DATA_DIR = HCatUtil.makePathASafeFileName(System.getProperty("java.io.tmpdir") + + File.separator + TestHCatLoader.class.getCanonicalName() + "-" + System.currentTimeMillis()); private static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse"; private static final String BASIC_FILE_NAME = TEST_DATA_DIR + "/basic.input.data"; private static final String COMPLEX_FILE_NAME = TEST_DATA_DIR + "/complex.input.data"; @@ -420,7 +421,7 @@ public class TestHCatLoader { assertEquals(0, driver.run("drop table if exists " + tbl).getResponseCode()); assertEquals(0, driver.run("create external table " + tbl + " (a string, b boolean) row format delimited fields terminated by '\t'" + - " stored as textfile location 'file://" + + " stored as textfile location 'file:///" + inputDataDir.getPath().replaceAll("\\\\", "/") + "'").getResponseCode()); Properties properties = new Properties(); Modified: hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java?rev=1552815&r1=1552814&r2=1552815&view=diff ============================================================================== --- hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java (original) +++ hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java Fri Dec 20 21:56:10 2013 @@ -33,13 +33,15 @@ import org.apache.hadoop.hive.conf.HiveC import org.apache.hadoop.hive.ql.CommandNeedRetryException; import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hive.hcatalog.common.HCatUtil; import org.apache.hive.hcatalog.data.Pair; import org.apache.pig.ExecType; import org.apache.pig.PigServer; public class TestHCatStorerMulti extends TestCase { - private static final String TEST_DATA_DIR = - "/tmp/build/test/data/" + TestHCatStorerMulti.class.getCanonicalName(); + public static final String TEST_DATA_DIR = HCatUtil.makePathASafeFileName( + System.getProperty("user.dir") + "/build/test/data/" + + TestHCatStorerMulti.class.getCanonicalName() + "-" + System.currentTimeMillis()); private static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse"; private static final String INPUT_FILE_NAME = TEST_DATA_DIR + "/input.data"; Modified: hive/trunk/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java?rev=1552815&r1=1552814&r2=1552815&view=diff ============================================================================== --- hive/trunk/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java (original) +++ hive/trunk/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java Fri Dec 20 21:56:10 2013 @@ -165,12 +165,8 @@ public abstract class SkeletonHBaseTest protected int usageCount = 0; public Context(String handle) { - try { - testDir = new File(TEST_DIR + "/test_" + handle + "_" + Math.abs(new Random().nextLong()) + "/").getCanonicalPath(); - System.out.println("Cluster work directory: " + testDir); - } catch (IOException e) { - throw new IllegalStateException("Failed to generate testDir", e); - } + testDir = new File(TEST_DIR + "/test_" + handle + "_" + Math.abs(new Random().nextLong()) + "/").getPath(); + System.out.println("Cluster work directory: " + testDir); } public void start() { Modified: hive/trunk/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHCatHBaseInputFormat.java URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHCatHBaseInputFormat.java?rev=1552815&r1=1552814&r2=1552815&view=diff ============================================================================== --- hive/trunk/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHCatHBaseInputFormat.java (original) +++ hive/trunk/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHCatHBaseInputFormat.java Fri Dec 20 21:56:10 2013 @@ -64,7 +64,7 @@ import org.apache.hcatalog.cli.HCatDrive import org.apache.hcatalog.cli.SemanticAnalysis.HCatSemanticAnalyzer; import org.apache.hcatalog.common.HCatConstants; import org.apache.hcatalog.common.HCatException; -import org.apache.hcatalog.common.HCatUtil; +import org.apache.hive.hcatalog.common.HCatUtil; import org.apache.hcatalog.data.HCatRecord; import org.apache.hcatalog.data.schema.HCatFieldSchema; import org.apache.hcatalog.data.schema.HCatSchema; @@ -188,7 +188,7 @@ public class TestHCatHBaseInputFormat ex String databaseName = newTableName("MyDatabase"); //Table name will be lower case unless specified by hbase.table.name property String hbaseTableName = (databaseName + "." + tableName).toLowerCase(); - String db_dir = getTestDir() + "/hbasedb"; + String db_dir = HCatUtil.makePathASafeFileName(getTestDir() + "/hbasedb"); String dbquery = "CREATE DATABASE IF NOT EXISTS " + databaseName + " LOCATION '" + db_dir + "'"; Modified: hive/trunk/hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/SkeletonHBaseTest.java URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/SkeletonHBaseTest.java?rev=1552815&r1=1552814&r2=1552815&view=diff ============================================================================== --- hive/trunk/hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/SkeletonHBaseTest.java (original) +++ hive/trunk/hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/SkeletonHBaseTest.java Fri Dec 20 21:56:10 2013 @@ -172,12 +172,8 @@ public abstract class SkeletonHBaseTest protected int usageCount = 0; public Context(String handle) { - try { - testDir = new File(TEST_DIR + "/test_" + handle + "_" + Math.abs(new Random().nextLong()) + "/").getCanonicalPath(); - System.out.println("Cluster work directory: " + testDir); - } catch (IOException e) { - throw new IllegalStateException("Failed to generate testDir", e); - } + testDir = new File(TEST_DIR + "/test_" + handle + "_" + Math.abs(new Random().nextLong()) + "/").getPath(); + System.out.println("Cluster work directory: " + testDir); } public void start() { Modified: hive/trunk/hcatalog/webhcat/java-client/src/test/java/org/apache/hcatalog/api/TestHCatClient.java URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/java-client/src/test/java/org/apache/hcatalog/api/TestHCatClient.java?rev=1552815&r1=1552814&r2=1552815&view=diff ============================================================================== --- hive/trunk/hcatalog/webhcat/java-client/src/test/java/org/apache/hcatalog/api/TestHCatClient.java (original) +++ hive/trunk/hcatalog/webhcat/java-client/src/test/java/org/apache/hcatalog/api/TestHCatClient.java Fri Dec 20 21:56:10 2013 @@ -128,7 +128,8 @@ public class TestHCatClient { assertTrue(testDb.getProperties().size() == 0); String warehouseDir = System .getProperty("test.warehouse.dir", "/user/hive/warehouse"); - String expectedDir = warehouseDir.replaceAll("\\\\", "/").replaceFirst("pfile:///", "pfile:/"); + String expectedDir = org.apache.hive.hcatalog.api.TestHCatClient.fixPath(warehouseDir). + replaceFirst("pfile:///", "pfile:/"); assertEquals(expectedDir + "/" + db + ".db", testDb.getLocation()); ArrayList<HCatFieldSchema> cols = new ArrayList<HCatFieldSchema>(); cols.add(new HCatFieldSchema("id", Type.INT, "id comment")); Modified: hive/trunk/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java?rev=1552815&r1=1552814&r2=1552815&view=diff ============================================================================== --- hive/trunk/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java (original) +++ hive/trunk/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java Fri Dec 20 21:56:10 2013 @@ -53,6 +53,8 @@ import static org.junit.Assert.assertFal import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertArrayEquals; +import org.apache.hadoop.util.Shell; + public class TestHCatClient { private static final Logger LOG = LoggerFactory.getLogger(TestHCatClient.class); private static final String msPort = "20101"; @@ -100,7 +102,16 @@ public class TestHCatClient { System.setProperty(HiveConf.ConfVars.PREEXECHOOKS.varname, " "); System.setProperty(HiveConf.ConfVars.POSTEXECHOOKS.varname, " "); } - + public static String fixPath(String path) { + if(!Shell.WINDOWS) { + return path; + } + String expectedDir = path.replaceAll("\\\\", "/"); + if (!expectedDir.startsWith("/")) { + expectedDir = "/" + expectedDir; + } + return expectedDir; + } @Test public void testBasicDDLCommands() throws Exception { String db = "testdb"; @@ -121,7 +132,7 @@ public class TestHCatClient { assertTrue(testDb.getProperties().size() == 0); String warehouseDir = System .getProperty("test.warehouse.dir", "/user/hive/warehouse"); - String expectedDir = warehouseDir.replaceAll("\\\\", "/").replaceFirst("pfile:///", "pfile:/"); + String expectedDir = fixPath(warehouseDir).replaceFirst("pfile:///", "pfile:/"); assertEquals(expectedDir + "/" + db + ".db", testDb.getLocation()); ArrayList<HCatFieldSchema> cols = new ArrayList<HCatFieldSchema>(); cols.add(new HCatFieldSchema("id", Type.INT, "id comment")); Modified: hive/trunk/itests/hcatalog-unit/src/test/java/org/apache/hcatalog/mapreduce/TestSequenceFileReadWrite.java URL: http://svn.apache.org/viewvc/hive/trunk/itests/hcatalog-unit/src/test/java/org/apache/hcatalog/mapreduce/TestSequenceFileReadWrite.java?rev=1552815&r1=1552814&r2=1552815&view=diff ============================================================================== --- hive/trunk/itests/hcatalog-unit/src/test/java/org/apache/hcatalog/mapreduce/TestSequenceFileReadWrite.java (original) +++ hive/trunk/itests/hcatalog-unit/src/test/java/org/apache/hcatalog/mapreduce/TestSequenceFileReadWrite.java Fri Dec 20 21:56:10 2013 @@ -43,7 +43,7 @@ import org.apache.hadoop.mapreduce.lib.i import org.apache.hcatalog.HcatTestUtils; import org.apache.hcatalog.common.HCatConstants; import org.apache.hcatalog.common.HCatException; -import org.apache.hcatalog.common.HCatUtil; +import org.apache.hive.hcatalog.common.HCatUtil; import org.apache.hcatalog.data.DefaultHCatRecord; import org.apache.hcatalog.data.schema.HCatFieldSchema; import org.apache.hcatalog.data.schema.HCatSchema; @@ -72,8 +72,8 @@ public class TestSequenceFileReadWrite { dataDir = new File(System.getProperty("java.io.tmpdir") + File.separator + TestSequenceFileReadWrite.class.getCanonicalName() + "-" + System.currentTimeMillis()); hiveConf = new HiveConf(this.getClass()); - warehouseDir = new File(dataDir, "warehouse").getAbsolutePath(); - inputFileName = new File(dataDir, "input.data").getAbsolutePath(); + warehouseDir = HCatUtil.makePathASafeFileName(dataDir + File.separator + "warehouse"); + inputFileName = HCatUtil.makePathASafeFileName(dataDir + File.separator + "input.data"); hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, ""); hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, ""); hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false"); Modified: hive/trunk/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/hbase/TestPigHBaseStorageHandler.java URL: http://svn.apache.org/viewvc/hive/trunk/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/hbase/TestPigHBaseStorageHandler.java?rev=1552815&r1=1552814&r2=1552815&view=diff ============================================================================== --- hive/trunk/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/hbase/TestPigHBaseStorageHandler.java (original) +++ hive/trunk/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/hbase/TestPigHBaseStorageHandler.java Fri Dec 20 21:56:10 2013 @@ -47,6 +47,7 @@ import org.apache.hadoop.hive.conf.HiveC import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hive.hcatalog.common.HCatUtil; import org.apache.pig.ExecType; import org.apache.pig.PigServer; import org.apache.pig.data.DataType; @@ -138,7 +139,7 @@ public class TestPigHBaseStorageHandler String databaseName = newTableName("MyDatabase"); //Table name will be lower case unless specified by hbase.table.name property String hbaseTableName = "testTable"; - String db_dir = getTestDir() + "/hbasedb"; + String db_dir = HCatUtil.makePathASafeFileName(getTestDir() + "/hbasedb"); String dbQuery = "CREATE DATABASE IF NOT EXISTS " + databaseName + " LOCATION '" + db_dir + "'"; @@ -193,7 +194,7 @@ public class TestPigHBaseStorageHandler String databaseName = newTableName("MyDatabase"); //Table name will be lower case unless specified by hbase.table.name property String hbaseTableName = (databaseName + "." + tableName).toLowerCase(); - String db_dir = getTestDir() + "/hbasedb"; + String db_dir = HCatUtil.makePathASafeFileName(getTestDir() + "/hbasedb"); String dbQuery = "CREATE DATABASE IF NOT EXISTS " + databaseName + " LOCATION '" + db_dir + "'"; @@ -255,7 +256,7 @@ public class TestPigHBaseStorageHandler String databaseName = newTableName("MyDatabase"); //Table name will be lower case unless specified by hbase.table.name property String hbaseTableName = (databaseName + "." + tableName).toLowerCase(); - String db_dir = getTestDir() + "/hbasedb"; + String db_dir = HCatUtil.makePathASafeFileName(getTestDir() + "/hbasedb"); String POPTXT_FILE_NAME = db_dir+"testfile.txt"; float f = -100.1f; Modified: hive/trunk/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/mapreduce/TestSequenceFileReadWrite.java URL: http://svn.apache.org/viewvc/hive/trunk/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/mapreduce/TestSequenceFileReadWrite.java?rev=1552815&r1=1552814&r2=1552815&view=diff ============================================================================== --- hive/trunk/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/mapreduce/TestSequenceFileReadWrite.java (original) +++ hive/trunk/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/mapreduce/TestSequenceFileReadWrite.java Fri Dec 20 21:56:10 2013 @@ -69,8 +69,8 @@ public class TestSequenceFileReadWrite { dataDir = new File(System.getProperty("java.io.tmpdir") + File.separator + TestSequenceFileReadWrite.class.getCanonicalName() + "-" + System.currentTimeMillis()); hiveConf = new HiveConf(this.getClass()); - warehouseDir = new File(dataDir, "warehouse").getAbsolutePath(); - inputFileName = new File(dataDir, "input.data").getAbsolutePath(); + warehouseDir = HCatUtil.makePathASafeFileName(dataDir + File.separator + "warehouse"); + inputFileName = HCatUtil.makePathASafeFileName(dataDir + File.separator + "input.data"); hiveConf = new HiveConf(this.getClass()); hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, ""); hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");