svn commit: r1530667 - /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
Author: daryn Date: Wed Oct 9 15:09:37 2013 New Revision: 1530667 URL: http://svn.apache.org/r1530667 Log: HADOOP-9470. eliminate duplicate FQN tests in different Hadoop modules (Ivan A. Veselovsky via daryn) Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1530667r1=1530666r2=1530667view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Wed Oct 9 15:09:37 2013 @@ -351,6 +351,9 @@ Release 2.3.0 - UNRELEASED HADOOP-9199. Cover package org.apache.hadoop.io with unit tests (Andrey Klochkov via jeagles) +HADOOP-9470. eliminate duplicate FQN tests in different Hadoop modules +(Ivan A. Veselovsky via daryn) + OPTIMIZATIONS HADOOP-9748. Reduce blocking on UGI.ensureInitialized (daryn)
svn commit: r1530671 - /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Author: daryn Date: Wed Oct 9 15:10:43 2013 New Revision: 1530671 URL: http://svn.apache.org/r1530671 Log: HADOOP-9470. eliminate duplicate FQN tests in different Hadoop modules (Ivan A. Veselovsky via daryn) Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1530671r1=1530670r2=1530671view=diff == --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Wed Oct 9 15:10:43 2013 @@ -64,6 +64,9 @@ Release 2.3.0 - UNRELEASED HADOOP-9199. Cover package org.apache.hadoop.io with unit tests (Andrey Klochkov via jeagles) +HADOOP-9470. eliminate duplicate FQN tests in different Hadoop modules +(Ivan A. Veselovsky via daryn) + OPTIMIZATIONS HADOOP-9748. Reduce blocking on UGI.ensureInitialized (daryn)
svn commit: r1530710 - in /hadoop/common/branches/branch-1: CHANGES.txt src/mapred/org/apache/hadoop/mapreduce/lib/db/FloatSplitter.java src/test/org/apache/hadoop/mapreduce/lib/db/TestSplitters.java
Author: jlowe Date: Wed Oct 9 16:53:52 2013 New Revision: 1530710 URL: http://svn.apache.org/r1530710 Log: MAPREDUCE-5569. FloatSplitter is not generating correct splits. Contributed by Nathan Roberts Added: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapreduce/lib/db/TestSplitters.java (with props) Modified: hadoop/common/branches/branch-1/CHANGES.txt hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapreduce/lib/db/FloatSplitter.java Modified: hadoop/common/branches/branch-1/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1530710r1=1530709r2=1530710view=diff == --- hadoop/common/branches/branch-1/CHANGES.txt (original) +++ hadoop/common/branches/branch-1/CHANGES.txt Wed Oct 9 16:53:52 2013 @@ -148,6 +148,9 @@ Release 1.3.0 - unreleased HADOOP-10009. Backport HADOOP-7808 to branch-1: fix NPE in SecurityUtil::setTokenService(). (Haohui Mai via jing9) +MAPREDUCE-5569. FloatSplitter is not generating correct splits (Nathan +Roberts via jlowe) + Release 1.2.2 - unreleased INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapreduce/lib/db/FloatSplitter.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapreduce/lib/db/FloatSplitter.java?rev=1530710r1=1530709r2=1530710view=diff == --- hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapreduce/lib/db/FloatSplitter.java (original) +++ hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapreduce/lib/db/FloatSplitter.java Wed Oct 9 16:53:52 2013 @@ -89,7 +89,7 @@ public class FloatSplitter implements DB // Catch any overage and create the closed interval for the last split. if (curLower = maxVal || splits.size() == 1) { splits.add(new DataDrivenDBInputFormat.DataDrivenDBInputSplit( - lowClausePrefix + Double.toString(curUpper), + lowClausePrefix + Double.toString(curLower), colName + = + Double.toString(maxVal))); } Added: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapreduce/lib/db/TestSplitters.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapreduce/lib/db/TestSplitters.java?rev=1530710view=auto == --- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapreduce/lib/db/TestSplitters.java (added) +++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapreduce/lib/db/TestSplitters.java Wed Oct 9 16:53:52 2013 @@ -0,0 +1,163 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapreduce.lib.db; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.math.BigDecimal; +import java.sql.ResultSet; +import java.util.List; +import java.util.regex.Pattern; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.lib.db.DataDrivenDBInputFormat.DataDrivenDBInputSplit; +import org.junit.Before; +import org.junit.Test; + +/** + * Test Splitters. Splitters should build parts of sql sentences for split result. + */ +public class TestSplitters { + + private Configuration configuration; + + @Before + public void setup() { +configuration = new Configuration(); +configuration.setInt(mapred.map.tasks, 2); + } + + @Test(timeout=2000) + public void testBooleanSplitter() throws Exception{ +BooleanSplitter splitter = new BooleanSplitter(); +ResultSet result = mock(ResultSet.class); +when(result.getString(1)).thenReturn(result1); + +ListInputSplit splits=splitter.split(configuration, result, column); +assertSplits(new String[] {column = FALSE column = FALSE, +column IS NULL column IS NULL}, splits); + +
svn commit: r1530823 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java src/test/java/org/apache/hadoop/fs/
Author: cnauroth Date: Wed Oct 9 22:57:59 2013 New Revision: 1530823 URL: http://svn.apache.org/r1530823 Log: HADOOP-10031. FsShell -get/copyToLocal/moveFromLocal should support Windows local path. Contributed by Chuan Liu. Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1530823r1=1530822r2=1530823view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Wed Oct 9 22:57:59 2013 @@ -413,6 +413,9 @@ Release 2.2.1 - UNRELEASED HADOOP-10030. FsShell -put/copyFromLocal should support Windows local path. (Chuan Liu via cnauroth) +HADOOP-10031. FsShell -get/copyToLocal/moveFromLocal should support Windows +local path. (Chuan Liu via cnauroth) + Release 2.2.0 - 2013-10-13 INCOMPATIBLE CHANGES Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java?rev=1530823r1=1530822r2=1530823view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java Wed Oct 9 22:57:59 2013 @@ -84,11 +84,16 @@ abstract class CommandWithDestination ex */ protected void getLocalDestination(LinkedListString args) throws IOException { +String pathString = (args.size() 2) ? Path.CUR_DIR : args.removeLast(); try { - String pathString = (args.size() 2) ? Path.CUR_DIR : args.removeLast(); dst = new PathData(new URI(pathString), getConf()); } catch (URISyntaxException e) { - throw new IOException(unexpected URISyntaxException, e); + if (Path.WINDOWS) { +// Unlike URI, PathData knows how to parse Windows drive-letter paths. +dst = new PathData(pathString, getConf()); + } else { +throw new IOException(unexpected URISyntaxException, e); + } } } Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java?rev=1530823r1=1530822r2=1530823view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java Wed Oct 9 22:57:59 2013 @@ -457,6 +457,34 @@ public class TestFsShellCopy { assertTrue(lfs.exists(srcDir)); } + @Test + public void testMoveFromWindowsLocalPath() throws Exception { +assumeTrue(Path.WINDOWS); +Path testRoot = new Path(testRootDir, testPutFile); +lfs.delete(testRoot, true); +lfs.mkdirs(testRoot); + +Path target = new Path(testRoot, target); +Path srcFile = new Path(testRoot, new Path(srcFile)); +lfs.createNewFile(srcFile); + +String winSrcFile = (new File(srcFile.toUri().getPath() +.toString())).getAbsolutePath(); +shellRun(0, -moveFromLocal, winSrcFile, target.toString()); +assertFalse(lfs.exists(srcFile)); +assertTrue(lfs.exists(target)); +assertTrue(lfs.isFile(target)); + } + + @Test + public void testGetWindowsLocalPath() throws Exception { +assumeTrue(Path.WINDOWS); +String winDstFile = (new File(dstPath.toUri().getPath() +.toString())).getAbsolutePath(); +shellRun(0, -get, srcPath.toString(), winDstFile); +checkPath(dstPath, false); + } + private void createFile(Path ... paths) throws IOException { for (Path path : paths) { FSDataOutputStream out = lfs.create(path);
svn commit: r1530827 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java src/test/java/org/apach
Author: cnauroth Date: Wed Oct 9 23:05:05 2013 New Revision: 1530827 URL: http://svn.apache.org/r1530827 Log: HADOOP-10031. Merging change r1530823 from trunk to branch-2. Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1530827r1=1530826r2=1530827view=diff == --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Wed Oct 9 23:05:05 2013 @@ -125,6 +125,9 @@ Release 2.2.1 - UNRELEASED HADOOP-10030. FsShell -put/copyFromLocal should support Windows local path. (Chuan Liu via cnauroth) +HADOOP-10031. FsShell -get/copyToLocal/moveFromLocal should support Windows +local path. (Chuan Liu via cnauroth) + Release 2.2.0 - 2013-10-13 INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java?rev=1530827r1=1530826r2=1530827view=diff == --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java Wed Oct 9 23:05:05 2013 @@ -84,11 +84,16 @@ abstract class CommandWithDestination ex */ protected void getLocalDestination(LinkedListString args) throws IOException { +String pathString = (args.size() 2) ? Path.CUR_DIR : args.removeLast(); try { - String pathString = (args.size() 2) ? Path.CUR_DIR : args.removeLast(); dst = new PathData(new URI(pathString), getConf()); } catch (URISyntaxException e) { - throw new IOException(unexpected URISyntaxException, e); + if (Path.WINDOWS) { +// Unlike URI, PathData knows how to parse Windows drive-letter paths. +dst = new PathData(pathString, getConf()); + } else { +throw new IOException(unexpected URISyntaxException, e); + } } } Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java?rev=1530827r1=1530826r2=1530827view=diff == --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java Wed Oct 9 23:05:05 2013 @@ -457,6 +457,34 @@ public class TestFsShellCopy { assertTrue(lfs.exists(srcDir)); } + @Test + public void testMoveFromWindowsLocalPath() throws Exception { +assumeTrue(Path.WINDOWS); +Path testRoot = new Path(testRootDir, testPutFile); +lfs.delete(testRoot, true); +lfs.mkdirs(testRoot); + +Path target = new Path(testRoot, target); +Path srcFile = new Path(testRoot, new Path(srcFile)); +lfs.createNewFile(srcFile); + +String winSrcFile = (new File(srcFile.toUri().getPath() +.toString())).getAbsolutePath(); +shellRun(0, -moveFromLocal, winSrcFile, target.toString()); +assertFalse(lfs.exists(srcFile)); +assertTrue(lfs.exists(target)); +assertTrue(lfs.isFile(target)); + } + + @Test + public void testGetWindowsLocalPath() throws Exception { +assumeTrue(Path.WINDOWS); +String winDstFile = (new File(dstPath.toUri().getPath() +.toString())).getAbsolutePath(); +shellRun(0, -get, srcPath.toString(), winDstFile); +checkPath(dstPath, false); + } + private void createFile(Path ... paths) throws IOException { for (Path path : paths) { FSDataOutputStream out = lfs.create(path);
svn commit: r1530828 - in /hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java src/test/java/org/apa
Author: cnauroth Date: Wed Oct 9 23:13:10 2013 New Revision: 1530828 URL: http://svn.apache.org/r1530828 Log: HADOOP-10031. Merging change r1530827 from branch-2 to branch-2.2 Modified: hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java Modified: hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1530828r1=1530827r2=1530828view=diff == --- hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/CHANGES.txt Wed Oct 9 23:13:10 2013 @@ -17,6 +17,9 @@ Release 2.2.1 - UNRELEASED HADOOP-10030. FsShell -put/copyFromLocal should support Windows local path. (Chuan Liu via cnauroth) +HADOOP-10031. FsShell -get/copyToLocal/moveFromLocal should support Windows +local path. (Chuan Liu via cnauroth) + Release 2.2.0 - 2013-10-13 INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java?rev=1530828r1=1530827r2=1530828view=diff == --- hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java (original) +++ hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java Wed Oct 9 23:13:10 2013 @@ -84,11 +84,16 @@ abstract class CommandWithDestination ex */ protected void getLocalDestination(LinkedListString args) throws IOException { +String pathString = (args.size() 2) ? Path.CUR_DIR : args.removeLast(); try { - String pathString = (args.size() 2) ? Path.CUR_DIR : args.removeLast(); dst = new PathData(new URI(pathString), getConf()); } catch (URISyntaxException e) { - throw new IOException(unexpected URISyntaxException, e); + if (Path.WINDOWS) { +// Unlike URI, PathData knows how to parse Windows drive-letter paths. +dst = new PathData(pathString, getConf()); + } else { +throw new IOException(unexpected URISyntaxException, e); + } } } Modified: hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java?rev=1530828r1=1530827r2=1530828view=diff == --- hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java (original) +++ hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java Wed Oct 9 23:13:10 2013 @@ -457,6 +457,34 @@ public class TestFsShellCopy { assertTrue(lfs.exists(srcDir)); } + @Test + public void testMoveFromWindowsLocalPath() throws Exception { +assumeTrue(Path.WINDOWS); +Path testRoot = new Path(testRootDir, testPutFile); +lfs.delete(testRoot, true); +lfs.mkdirs(testRoot); + +Path target = new Path(testRoot, target); +Path srcFile = new Path(testRoot, new Path(srcFile)); +lfs.createNewFile(srcFile); + +String winSrcFile = (new File(srcFile.toUri().getPath() +.toString())).getAbsolutePath(); +shellRun(0, -moveFromLocal, winSrcFile, target.toString()); +assertFalse(lfs.exists(srcFile)); +assertTrue(lfs.exists(target)); +assertTrue(lfs.isFile(target)); + } + + @Test + public void testGetWindowsLocalPath() throws Exception { +assumeTrue(Path.WINDOWS); +String winDstFile = (new File(dstPath.toUri().getPath() +.toString())).getAbsolutePath(); +shellRun(0, -get, srcPath.toString(), winDstFile); +checkPath(dstPath, false); + } + private void createFile(Path ... paths) throws IOException { for (Path path : paths) { FSDataOutputStream out = lfs.create(path);