svn commit: r1529238 - /hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java
Author: wang
Date: Fri Oct 4 17:46:18 2013
New Revision: 1529238
URL: http://svn.apache.org/r1529238
Log:
HDFS-5119. Persist CacheManager state in the edit log. (Contributed by Andrew
Wang)
Modified:
hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java
Modified:
hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java?rev=1529238r1=1529237r2=1529238view=diff
==
---
hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java
(original)
+++
hadoop/common/branches/HDFS-4949/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java
Fri Oct 4 17:46:18 2013
@@ -454,10 +454,7 @@ public class Text extends BinaryComparab
/** Read a UTF8 encoded string from in
*/
public static String readString(DataInput in) throws IOException {
-int length = WritableUtils.readVInt(in);
-byte [] bytes = new byte[length];
-in.readFully(bytes, 0, length);
-return decode(bytes);
+return readString(in, Integer.MAX_VALUE);
}
/** Read a UTF8 encoded string with a maximum size
svn commit: r1529258 - in /hadoop/common/branches/branch-1: ./ src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/ src/core/org/apache/hadoop/security/ src/test/org/apache/hadoop/security/
Author: jing9
Date: Fri Oct 4 18:54:48 2013
New Revision: 1529258
URL: http://svn.apache.org/r1529258
Log:
HADOOP-10009. Backport HADOOP-7808 to branch-1: fix NPE in
SecurityUtil::setTokenService(). Contributed by Haohui Mai.
Modified:
hadoop/common/branches/branch-1/CHANGES.txt
hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestRandomAlgorithm.java
hadoop/common/branches/branch-1/src/core/org/apache/hadoop/security/SecurityUtil.java
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/security/TestSecurityUtil.java
Modified: hadoop/common/branches/branch-1/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1529258r1=1529257r2=1529258view=diff
==
--- hadoop/common/branches/branch-1/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1/CHANGES.txt Fri Oct 4 18:54:48 2013
@@ -145,6 +145,9 @@ Release 1.3.0 - unreleased
MAPREDUCE-5508. JobTracker memory leak caused by unreleased FileSystem
objects in JobInProgress#cleanupJob. (Xi Fang via cnauroth)
+HADOOP-10009. Backport HADOOP-7808 to branch-1: fix NPE in
+SecurityUtil::setTokenService(). (Haohui Mai via jing9)
+
Release 1.2.2 - unreleased
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestRandomAlgorithm.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestRandomAlgorithm.java?rev=1529258r1=1529257r2=1529258view=diff
==
---
hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestRandomAlgorithm.java
(original)
+++
hadoop/common/branches/branch-1/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestRandomAlgorithm.java
Fri Oct 4 18:54:48 2013
@@ -17,7 +17,8 @@
*/
package org.apache.hadoop.mapred.gridmix;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import java.util.ArrayList;
import java.util.Arrays;
@@ -30,8 +31,6 @@ import java.util.Set;
import org.junit.Test;
-import com.sun.tools.javac.code.Attribute.Array;
-
public class TestRandomAlgorithm {
private static final int[][] parameters = new int[][] {
{5, 1, 1},
Modified:
hadoop/common/branches/branch-1/src/core/org/apache/hadoop/security/SecurityUtil.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/core/org/apache/hadoop/security/SecurityUtil.java?rev=1529258r1=1529257r2=1529258view=diff
==
---
hadoop/common/branches/branch-1/src/core/org/apache/hadoop/security/SecurityUtil.java
(original)
+++
hadoop/common/branches/branch-1/src/core/org/apache/hadoop/security/SecurityUtil.java
Fri Oct 4 18:54:48 2013
@@ -332,7 +332,13 @@ public class SecurityUtil {
* @param addr the socket for the rpc connection
*/
public static void setTokenService(Token? token, InetSocketAddress addr) {
-token.setService(buildTokenService(addr));
+Text service = buildTokenService(addr);
+if (token != null) {
+ token.setService(service);
+ LOG.info(Acquired token +token); // Token#toString() prints service
+} else {
+ LOG.warn(Failed to get token for service +service);
+}
}
/**
Modified:
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/security/TestSecurityUtil.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/security/TestSecurityUtil.java?rev=1529258r1=1529257r2=1529258view=diff
==
---
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/security/TestSecurityUtil.java
(original)
+++
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/security/TestSecurityUtil.java
Fri Oct 4 18:54:48 2013
@@ -16,7 +16,10 @@
*/
package org.apache.hadoop.security;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.net.InetAddress;
@@ -28,6 +31,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.io.Text;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
import org.junit.Test;
import org.mockito.Mockito;
@@ -233,10 +237,11 @@ public class TestSecurityUtil {
SecurityUtil.setTokenServiceUseIp(useIp);
String serviceHost = useIp ? ip : host.toLowerCase();
-Token token = new
svn commit: r1529296 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: CHANGES.txt src/test/java/org/apache/hadoop/io/compress/snappy/ src/test/java/org/apache/hadoop/io/compress/snappy/T
Author: jlowe
Date: Fri Oct 4 20:56:21 2013
New Revision: 1529296
URL: http://svn.apache.org/r1529296
Log:
HADOOP-9225. Cover package org.apache.hadoop.compress.Snappy. Contributed by
Vadim Bondarev, Andrey Klochkov and Nathan Roberts
Added:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/snappy/
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/snappy/TestSnappyCompressorDecompressor.java
(with props)
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1529296r1=1529295r2=1529296view=diff
==
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Fri Oct
4 20:56:21 2013
@@ -345,6 +345,9 @@ Release 2.3.0 - UNRELEASED
HADOOP-9254. Cover packages org.apache.hadoop.util.bloom,
org.apache.hadoop.util.hash (Vadim Bondarev via jlowe)
+HADOOP-9225. Cover package org.apache.hadoop.compress.Snappy (Vadim
+Bondarev, Andrey Klochkov and Nathan Roberts via jlowe)
+
OPTIMIZATIONS
HADOOP-9748. Reduce blocking on UGI.ensureInitialized (daryn)
Added:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/snappy/TestSnappyCompressorDecompressor.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/snappy/TestSnappyCompressorDecompressor.java?rev=1529296view=auto
==
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/snappy/TestSnappyCompressorDecompressor.java
(added)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/snappy/TestSnappyCompressorDecompressor.java
Fri Oct 4 20:56:21 2013
@@ -0,0 +1,342 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.compress.snappy;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.lang.reflect.Array;
+import java.util.Random;
+
+import org.apache.hadoop.io.DataInputBuffer;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.io.compress.BlockCompressorStream;
+import org.apache.hadoop.io.compress.BlockDecompressorStream;
+import org.apache.hadoop.io.compress.CompressionInputStream;
+import org.apache.hadoop.io.compress.CompressionOutputStream;
+import org.apache.hadoop.io.compress.SnappyCodec;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assume.*;
+
+public class TestSnappyCompressorDecompressor {
+
+ @Before
+ public void before() {
+assumeTrue(SnappyCodec.isNativeCodeLoaded());
+ }
+
+ @Test
+ public void testSnappyCompressorSetInputNullPointerException() {
+try {
+ SnappyCompressor compressor = new SnappyCompressor();
+ compressor.setInput(null, 0, 10);
+ fail(testSnappyCompressorSetInputNullPointerException error !!!);
+} catch (NullPointerException ex) {
+ // excepted
+} catch (Exception ex) {
+ fail(testSnappyCompressorSetInputNullPointerException ex error !!!);
+}
+ }
+
+ @Test
+ public void testSnappyDecompressorSetInputNullPointerException() {
+try {
+ SnappyDecompressor decompressor = new SnappyDecompressor();
+ decompressor.setInput(null, 0, 10);
+ fail(testSnappyDecompressorSetInputNullPointerException error !!!);
+} catch (NullPointerException ex) {
+
svn commit: r1529301 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: CHANGES.txt src/test/java/org/apache/hadoop/io/compress/snappy/ src/test/java/org/apache/hadoop/io/compr
Author: jlowe Date: Fri Oct 4 21:03:16 2013 New Revision: 1529301 URL: http://svn.apache.org/r1529301 Log: svn merge -c 1529296 FIXES: HADOOP-9225. Cover package org.apache.hadoop.compress.Snappy. Contributed by Vadim Bondarev, Andrey Klochkov and Nathan Roberts Added: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/snappy/ - copied from r1529296, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/snappy/ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/snappy/TestSnappyCompressorDecompressor.java - copied unchanged from r1529296, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/snappy/TestSnappyCompressorDecompressor.java Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1529301r1=1529300r2=1529301view=diff == --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Fri Oct 4 21:03:16 2013 @@ -58,6 +58,9 @@ Release 2.3.0 - UNRELEASED HADOOP-9254. Cover packages org.apache.hadoop.util.bloom, org.apache.hadoop.util.hash (Vadim Bondarev via jlowe) +HADOOP-9225. Cover package org.apache.hadoop.compress.Snappy (Vadim +Bondarev, Andrey Klochkov and Nathan Roberts via jlowe) + OPTIMIZATIONS HADOOP-9748. Reduce blocking on UGI.ensureInitialized (daryn)
svn commit: r1529325 - /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm
Author: tucu Date: Fri Oct 4 21:59:43 2013 New Revision: 1529325 URL: http://svn.apache.org/r1529325 Log: YARN-1253. Changes to LinuxContainerExecutor to run containers as a single dedicated user in non-secure mode. (rvs via tucu) Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm?rev=1529325r1=1529324r2=1529325view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm Fri Oct 4 21:59:43 2013 @@ -854,8 +854,10 @@ KVNO Timestamp Principal | | The container process has the same Unix user as the NodeManager. | *--+--+ | LinuxContainerExecutor | | -| | Supported only on GNU/Linux, this executor runs the containers as the | -| | user who submitted the application. It requires all user accounts to be | +| | Supported only on GNU/Linux, this executor runs the containers as either the | +| | YARN user who submitted the application (when full security is enabled) or | +| | as a dedicated user (defaults to nobody) when full security is not enabled. | +| | When full security is enabled, this executor requires all user accounts to be | | | created on the cluster nodes where the containers are launched. It uses | | | a setuid executable that is included in the Hadoop distribution. | | | The NodeManager uses this executable to launch and kill containers. |
svn commit: r1529326 - /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm
Author: tucu Date: Fri Oct 4 22:00:01 2013 New Revision: 1529326 URL: http://svn.apache.org/r1529326 Log: YARN-1253. Changes to LinuxContainerExecutor to run containers as a single dedicated user in non-secure mode. (rvs via tucu) Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm?rev=1529326r1=1529325r2=1529326view=diff == --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm Fri Oct 4 22:00:01 2013 @@ -854,8 +854,10 @@ KVNO Timestamp Principal | | The container process has the same Unix user as the NodeManager. | *--+--+ | LinuxContainerExecutor | | -| | Supported only on GNU/Linux, this executor runs the containers as the | -| | user who submitted the application. It requires all user accounts to be | +| | Supported only on GNU/Linux, this executor runs the containers as either the | +| | YARN user who submitted the application (when full security is enabled) or | +| | as a dedicated user (defaults to nobody) when full security is not enabled. | +| | When full security is enabled, this executor requires all user accounts to be | | | created on the cluster nodes where the containers are launched. It uses | | | a setuid executable that is included in the Hadoop distribution. | | | The NodeManager uses this executable to launch and kill containers. |
