[2/2] git commit: HADOOP-10400. Incorporate new S3A FileSystem implementation. Contributed by Jordan Mendelson and Dave Wang.
HADOOP-10400. Incorporate new S3A FileSystem implementation. Contributed by
Jordan Mendelson and Dave Wang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/24d920b8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/24d920b8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/24d920b8
Branch: refs/heads/trunk
Commit: 24d920b80eb3626073925a1d0b6dcf148add8cc0
Parents: fc741b5
Author: Aaron T. Myers
Authored: Mon Sep 15 08:27:07 2014 -0700
Committer: Aaron T. Myers
Committed: Mon Sep 15 08:27:07 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt |3 +
.../src/main/conf/log4j.properties |5 +
.../src/main/resources/core-default.xml | 86 ++
hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml |8 +
hadoop-project/pom.xml | 26 +-
hadoop-tools/hadoop-aws/pom.xml | 10 +
.../fs/s3a/AnonymousAWSCredentialsProvider.java | 37 +
.../fs/s3a/BasicAWSCredentialsProvider.java | 51 +
.../org/apache/hadoop/fs/s3a/Constants.java | 90 ++
.../org/apache/hadoop/fs/s3a/S3AFileStatus.java | 62 ++
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 1019 ++
.../apache/hadoop/fs/s3a/S3AInputStream.java| 207
.../apache/hadoop/fs/s3a/S3AOutputStream.java | 208
.../services/org.apache.hadoop.fs.FileSystem|1 +
.../hadoop/fs/contract/s3a/S3AContract.java | 43 +
.../fs/contract/s3a/TestS3AContractCreate.java | 38 +
.../fs/contract/s3a/TestS3AContractDelete.java | 31 +
.../fs/contract/s3a/TestS3AContractMkdir.java | 34 +
.../fs/contract/s3a/TestS3AContractOpen.java| 31 +
.../fs/contract/s3a/TestS3AContractRename.java | 64 ++
.../fs/contract/s3a/TestS3AContractRootDir.java | 35 +
.../fs/contract/s3a/TestS3AContractSeek.java| 31 +
.../fs/s3a/S3AFileSystemContractBaseTest.java | 327 ++
.../src/test/resources/contract/s3a.xml | 105 ++
.../src/test/resources/contract/s3n.xml |7 +-
hadoop-tools/hadoop-azure/pom.xml | 10 +-
26 files changed, 2552 insertions(+), 17 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/24d920b8/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 051eac1..c2ae5ed 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -342,6 +342,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-10893. isolated classloader on the client side (Sangjin Lee via
jlowe)
+HADOOP-10400. Incorporate new S3A FileSystem implementation. (Jordan
+Mendelson and Dave Wang via atm)
+
IMPROVEMENTS
HADOOP-10808. Remove unused native code for munlock. (cnauroth)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/24d920b8/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
--
diff --git a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
index ef9acbf..5fa21fa 100644
--- a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
+++ b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
@@ -174,6 +174,11 @@
log4j.appender.MRAUDIT.MaxBackupIndex=${mapred.audit.log.maxbackupindex}
# Jets3t library
log4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR
+# AWS SDK & S3A FileSystem
+log4j.logger.com.amazonaws=ERROR
+log4j.logger.com.amazonaws.http.AmazonHttpClient=ERROR
+log4j.logger.org.apache.hadoop.fs.s3a.S3AFileSystem=WARN
+
#
# Event Counter Appender
# Sends counts of logging messages at different severity levels to Hadoop
Metrics.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/24d920b8/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 3cc7545..828dec2 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -690,6 +690,92 @@ for ldap providers in the same way as above does.
+ fs.s3a.access.key
+ AWS access key ID. Omit for Role-based
authentication.
+
+
+
+ fs.s3a.secret.key
+ AWS secret key. Omit for Role-based
authentication.
+
+
+
+ fs.s3a.connection.maximum
+ 15
+ Controls the maximum number of simultaneous conn
[1/2] HADOOP-10400. Incorporate new S3A FileSystem implementation. Contributed by Jordan Mendelson and Dave Wang.
Repository: hadoop
Updated Branches:
refs/heads/trunk fc741b5d7 -> 24d920b80
http://git-wip-us.apache.org/repos/asf/hadoop/blob/24d920b8/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractSeek.java
--
diff --git
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractSeek.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractSeek.java
new file mode 100644
index 000..d677ec4
--- /dev/null
+++
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractSeek.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3a;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractSeekTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+public class TestS3AContractSeek extends AbstractContractSeekTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+return new S3AContract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/24d920b8/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3AFileSystemContractBaseTest.java
--
diff --git
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3AFileSystemContractBaseTest.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3AFileSystemContractBaseTest.java
new file mode 100644
index 000..8455233
--- /dev/null
+++
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3AFileSystemContractBaseTest.java
@@ -0,0 +1,327 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import static org.junit.Assume.*;
+
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystemContractBaseTest;
+import org.apache.hadoop.fs.Path;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.util.UUID;
+
+/**
+ * Tests a live S3 system. If you keys and bucket aren't specified, all tests
+ * are marked as passed
+ *
+ * This uses BlockJUnit4ClassRunner because FileSystemContractBaseTest from
+ * TestCase which uses the old Junit3 runner that doesn't ignore assumptions
+ * properly making it impossible to skip the tests if we don't have a valid
+ * bucket.
+ **/
+public class S3AFileSystemContractBaseTest extends FileSystemContractBaseTest {
+ private static final int TEST_BUFFER_SIZE = 128;
+ private static final int MODULUS = 128;
+
+ protected static final Logger LOG =
LoggerFactory.getLogger(S3AFileSystemContractBaseTest.class);
+
+ @Override
+ public void setUp() throws Exception {
+Configuration conf = new Configuration();
+
+URI testURI = URI.create(conf.get("test.fs.s3a.name"));
+
+boolean liveTest = testURI != null && !testURI.equals("s3a:///");
+
+// This doesn't work with our JUnit 3 style test cases, so instead we'll
+// make this whole class not run by default
+assumeTrue(liveTest);
+
+fs = new S3AFileSystem();
+fs.initialize(t
[1/2] HADOOP-10400. Incorporate new S3A FileSystem implementation. Contributed by Jordan Mendelson and Dave Wang.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 dd3e28d43 -> a0c54aeb0
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0c54aeb/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractSeek.java
--
diff --git
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractSeek.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractSeek.java
new file mode 100644
index 000..d677ec4
--- /dev/null
+++
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractSeek.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3a;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractSeekTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+public class TestS3AContractSeek extends AbstractContractSeekTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+return new S3AContract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0c54aeb/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3AFileSystemContractBaseTest.java
--
diff --git
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3AFileSystemContractBaseTest.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3AFileSystemContractBaseTest.java
new file mode 100644
index 000..8455233
--- /dev/null
+++
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3AFileSystemContractBaseTest.java
@@ -0,0 +1,327 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import static org.junit.Assume.*;
+
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystemContractBaseTest;
+import org.apache.hadoop.fs.Path;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.util.UUID;
+
+/**
+ * Tests a live S3 system. If you keys and bucket aren't specified, all tests
+ * are marked as passed
+ *
+ * This uses BlockJUnit4ClassRunner because FileSystemContractBaseTest from
+ * TestCase which uses the old Junit3 runner that doesn't ignore assumptions
+ * properly making it impossible to skip the tests if we don't have a valid
+ * bucket.
+ **/
+public class S3AFileSystemContractBaseTest extends FileSystemContractBaseTest {
+ private static final int TEST_BUFFER_SIZE = 128;
+ private static final int MODULUS = 128;
+
+ protected static final Logger LOG =
LoggerFactory.getLogger(S3AFileSystemContractBaseTest.class);
+
+ @Override
+ public void setUp() throws Exception {
+Configuration conf = new Configuration();
+
+URI testURI = URI.create(conf.get("test.fs.s3a.name"));
+
+boolean liveTest = testURI != null && !testURI.equals("s3a:///");
+
+// This doesn't work with our JUnit 3 style test cases, so instead we'll
+// make this whole class not run by default
+assumeTrue(liveTest);
+
+fs = new S3AFileSystem();
+fs.initializ
[2/2] git commit: HADOOP-10400. Incorporate new S3A FileSystem implementation. Contributed by Jordan Mendelson and Dave Wang.
HADOOP-10400. Incorporate new S3A FileSystem implementation. Contributed by
Jordan Mendelson and Dave Wang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a0c54aeb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a0c54aeb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a0c54aeb
Branch: refs/heads/branch-2
Commit: a0c54aeb00c0bc38f7dfa3615ce6866023d1ef74
Parents: dd3e28d
Author: Aaron T. Myers
Authored: Mon Sep 15 08:30:42 2014 -0700
Committer: Aaron T. Myers
Committed: Mon Sep 15 08:30:42 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt |3 +
.../src/main/conf/log4j.properties |5 +
.../src/main/resources/core-default.xml | 86 ++
hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml |8 +
hadoop-project/pom.xml | 26 +-
hadoop-tools/hadoop-aws/pom.xml | 10 +
.../fs/s3a/AnonymousAWSCredentialsProvider.java | 37 +
.../fs/s3a/BasicAWSCredentialsProvider.java | 51 +
.../org/apache/hadoop/fs/s3a/Constants.java | 90 ++
.../org/apache/hadoop/fs/s3a/S3AFileStatus.java | 62 ++
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 1019 ++
.../apache/hadoop/fs/s3a/S3AInputStream.java| 207
.../apache/hadoop/fs/s3a/S3AOutputStream.java | 208
.../services/org.apache.hadoop.fs.FileSystem|1 +
.../hadoop/fs/contract/s3a/S3AContract.java | 43 +
.../fs/contract/s3a/TestS3AContractCreate.java | 38 +
.../fs/contract/s3a/TestS3AContractDelete.java | 31 +
.../fs/contract/s3a/TestS3AContractMkdir.java | 34 +
.../fs/contract/s3a/TestS3AContractOpen.java| 31 +
.../fs/contract/s3a/TestS3AContractRename.java | 64 ++
.../fs/contract/s3a/TestS3AContractRootDir.java | 35 +
.../fs/contract/s3a/TestS3AContractSeek.java| 31 +
.../fs/s3a/S3AFileSystemContractBaseTest.java | 327 ++
.../src/test/resources/contract/s3a.xml | 105 ++
.../src/test/resources/contract/s3n.xml |7 +-
25 files changed, 2550 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0c54aeb/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 2dababb..e3dd7d1 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -11,6 +11,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-10893. isolated classloader on the client side (Sangjin Lee via
jlowe)
+HADOOP-10400. Incorporate new S3A FileSystem implementation. (Jordan
+Mendelson and Dave Wang via atm)
+
IMPROVEMENTS
HADOOP-10808. Remove unused native code for munlock. (cnauroth)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0c54aeb/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
--
diff --git a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
index ef9acbf..5fa21fa 100644
--- a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
+++ b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
@@ -174,6 +174,11 @@
log4j.appender.MRAUDIT.MaxBackupIndex=${mapred.audit.log.maxbackupindex}
# Jets3t library
log4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR
+# AWS SDK & S3A FileSystem
+log4j.logger.com.amazonaws=ERROR
+log4j.logger.com.amazonaws.http.AmazonHttpClient=ERROR
+log4j.logger.org.apache.hadoop.fs.s3a.S3AFileSystem=WARN
+
#
# Event Counter Appender
# Sends counts of logging messages at different severity levels to Hadoop
Metrics.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0c54aeb/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index ee3cbf0..cd953e3 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -682,6 +682,92 @@ for ldap providers in the same way as above does.
+ fs.s3a.access.key
+ AWS access key ID. Omit for Role-based
authentication.
+
+
+
+ fs.s3a.secret.key
+ AWS secret key. Omit for Role-based
authentication.
+
+
+
+ fs.s3a.connection.maximum
+ 15
+ Controls the maximum number of simultaneous connections to
S3.
+
+
+
+ fs.s3a.connection.ssl.enabled
+ t
git commit: HDFS-7062. Archival Storage: skip under construction block for migration. Contributed by Jing Zhao.
Repository: hadoop
Updated Branches:
refs/heads/HDFS-6584 dba52ce0b -> 2689b6ca7
HDFS-7062. Archival Storage: skip under construction block for migration.
Contributed by Jing Zhao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2689b6ca
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2689b6ca
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2689b6ca
Branch: refs/heads/HDFS-6584
Commit: 2689b6ca727fff8a13347b811eb4cf79b9d30f48
Parents: dba52ce
Author: Jing Zhao
Authored: Mon Sep 15 10:16:56 2014 -0700
Committer: Jing Zhao
Committed: Mon Sep 15 10:16:56 2014 -0700
--
.../apache/hadoop/hdfs/server/mover/Mover.java | 10 ++-
.../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 4 +-
.../hdfs/server/mover/TestStorageMover.java | 77
3 files changed, 88 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2689b6ca/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
index 96588ff..e336ebc 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
@@ -321,7 +321,14 @@ public class Mover {
final LocatedBlocks locatedBlocks = status.getBlockLocations();
boolean hasRemaining = false;
- for(LocatedBlock lb : locatedBlocks.getLocatedBlocks()) {
+ final boolean lastBlkComplete = locatedBlocks.isLastBlockComplete();
+ List lbs = locatedBlocks.getLocatedBlocks();
+ for(int i = 0; i < lbs.size(); i++) {
+if (i == lbs.size() - 1 && !lastBlkComplete) {
+ // last block is incomplete, skip it
+ continue;
+}
+LocatedBlock lb = lbs.get(i);
final StorageTypeDiff diff = new StorageTypeDiff(types,
lb.getStorageTypes());
if (!diff.removeOverlap()) {
@@ -472,6 +479,7 @@ public class Mover {
final ExitStatus r = m.run();
if (r == ExitStatus.SUCCESS) {
+IOUtils.cleanup(LOG, nnc);
iter.remove();
} else if (r != ExitStatus.IN_PROGRESS) {
// must be an error statue, return
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2689b6ca/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
index 10012c6..556eca6 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
@@ -387,8 +387,8 @@ public class DFSAdmin extends FsShell {
"\t[-shutdownDatanode [upgrade]]\n" +
"\t[-getDatanodeInfo ]\n" +
"\t[-metasave filename]\n" +
-"\t[-setStoragePolicy path policyName\n" +
-"\t[-getStoragePolicy path\n" +
+"\t[-setStoragePolicy path policyName]\n" +
+"\t[-getStoragePolicy path]\n" +
"\t[-help [cmd]]\n";
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2689b6ca/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
index d5d5cab..ceedfc2 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
@@ -30,9 +30,13 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.BlockStoragePolicy;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSOutputStream;
import org.apache.hadoop.hdfs
git commit: HDFS-7032. Add WebHDFS support for reading and writing to encryption zones. Contributed by Charles Lamb.
Repository: hadoop
Updated Branches:
refs/heads/trunk 24d920b80 -> 43b030300
HDFS-7032. Add WebHDFS support for reading and writing to encryption zones.
Contributed by Charles Lamb.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/43b03030
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/43b03030
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/43b03030
Branch: refs/heads/trunk
Commit: 43b03030084839db041d0337013806aaeef12aaa
Parents: 24d920b
Author: Andrew Wang
Authored: Mon Sep 15 10:23:57 2014 -0700
Committer: Andrew Wang
Committed: Mon Sep 15 10:23:57 2014 -0700
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../web/resources/DatanodeWebHdfsMethods.java | 13 +++--
.../apache/hadoop/hdfs/TestEncryptionZones.java | 52
3 files changed, 63 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/43b03030/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 57a4a0f..209c7c1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -647,6 +647,9 @@ Release 2.6.0 - UNRELEASED
HDFS-7045. Fix NameNode deadlock when opening file under /.reserved path.
(Yi Liu via wang)
+HDFS-7032. Add WebHDFS support for reading and writing to encryption zones.
+(clamb via wang)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HDFS-6387. HDFS CLI admin tool for creating & deleting an
http://git-wip-us.apache.org/repos/asf/hadoop/blob/43b03030/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java
index 51731c8..0f0f3be 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java
@@ -231,11 +231,13 @@ public class DatanodeWebHdfsMethods {
DFSClient dfsclient = newDfsClient(nnId, conf);
FSDataOutputStream out = null;
try {
-out = new FSDataOutputStream(dfsclient.create(
+out = dfsclient.createWrappedOutputStream(dfsclient.create(
fullpath, permission.getFsPermission(),
-overwrite.getValue() ? EnumSet.of(CreateFlag.CREATE,
CreateFlag.OVERWRITE)
-: EnumSet.of(CreateFlag.CREATE),
-replication.getValue(conf), blockSize.getValue(conf), null, b,
null), null);
+overwrite.getValue() ?
+EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE) :
+EnumSet.of(CreateFlag.CREATE),
+replication.getValue(conf), blockSize.getValue(conf), null,
+b, null), null);
IOUtils.copyBytes(in, out, b);
out.close();
out = null;
@@ -418,7 +420,8 @@ public class DatanodeWebHdfsMethods {
final DFSClient dfsclient = newDfsClient(nnId, conf);
HdfsDataInputStream in = null;
try {
-in = new HdfsDataInputStream(dfsclient.open(fullpath, b, true));
+in = dfsclient.createWrappedInputStream(
+dfsclient.open(fullpath, b, true));
in.seek(offset.getValue());
} catch(IOException ioe) {
IOUtils.cleanup(LOG, in);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/43b03030/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
index b3bf5d9..68fc850 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.crypto.key.JavaKeyStoreProvider;
import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
import org.apache.hadoop.crypto.key.KeyProviderFactory;
+import org.apache.hadoop.fs
git commit: HDFS-7032. Add WebHDFS support for reading and writing to encryption zones. Contributed by Charles Lamb.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 a0c54aeb0 -> d59555785
HDFS-7032. Add WebHDFS support for reading and writing to encryption zones.
Contributed by Charles Lamb.
(cherry picked from commit 43b03030084839db041d0337013806aaeef12aaa)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d5955578
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d5955578
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d5955578
Branch: refs/heads/branch-2
Commit: d59555785935a25526f0b294038294ac6293fdc6
Parents: a0c54ae
Author: Andrew Wang
Authored: Mon Sep 15 10:23:57 2014 -0700
Committer: Andrew Wang
Committed: Mon Sep 15 10:24:38 2014 -0700
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../web/resources/DatanodeWebHdfsMethods.java | 13 +++--
.../apache/hadoop/hdfs/TestEncryptionZones.java | 52
3 files changed, 63 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5955578/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 7a42ea9..9afe524 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -389,6 +389,9 @@ Release 2.6.0 - UNRELEASED
HDFS-7045. Fix NameNode deadlock when opening file under /.reserved path.
(Yi Liu via wang)
+HDFS-7032. Add WebHDFS support for reading and writing to encryption zones.
+(clamb via wang)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HDFS-6387. HDFS CLI admin tool for creating & deleting an
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5955578/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java
index 51731c8..0f0f3be 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java
@@ -231,11 +231,13 @@ public class DatanodeWebHdfsMethods {
DFSClient dfsclient = newDfsClient(nnId, conf);
FSDataOutputStream out = null;
try {
-out = new FSDataOutputStream(dfsclient.create(
+out = dfsclient.createWrappedOutputStream(dfsclient.create(
fullpath, permission.getFsPermission(),
-overwrite.getValue() ? EnumSet.of(CreateFlag.CREATE,
CreateFlag.OVERWRITE)
-: EnumSet.of(CreateFlag.CREATE),
-replication.getValue(conf), blockSize.getValue(conf), null, b,
null), null);
+overwrite.getValue() ?
+EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE) :
+EnumSet.of(CreateFlag.CREATE),
+replication.getValue(conf), blockSize.getValue(conf), null,
+b, null), null);
IOUtils.copyBytes(in, out, b);
out.close();
out = null;
@@ -418,7 +420,8 @@ public class DatanodeWebHdfsMethods {
final DFSClient dfsclient = newDfsClient(nnId, conf);
HdfsDataInputStream in = null;
try {
-in = new HdfsDataInputStream(dfsclient.open(fullpath, b, true));
+in = dfsclient.createWrappedInputStream(
+dfsclient.open(fullpath, b, true));
in.seek(offset.getValue());
} catch(IOException ioe) {
IOUtils.cleanup(LOG, in);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5955578/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
index b3bf5d9..68fc850 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.crypto.key.JavaKeyStoreProvider;
import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
import o
[05/52] [abbrv] git commit: MAPREDUCE-5996. native-task: Rename system tests into standard directory layout. Contributed by Todd Lipcon.
MAPREDUCE-5996. native-task: Rename system tests into standard directory layout. Contributed by Todd Lipcon. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/MR-2841@1613004 13f79535-47bb-0310-9956-ffa450edef68 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b2cba48f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b2cba48f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b2cba48f Branch: refs/heads/HDFS-6584 Commit: b2cba48f36c96f190f6a25e65291873be7aee322 Parents: d468a92 Author: Todd Lipcon Authored: Thu Jul 24 06:17:33 2014 + Committer: Todd Lipcon Committed: Thu Jul 24 06:17:33 2014 + -- .../CHANGES.MAPREDUCE-2841.txt | 1 + .../nativetask/combinertest/CombinerTest.java | 122 +++ .../combinertest/LargeKVCombinerTest.java | 96 + .../combinertest/OldAPICombinerTest.java| 107 ++ .../nativetask/combinertest/WordCount.java | 89 .../combinertest/WordCountWithOldAPI.java | 65 ++ .../nativetask/compresstest/CompressMapper.java | 69 ++ .../nativetask/compresstest/CompressTest.java | 134 .../nativetask/kvtest/HashSumReducer.java | 48 + .../hadoop/mapred/nativetask/kvtest/KVJob.java | 97 + .../hadoop/mapred/nativetask/kvtest/KVTest.java | 181 .../mapred/nativetask/kvtest/LargeKVTest.java | 129 +++ .../mapred/nativetask/kvtest/TestInputFile.java | 213 +++ .../nativetask/nonsorttest/NonSortTest.java | 99 + .../nativetask/nonsorttest/NonSortTestMR.java | 71 +++ .../nativetask/testutil/BytesFactory.java | 104 + .../EnforceNativeOutputCollectorDelegator.java | 48 + .../nativetask/testutil/MockValueClass.java | 72 +++ .../nativetask/testutil/ResultVerifier.java | 141 .../testutil/ScenarioConfiguration.java | 58 + .../nativetask/testutil/TestConstants.java | 67 ++ .../test/java/system/data/testGlibcBugSpill.out | 2 - .../nativetask/combinertest/CombinerTest.java | 122 --- .../combinertest/LargeKVCombinerTest.java | 96 - .../combinertest/OldAPICombinerTest.java| 107 -- .../nativetask/combinertest/WordCount.java | 89 .../combinertest/WordCountWithOldAPI.java | 65 -- .../nativetask/compresstest/CompressMapper.java | 69 -- .../nativetask/compresstest/CompressTest.java | 134 .../nativetask/kvtest/HashSumReducer.java | 48 - .../hadoop/mapred/nativetask/kvtest/KVJob.java | 97 - .../hadoop/mapred/nativetask/kvtest/KVTest.java | 181 .../mapred/nativetask/kvtest/LargeKVTest.java | 129 --- .../mapred/nativetask/kvtest/TestInputFile.java | 213 --- .../nativetask/nonsorttest/NonSortTest.java | 99 - .../nativetask/nonsorttest/NonSortTestMR.java | 71 --- .../nativetask/testutil/BytesFactory.java | 104 - .../EnforceNativeOutputCollectorDelegator.java | 48 - .../nativetask/testutil/MockValueClass.java | 72 --- .../nativetask/testutil/ResultVerifier.java | 141 .../testutil/ScenarioConfiguration.java | 58 - .../nativetask/testutil/TestConstants.java | 67 -- 42 files changed, 2011 insertions(+), 2012 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt -- diff --git a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt index 166d68d..cea5a76 100644 --- a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt +++ b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt @@ -3,3 +3,4 @@ Changes for Hadoop Native Map Output Collector MAPREDUCE-5985. native-task: Fix build on macosx. Contributed by Binglin Chang MAPREDUCE-5994. Simplify ByteUtils and fix failing test. (todd) +MAPREDUCE-5996. native-task: Rename system tests into standard directory layout (todd) http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/native
[26/52] [abbrv] MAPREDUCE-6056. native-task: move system test working dir to target dir and cleanup test config xml files (Manu Zhang via bchang)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ade9b04/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-snappy-compress-conf.xml -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-snappy-compress-conf.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-snappy-compress-conf.xml deleted file mode 100644 index 46b3ea7..000 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-snappy-compress-conf.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - - - - - -mapred.output.compress -false - - -mapreduce.reduce.class -org.apache.hadoop.mapred.nativetask.kvtest.HashSumReducer - - - -mapred.output.value.class -org.apache.hadoop.io.IntWritable - - -nativetask.compress.filesize -10 - - - -mapred.compress.map.output -true - - -mapred.output.compression.type -BLOCK - - -mapred.map.output.compression.codec -org.apache.hadoop.io.compress.SnappyCodec - - -hadoop.native.lib -true - -
[25/52] [abbrv] git commit: MAPREDUCE-6058. native-task: KVTest and LargeKVTest should check mr job is sucessful (Contributed by Binglin Chang)
MAPREDUCE-6058. native-task: KVTest and LargeKVTest should check mr job is sucessful (Contributed by Binglin Chang) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/17cd0faa Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/17cd0faa Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/17cd0faa Branch: refs/heads/HDFS-6584 Commit: 17cd0faaceb8f9ce00b8c2c200e810f78f36c671 Parents: bfd1d75 Author: Binglin Chang Authored: Tue Sep 2 11:00:48 2014 +0800 Committer: Binglin Chang Committed: Tue Sep 2 11:00:48 2014 +0800 -- .../CHANGES.MAPREDUCE-2841.txt | 1 + .../hadoop-mapreduce-client-nativetask/pom.xml | 28 ++-- .../nativetask/combinertest/CombinerTest.java | 10 +-- .../combinertest/LargeKVCombinerTest.java | 15 +-- .../combinertest/OldAPICombinerTest.java| 5 .../nativetask/compresstest/CompressTest.java | 18 - .../hadoop/mapred/nativetask/kvtest/KVJob.java | 5 ++-- .../hadoop/mapred/nativetask/kvtest/KVTest.java | 15 +-- .../mapred/nativetask/kvtest/LargeKVTest.java | 15 +-- .../nativetask/nonsorttest/NonSortTest.java | 5 10 files changed, 74 insertions(+), 43 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/17cd0faa/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt -- diff --git a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt index 6384757..baa88c1 100644 --- a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt +++ b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt @@ -17,3 +17,4 @@ MAPREDUCE-6026. native-task: fix logging (Manu Zhang via todd) MAPREDUCE-6035. native-task: sources/test-sources jar distribution (Manu Zhang via todd) MAPREDUCE-5977. Fix or suppress native-task gcc warnings (Manu Zhang via todd) MAPREDUCE-6054. native-task: Speed up tests (todd) +MAPREDUCE-6058. native-task: KVTest and LargeKVTest should check mr job is sucessful (Binglin Chang) http://git-wip-us.apache.org/repos/asf/hadoop/blob/17cd0faa/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml index bb7d7bb..f62743e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml @@ -83,10 +83,8 @@ -**/TestTaskContext.java -**/buffer/Test*.java -**/handlers/Test*.java -**/serde/Test*.java +**/*Test.java +**/Test*.java @@ -201,28 +199,6 @@ - -org.apache.maven.plugins -maven-surefire-plugin - - - - listener - org.apache.hadoop.test.TimedOutTestsListener - - - -**/TestTaskContext.java -**/buffer/Test*.java -**/handlers/Test*.java -**/serde/Test*.java -**/combinertest/*Test.java -**/compresstest/*Test.java -**/nonsorttest/*Test.java -**/kvtest/*Test.java - - - http://git-wip-us.apache.org/repos/asf/hadoop/blob/17cd0faa/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java index 8a4aa6f..abbe28e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java @@ -18,6 +18,7 @@
[41/52] [abbrv] git commit: HADOOP-11088. Unittest TestKeyShell, TestCredShell and TestKMS assume UNIX path separator for JECKS key store path. Contributed by Xiaoyu Yao.
HADOOP-11088. Unittest TestKeyShell, TestCredShell and TestKMS assume UNIX path
separator for JECKS key store path. Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/957414d4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/957414d4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/957414d4
Branch: refs/heads/HDFS-6584
Commit: 957414d4cb57cb8172070cc53530b7da78b8c9ca
Parents: 54e5794
Author: cnauroth
Authored: Fri Sep 12 14:50:07 2014 -0700
Committer: cnauroth
Committed: Fri Sep 12 14:50:07 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../apache/hadoop/crypto/key/TestKeyShell.java | 4 +++-
.../hadoop/security/alias/TestCredShell.java| 22 +---
.../crypto/key/kms/server/KMSConfiguration.java | 15 ++---
.../hadoop/crypto/key/kms/server/MiniKMS.java | 3 ++-
.../hadoop/crypto/key/kms/server/TestKMS.java | 3 ++-
6 files changed, 30 insertions(+), 20 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/957414d4/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 1176a21..051eac1 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -793,6 +793,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11083. After refactoring of HTTP proxyuser to common, doAs param is
case sensitive. (tucu)
+HADOOP-11088. Unittest TestKeyShell, TestCredShell and TestKMS assume UNIX
+path separator for JECKS key store path. (Xiaoyu Yao via cnauroth)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/957414d4/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java
index d65d9b2..3407eb7 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java
@@ -24,6 +24,7 @@ import java.io.PrintStream;
import java.util.UUID;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -51,7 +52,8 @@ public class TestKeyShell {
if (!tmpDir.mkdirs()) {
throw new IOException("Unable to create " + tmpDir);
}
-jceksProvider = "jceks://file" + tmpDir + "/keystore.jceks";
+final Path jksPath = new Path(tmpDir.toString(), "keystore.jceks");
+jceksProvider = "jceks://file" + jksPath.toUri();
initialStdOut = System.out;
initialStdErr = System.err;
System.setOut(new PrintStream(outContent));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/957414d4/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredShell.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredShell.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredShell.java
index b9f0dc9..c890362 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredShell.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredShell.java
@@ -29,6 +29,7 @@ import java.util.Arrays;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
import org.junit.Before;
import org.junit.Test;
@@ -37,18 +38,23 @@ public class TestCredShell {
private final ByteArrayOutputStream errContent = new ByteArrayOutputStream();
private static final File tmpDir =
new File(System.getProperty("test.build.data", "/tmp"), "creds");
-
+
+ /* The default JCEKS provider - for testing purposes */
+ private String jceksProvider;
+
@Before
public void setup() throws Exception {
System.setOut(new PrintStream(outContent));
System.setErr(new PrintStream(errContent));
+final Path jksPath = new Path(tmpDir.toString(), "keystore.jceks");
+jceksProvider = "jceks://file" + jksPath.toUri();
}
@Test
public void testCredentialSuccessfulLifecycle() throws Exception {
outContent.reset();
Str
[11/52] [abbrv] git commit: MAPREDUCE-5995. native-task: Revert changes to Text internals. Contributed by Todd Lipcon.
MAPREDUCE-5995. native-task: Revert changes to Text internals. Contributed by
Todd Lipcon.
git-svn-id:
https://svn.apache.org/repos/asf/hadoop/common/branches/MR-2841@1613828
13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/78d86a98
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/78d86a98
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/78d86a98
Branch: refs/heads/HDFS-6584
Commit: 78d86a983960f231f51e47579c0b1ae73f9645fd
Parents: 43917e5
Author: Todd Lipcon
Authored: Sun Jul 27 19:04:14 2014 +
Committer: Todd Lipcon
Committed: Sun Jul 27 19:04:14 2014 +
--
.../src/main/java/org/apache/hadoop/io/Text.java| 6 +-
hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt | 1 +
.../hadoop/mapred/nativetask/serde/TextSerializer.java | 9 +
3 files changed, 3 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78d86a98/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java
index 73cd1e6..3dc5076 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java
@@ -226,10 +226,6 @@ public class Text extends BinaryComparable
this.length = len;
}
- public void setLength(int len) {
-this.length = len;
- }
-
/**
* Append a range of bytes to the end of the given text
* @param utf8 the data to copy from
@@ -264,7 +260,7 @@ public class Text extends BinaryComparable
* @param len the number of bytes we need
* @param keepData should the old data be kept
*/
- public void setCapacity(int len, boolean keepData) {
+ private void setCapacity(int len, boolean keepData) {
if (bytes == null || bytes.length < len) {
if (bytes != null && keepData) {
bytes = Arrays.copyOf(bytes, Math.max(len,length << 1));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78d86a98/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
index 92c94a8..42e0af3 100644
--- a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
+++ b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
@@ -7,3 +7,4 @@ MAPREDUCE-5996. native-task: Rename system tests into standard
directory layout
MAPREDUCE-5997. native-task: Use DirectBufferPool from Hadoop Common (todd)
MAPREDUCE-6000. native-task: Simplify ByteBufferDataReader/Writer (todd)
MAPREDUCE-5991. native-task should not run unit tests if native profile is not
enabled. (Binglin Chang)
+MAPREDUCE-5995. native-task: Revert changes to Text internals (todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78d86a98/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/TextSerializer.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/TextSerializer.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/TextSerializer.java
index 63a64de..cde4c7b 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/TextSerializer.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/TextSerializer.java
@@ -42,13 +42,6 @@ public class TextSerializer implements
INativeSerializer, INativeComparabl
@Override
public void deserialize(DataInput in, int length, Text w) throws IOException
{
-try {
- w.setCapacity(length, true);
- w.setLength(length);
-} catch (final Exception e) {
- throw new IOException(e);
-}
-final byte[] bytes = w.getBytes();
-in.readFully(bytes, 0, length);
+w.readWithKnownLength(in, length);
}
}
[30/52] [abbrv] git commit: MAPREDUCE-6055. native-task: findbugs, interface annotations, and other misc cleanup
MAPREDUCE-6055. native-task: findbugs, interface annotations, and other misc cleanup Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1081d9ce Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1081d9ce Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1081d9ce Branch: refs/heads/HDFS-6584 Commit: 1081d9cee23aa661d7c9165bc9855865a38b528e Parents: cce7d1e Author: Todd Lipcon Authored: Wed Sep 3 12:02:47 2014 -0700 Committer: Todd Lipcon Committed: Wed Sep 3 12:22:38 2014 -0700 -- .../CHANGES.MAPREDUCE-2841.txt | 1 + .../hadoop/mapred/nativetask/Command.java | 8 + .../mapred/nativetask/CommandDispatcher.java| 11 ++- .../hadoop/mapred/nativetask/Constants.java | 18 ++- .../hadoop/mapred/nativetask/DataChannel.java | 3 ++ .../hadoop/mapred/nativetask/DataReceiver.java | 3 ++ .../mapred/nativetask/HadoopPlatform.java | 6 ++-- .../mapred/nativetask/ICombineHandler.java | 3 ++ .../mapred/nativetask/INativeComparable.java| 5 +++ .../mapred/nativetask/INativeHandler.java | 2 ++ .../mapred/nativetask/NativeBatchProcessor.java | 3 +- .../mapred/nativetask/NativeDataSource.java | 10 ++ .../mapred/nativetask/NativeDataTarget.java | 14 +++-- .../NativeMapOutputCollectorDelegator.java | 11 --- .../hadoop/mapred/nativetask/NativeRuntime.java | 10 +++--- .../hadoop/mapred/nativetask/Platform.java | 15 - .../hadoop/mapred/nativetask/Platforms.java | 7 +++-- .../mapred/nativetask/StatusReportChecker.java | 33 ++-- .../hadoop/mapred/nativetask/TaskContext.java | 29 + .../mapred/nativetask/buffer/BufferType.java| 5 +-- .../nativetask/buffer/ByteBufferDataReader.java | 10 ++ .../nativetask/buffer/ByteBufferDataWriter.java | 10 +++--- .../nativetask/buffer/DataInputStream.java | 2 ++ .../nativetask/buffer/DataOutputStream.java | 11 +++ .../mapred/nativetask/buffer/InputBuffer.java | 2 ++ .../mapred/nativetask/buffer/OutputBuffer.java | 2 ++ .../nativetask/handlers/BufferPullee.java | 4 ++- .../nativetask/handlers/BufferPuller.java | 26 --- .../nativetask/handlers/BufferPushee.java | 2 ++ .../nativetask/handlers/BufferPusher.java | 3 +- .../nativetask/handlers/CombinerHandler.java| 19 +-- .../mapred/nativetask/handlers/IDataLoader.java | 3 ++ .../handlers/NativeCollectorOnlyHandler.java| 22 - .../serde/BoolWritableSerializer.java | 2 ++ .../serde/ByteWritableSerializer.java | 2 ++ .../serde/BytesWritableSerializer.java | 2 ++ .../nativetask/serde/DefaultSerializer.java | 4 +++ .../serde/DoubleWritableSerializer.java | 2 ++ .../serde/FloatWritableSerializer.java | 3 ++ .../mapred/nativetask/serde/IKVSerializer.java | 12 --- .../nativetask/serde/INativeSerializer.java | 4 +++ .../nativetask/serde/IntWritableSerializer.java | 2 ++ .../mapred/nativetask/serde/KVSerializer.java | 17 ++ .../serde/LongWritableSerializer.java | 2 ++ .../nativetask/serde/NativeSerialization.java | 2 ++ .../serde/NullWritableSerializer.java | 2 ++ .../serde/SerializationFramework.java | 3 ++ .../mapred/nativetask/serde/TextSerializer.java | 2 ++ .../serde/VIntWritableSerializer.java | 3 +- .../serde/VLongWritableSerializer.java | 3 +- .../mapred/nativetask/util/BytesUtil.java | 2 ++ .../mapred/nativetask/util/ConfigUtil.java | 5 ++- .../nativetask/util/LocalJobOutputFiles.java| 2 ++ .../nativetask/util/NativeTaskOutput.java | 2 ++ .../nativetask/util/NativeTaskOutputFiles.java | 3 +- .../mapred/nativetask/util/OutputUtil.java | 5 +-- .../mapred/nativetask/util/ReadWriteBuffer.java | 5 +-- .../mapred/nativetask/util/SizedWritable.java | 4 ++- .../mapred/nativetask/TestTaskContext.java | 4 +-- 59 files changed, 264 insertions(+), 148 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt -- diff --git a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt index 7c9558e..269a2f6 100644 --- a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt +++ b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt @@ -19,3 +19,4 @@ MAPREDUCE-5977. Fix or suppress native-task gcc warnings (Manu Zhang via todd) MAPREDUCE-6054. native-task: Speed up tests (todd) MAPREDUCE-6058. native-task: KVTest and LargeKVTest should check mr job is sucessful (Binglin Chang) MAPREDUCE-6
[07/52] [abbrv] git commit: MAPREDUCE-5997. native-task: Use DirectBufferPool from Hadoop Common. Contributed by Todd Lipcon.
MAPREDUCE-5997. native-task: Use DirectBufferPool from Hadoop Common.
Contributed by Todd Lipcon.
git-svn-id:
https://svn.apache.org/repos/asf/hadoop/common/branches/MR-2841@1613034
13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/77acc70d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/77acc70d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/77acc70d
Branch: refs/heads/HDFS-6584
Commit: 77acc70df5d64055ea809222f3d2f0d66c611196
Parents: 5149a8a
Author: Todd Lipcon
Authored: Thu Jul 24 08:20:25 2014 +
Committer: Todd Lipcon
Committed: Thu Jul 24 08:20:25 2014 +
--
.../CHANGES.MAPREDUCE-2841.txt | 1 +
.../mapred/nativetask/NativeBatchProcessor.java | 8 +-
.../nativetask/buffer/DirectBufferPool.java | 93 -
.../mapred/nativetask/buffer/InputBuffer.java | 17 +-
.../nativetask/buffer/TestDirectBufferPool.java | 201 ---
5 files changed, 20 insertions(+), 300 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77acc70d/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
index cea5a76..e12f743 100644
--- a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
+++ b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
@@ -4,3 +4,4 @@ Changes for Hadoop Native Map Output Collector
MAPREDUCE-5985. native-task: Fix build on macosx. Contributed by Binglin Chang
MAPREDUCE-5994. Simplify ByteUtils and fix failing test. (todd)
MAPREDUCE-5996. native-task: Rename system tests into standard directory
layout (todd)
+MAPREDUCE-5997. native-task: Use DirectBufferPool from Hadoop Common (todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77acc70d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeBatchProcessor.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeBatchProcessor.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeBatchProcessor.java
index fd68ea6..837da0e 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeBatchProcessor.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeBatchProcessor.java
@@ -24,12 +24,13 @@ import java.nio.ByteBuffer;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.mapred.nativetask.buffer.BufferType;
-import org.apache.hadoop.mapred.nativetask.buffer.DirectBufferPool;
import org.apache.hadoop.mapred.nativetask.buffer.InputBuffer;
import org.apache.hadoop.mapred.nativetask.buffer.OutputBuffer;
import org.apache.hadoop.mapred.nativetask.util.ReadWriteBuffer;
import org.apache.hadoop.mapred.nativetask.util.ConfigUtil;
+import org.apache.hadoop.util.DirectBufferPool;
/**
* used to create channel, transfer data and command between Java and native
@@ -126,9 +127,8 @@ public class NativeBatchProcessor implements INativeHandler
{
NativeRuntime.releaseNativeObject(nativeHandlerAddr);
nativeHandlerAddr = 0;
}
-if (null != in && null != in.getByteBuffer() &&
in.getByteBuffer().isDirect()) {
- DirectBufferPool.getInstance().returnBuffer(in.getByteBuffer());
-}
+IOUtils.cleanup(LOG, in);
+in = null;
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77acc70d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/DirectBufferPool.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/DirectBufferPool.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/DirectBufferPool.java
deleted file mode 100644
index bd3c6bb..000
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/ha
[12/52] [abbrv] git commit: MAPREDUCE-6005. native-task: Fix some valgrind errors (Binglin Chang)
MAPREDUCE-6005. native-task: Fix some valgrind errors (Binglin Chang)
git-svn-id:
https://svn.apache.org/repos/asf/hadoop/common/branches/MR-2841@1615489
13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/83a39673
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/83a39673
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/83a39673
Branch: refs/heads/HDFS-6584
Commit: 83a396733ea43564beee928f41041a341e26db05
Parents: 78d86a9
Author: Binglin Chang
Authored: Mon Aug 4 06:19:51 2014 +
Committer: Binglin Chang
Committed: Mon Aug 4 06:19:51 2014 +
--
.../CHANGES.MAPREDUCE-2841.txt | 1 +
.../src/main/native/src/codec/GzipCodec.cc | 11 +---
.../src/main/native/src/lib/Buffers.cc | 2 +-
.../src/main/native/src/lib/FileSystem.cc | 54 +++-
.../src/main/native/src/lib/FileSystem.h| 5 --
.../src/main/native/src/lib/MemoryBlock.h | 4 ++
.../main/native/src/lib/NativeObjectFactory.cc | 2 +-
.../main/native/src/lib/NativeObjectFactory.h | 15 +-
.../native/src/lib/PartitionBucketIterator.cc | 7 ++-
.../src/main/native/src/util/StringUtil.cc | 15 ++
.../src/main/native/src/util/StringUtil.h | 2 +-
.../src/main/native/src/util/WritableUtils.cc | 2 +-
.../src/main/native/test/TestCommand.cc | 12 ++---
.../src/main/native/test/TestCounter.cc | 15 +-
.../src/main/native/test/TestFileSystem.cc | 31 ---
.../src/main/native/test/TestIFile.cc | 9 ++--
.../src/main/native/test/TestMain.cc| 8 ++-
.../src/main/native/test/lib/TestByteBuffer.cc | 1 +
.../lib/TestComparatorForDualPivotQuickSort.cc | 1 +
.../native/test/lib/TestComparatorForStdSort.cc | 1 +
.../native/test/lib/TestFixSizeContainer.cc | 3 ++
.../src/main/native/test/lib/TestIterator.cc| 40 +++
.../src/main/native/test/lib/TestKVBuffer.cc| 1 +
.../native/test/lib/TestMemBlockIterator.cc | 8 +++
.../src/main/native/test/lib/TestMemoryBlock.cc | 3 ++
.../main/native/test/lib/TestPartitionBucket.cc | 16 +-
.../src/main/native/test/lib/TestReadBuffer.cc | 4 +-
.../main/native/test/lib/TestReadWriteBuffer.cc | 4 +-
.../src/main/native/test/util/TestStringUtil.cc | 5 ++
29 files changed, 146 insertions(+), 136 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83a39673/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
index 42e0af3..c70df12 100644
--- a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
+++ b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
@@ -8,3 +8,4 @@ MAPREDUCE-5997. native-task: Use DirectBufferPool from Hadoop
Common (todd)
MAPREDUCE-6000. native-task: Simplify ByteBufferDataReader/Writer (todd)
MAPREDUCE-5991. native-task should not run unit tests if native profile is not
enabled. (Binglin Chang)
MAPREDUCE-5995. native-task: Revert changes to Text internals (todd)
+MAPREDUCE-6005. native-task: Fix some valgrind errors (Binglin Chang)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83a39673/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/GzipCodec.cc
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/GzipCodec.cc
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/GzipCodec.cc
index 79eac8b..bb699e9 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/GzipCodec.cc
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/GzipCodec.cc
@@ -28,7 +28,6 @@ GzipCompressStream::GzipCompressStream(OutputStream * stream,
uint32_t bufferSiz
: CompressStream(stream), _compressedBytesWritten(0), _zstream(NULL),
_finished(false) {
_buffer = new char[bufferSizeHint];
_capacity = bufferSizeHint;
- std::cout << "gzip capacity " << _capacity << std::endl;
_zstream = malloc(sizeof(z_stream));
z_stream * zstream = (z_stream*)_zstream;
memset(zstream, 0, sizeof(z_stream));
@@ -44,6 +43,7 @@ GzipCompressStream::GzipCompressStream(OutputStream * stream,
uint32_t bufferSiz
GzipCompressStream::~GzipCompressStream() {
if (_zstream != NULL) {
+deflateEnd((z_stream*)_zstream);
free(_zstream);
_zstream = NULL;
}
@@ -52,17 +52,13 @
[49/52] [abbrv] git commit: HDFS-7061. Add test to verify encryption zone creation after NameNode restart without saving namespace. Contributed by Stephen Chu.
HDFS-7061. Add test to verify encryption zone creation after NameNode restart
without saving namespace. Contributed by Stephen Chu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fc741b5d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fc741b5d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fc741b5d
Branch: refs/heads/HDFS-6584
Commit: fc741b5d78e7e006355e17b1b5839f502e37261b
Parents: 14e2639
Author: Andrew Wang
Authored: Sun Sep 14 23:48:24 2014 -0700
Committer: Andrew Wang
Committed: Sun Sep 14 23:49:01 2014 -0700
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/hdfs/TestEncryptionZones.java | 10 ++
2 files changed, 13 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc741b5d/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index bddf303..57a4a0f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -455,6 +455,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6482. Use block ID-based block layout on datanodes (James Thomas via
Colin Patrick McCabe)
+HDFS-7061. Add test to verify encryption zone creation after NameNode
+restart without saving namespace. (Stephen Chu via wang)
+
OPTIMIZATIONS
HDFS-6690. Deduplicate xattr names in memory. (wang)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc741b5d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
index db3c085..b3bf5d9 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
@@ -338,6 +338,16 @@ public class TestEncryptionZones {
cluster.restartNameNode(true);
assertNumZones(numZones);
assertZonePresent(null, zone1.toString());
+
+// Verify newly added ez is present after restarting the NameNode
+// without persisting the namespace.
+Path nonpersistZone = new Path("/nonpersistZone");
+fsWrapper.mkdir(nonpersistZone, FsPermission.getDirDefault(), false);
+dfsAdmin.createEncryptionZone(nonpersistZone, TEST_KEY);
+numZones++;
+cluster.restartNameNode(true);
+assertNumZones(numZones);
+assertZonePresent(null, nonpersistZone.toString());
}
/**
[39/52] [abbrv] git commit: Merge branch 'trunk' into MR-2841
Merge branch 'trunk' into MR-2841 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b160707a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b160707a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b160707a Branch: refs/heads/HDFS-6584 Commit: b160707ace7900f360fa914cd291675c1d9703ec Parents: 52a8b4d d1fa582 Author: Sean Zhong Authored: Sun Sep 7 13:01:27 2014 +0800 Committer: Sean Zhong Committed: Sun Sep 7 13:01:27 2014 +0800 -- hadoop-common-project/hadoop-common/CHANGES.txt | 13 + .../src/main/bin/hadoop-functions.sh| 4 +- .../crypto/key/kms/KMSClientProvider.java | 6 +- .../security/ssl/FileBasedKeyStoresFactory.java | 2 +- hadoop-common-project/hadoop-kms/pom.xml| 4 +- .../hadoop/crypto/key/kms/server/MiniKMS.java | 197 + .../hadoop/crypto/key/kms/server/TestKMS.java | 88 +- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 16 ++ hadoop-hdfs-project/hadoop-hdfs/pom.xml | 6 +- .../hadoop-hdfs/src/main/bin/hdfs | 15 +- .../java/org/apache/hadoop/hdfs/DFSClient.java | 4 + .../hadoop/hdfs/DistributedFileSystem.java | 24 ++ .../hadoop/hdfs/protocol/HdfsConstants.java | 2 +- .../server/blockmanagement/BlockManager.java| 23 +- .../server/blockmanagement/DatanodeManager.java | 6 +- .../server/blockmanagement/HostFileManager.java | 4 + .../server/datanode/ReplicaBeingWritten.java| 12 +- .../hdfs/server/datanode/ReplicaInPipeline.java | 33 ++- .../hdfs/server/datanode/ReplicaInfo.java | 7 + .../server/datanode/fsdataset/FsVolumeSpi.java | 11 + .../datanode/fsdataset/impl/BlockPoolSlice.java | 6 +- .../datanode/fsdataset/impl/FsDatasetImpl.java | 15 +- .../datanode/fsdataset/impl/FsVolumeImpl.java | 58 +++- .../hdfs/server/namenode/FSNamesystem.java | 46 +-- .../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 175 +-- .../apache/hadoop/hdfs/TestEncryptionZones.java | 43 +++ .../apache/hadoop/hdfs/TestHDFSServerPorts.java | 24 +- .../server/datanode/TestDirectoryScanner.java | 8 + .../fsdataset/impl/TestRbwSpaceReservation.java | 288 +++ .../fsdataset/impl/TestWriteToReplica.java | 2 +- .../hdfs/server/namenode/NameNodeAdapter.java | 2 +- .../TestValidateConfigurationSettings.java | 16 +- .../namenode/ha/TestDelegationTokensWithHA.java | 63 ++-- .../hdfs/server/namenode/ha/TestHAMetrics.java | 10 +- .../namenode/ha/TestHAStateTransitions.java | 52 ++-- .../namenode/ha/TestStandbyCheckpoints.java | 54 ++-- .../java/org/apache/hadoop/tools/TestTools.java | 2 +- hadoop-yarn-project/CHANGES.txt | 6 + .../yarn/webapp/util/TestWebAppUtils.java | 4 +- .../webapp/CrossOriginFilterInitializer.java| 12 +- .../TestCrossOriginFilterInitializer.java | 7 +- pom.xml | 1 + 42 files changed, 998 insertions(+), 373 deletions(-) --
[14/52] [abbrv] git commit: MAPREDUCE-5976. native-task: should not fail to build if snappy is missing (Manu Zhang)
MAPREDUCE-5976. native-task: should not fail to build if snappy is missing
(Manu Zhang)
git-svn-id:
https://svn.apache.org/repos/asf/hadoop/common/branches/MR-2841@1616115
13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/432f641b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/432f641b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/432f641b
Branch: refs/heads/HDFS-6584
Commit: 432f641bc2f6c7fdb67da578233cbe07fdfbf1ea
Parents: 7ecaa81
Author: Sean Zhong
Authored: Wed Aug 6 07:32:49 2014 +
Committer: Sean Zhong
Committed: Wed Aug 6 07:32:49 2014 +
--
.../CHANGES.MAPREDUCE-2841.txt | 1 +
.../src/CMakeLists.txt | 8 +-
.../NativeMapOutputCollectorDelegator.java | 34 ++--
.../hadoop/mapred/nativetask/NativeRuntime.java | 11 +-
.../mapred/nativetask/util/SnappyUtil.java | 32
.../src/main/native/src/codec/SnappyCodec.cc| 9 +-
.../src/main/native/src/codec/snappy-c.h| 138
.../src/main/native/src/codec/snappy.h | 163 ---
.../src/main/native/src/lib/Compressions.cc | 9 +
.../main/native/src/lib/NativeRuntimeJniImpl.cc | 25 +++
...che_hadoop_mapred_nativetask_NativeRuntime.h | 8 +
.../src/main/native/test/TestCompressions.cc| 69
.../src/main/native/test/TestIFile.cc | 3 +
.../src/main/native/test/lib/TestReadBuffer.cc | 4 +-
14 files changed, 117 insertions(+), 397 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/432f641b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
index 9dddcd5..4b77262 100644
--- a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
+++ b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
@@ -10,3 +10,4 @@ MAPREDUCE-5991. native-task should not run unit tests if
native profile is not e
MAPREDUCE-5995. native-task: Revert changes to Text internals (todd)
MAPREDUCE-6005. native-task: Fix some valgrind errors (Binglin Chang)
MAPREDUCE-5984. native-task: Reuse lz4 sources in hadoop-common (Binglin Chang)
+MAPREDUCE-5976. native-task: should not fail to build if snappy is missing
(Manu Zhang)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/432f641b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
index 36dbd9c..f38021d 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
@@ -122,7 +122,6 @@ CHECK_FUNCTION_EXISTS(memset HAVE_MEMSET)
CHECK_FUNCTION_EXISTS(strchr HAVE_STRCHR)
CHECK_FUNCTION_EXISTS(strtoul HAVE_STRTOUL)
-
SET(STORED_CMAKE_FIND_LIBRARY_SUFFIXES CMAKE_FIND_LIBRARY_SUFFIXES)
set_find_shared_library_version("1")
find_library(SNAPPY_LIBRARY
@@ -139,6 +138,7 @@ if (SNAPPY_LIBRARY AND SNAPPY_INCLUDE_DIR)
set(SNAPPY_SOURCE_FILES
"${D}/src/codec/SnappyCodec.cc")
else (SNAPPY_LIBRARY AND SNAPPY_INCLUDE_DIR)
+set(SNAPPY_LIBRARY "")
set(SNAPPY_INCLUDE_DIR "")
set(SNAPPY_SOURCE_FILES "")
IF(REQUIRE_SNAPPY)
@@ -146,6 +146,8 @@ else (SNAPPY_LIBRARY AND SNAPPY_INCLUDE_DIR)
ENDIF(REQUIRE_SNAPPY)
endif (SNAPPY_LIBRARY AND SNAPPY_INCLUDE_DIR)
+CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h)
+
include_directories(
${GENERATED_JAVAH}
${D}
@@ -154,15 +156,11 @@ include_directories(
${D}/src/lib
${D}/test
${CMAKE_CURRENT_SOURCE_DIR}
-#${CMAKE_CURRENT_SOURCE_DIR}/src
${CMAKE_BINARY_DIR}
${JNI_INCLUDE_DIRS}
${SNAPPY_INCLUDE_DIR}
)
-#SET(CMAKE_SOURCE_DIR
"/cygdrive/c/Users/tianlunz/repo/hadoop-2.2.0-src/hadoop-common-project/hadoop-common/src")
-CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h)
-
SET(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/432f641b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeMapOutputCollectorDelegator.java
--
diff --g
[04/52] [abbrv] MAPREDUCE-5996. native-task: Rename system tests into standard directory layout. Contributed by Todd Lipcon.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/MockValueClass.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/MockValueClass.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/MockValueClass.java
new file mode 100644
index 000..b27f00f
--- /dev/null
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/MockValueClass.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.nativetask.testutil;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.Random;
+
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapred.nativetask.util.BytesUtil;
+
+public class MockValueClass implements Writable {
+ private final static int DEFAULT_ARRAY_LENGTH = 16;
+ private int a = 0;
+ private byte[] array;
+ private final LongWritable longWritable;
+ private final Text txt;
+ private final Random rand = new Random();
+
+ public MockValueClass() {
+a = rand.nextInt();
+array = new byte[DEFAULT_ARRAY_LENGTH];
+rand.nextBytes(array);
+longWritable = new LongWritable(rand.nextLong());
+txt = new Text(BytesUtil.toStringBinary(array));
+ }
+
+ public MockValueClass(byte[] seed) {
+a = seed.length;
+array = new byte[seed.length];
+System.arraycopy(seed, 0, array, 0, seed.length);
+longWritable = new LongWritable(a);
+txt = new Text(BytesUtil.toStringBinary(array));
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+out.writeInt(a);
+out.writeInt(array.length);
+out.write(array);
+longWritable.write(out);
+txt.write(out);
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+a = in.readInt();
+final int length = in.readInt();
+array = new byte[length];
+in.readFully(array);
+longWritable.readFields(in);
+txt.readFields(in);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/ResultVerifier.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/ResultVerifier.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/ResultVerifier.java
new file mode 100644
index 000..b665971
--- /dev/null
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/ResultVerifier.java
@@ -0,0 +1,141 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
[24/52] [abbrv] git commit: MAPREDUCE-6054. native-task: Speed up tests. Contributed by Todd Lipcon.
MAPREDUCE-6054. native-task: Speed up tests. Contributed by Todd Lipcon.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bfd1d75d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bfd1d75d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bfd1d75d
Branch: refs/heads/HDFS-6584
Commit: bfd1d75d875b6ba261fdb1825d0f151b026c2d24
Parents: fad4524
Author: Todd Lipcon
Authored: Wed Aug 27 12:25:07 2014 -0700
Committer: Todd Lipcon
Committed: Wed Aug 27 12:25:49 2014 -0700
--
.../CHANGES.MAPREDUCE-2841.txt | 2 +-
.../mapred/nativetask/util/BytesUtil.java | 7 +-
.../hadoop/mapred/nativetask/kvtest/KVJob.java | 8 +-
.../hadoop/mapred/nativetask/kvtest/KVTest.java | 94 +---
.../mapred/nativetask/kvtest/TestInputFile.java | 13 ++-
.../nativetask/testutil/BytesFactory.java | 71 +--
.../nativetask/testutil/MockValueClass.java | 8 ++
.../mapred/nativetask/utils/TestBytesUtil.java | 7 ++
8 files changed, 123 insertions(+), 87 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfd1d75d/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
index 4dc08cb..6384757 100644
--- a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
+++ b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
@@ -16,4 +16,4 @@ MAPREDUCE-6006. native-task: add native tests to maven and
fix bug in pom.xml (B
MAPREDUCE-6026. native-task: fix logging (Manu Zhang via todd)
MAPREDUCE-6035. native-task: sources/test-sources jar distribution (Manu Zhang
via todd)
MAPREDUCE-5977. Fix or suppress native-task gcc warnings (Manu Zhang via todd)
-
+MAPREDUCE-6054. native-task: Speed up tests (todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfd1d75d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/BytesUtil.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/BytesUtil.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/BytesUtil.java
index d90ae8d..e33b23e 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/BytesUtil.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/BytesUtil.java
@@ -23,6 +23,9 @@ import com.google.common.primitives.Longs;
public class BytesUtil {
+ private static final char[] HEX_CHARS =
+ "0123456789abcdef".toCharArray();
+
/**
* Converts a big-endian byte array to a long value.
*
@@ -124,7 +127,9 @@ public class BytesUtil {
|| " `~!@#$%^&*()-_=+[]{}|;:'\",.<>/?".indexOf(ch) >= 0 ) {
result.append((char)ch);
} else {
-result.append(String.format("\\x%02X", ch));
+result.append("\\x");
+result.append(HEX_CHARS[(ch >> 4) & 0x0F]);
+result.append(HEX_CHARS[ch & 0x0F]);
}
}
return result.toString();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfd1d75d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java
index 6d683f8..3215d0b 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java
@@ -20,7 +20,11 @@ package org.apache.hadoop.mapred.nativetask.kvtest;
import java.io.IOException;
import java.util.zip.CRC32;
+import com.google.common.base.Stopwatch;
import com.google.common.primitives.Longs;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
im
[36/52] [abbrv] git commit: Merge remote-tracking branch 'apache/trunk' into MR-2841
Merge remote-tracking branch 'apache/trunk' into MR-2841 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4b3f1e2c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4b3f1e2c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4b3f1e2c Branch: refs/heads/HDFS-6584 Commit: 4b3f1e2ce48124b7406d77cc2ae1d0914311b0d4 Parents: 683987b c6107f5 Author: Todd Lipcon Authored: Fri Sep 5 10:47:27 2014 -0700 Committer: Todd Lipcon Committed: Fri Sep 5 10:47:27 2014 -0700 -- LICENSE.txt | 290 NOTICE.txt | 2 + README.txt | 31 ++ dev-support/create-release.sh | 24 +- .../main/resources/assemblies/hadoop-src.xml| 8 + hadoop-common-project/hadoop-common/CHANGES.txt | 20 ++ hadoop-common-project/hadoop-common/LICENSE.txt | 290 hadoop-common-project/hadoop-common/NOTICE.txt | 2 - hadoop-common-project/hadoop-common/README.txt | 31 -- .../dev-support/findbugsExcludeFile.xml | 2 +- .../hadoop/crypto/key/KeyProviderFactory.java | 36 +- .../crypto/key/kms/KMSClientProvider.java | 57 +--- .../hadoop/crypto/random/OsSecureRandom.java| 21 +- .../security/authorize/AccessControlList.java | 12 +- .../DelegationTokenAuthenticationFilter.java| 15 +- .../DelegationTokenAuthenticationHandler.java | 6 +- .../web/DelegationTokenAuthenticator.java | 20 +- .../apache/hadoop/util/HttpExceptionUtils.java | 185 ++ .../apache/hadoop/crypto/TestCryptoCodec.java | 69 +++- .../apache/hadoop/crypto/TestCryptoStreams.java | 2 +- .../crypto/key/TestKeyProviderFactory.java | 13 + .../crypto/random/TestOsSecureRandom.java | 15 + ...tionTokenAuthenticationHandlerWithMocks.java | 35 +- .../hadoop/util/TestHttpExceptionUtils.java | 167 + hadoop-common-project/hadoop-kms/pom.xml| 1 + .../hadoop/crypto/key/kms/server/KMS.java | 27 +- .../hadoop/crypto/key/kms/server/KMSACLs.java | 55 ++- .../key/kms/server/KMSExceptionsProvider.java | 12 +- .../hadoop-kms/src/site/apt/index.apt.vm| 88 - .../hadoop/crypto/key/kms/server/TestKMS.java | 100 +- .../crypto/key/kms/server/TestKMSACLs.java | 2 +- hadoop-dist/pom.xml | 3 + .../hadoop/fs/http/client/HttpFSFileSystem.java | 70 ++-- .../hadoop/fs/http/client/HttpFSUtils.java | 50 --- .../hadoop/lib/wsrs/ExceptionProvider.java | 14 +- .../fs/http/client/BaseTestHttpFSWith.java | 4 +- .../fs/http/server/TestHttpFSServerNoACLs.java | 10 +- .../hadoop/hdfs/nfs/nfs3/OpenFileCtx.java | 6 +- .../hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java| 2 +- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 25 ++ hadoop-hdfs-project/hadoop-hdfs/LICENSE.txt | 271 --- hadoop-hdfs-project/hadoop-hdfs/NOTICE.txt | 2 - .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 3 + .../java/org/apache/hadoop/hdfs/DFSUtil.java| 80 - .../hadoop/hdfs/DistributedFileSystem.java | 11 +- .../org/apache/hadoop/hdfs/inotify/Event.java | 12 + .../apache/hadoop/hdfs/protocolPB/PBHelper.java | 6 +- .../hdfs/server/datanode/BlockPoolManager.java | 12 +- .../hadoop/hdfs/server/datanode/DataNode.java | 2 +- .../hdfs/server/datanode/DataStorage.java | 2 +- .../hdfs/server/namenode/FSDirectory.java | 44 ++- .../hadoop/hdfs/server/namenode/FSEditLog.java | 6 +- .../hdfs/server/namenode/FSEditLogLoader.java | 8 +- .../hdfs/server/namenode/FSEditLogOp.java | 18 + .../server/namenode/FSImageSerialization.java | 17 + .../hdfs/server/namenode/FSNamesystem.java | 60 ++-- .../hdfs/server/namenode/INodeReference.java| 4 +- .../namenode/InotifyFSEditLogOpTranslator.java | 1 + .../server/namenode/NameNodeLayoutVersion.java | 4 +- .../snapshot/DirectorySnapshottableFeature.java | 2 +- .../org/apache/hadoop/hdfs/tools/GetConf.java | 4 +- .../hadoop-hdfs/src/main/proto/inotify.proto| 1 + .../src/main/resources/hdfs-default.xml | 18 + .../hadoop/hdfs/TestBlocksScheduledCounter.java | 18 +- .../hdfs/TestDFSInotifyEventInputStream.java| 8 +- .../org/apache/hadoop/hdfs/TestDFSRename.java | 6 + .../org/apache/hadoop/hdfs/TestDFSUtil.java | 26 ++ .../org/apache/hadoop/hdfs/TestFileAppend4.java | 2 +- .../apache/hadoop/hdfs/TestFileCreation.java| 119 +++ .../org/apache/hadoop/hdfs/TestLocalDFS.java| 29 +- .../server/datanode/TestBlockPoolManager.java | 22 ++ .../hdfs/server/namenode/CreateEditsLog.java| 2 +- .../hdfs/server/namenode/TestEditLog.java | 2 +- .../hdfs/server/namenode/TestStartup.java | 2 +- .../snapshot/Test
[03/52] [abbrv] MAPREDUCE-5996. native-task: Rename system tests into standard directory layout. Contributed by Todd Lipcon.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/data/testGlibcBugSpill.out
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/data/testGlibcBugSpill.out
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/data/testGlibcBugSpill.out
deleted file mode 100644
index 168a65d..000
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/data/testGlibcBugSpill.out
+++ /dev/null
@@ -1,2 +0,0 @@
-��*�PKM��&:\xBD\xC8\xCA\xCB\xAA\xB4c`\xD5|A1+\xCB\xDC\xAA\x7F\x1D\xA4\x1Bx\x1B\xD1k\xE1\x01\x0E\xEA\xC82\xB7\xF8
\xBD.z\xEFH\xFB*\xE8\xBD\xD4\x15o\x82\xA1\x0F}\x03\x00\x17\xC7\xF7Ce*T\x1D\xC4tk\xE7\x0CS\xBF["\xADW{\xB9\xFFLM\xB2\x87\x03\xFC\xC2\xE2\x9C%\xCB\xFF\x9A\x97]\x15_N\xA3\x8F&Db\xA6\x172\x99\xF8+\xA4\xA5\xAC4n\x0D}\xA0/[\xBD\xFD\xC4\xA7\xBCj\xF8\x85\xC6V\x87B\xBA\x85\x8A\xCDZ\xFBU\x84\xD7\xBC\xAB\x84H}_v\xC4\xB2\x11\x19\xDB\x06\x93\xB5\xBE\x92E\xCE\xFA\x02RG\xE4\xA3HcI4\xAB.X\xA5\xB8a\x06\x1E.\xADS\xFB\x8A&W\xDA\xCA2^\\x90\x87\x95l]r\x99\x88b\x1EU\xC8\xE0l\x1F#h\x08\xA8\x80I\x9B\xC0E\x19%\xDE\xE5\xA6?\xC2\x83\xA0\xD6O1\xFB_\x94\xF8\x8F\xD6\xD9\x16\xE2\x91\x1E\x8EL\x1F\xA2Y\x05\x9CoMN\x0C\x86\xE2\xE9\x9A\xB16o\xF6\xF3Q\xA2\xD8\xB8\xD0\xDB\x86\x93\xAFesc\xA0h\x12\xB9E\x1BU\x12\x02OXS\x8BXn\x0EQ\xABJ\xCE\xB2k\xB1\xD7Z'\xE4\xB7[`~4\x11\xB4\xCD`\xF5%\x1F\xF9\xEEW\x88\xC5\x14<+O\x1B9B>\x11\x95<\xDE\xD5\xA4C]\\xF1\x0C\xD5\xF4\x0E#\x87}\xE3\x89\xA2\xC2\xEA\x86\x9D\xE7\xAF\xA1\xC3;\xD2\xFF\xA6\xB2!\xAB\\x90i|n\xDE\xBB>:\xC6\x08\x1D,Q\xC1;\x15\x9D>UV\x8F\xD3;\xFA\x12\xD0U\xA9\xD7<\xC6\xFDX\x8F\x10vX\xF1\xF9H7\x12Z\x1FoB\x8B\xD6\xF0\x04\xEB\xA2\xC1\xACH\xFE\xC9_R\xF5\x1D\xA8\x82\xBD\xC6\x11\x19#\x9A\xA8\x09\x95\x06\x98\xA9f\xD9\x12\xA0i\xFA\xD3\xB2\x01\xE9\x06\x14"~."z\xDF\xF2YV
#z\xEB-\xFA$[U\xA6\xE2.\xD6\xD0kf\xA3\x83\xE0\x86\xD9\xC1\x7FO\xE9\xF4\xEF\x81\x06\xA3\xDE\xC8\xAC"t\x8B\xCAr\xE6\x82$8\x0F\xE1\x06\xF9\xBB\x0C\x9F\xAA"8\x94\xBBU\x8DQ\xC3\xE9@'\xF9\xBF\xCF\x8B\xD4\x995\xEB\xE9t3Q\x10\xD7\x0D\x9D\x94\xCA\x84\xC2\xE2{\x0B\x89r\xA6\xC6\xAA\xE5C\xC6U\xA6\xD1tqa\xA0\xD7RO\x92\xC9\xBE"\xF9\xD1\xDE\x93b\x06\xD3ae:\xB7\x8C\x99\xD6\xFFI\x86\x8CvtX@k\xE4m\x93wz\x8A\xC2U\xFBb\xA2\x9Ao\xAF\x8D\x19k\xA2pP\x83s\xFE\x0E\x0FY\xA0\xA7E'\xC0\x02\xF4\x98A5\xF2\x8A?\x04$\x89\xC7]\x0A\xFBX\x97*\xAEN\x13#\xB3a\xD2y\xD3#_B\xAC\x05:\xAC\x92\xEAO\x08H\x88N\x1A\xB9\xDC\xFA\x11ikJ\x12!\xE8u\xCD+\x88\x98\xE3c\xCB\xD9<1%\x98KDN\xC6\xF2\xB7\x86o6\x91P`\x9B\xA1\x0B\x82\xEB\x15H\xA0\xC4\x85\xDF\xAC"\xA1b\xD9\xA3b\xB8E\xB59_\xF4\xCCWH\x09&5\xE6\xBE\xF2\x19FC\x0E\xAB\xEA6\x0C\xAD5\x90/$D\xB3\x9E\x81[9j\x8A\xC4\x85\xAAA\x7Fe\xDCG8\x00\xDA\xCFi\xBDp\x18?\xF5\xA8~@\xC1\x08\xDF\xE5\xAE,\xDF0t\xCB\x92W8&V\x01F\x1A./\x8D\xAF\xD8\x87\xCE\x80w*\x18Is\x17\x15\x17DI\xB4a_\N\xB77\xA7n\x16\xDF
IE\xEF\x9E\x8Cd7\x1B\xF9\x97\xF9E\x86\x98\x9F\x1D\xB6\x9F\x94\xF7\x8AJ\x1A\xCD\x88\xD3\xD3\xDEw\x92Q\\xF5\xC6\xD6\x11c\x81\x00\xE8\xD9"'\xE1\x9D5\xFC\x11},\xB8\xB2V\xE6\xC0\xB7a/\x18~=G\xAC\x9EGxR,\x9B\x91\xA0\xE9\x85\x14J\xB3\xB2O\xEA3\xB2F\xA7vo\x88\xFEm\x18*g%\xA4l\x9B\xF0\xA5`$\xEBo\xFC?\x13"s\x0D\x91y\x92\xE0u\xFA\xD1p+)\xACpi\xE3\xB4L-\x0A\xF1#\xCF\x1A\x82\x8A\xE5\xEF80\xC9\x17z@\xD1\x9AoK\xCE\xE42\x92M\xEF\x85\xBE(z\x860\xC6\x03t\x02\xA1\xD2\x09\x1E\xB3\x80t\x86|\x8E~F\xFD1i|\x84^\x07\xC9Z\xBE\x91\xA7\x06\x9B\xC7\x8F\xFB\xD4\xB84\xED\xA2\x108/X\x89\xF4W\xF6\xE3\xEE\x94Q,H\xFF>o3E\xA7q\xE5\x15\x86\xCF\x0F_\xF9\xE8N\xCD}\xEB\xFD\x0E\x03EZi\x83
\xA7D|{]\xEE\xBA\xE4\x00RR\x1C\xFBj\x81\xF0{w\x9F\xA6F\xBB\x00\x0Cw\x01\xE1\xFE\xC8\xE8\xAC\xD18\x19,\xE9\x9E\xFE\xF0\xA45ov[K\x86UT\x00\xC0*\xEF\x9De\xE7pN[\xA5~\xF3\xDA\xAD\xE0\x85\xB0Nb\x09I_\xA8B:b\x9A\x10\xC2\xCF\xE83|\xB1\xCD\x17\xE8\x95\xA6!"\xD72DR\x03\xB7\xF4"\xC2\x88\xF1Rl+t\xA7x\x04\x10\xC1@\\xC3\xE5}\xDD`\xA2\x91W\x7F%S\xB7\x1
3\xFF\xF2\x82~e\xDC\xDD\xA1\xC5\xDF\xC6xx\xAC\xF3!\xDC\xFE)\xCF@\x8E\x9CU\xE9\x03m\xF7\xB5\xC8\xAB\xB9\x94\x02\x8C\x910\xE6\xBC\xA3\xFAc\x18\x83v\x09\xAF\xE9u\xE1\xB6N4\x16\x8E\xD0\xFA;\xBF\xDEt:\xDAq\xC6~\x979\x0C\x9C%=\x19\x09\xFB\xC7\xB2,yV\xED\xC6}\x0A\\xFB,*\xEF+V\xA8\xA9\x81\xADOD\xF4\@\xFE\xAEV\xED\xC8\xC5O[\xE6\xC7\xEE\xC7\xA0\x10L\xFFiY\xAE\xD4\xDE\xE89\xF6\x83\xC8\xF7\xB9-\xB8\x0A8{A8\xF2k\xF0U\xADh\x1F\xB5\xEF\x14"\x8E\xB0Cl'\xAE\xC6Qn\xD4V\xB6i\x07\xBA(+\xEFZ\xE7\xB2\xCC[p\xC0j\x90\x8C.l*\xE2\x97;&\xD9\xB7\x8D\x8A\xE5S\xCEtT|\xAC\xAEp\xACg\xF2\xFC\xB5\xECac\x0E]\x09,\x09n>#\x1Akg\xC2\xD5|\xB9\x8Df\x17\x851v'|*=\x8D\xFA\x00>\xB2\xD9\x04&QI\xB0\xC4{\x18\x1A}\x8DV:\xA8\xECN4n*\xAE\xE4\x07YK\x03\xF8ms\x17/\x17J\xFD\xFD\x14/\x86X\xCE3\x1C^\xC4\x8D\x9Bi\xF1\xCA\xE6\xB2\xE9\x83\xFD\x0F\xCD\x8A\x9F\x1D\xB5\xC70\xBAU\xFD\xDCOR\xF7Y2I$^\xBD\xFC;"\xE6\x10\xF4\xA5\x80\x8B\x13\xF7'm}\xD3\x8C\xAB\xED\xE0\xE7\
[13/52] [abbrv] git commit: MAPREDUCE-5984. native-task: Reuse lz4 sources in hadoop-common. Contributed by Binglin Chang
MAPREDUCE-5984. native-task: Reuse lz4 sources in hadoop-common. Contributed by
Binglin Chang
git-svn-id:
https://svn.apache.org/repos/asf/hadoop/common/branches/MR-2841@1616105
13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7ecaa81d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7ecaa81d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7ecaa81d
Branch: refs/heads/HDFS-6584
Commit: 7ecaa81d2746a805b9937fcd134ceaa8607ed7b5
Parents: 83a3967
Author: Binglin Chang
Authored: Wed Aug 6 06:01:12 2014 +
Committer: Binglin Chang
Committed: Wed Aug 6 06:01:12 2014 +
--
.../CHANGES.MAPREDUCE-2841.txt | 1 +
.../hadoop-mapreduce-client-nativetask/pom.xml | 6 +
.../src/CMakeLists.txt | 4 +-
.../src/main/native/COPYING | 12 +-
.../src/main/native/lz4/lz4.c | 740 ---
.../src/main/native/lz4/lz4.h | 96 ---
.../src/main/native/src/codec/Lz4Codec.cc | 22 +-
.../src/main/native/test/TestCompressions.cc| 6 +-
.../src/main/native/test/TestIFile.cc | 2 +-
9 files changed, 21 insertions(+), 868 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ecaa81d/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
index c70df12..9dddcd5 100644
--- a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
+++ b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
@@ -9,3 +9,4 @@ MAPREDUCE-6000. native-task: Simplify
ByteBufferDataReader/Writer (todd)
MAPREDUCE-5991. native-task should not run unit tests if native profile is not
enabled. (Binglin Chang)
MAPREDUCE-5995. native-task: Revert changes to Text internals (todd)
MAPREDUCE-6005. native-task: Fix some valgrind errors (Binglin Chang)
+MAPREDUCE-5984. native-task: Reuse lz4 sources in hadoop-common (Binglin Chang)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ecaa81d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
index 2cb483e..9727800 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
@@ -163,6 +163,12 @@
+
+
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ecaa81d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
index 3094162..36dbd9c 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
@@ -155,7 +155,7 @@ include_directories(
${D}/test
${CMAKE_CURRENT_SOURCE_DIR}
#${CMAKE_CURRENT_SOURCE_DIR}/src
-#${CMAKE_BINARY_DIR}
+${CMAKE_BINARY_DIR}
${JNI_INCLUDE_DIRS}
${SNAPPY_INCLUDE_DIR}
)
@@ -174,7 +174,7 @@ else (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
endif (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
add_dual_library(nativetask
-${D}/lz4/lz4.c
+${CMAKE_BINARY_DIR}/lz4.c
${D}/cityhash/city.cc
${D}/src/codec/BlockCodec.cc
${D}/src/codec/GzipCodec.cc
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ecaa81d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/COPYING
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/COPYING
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/COPYING
index c5d3420..1d60d1d 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/COP
[44/52] [abbrv] git commit: YARN-2542. Fixed NPE when retrieving ApplicationReport from TimeLineServer. Contributed by Zhijie Shen
YARN-2542. Fixed NPE when retrieving ApplicationReport from TimeLineServer.
Contributed by Zhijie Shen
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a0ad975e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a0ad975e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a0ad975e
Branch: refs/heads/HDFS-6584
Commit: a0ad975ea1e70f9532cf6cb6c1d9d92736ca0ebc
Parents: e65ae57
Author: Jian He
Authored: Fri Sep 12 15:24:17 2014 -0700
Committer: Jian He
Committed: Fri Sep 12 15:27:13 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../hadoop/yarn/client/cli/ApplicationCLI.java | 12 ++-
.../hadoop/yarn/client/cli/TestYarnCLI.java | 80 ++--
3 files changed, 54 insertions(+), 41 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0ad975e/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index efc3e09..06d94ca 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -348,6 +348,9 @@ Release 2.6.0 - UNRELEASED
YARN-2456. Possible livelock in CapacityScheduler when RM is recovering
apps.
(Jian He via xgong)
+YARN-2542. Fixed NPE when retrieving ApplicationReport from TimeLineServer.
+(Zhijie Shen via jianhe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0ad975e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
index 54cfe91..a847cd5 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
@@ -463,9 +463,15 @@ public class ApplicationCLI extends YarnCLI {
appReportStr.println(appReport.getHost());
appReportStr.print("\tAggregate Resource Allocation : ");
- ApplicationResourceUsageReport usageReport =
appReport.getApplicationResourceUsageReport();
- appReportStr.print(usageReport.getMemorySeconds() + " MB-seconds, ");
- appReportStr.println(usageReport.getVcoreSeconds() + " vcore-seconds");
+ ApplicationResourceUsageReport usageReport =
+ appReport.getApplicationResourceUsageReport();
+ if (usageReport != null) {
+//completed app report in the timeline server doesn't have usage report
+appReportStr.print(usageReport.getMemorySeconds() + " MB-seconds, ");
+appReportStr.println(usageReport.getVcoreSeconds() + " vcore-seconds");
+ } else {
+appReportStr.println("N/A");
+ }
appReportStr.print("\tDiagnostics : ");
appReportStr.print(appReport.getDiagnostics());
} else {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0ad975e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
index 1a593d2..980517f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
@@ -86,44 +86,48 @@ public class TestYarnCLI {
@Test
public void testGetApplicationReport() throws Exception {
-ApplicationCLI cli = createAndGetAppCLI();
-ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
-ApplicationResourceUsageReport usageReport =
-ApplicationResourceUsageReport.newInstance(
-2, 0, null, null, null, 123456, 4567);
-ApplicationReport newApplicationReport = ApplicationReport.newInstance(
-applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
-"user", "queue", "appname", "host", 124, null,
-YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0,
-FinalApplicationStatus.SUCCEEDED, usageReport, "N/A", 0.53789f, "YARN",
-null);
-when(client.ge
[38/52] [abbrv] git commit: MAPREDUCE-6077. native-task: Remove CustomModule examples in nativetask (seanzhong)
MAPREDUCE-6077. native-task: Remove CustomModule examples in nativetask (seanzhong) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/52a8b4db Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/52a8b4db Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/52a8b4db Branch: refs/heads/HDFS-6584 Commit: 52a8b4db92a35598006103c59a2ff93afc5312ee Parents: 7c91f9b Author: Sean Zhong Authored: Sat Sep 6 11:46:07 2014 +0800 Committer: Sean Zhong Committed: Sat Sep 6 11:46:07 2014 +0800 -- .../CHANGES.MAPREDUCE-2841.txt | 1 + .../sdk/example/CustomModule/README.txt | 1 - .../sdk/example/CustomModule/pom.xml| 131 --- .../platform/custom/CustomPlatform.java | 39 -- .../nativetask/serde/custom/CustomWritable.java | 75 --- .../serde/custom/CustomWritableSerializer.java | 33 - .../src/main/native/src/CustomComparator.cpp| 88 - 7 files changed, 1 insertion(+), 367 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/52a8b4db/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt -- diff --git a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt index 539e7be..cfc9412 100644 --- a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt +++ b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt @@ -23,3 +23,4 @@ MAPREDUCE-6055. native-task: findbugs, interface annotations, and other misc cle MAPREDUCE-6067. native-task: fix some counter issues (Binglin Chang) MAPREDUCE-6069. native-task: Lint/style fixes and removal of unused code (todd) MAPREDUCE-6074. native-task: fix release audit, javadoc, javac warnings (todd) +MAPREDUCE-6077. native-task: Remove CustomModule examples in nativetask (seanzhong) http://git-wip-us.apache.org/repos/asf/hadoop/blob/52a8b4db/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/README.txt -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/README.txt b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/README.txt deleted file mode 100644 index 0ad6f1e..000 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/README.txt +++ /dev/null @@ -1 +0,0 @@ -This project is depend on hadoop and hadoop-nativetask, so in order to complete the build, you have to specify the hadoop dir first. \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/52a8b4db/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/pom.xml -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/pom.xml deleted file mode 100644 index 5bf67c7..000 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/pom.xml +++ /dev/null @@ -1,131 +0,0 @@ - - -http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 - http://maven.apache.org/xsd/maven-4.0.0.xsd";> - 4.0.0 - org.apache.hadoop - nativetask-sdk - 2.2.0 - nativetask-sdk - - - - - - - org.apache.hadoop - hadoop-mapreduce-client-nativetask - 2.2.0 - - - - - - -org.apache.maven.plugins -maven-jar-plugin - - -org.apache.maven.plugins -maven-surefire-plugin - - - - - - - - - native - -false - - - - -org.apache.maven.plugins -maven-enforcer-plugin - - -enforce-os - - enforce - - - - - mac - unix - native build only supported on Mac or -Unix - - - true - - - - - -org.apache.maven.plugins -maven-antrun-plugin - - -
[10/52] [abbrv] git commit: Add entry to branch's CHANGES.txt for previous commit
Add entry to branch's CHANGES.txt for previous commit git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/MR-2841@1613827 13f79535-47bb-0310-9956-ffa450edef68 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/43917e56 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/43917e56 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/43917e56 Branch: refs/heads/HDFS-6584 Commit: 43917e564d266b67c323f4657d11a8b70ae1d706 Parents: ea9e5b7 Author: Todd Lipcon Authored: Sun Jul 27 18:54:02 2014 + Committer: Todd Lipcon Committed: Sun Jul 27 18:54:02 2014 + -- hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/43917e56/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt -- diff --git a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt index aa695cf..92c94a8 100644 --- a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt +++ b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt @@ -6,3 +6,4 @@ MAPREDUCE-5994. Simplify ByteUtils and fix failing test. (todd) MAPREDUCE-5996. native-task: Rename system tests into standard directory layout (todd) MAPREDUCE-5997. native-task: Use DirectBufferPool from Hadoop Common (todd) MAPREDUCE-6000. native-task: Simplify ByteBufferDataReader/Writer (todd) +MAPREDUCE-5991. native-task should not run unit tests if native profile is not enabled. (Binglin Chang)
[29/52] [abbrv] MAPREDUCE-6055. native-task: findbugs, interface annotations, and other misc cleanup
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/LongWritableSerializer.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/LongWritableSerializer.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/LongWritableSerializer.java
index ec326ca..d8a6595 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/LongWritableSerializer.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/LongWritableSerializer.java
@@ -20,9 +20,11 @@ package org.apache.hadoop.mapred.nativetask.serde;
import java.io.IOException;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.nativetask.INativeComparable;
+@InterfaceAudience.Private
public class LongWritableSerializer extends DefaultSerializer implements
INativeComparable {
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/NativeSerialization.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/NativeSerialization.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/NativeSerialization.java
index f5a033d..5881a46 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/NativeSerialization.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/NativeSerialization.java
@@ -21,8 +21,10 @@ package org.apache.hadoop.mapred.nativetask.serde;
import java.io.IOException;
import java.util.concurrent.ConcurrentHashMap;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.io.Writable;
+@InterfaceAudience.Private
public class NativeSerialization {
private final ConcurrentHashMap> map = new
ConcurrentHashMap>();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/NullWritableSerializer.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/NullWritableSerializer.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/NullWritableSerializer.java
index afa4e8e..f6e7cf5 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/NullWritableSerializer.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/NullWritableSerializer.java
@@ -20,9 +20,11 @@ package org.apache.hadoop.mapred.nativetask.serde;
import java.io.IOException;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.nativetask.INativeComparable;
+@InterfaceAudience.Private
public class NullWritableSerializer extends DefaultSerializer implements
INativeComparable {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1081d9ce/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/SerializationFramework.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/SerializationFramework.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/SerializationFramework.java
index e95a0c4..8dee58f 100644
---
a/hadoop-mapreduce-project/hadoop-ma
[09/52] [abbrv] git commit: MAPREDUCE-5991. native-task should not run unit tests if native profile is not enabled. Contributed by Binglin Chang.
MAPREDUCE-5991. native-task should not run unit tests if native profile is not
enabled. Contributed by Binglin Chang.
git-svn-id:
https://svn.apache.org/repos/asf/hadoop/common/branches/MR-2841@1613072
13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ea9e5b7b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ea9e5b7b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ea9e5b7b
Branch: refs/heads/HDFS-6584
Commit: ea9e5b7baae34c7ba660681d1e5aceb9c1047e6f
Parents: 6635180
Author: Binglin Chang
Authored: Thu Jul 24 11:44:09 2014 +
Committer: Binglin Chang
Committed: Thu Jul 24 11:44:09 2014 +
--
.../hadoop-mapreduce-client-common/pom.xml | 11 ++
.../hadoop-mapreduce-client-nativetask/pom.xml | 35
hadoop-project/pom.xml | 6
3 files changed, 45 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea9e5b7b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
index 7ef7d3b..48db5c8 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
@@ -84,6 +84,17 @@
+
+maven-jar-plugin
+
+
+
+ test-jar
+
+test-compile
+
+
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea9e5b7b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
index 40b6520..2cb483e 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
@@ -63,6 +63,12 @@
test-jar
test
+
+ org.apache.hadoop
+ hadoop-mapreduce-client-common
+ test-jar
+ test
+
@@ -82,14 +88,7 @@
**/buffer/Test*.java
**/handlers/Test*.java
**/serde/Test*.java
-**/combinertest/*Test.java
-**/compresstest/*Test.java
-**/nonsorttest/*Test.java
-**/kvtest/*Test.java
-
-
${basedir}/../hadoop-mapreduce-client-common/target/classes
-
@@ -184,6 +183,28 @@
+
+org.apache.maven.plugins
+maven-surefire-plugin
+
+
+
+ listener
+ org.apache.hadoop.test.TimedOutTestsListener
+
+
+
+**/TestTaskContext.java
+**/buffer/Test*.java
+**/handlers/Test*.java
+**/serde/Test*.java
+**/combinertest/*Test.java
+**/compresstest/*Test.java
+**/nonsorttest/*Test.java
+**/kvtest/*Test.java
+
+
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea9e5b7b/hadoop-project/pom.xml
--
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 2dc83dd..947be86 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -136,6 +136,12 @@
org.apache.hadoop
+hadoop-mapreduce-client-common
+${project.version}
+test-jar
+
+
+org.apache.hadoop
hadoop-yarn-api
${project.version}
[27/52] [abbrv] git commit: MAPREDUCE-6056. native-task: move system test working dir to target dir and cleanup test config xml files (Manu Zhang via bchang)
MAPREDUCE-6056. native-task: move system test working dir to target dir and
cleanup test config xml files (Manu Zhang via bchang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7ade9b04
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7ade9b04
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7ade9b04
Branch: refs/heads/HDFS-6584
Commit: 7ade9b04ea0a46a51564d23c8d715ddb1b9ca6d0
Parents: 17cd0fa
Author: Binglin Chang
Authored: Tue Sep 2 16:26:09 2014 +0800
Committer: Binglin Chang
Committed: Tue Sep 2 16:26:09 2014 +0800
--
.../CHANGES.MAPREDUCE-2841.txt | 1 +
.../nativetask/combinertest/CombinerTest.java | 65
.../combinertest/LargeKVCombinerTest.java | 94 +--
.../combinertest/OldAPICombinerTest.java| 27 ++--
.../nativetask/compresstest/CompressMapper.java | 38 ++---
.../nativetask/compresstest/CompressTest.java | 90 +++
.../TestNativeCollectorOnlyHandler.java | 20 ++-
.../hadoop/mapred/nativetask/kvtest/KVTest.java | 83 +-
.../mapred/nativetask/kvtest/LargeKVTest.java | 104 ++--
.../nativetask/nonsorttest/NonSortTest.java | 30 ++--
.../nativetask/testutil/TestConstants.java | 56 ---
.../src/test/resources/common_conf.xml | 71 +
.../src/test/resources/kvtest-conf.xml | 158 +--
.../src/test/resources/native_conf.xml | 33 ++--
.../src/test/resources/normal_conf.xml | 26 ++-
.../src/test/resources/test-combiner-conf.xml | 54 +++
.../src/test/resources/test-compress-conf.xml | 50 ++
.../test/resources/test-gzip-compress-conf.xml | 39 -
.../test/resources/test-lz4-compress-conf.xml | 39 -
.../src/test/resources/test-nonsort-conf.xml| 49 +++---
.../resources/test-snappy-compress-conf.xml | 41 -
21 files changed, 588 insertions(+), 580 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ade9b04/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
index baa88c1..7c9558e 100644
--- a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
+++ b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
@@ -18,3 +18,4 @@ MAPREDUCE-6035. native-task: sources/test-sources jar
distribution (Manu Zhang v
MAPREDUCE-5977. Fix or suppress native-task gcc warnings (Manu Zhang via todd)
MAPREDUCE-6054. native-task: Speed up tests (todd)
MAPREDUCE-6058. native-task: KVTest and LargeKVTest should check mr job is
sucessful (Binglin Chang)
+MAPREDUCE-6056. native-task: move system test working dir to target dir and
cleanup test config xml files (Manu Zhang via bchang)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ade9b04/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java
index abbe28e..d7f05be 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java
@@ -38,11 +38,14 @@ import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.junit.AfterClass;
import org.apache.hadoop.util.NativeCodeLoader;
import org.junit.Assume;
import org.junit.Before;
import org.junit.Test;
+import java.io.IOException;
+
public class CombinerTest {
private FileSystem fs;
private String inputpath;
@@ -50,33 +53,25 @@ public class CombinerTest {
private String hadoopoutputpath;
@Test
- public void testWordCountCombiner() {
-try {
-
- final Configuration nativeConf =
ScenarioConfiguration.getNativeConfiguration();
- nativeConf.addResource(TestConstants.COMBINER_CONF_PATH);
- final Job nativejob = getJob("nativewordcount", nativeConf, inputpath,
nativeout
[23/52] [abbrv] git commit: MAPREDUCE-5977. Fix or suppress native-task gcc warnings. Contributed by Manu Zhang.
MAPREDUCE-5977. Fix or suppress native-task gcc warnings. Contributed by Manu
Zhang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fad4524c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fad4524c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fad4524c
Branch: refs/heads/HDFS-6584
Commit: fad4524c85848b8efe27dcf4a3e3a9ac1725bd2a
Parents: 6d39367
Author: Todd Lipcon
Authored: Wed Aug 27 12:23:03 2014 -0700
Committer: Todd Lipcon
Committed: Wed Aug 27 12:25:42 2014 -0700
--
.../CHANGES.MAPREDUCE-2841.txt | 2 +
.../src/CMakeLists.txt | 9 +-
.../src/main/native/gtest/gtest.h | 19541 -
.../src/main/native/gtest/include/gtest/gtest.h | 19541 +
.../src/main/native/src/NativeTask.h| 2 -
.../src/main/native/src/codec/BlockCodec.cc | 4 +-
.../src/main/native/src/handler/BatchHandler.h | 4 +-
.../main/native/src/handler/CombineHandler.cc |10 +-
.../src/main/native/src/lib/Buffers.h | 4 +-
.../src/main/native/src/lib/IFile.cc|11 +-
.../main/native/src/lib/MapOutputCollector.cc |14 +-
.../main/native/src/lib/MapOutputCollector.h| 4 +-
.../src/main/native/src/lib/Merge.cc| 4 +-
.../src/main/native/src/lib/PartitionBucket.h |12 +-
.../src/main/native/src/lib/commons.h | 3 +
.../src/main/native/src/util/StringUtil.cc | 6 +-
.../src/main/native/test/TestCompressions.cc| 7 +-
.../src/main/native/test/TestIFile.cc | 4 +
.../src/main/native/test/lib/TestByteBuffer.cc | 2 +-
.../native/test/lib/TestFixSizeContainer.cc | 2 +-
.../src/main/native/test/lib/TestIterator.cc| 2 -
.../native/test/lib/TestMemBlockIterator.cc | 6 +-
.../src/main/native/test/lib/TestMemoryBlock.cc | 1 +
.../src/main/native/test/util/TestHash.cc | 2 +-
24 files changed, 19600 insertions(+), 19597 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
index 95f7858..4dc08cb 100644
--- a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
+++ b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
@@ -15,3 +15,5 @@ MAPREDUCE-5978. native-task: remove test case for not
supported codec Bzip2Codec
MAPREDUCE-6006. native-task: add native tests to maven and fix bug in pom.xml
(Binglin Chang via todd)
MAPREDUCE-6026. native-task: fix logging (Manu Zhang via todd)
MAPREDUCE-6035. native-task: sources/test-sources jar distribution (Manu Zhang
via todd)
+MAPREDUCE-5977. Fix or suppress native-task gcc warnings (Manu Zhang via todd)
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
index 77b6109..0ab99db 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
@@ -90,7 +90,7 @@ SET(CMAKE_FIND_LIBRARY_SUFFIXES
STORED_CMAKE_FIND_LIBRARY_SUFFIXES)
# primitive configs
set(PRFLAGS "-DSIMPLE_MEMCPY")
-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${PRFLAGS} -Wall")
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${PRFLAGS} -fno-strict-aliasing -Wall
-Wno-sign-compare")
set(CMAKE_LD_FLAGS "${CMAKE_LD_FLAGS} -no-undefined -version-info 0:1:0
-L${_JAVA_HOME}/jre/lib/amd64/server -ljvm")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_C_FLAGS} -g -O2 -DNDEBUG
-fPIC")
@@ -150,7 +150,6 @@ CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/config.h.cmake
${CMAKE_BINARY_DIR}/config.h)
include_directories(
${GENERATED_JAVAH}
-${D}
${D}/src
${D}/src/util
${D}/src/lib
@@ -160,6 +159,8 @@ include_directories(
${JNI_INCLUDE_DIRS}
${SNAPPY_INCLUDE_DIR}
)
+# add gtest as system library to suppress gcc warnings
+include_directories(SYSTEM ${D}/gtest/include)
SET(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
@@ -220,8 +221,9 @@ add_dual_library(nativetask
target_link_libraries(nativetask ${NT_DEPEND_LIBRARY})
+add_library(gtest ${D}/gtest/gtest-all.cc)
+set_target_properties(gtest PROPERTIES COMPILE_F
[47/52] [abbrv] git commit: YARN-2528. Relaxed http response split vulnerability protection for the origins header and made it accept multiple origins in CrossOriginFilter. Contributed by Jonathan Eag
YARN-2528. Relaxed http response split vulnerability protection for the origins
header and made it accept multiple origins in CrossOriginFilter. Contributed by
Jonathan Eagles.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/98588cf0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/98588cf0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/98588cf0
Branch: refs/heads/HDFS-6584
Commit: 98588cf044d9908ecf767257c09a52cf17aa2ec2
Parents: 3c91817
Author: Zhijie Shen
Authored: Fri Sep 12 21:33:01 2014 -0700
Committer: Zhijie Shen
Committed: Fri Sep 12 21:33:01 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 4 ++
.../timeline/webapp/CrossOriginFilter.java | 47 ++--
.../timeline/webapp/TestCrossOriginFilter.java | 41 ++---
3 files changed, 61 insertions(+), 31 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/98588cf0/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 06d94ca..065869a 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -351,6 +351,10 @@ Release 2.6.0 - UNRELEASED
YARN-2542. Fixed NPE when retrieving ApplicationReport from TimeLineServer.
(Zhijie Shen via jianhe)
+YARN-2528. Relaxed http response split vulnerability protection for the
origins
+header and made it accept multiple origins in CrossOriginFilter. (Jonathan
+Eagles via zjshen)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/98588cf0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
index cceee54..d5fab7a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
@@ -19,8 +19,6 @@
package org.apache.hadoop.yarn.server.timeline.webapp;
import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@@ -106,12 +104,12 @@ public class CrossOriginFilter implements Filter {
private void doCrossFilter(HttpServletRequest req, HttpServletResponse res) {
-String origin = encodeHeader(req.getHeader(ORIGIN));
-if (!isCrossOrigin(origin)) {
+String originsList = encodeHeader(req.getHeader(ORIGIN));
+if (!isCrossOrigin(originsList)) {
return;
}
-if (!isOriginAllowed(origin)) {
+if (!areOriginsAllowed(originsList)) {
return;
}
@@ -127,7 +125,7 @@ public class CrossOriginFilter implements Filter {
return;
}
-res.setHeader(ACCESS_CONTROL_ALLOW_ORIGIN, origin);
+res.setHeader(ACCESS_CONTROL_ALLOW_ORIGIN, originsList);
res.setHeader(ACCESS_CONTROL_ALLOW_CREDENTIALS, Boolean.TRUE.toString());
res.setHeader(ACCESS_CONTROL_ALLOW_METHODS, getAllowedMethodsHeader());
res.setHeader(ACCESS_CONTROL_ALLOW_HEADERS, getAllowedHeadersHeader());
@@ -191,35 +189,36 @@ public class CrossOriginFilter implements Filter {
if (header == null) {
return null;
}
-try {
- // Protect against HTTP response splitting vulnerability
- // since value is written as part of the response header
- return URLEncoder.encode(header, "ASCII");
-} catch (UnsupportedEncodingException e) {
- return null;
-}
+// Protect against HTTP response splitting vulnerability
+// since value is written as part of the response header
+// Ensure this header only has one header by removing
+// CRs and LFs
+return header.split("\n|\r")[0].trim();
}
- static boolean isCrossOrigin(String origin) {
-return origin != null;
+ static boolean isCrossOrigin(String originsList) {
+return originsList != null;
}
@VisibleForTesting
- boolean isOriginAllowed(String origin) {
+ boolean areO
[50/52] [abbrv] HADOOP-10400. Incorporate new S3A FileSystem implementation. Contributed by Jordan Mendelson and Dave Wang.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/24d920b8/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractSeek.java
--
diff --git
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractSeek.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractSeek.java
new file mode 100644
index 000..d677ec4
--- /dev/null
+++
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractSeek.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3a;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractSeekTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+public class TestS3AContractSeek extends AbstractContractSeekTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+return new S3AContract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/24d920b8/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3AFileSystemContractBaseTest.java
--
diff --git
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3AFileSystemContractBaseTest.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3AFileSystemContractBaseTest.java
new file mode 100644
index 000..8455233
--- /dev/null
+++
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3AFileSystemContractBaseTest.java
@@ -0,0 +1,327 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import static org.junit.Assume.*;
+
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystemContractBaseTest;
+import org.apache.hadoop.fs.Path;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.util.UUID;
+
+/**
+ * Tests a live S3 system. If you keys and bucket aren't specified, all tests
+ * are marked as passed
+ *
+ * This uses BlockJUnit4ClassRunner because FileSystemContractBaseTest from
+ * TestCase which uses the old Junit3 runner that doesn't ignore assumptions
+ * properly making it impossible to skip the tests if we don't have a valid
+ * bucket.
+ **/
+public class S3AFileSystemContractBaseTest extends FileSystemContractBaseTest {
+ private static final int TEST_BUFFER_SIZE = 128;
+ private static final int MODULUS = 128;
+
+ protected static final Logger LOG =
LoggerFactory.getLogger(S3AFileSystemContractBaseTest.class);
+
+ @Override
+ public void setUp() throws Exception {
+Configuration conf = new Configuration();
+
+URI testURI = URI.create(conf.get("test.fs.s3a.name"));
+
+boolean liveTest = testURI != null && !testURI.equals("s3a:///");
+
+// This doesn't work with our JUnit 3 style test cases, so instead we'll
+// make this whole class not run by default
+assumeTrue(liveTest);
+
+fs = new S3AFileSystem();
+fs.initialize(testURI, conf);
+super.setUp();
+ }
+
+ @Override
+ protected void tearDown
[33/52] [abbrv] MAPREDUCE-6069. Lint/style fixes and removal of unused code. Contributed by Todd Lipcon.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Hash.cc
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Hash.cc
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Hash.cc
deleted file mode 100644
index eec8091..000
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Hash.cc
+++ /dev/null
@@ -1,24 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "Hash.h"
-
-namespace NativeTask {
-
-} // namespace NativeTask
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Hash.h
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Hash.h
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Hash.h
deleted file mode 100644
index 4f99d26..000
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Hash.h
+++ /dev/null
@@ -1,73 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef HASH_H_
-#define HASH_H_
-
-#include
-#include
-
-// Hash function for a byte array.
-extern uint64_t CityHash64(const char *buf, size_t len);
-
-// Hash function for a byte array. For convenience, a 64-bit seed is also
-// hashed into the result.
-extern uint64_t CityHash64WithSeed(const char *buf, size_t len, uint64_t seed);
-
-namespace NativeTask {
-
-class Hash {
-public:
- /**
- * Compatible with hadoop Text & BytesWritable hash
- */
- inline static int32_t BytesHash(const char * bytes, uint32_t length) {
-int32_t hash = 1;
-for (uint32_t i = 0; i < length; i++)
- hash = (31 * hash) + (int32_t)bytes[i];
-return hash;
- }
-
- /**
- * Unsigned version of BytesHash
- */
- inline static uint32_t BytesHashU(const char * bytes, uint32_t length) {
-uint32_t hash = 1;
-for (uint32_t i = 0; i < length; i++)
- hash = (31U * hash) + (uint32_t)bytes[i];
-return hash;
- }
-
- /**
- * City hash, faster for longer input
- */
- inline static uint64_t CityHash(const char * bytes, uint32_t length) {
-return CityHash64(bytes, length);
- }
-
- /**
- * City hash, faster for longer input
- */
- inline static uint64_t CityHashWithSeed(const char * bytes, uint32_t length,
uint64_t seed) {
-return CityHash64WithSeed(bytes, length, seed);
- }
-};
-
-} // namespace NativeTask
-
-#endif /* HASH_H_ */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Random.cc
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Random.cc
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/nati
[02/52] [abbrv] MAPREDUCE-5996. native-task: Rename system tests into standard directory layout. Contributed by Todd Lipcon.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java
deleted file mode 100644
index 8a4aa6f..000
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.combinertest;
-
-import static org.junit.Assert.assertEquals;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.Task;
-import
org.apache.hadoop.mapred.nativetask.combinertest.WordCount.IntSumReducer;
-import
org.apache.hadoop.mapred.nativetask.combinertest.WordCount.TokenizerMapper;
-import org.apache.hadoop.mapred.nativetask.kvtest.TestInputFile;
-import org.apache.hadoop.mapred.nativetask.testutil.ResultVerifier;
-import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
-import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
-import org.apache.hadoop.mapreduce.Counter;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.junit.Before;
-import org.junit.Test;
-
-public class CombinerTest {
- private FileSystem fs;
- private String inputpath;
- private String nativeoutputpath;
- private String hadoopoutputpath;
-
- @Test
- public void testWordCountCombiner() {
-try {
-
- final Configuration nativeConf =
ScenarioConfiguration.getNativeConfiguration();
- nativeConf.addResource(TestConstants.COMBINER_CONF_PATH);
- final Job nativejob = getJob("nativewordcount", nativeConf, inputpath,
nativeoutputpath);
-
- final Configuration commonConf =
ScenarioConfiguration.getNormalConfiguration();
- commonConf.addResource(TestConstants.COMBINER_CONF_PATH);
-
- final Job normaljob = getJob("normalwordcount", commonConf, inputpath,
hadoopoutputpath);
-
- nativejob.waitForCompletion(true);
-
- Counter nativeReduceGroups =
nativejob.getCounters().findCounter(Task.Counter.REDUCE_INPUT_RECORDS);
-
- normaljob.waitForCompletion(true);
- Counter normalReduceGroups =
normaljob.getCounters().findCounter(Task.Counter.REDUCE_INPUT_RECORDS);
-
- assertEquals(true, ResultVerifier.verify(nativeoutputpath,
hadoopoutputpath));
- assertEquals("Native Reduce reduce group counter should equal orignal
reduce group counter",
- nativeReduceGroups.getValue(), normalReduceGroups.getValue());
-
-} catch (final Exception e) {
- e.printStackTrace();
- assertEquals("run exception", true, false);
-}
- }
-
- @Before
- public void startUp() throws Exception {
-final ScenarioConfiguration conf = new ScenarioConfiguration();
-conf.addcombinerConf();
-
-this.fs = FileSystem.get(conf);
-
-this.inputpath =
conf.get(TestConstants.NATIVETASK_TEST_COMBINER_INPUTPATH_KEY,
-TestConstants.NATIVETASK_TEST_COMBINER_INPUTPATH_DEFAULTV) +
"/wordcount";
-
-if (!fs.exists(new Path(inputpath))) {
- new TestInputFile(
- conf.getInt(TestConstants.NATIVETASK_COMBINER_WORDCOUNT_FILESIZE,
100),
- Text.class.getName(),
- Text.class.getName(), conf).createSequenceTestFile(i
[21/52] [abbrv] MAPREDUCE-5977. Fix or suppress native-task gcc warnings. Contributed by Manu Zhang.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/gtest/include/gtest/gtest.h -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/gtest/include/gtest/gtest.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/gtest/include/gtest/gtest.h new file mode 100644 index 000..b467822 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/gtest/include/gtest/gtest.h @@ -0,0 +1,19541 @@ +// Copyright 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: w...@google.com (Zhanyong Wan) +// +// The Google C++ Testing Framework (Google Test) +// +// This header file defines the public API for Google Test. It should be +// included by any test program that uses Google Test. +// +// IMPORTANT NOTE: Due to limitation of the C++ language, we have to +// leave some internal implementation details in this header file. +// They are clearly marked by comments like this: +// +// // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. +// +// Such code is NOT meant to be used by a user directly, and is subject +// to CHANGE WITHOUT NOTICE. Therefore DO NOT DEPEND ON IT in a user +// program! +// +// Acknowledgment: Google Test borrowed the idea of automatic test +// registration from Barthelemy Dagenais' (barthel...@prologique.com) +// easyUnit framework. + +#ifndef GTEST_INCLUDE_GTEST_GTEST_H_ +#define GTEST_INCLUDE_GTEST_GTEST_H_ + +#include +#include + +// Copyright 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Authors: w...@google.com (Zhanyong Wan), eef...@gmail.com (Sean Mcafee) +// +// The Google C++ Testing Framework (Google Test) +// +// This header
[08/52] [abbrv] git commit: MAPREDUCE-6000. native-task: Simplify ByteBufferDataReader/Writer. Contributed by Todd Lipcon.
MAPREDUCE-6000. native-task: Simplify ByteBufferDataReader/Writer. Contributed
by Todd Lipcon.
git-svn-id:
https://svn.apache.org/repos/asf/hadoop/common/branches/MR-2841@1613036
13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/66351803
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/66351803
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/66351803
Branch: refs/heads/HDFS-6584
Commit: 663518032449940f136ae8a9f149ed453837a8d8
Parents: 77acc70
Author: Todd Lipcon
Authored: Thu Jul 24 08:24:05 2014 +
Committer: Todd Lipcon
Committed: Thu Jul 24 08:24:05 2014 +
--
.../CHANGES.MAPREDUCE-2841.txt | 1 +
.../nativetask/buffer/ByteBufferDataReader.java | 126 +
.../nativetask/buffer/ByteBufferDataWriter.java | 141 ---
.../buffer/TestByteBufferReadWrite.java | 93 +---
4 files changed, 75 insertions(+), 286 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/66351803/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
index e12f743..aa695cf 100644
--- a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
+++ b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
@@ -5,3 +5,4 @@ MAPREDUCE-5985. native-task: Fix build on macosx. Contributed
by Binglin Chang
MAPREDUCE-5994. Simplify ByteUtils and fix failing test. (todd)
MAPREDUCE-5996. native-task: Rename system tests into standard directory
layout (todd)
MAPREDUCE-5997. native-task: Use DirectBufferPool from Hadoop Common (todd)
+MAPREDUCE-6000. native-task: Simplify ByteBufferDataReader/Writer (todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/66351803/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/ByteBufferDataReader.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/ByteBufferDataReader.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/ByteBufferDataReader.java
index 5af7180..24f402d 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/ByteBufferDataReader.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/ByteBufferDataReader.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.mapred.nativetask.buffer;
+import com.google.common.base.Charsets;
+
import java.io.DataInput;
import java.io.EOFException;
import java.io.IOException;
@@ -31,11 +33,13 @@ import java.nio.ByteBuffer;
public class ByteBufferDataReader extends DataInputStream {
private ByteBuffer byteBuffer;
private char lineCache[];
+ private java.io.DataInputStream javaReader;
public ByteBufferDataReader(InputBuffer buffer) {
if (buffer != null) {
- this.byteBuffer = buffer.getByteBuffer();
+ reset(buffer);
}
+javaReader = new java.io.DataInputStream(this);
}
public void reset(InputBuffer buffer) {
@@ -128,128 +132,12 @@ public class ByteBufferDataReader extends
DataInputStream {
@Override
public String readLine() throws IOException {
-
-InputStream in = this;
-
-char buf[] = lineCache;
-
-if (buf == null) {
- buf = lineCache = new char[128];
-}
-
-int room = buf.length;
-int offset = 0;
-int c;
-
-loop: while (true) {
- switch (c = in.read()) {
- case -1:
- case '\n':
-break loop;
-
- case '\r':
-final int c2 = in.read();
-if ((c2 != '\n') && (c2 != -1)) {
- if (!(in instanceof PushbackInputStream)) {
-in = new PushbackInputStream(in);
- }
- ((PushbackInputStream) in).unread(c2);
-}
-break loop;
-
- default:
-if (--room < 0) {
- buf = new char[offset + 128];
- room = buf.length - offset - 1;
- System.arraycopy(lineCache, 0, buf, 0, offset);
- lineCache = buf;
-}
-buf[offset++] = (char) c;
-break;
- }
-}
-if ((c == -1) && (offset == 0)) {
- return null;
-}
-return String.copyValueOf(buf, 0, offset);
+return javaReader.readLine();
}
@Override
public final String readUTF() t
[46/52] [abbrv] git commit: Merge MR-2841 changes into main MapReduce CHANGES.txt file
Merge MR-2841 changes into main MapReduce CHANGES.txt file Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3c918172 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3c918172 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3c918172 Branch: refs/heads/HDFS-6584 Commit: 3c9181722b05a9192f5440ea8f3f77231f84eac6 Parents: 8a3a327 Author: Todd Lipcon Authored: Fri Sep 12 18:35:13 2014 -0700 Committer: Todd Lipcon Committed: Fri Sep 12 18:35:13 2014 -0700 -- .../CHANGES.MAPREDUCE-2841.txt | 26 hadoop-mapreduce-project/CHANGES.txt| 65 2 files changed, 65 insertions(+), 26 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c918172/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt -- diff --git a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt deleted file mode 100644 index cfc9412..000 --- a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt +++ /dev/null @@ -1,26 +0,0 @@ -Changes for Hadoop Native Map Output Collector - - -MAPREDUCE-5985. native-task: Fix build on macosx. Contributed by Binglin Chang -MAPREDUCE-5994. Simplify ByteUtils and fix failing test. (todd) -MAPREDUCE-5996. native-task: Rename system tests into standard directory layout (todd) -MAPREDUCE-5997. native-task: Use DirectBufferPool from Hadoop Common (todd) -MAPREDUCE-6000. native-task: Simplify ByteBufferDataReader/Writer (todd) -MAPREDUCE-5991. native-task should not run unit tests if native profile is not enabled. (Binglin Chang) -MAPREDUCE-5995. native-task: Revert changes to Text internals (todd) -MAPREDUCE-6005. native-task: Fix some valgrind errors (Binglin Chang) -MAPREDUCE-5984. native-task: Reuse lz4 sources in hadoop-common (Binglin Chang) -MAPREDUCE-5976. native-task: should not fail to build if snappy is missing (Manu Zhang) -MAPREDUCE-5978. native-task: remove test case for not supported codec Bzip2Codec and DefaultCodec (Manu Zhang) -MAPREDUCE-6006. native-task: add native tests to maven and fix bug in pom.xml (Binglin Chang via todd) -MAPREDUCE-6026. native-task: fix logging (Manu Zhang via todd) -MAPREDUCE-6035. native-task: sources/test-sources jar distribution (Manu Zhang via todd) -MAPREDUCE-5977. Fix or suppress native-task gcc warnings (Manu Zhang via todd) -MAPREDUCE-6054. native-task: Speed up tests (todd) -MAPREDUCE-6058. native-task: KVTest and LargeKVTest should check mr job is sucessful (Binglin Chang) -MAPREDUCE-6056. native-task: move system test working dir to target dir and cleanup test config xml files (Manu Zhang via bchang) -MAPREDUCE-6055. native-task: findbugs, interface annotations, and other misc cleanup (todd) -MAPREDUCE-6067. native-task: fix some counter issues (Binglin Chang) -MAPREDUCE-6069. native-task: Lint/style fixes and removal of unused code (todd) -MAPREDUCE-6074. native-task: fix release audit, javadoc, javac warnings (todd) -MAPREDUCE-6077. native-task: Remove CustomModule examples in nativetask (seanzhong) http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c918172/hadoop-mapreduce-project/CHANGES.txt -- diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index f1435d2..5d1e5f5 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -20,6 +20,9 @@ Trunk (Unreleased) MAPREDUCE-5910. Make MR AM resync with RM in case of work-preserving RM-restart. (Rohith via jianhe) +MAPREDUCE-2841. Add a native implementation of MapOutputCollector. +(see section below for detailed breakdown) + IMPROVEMENTS MAPREDUCE-3481. [Gridmix] Improve Gridmix STRESS mode. (amarrk) @@ -157,6 +160,68 @@ Trunk (Unreleased) MAPREDUCE-5972. Fix typo 'programatically' in job.xml (and a few other places) (Akira AJISAKA via aw) + BREAKDOWN OF MAPREDUCE-2841 (NATIVE TASK) SUBTASKS + +MAPREDUCE-5985. native-task: Fix build on macosx. Contributed by +Binglin Chang + +MAPREDUCE-5994. Simplify ByteUtils and fix failing test. (todd) + +MAPREDUCE-5996. native-task: Rename system tests into standard directory +layout (todd) + +MAPREDUCE-5997. native-task: Use DirectBufferPool from Hadoop Common (todd) + +MAPREDUCE-6000. native-task: Simplify ByteBufferDataReader/Writer (todd) + +MAPREDUCE-5991. native-task should not run unit tests if native profile is +not enabled. (Binglin Chang) + +MAPREDUCE-5995. native-task: Revert changes to Text internals (todd) + +MAPREDUCE-6005. native-task: Fix some valgrind errors (Bin
[40/52] [abbrv] git commit: YARN-2547. Cross Origin Filter throws UnsupportedOperationException upon destroy (Mit Desai via jeagles)
YARN-2547. Cross Origin Filter throws UnsupportedOperationException upon
destroy (Mit Desai via jeagles)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/54e57948
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/54e57948
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/54e57948
Branch: refs/heads/HDFS-6584
Commit: 54e5794806bd856da0277510efe63656eed23146
Parents: 3122daa
Author: Jonathan Eagles
Authored: Fri Sep 12 15:35:17 2014 -0500
Committer: Jonathan Eagles
Committed: Fri Sep 12 15:35:17 2014 -0500
--
hadoop-yarn-project/CHANGES.txt | 3 ++
.../timeline/webapp/CrossOriginFilter.java | 12 ++---
.../timeline/webapp/TestCrossOriginFilter.java | 49
3 files changed, 58 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54e57948/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index bb73dc7..3993f5e 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -214,6 +214,9 @@ Release 2.6.0 - UNRELEASED
YARN-2229. Changed the integer field of ContainerId to be long type.
(Tsuyoshi OZAWA via jianhe)
+YARN-2547. Cross Origin Filter throws UnsupportedOperationException upon
+destroy (Mit Desai via jeagles)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54e57948/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
index 5a0703d..cceee54 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
@@ -150,8 +150,8 @@ public class CrossOriginFilter implements Filter {
if (allowedMethodsConfig == null) {
allowedMethodsConfig = ALLOWED_METHODS_DEFAULT;
}
-allowedMethods =
-Arrays.asList(allowedMethodsConfig.trim().split("\\s*,\\s*"));
+allowedMethods.addAll(
+Arrays.asList(allowedMethodsConfig.trim().split("\\s*,\\s*")));
LOG.info("Allowed Methods: " + getAllowedMethodsHeader());
}
@@ -161,8 +161,8 @@ public class CrossOriginFilter implements Filter {
if (allowedHeadersConfig == null) {
allowedHeadersConfig = ALLOWED_HEADERS_DEFAULT;
}
-allowedHeaders =
-Arrays.asList(allowedHeadersConfig.trim().split("\\s*,\\s*"));
+allowedHeaders.addAll(
+Arrays.asList(allowedHeadersConfig.trim().split("\\s*,\\s*")));
LOG.info("Allowed Headers: " + getAllowedHeadersHeader());
}
@@ -172,8 +172,8 @@ public class CrossOriginFilter implements Filter {
if (allowedOriginsConfig == null) {
allowedOriginsConfig = ALLOWED_ORIGINS_DEFAULT;
}
-allowedOrigins =
-Arrays.asList(allowedOriginsConfig.trim().split("\\s*,\\s*"));
+allowedOrigins.addAll(
+Arrays.asList(allowedOriginsConfig.trim().split("\\s*,\\s*")));
allowAllOrigins = allowedOrigins.contains("*");
LOG.info("Allowed Origins: " + StringUtils.join(allowedOrigins, ','));
LOG.info("Allow All Origins: " + allowAllOrigins);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54e57948/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilter.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilter.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilter.java
index ccc9bbf..e0990f9 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/s
[16/52] [abbrv] git commit: MAPREDUCE-6006. native-task: add native tests to maven and fix bug in pom.xml. Contributed by Binglin Chang.
MAPREDUCE-6006. native-task: add native tests to maven and fix bug in pom.xml. Contributed by Binglin Chang. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/MR-2841@1617877 13f79535-47bb-0310-9956-ffa450edef68 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/886338ff Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/886338ff Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/886338ff Branch: refs/heads/HDFS-6584 Commit: 886338ff80438b2daea3755f165cc411e39ef560 Parents: 4d4fb17 Author: Todd Lipcon Authored: Thu Aug 14 04:47:20 2014 + Committer: Todd Lipcon Committed: Thu Aug 14 04:47:20 2014 + -- .../CHANGES.MAPREDUCE-2841.txt | 3 +- .../hadoop-mapreduce-client-common/pom.xml | 11 .../hadoop-mapreduce-client-nativetask/pom.xml | 16 +-- .../src/CMakeLists.txt | 4 +++ .../src/main/native/test.sh | 29 hadoop-project/pom.xml | 6 6 files changed, 38 insertions(+), 31 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/886338ff/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt -- diff --git a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt index 462ac8b..c3c6ab9 100644 --- a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt +++ b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt @@ -11,4 +11,5 @@ MAPREDUCE-5995. native-task: Revert changes to Text internals (todd) MAPREDUCE-6005. native-task: Fix some valgrind errors (Binglin Chang) MAPREDUCE-5984. native-task: Reuse lz4 sources in hadoop-common (Binglin Chang) MAPREDUCE-5976. native-task: should not fail to build if snappy is missing (Manu Zhang) -MAPREDUCE-5978. native-task: remove test case for not supported codec Bzip2Codec and DefaultCodec (Manu Zhang) \ No newline at end of file +MAPREDUCE-5978. native-task: remove test case for not supported codec Bzip2Codec and DefaultCodec (Manu Zhang) +MAPREDUCE-6006. native-task: add native tests to maven and fix bug in pom.xml (Binglin Chang via todd) http://git-wip-us.apache.org/repos/asf/hadoop/blob/886338ff/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml index 48db5c8..7ef7d3b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml @@ -84,17 +84,6 @@ - -maven-jar-plugin - - - - test-jar - -test-compile - - - http://git-wip-us.apache.org/repos/asf/hadoop/blob/886338ff/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml index 9727800..bb7d7bb 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml @@ -66,7 +66,6 @@ org.apache.hadoop hadoop-mapreduce-client-common - test-jar test @@ -173,7 +172,6 @@ overwrite="true"> - @@ -187,6 +185,20 @@ + +native_tests +test +run + + + + + + + + + + http://git-wip-us.apache.org/repos/asf/hadoop/blob/886338ff/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMak
[35/52] [abbrv] git commit: MAPREDUCE-6069. Lint/style fixes and removal of unused code. Contributed by Todd Lipcon.
MAPREDUCE-6069. Lint/style fixes and removal of unused code. Contributed by Todd Lipcon. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/683987be Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/683987be Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/683987be Branch: refs/heads/HDFS-6584 Commit: 683987be7c160e67ddb8534eeb3c464bbe2796dd Parents: 0032216 Author: Todd Lipcon Authored: Wed Sep 3 13:07:24 2014 -0700 Committer: Todd Lipcon Committed: Fri Sep 5 10:44:38 2014 -0700 -- .../CHANGES.MAPREDUCE-2841.txt | 1 + .../hadoop-mapreduce-client-nativetask/LICENSE | 22 -- .../src/CMakeLists.txt | 6 - .../hadoop/mapred/nativetask/Constants.java | 13 - .../hadoop/mapred/nativetask/DataReceiver.java | 3 - .../mapred/nativetask/ICombineHandler.java | 2 - .../mapred/nativetask/INativeHandler.java | 8 - .../mapred/nativetask/NativeBatchProcessor.java | 11 - .../NativeMapOutputCollectorDelegator.java | 10 +- .../hadoop/mapred/nativetask/NativeRuntime.java | 32 +- .../hadoop/mapred/nativetask/Platform.java | 11 +- .../nativetask/handlers/BufferPullee.java | 3 +- .../nativetask/handlers/BufferPuller.java | 2 - .../nativetask/handlers/BufferPushee.java | 6 +- .../nativetask/handlers/BufferPusher.java | 3 +- .../nativetask/handlers/CombinerHandler.java| 16 +- .../mapred/nativetask/handlers/IDataLoader.java | 1 - .../handlers/NativeCollectorOnlyHandler.java| 6 +- .../serde/BytesWritableSerializer.java | 3 +- .../mapred/nativetask/serde/IKVSerializer.java | 31 +- .../mapred/nativetask/serde/KVSerializer.java | 6 +- .../nativetask/serde/NativeSerialization.java | 11 +- .../mapred/nativetask/util/BytesUtil.java | 2 +- .../nativetask/util/LocalJobOutputFiles.java| 58 +--- .../nativetask/util/NativeTaskOutput.java | 55 +--- .../nativetask/util/NativeTaskOutputFiles.java | 72 ++--- .../mapred/nativetask/util/ReadWriteBuffer.java | 16 +- .../src/main/native/cityhash/city.cc| 307 --- .../src/main/native/cityhash/city.h | 90 -- .../src/main/native/src/NativeTask.h| 137 - .../src/main/native/src/codec/BlockCodec.cc | 5 +- .../src/main/native/src/codec/BlockCodec.h | 2 +- .../src/main/native/src/codec/GzipCodec.cc | 2 +- .../src/main/native/src/codec/GzipCodec.h | 2 +- .../src/main/native/src/codec/Lz4Codec.cc | 2 +- .../src/main/native/src/codec/Lz4Codec.h| 2 +- .../src/main/native/src/codec/SnappyCodec.cc| 2 +- .../src/main/native/src/codec/SnappyCodec.h | 2 +- .../native/src/handler/AbstractMapHandler.cc| 9 +- .../src/main/native/src/handler/BatchHandler.cc | 9 +- .../main/native/src/handler/CombineHandler.cc | 4 +- .../main/native/src/handler/CombineHandler.h| 2 +- .../src/handler/MCollectorOutputHandler.cc | 6 +- .../src/main/native/src/lib/BufferStream.cc | 116 +-- .../src/main/native/src/lib/BufferStream.h | 39 +-- .../src/main/native/src/lib/Buffers.cc | 77 + .../src/main/native/src/lib/Buffers.h | 63 +--- .../src/main/native/src/lib/Combiner.cc | 73 - .../src/main/native/src/lib/Combiner.h | 18 +- .../src/main/native/src/lib/Compressions.cc | 6 +- .../src/main/native/src/lib/Compressions.h | 2 +- .../src/main/native/src/lib/FileSystem.cc | 23 +- .../src/main/native/src/lib/FileSystem.h| 2 +- .../src/main/native/src/lib/IFile.cc| 6 +- .../src/main/native/src/lib/IFile.h | 10 +- .../src/main/native/src/lib/Iterator.cc | 6 +- .../src/main/native/src/lib/Log.cc | 2 +- .../main/native/src/lib/MapOutputCollector.cc | 58 ++-- .../main/native/src/lib/MapOutputCollector.h| 16 +- .../src/main/native/src/lib/MapOutputSpec.cc| 4 +- .../src/main/native/src/lib/MapOutputSpec.h | 4 +- .../src/main/native/src/lib/MemoryBlock.cc | 24 +- .../src/main/native/src/lib/MemoryPool.h| 4 +- .../src/main/native/src/lib/Merge.cc| 5 +- .../src/main/native/src/lib/Merge.h | 8 +- .../src/main/native/src/lib/MinHeap.h | 2 +- .../src/main/native/src/lib/NativeLibrary.cc| 7 +- .../main/native/src/lib/NativeObjectFactory.cc | 19 +- .../main/native/src/lib/NativeRuntimeJniImpl.cc | 27 +- .../src/main/native/src/lib/NativeTask.cc | 44 +-- .../src/main/native/src/lib/PartitionBucket.cc | 25 +- .../src/main/native/src/lib/PartitionBucket.h | 16 +- .../native/src/lib/PartitionBucketIterator.cc | 22 +- .../native/src/lib/PartitionBucketIterator.h| 16 +- .../src/main/na
[31/52] [abbrv] git commit: MAPREDUCE-6067. native-task: fix some counter issues (Binglin Chang)
MAPREDUCE-6067. native-task: fix some counter issues (Binglin Chang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/00322161
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/00322161
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/00322161
Branch: refs/heads/HDFS-6584
Commit: 00322161b5d4d54770c2f0823e036537edecf5bf
Parents: 1081d9c
Author: Binglin Chang
Authored: Fri Sep 5 14:20:39 2014 +0800
Committer: Binglin Chang
Committed: Fri Sep 5 14:20:39 2014 +0800
--
.../CHANGES.MAPREDUCE-2841.txt | 1 +
.../NativeMapOutputCollectorDelegator.java | 4 +
.../mapred/nativetask/StatusReportChecker.java | 5 --
.../src/handler/MCollectorOutputHandler.cc | 3 +-
.../src/main/native/src/lib/IFile.cc| 6 +-
.../src/main/native/src/lib/IFile.h | 3 +-
.../main/native/src/lib/MapOutputCollector.cc | 85 ++--
.../main/native/src/lib/MapOutputCollector.h| 8 +-
.../src/main/native/src/lib/Merge.cc| 24 --
.../src/main/native/src/lib/PartitionBucket.h | 2 -
.../src/main/native/src/lib/TaskCounters.cc | 10 +--
.../src/main/native/src/lib/TaskCounters.h | 8 --
.../nativetask/combinertest/CombinerTest.java | 8 +-
.../combinertest/LargeKVCombinerTest.java | 6 +-
.../nativetask/compresstest/CompressTest.java | 3 +
.../hadoop/mapred/nativetask/kvtest/KVTest.java | 74 +++--
.../mapred/nativetask/kvtest/LargeKVTest.java | 82 +--
.../nativetask/nonsorttest/NonSortTest.java | 6 +-
.../nativetask/testutil/ResultVerifier.java | 24 +-
19 files changed, 179 insertions(+), 183 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00322161/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
index 269a2f6..279b960 100644
--- a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
+++ b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
@@ -20,3 +20,4 @@ MAPREDUCE-6054. native-task: Speed up tests (todd)
MAPREDUCE-6058. native-task: KVTest and LargeKVTest should check mr job is
sucessful (Binglin Chang)
MAPREDUCE-6056. native-task: move system test working dir to target dir and
cleanup test config xml files (Manu Zhang via bchang)
MAPREDUCE-6055. native-task: findbugs, interface annotations, and other misc
cleanup (todd)
+MAPREDUCE-6067. native-task: fix some counter issues (Binglin Chang)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00322161/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeMapOutputCollectorDelegator.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeMapOutputCollectorDelegator.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeMapOutputCollectorDelegator.java
index 224b95b..828d7df 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeMapOutputCollectorDelegator.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeMapOutputCollectorDelegator.java
@@ -34,6 +34,7 @@ import
org.apache.hadoop.mapred.nativetask.serde.INativeSerializer;
import org.apache.hadoop.mapred.nativetask.serde.NativeSerialization;
import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.TaskCounter;
import org.apache.hadoop.util.QuickSort;
/**
@@ -46,6 +47,7 @@ public class NativeMapOutputCollectorDelegator
implements MapOutputCollect
private JobConf job;
private NativeCollectorOnlyHandler handler;
+ private Context context;
private StatusReportChecker updater;
@Override
@@ -58,6 +60,7 @@ public class NativeMapOutputCollectorDelegator
implements MapOutputCollect
handler.close();
if (null != updater) {
updater.stop();
+ NativeRuntime.reportStatus(context.getReporter());
}
}
@@ -69,6 +72,7 @@ public class NativeMapOutputCollectorDelegator
implements MapOutputCollect
@SuppressWarnings("unchecked")
@Override
public void init(Context context) throws IOException, ClassNotFoundException
{
+this.context = context;
this.job = context.getJobCon
[22/52] [abbrv] MAPREDUCE-5977. Fix or suppress native-task gcc warnings. Contributed by Manu Zhang.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/gtest/gtest.h -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/gtest/gtest.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/gtest/gtest.h deleted file mode 100644 index b467822..000 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/gtest/gtest.h +++ /dev/null @@ -1,19541 +0,0 @@ -// Copyright 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Author: w...@google.com (Zhanyong Wan) -// -// The Google C++ Testing Framework (Google Test) -// -// This header file defines the public API for Google Test. It should be -// included by any test program that uses Google Test. -// -// IMPORTANT NOTE: Due to limitation of the C++ language, we have to -// leave some internal implementation details in this header file. -// They are clearly marked by comments like this: -// -// // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. -// -// Such code is NOT meant to be used by a user directly, and is subject -// to CHANGE WITHOUT NOTICE. Therefore DO NOT DEPEND ON IT in a user -// program! -// -// Acknowledgment: Google Test borrowed the idea of automatic test -// registration from Barthelemy Dagenais' (barthel...@prologique.com) -// easyUnit framework. - -#ifndef GTEST_INCLUDE_GTEST_GTEST_H_ -#define GTEST_INCLUDE_GTEST_GTEST_H_ - -#include -#include - -// Copyright 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Authors: w...@google.com (Zhanyong Wan), eef...@gmail.com (Sean Mcafee) -// -// The Google C++ Testing Framework (Google Test) -// -// This header file declares functions and macros used internally
[15/52] [abbrv] git commit: MAPREDUCE-5978. native-task: remove test case for not supported codec Bzip2Codec and DefaultCodec (Manu Zhang)
MAPREDUCE-5978. native-task: remove test case for not supported codec
Bzip2Codec and DefaultCodec (Manu Zhang)
git-svn-id:
https://svn.apache.org/repos/asf/hadoop/common/branches/MR-2841@1616116
13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4d4fb172
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4d4fb172
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4d4fb172
Branch: refs/heads/HDFS-6584
Commit: 4d4fb1723fd825df4ad2488e4ecde0c69359e83a
Parents: 432f641
Author: Sean Zhong
Authored: Wed Aug 6 07:40:24 2014 +
Committer: Sean Zhong
Committed: Wed Aug 6 07:40:24 2014 +
--
.../CHANGES.MAPREDUCE-2841.txt | 3 +-
.../nativetask/compresstest/CompressTest.java | 33 -
.../test/resources/test-bzip2-compress-conf.xml | 39
.../resources/test-default-compress-conf.xml| 39
4 files changed, 2 insertions(+), 112 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d4fb172/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
index 4b77262..462ac8b 100644
--- a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
+++ b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
@@ -10,4 +10,5 @@ MAPREDUCE-5991. native-task should not run unit tests if
native profile is not e
MAPREDUCE-5995. native-task: Revert changes to Text internals (todd)
MAPREDUCE-6005. native-task: Fix some valgrind errors (Binglin Chang)
MAPREDUCE-5984. native-task: Reuse lz4 sources in hadoop-common (Binglin Chang)
-MAPREDUCE-5976. native-task: should not fail to build if snappy is missing
(Manu Zhang)
\ No newline at end of file
+MAPREDUCE-5976. native-task: should not fail to build if snappy is missing
(Manu Zhang)
+MAPREDUCE-5978. native-task: remove test case for not supported codec
Bzip2Codec and DefaultCodec (Manu Zhang)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d4fb172/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressTest.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressTest.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressTest.java
index 0406375..b98e2de 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressTest.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/compresstest/CompressTest.java
@@ -68,23 +68,6 @@ public class CompressTest {
}
@Test
- public void testBzip2Compress() throws Exception {
-final Configuration nativeconf =
ScenarioConfiguration.getNativeConfiguration();
-nativeconf.addResource(TestConstants.BZIP2_COMPRESS_CONF_PATH);
-final Job nativejob = CompressMapper.getCompressJob("nativebzip2",
nativeconf);
-nativejob.waitForCompletion(true);
-
-final Configuration hadoopconf =
ScenarioConfiguration.getNormalConfiguration();
-hadoopconf.addResource(TestConstants.BZIP2_COMPRESS_CONF_PATH);
-final Job hadoopjob = CompressMapper.getCompressJob("hadoopbzip2",
hadoopconf);
-hadoopjob.waitForCompletion(true);
-
-final boolean compareRet =
ResultVerifier.verify(CompressMapper.outputFileDir + "nativebzip2",
-CompressMapper.outputFileDir + "hadoopbzip2");
-assertEquals("file compare result: if they are the same ,then return
true", true, compareRet);
- }
-
- @Test
public void testLz4Compress() throws Exception {
final Configuration nativeConf =
ScenarioConfiguration.getNativeConfiguration();
nativeConf.addResource(TestConstants.LZ4_COMPRESS_CONF_PATH);
@@ -100,22 +83,6 @@ public class CompressTest {
assertEquals("file compare result: if they are the same ,then return
true", true, compareRet);
}
- @Test
- public void testDefaultCompress() throws Exception {
-final Configuration nativeConf =
ScenarioConfiguration.getNativeConfiguration();
-nativeConf.addResource(TestConstants.DEFAULT_COMPRESS_CONF_PATH);
-final Job nativeJob = CompressMapper.getCompressJob("nativedefault",
nativeConf);
-nativeJob.waitForCompletion
[32/52] [abbrv] MAPREDUCE-6069. Lint/style fixes and removal of unused code. Contributed by Todd Lipcon.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTest.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTest.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTest.java
index 2258726..f81d94f 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTest.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTest.java
@@ -74,13 +74,14 @@ public class NonSortTest {
public void startUp() throws Exception {
Assume.assumeTrue(NativeCodeLoader.isNativeCodeLoaded());
Assume.assumeTrue(NativeRuntime.isNativeLibraryLoaded());
-final ScenarioConfiguration configuration = new ScenarioConfiguration();
-configuration.addNonSortTestConf();
-final FileSystem fs = FileSystem.get(configuration);
+final ScenarioConfiguration conf = new ScenarioConfiguration();
+conf.addNonSortTestConf();
+final FileSystem fs = FileSystem.get(conf);
final Path path = new Path(TestConstants.NATIVETASK_NONSORT_TEST_INPUTDIR);
if (!fs.exists(path)) {
- new
TestInputFile(configuration.getInt(TestConstants.NATIVETASK_NONSORTTEST_FILESIZE,
1000), Text.class.getName(),
- Text.class.getName(),
configuration).createSequenceTestFile(path.toString());
+ int filesize =
conf.getInt(TestConstants.NATIVETASK_NONSORTTEST_FILESIZE, 1000);
+ new TestInputFile(filesize, Text.class.getName(),
+ Text.class.getName(), conf).createSequenceTestFile(path.toString());
}
fs.close();
}
@@ -93,7 +94,8 @@ public class NonSortTest {
}
- private Job getJob(Configuration conf, String jobName, String inputpath,
String outputpath) throws IOException {
+ private Job getJob(Configuration conf, String jobName,
+ String inputpath, String outputpath) throws IOException {
final FileSystem fs = FileSystem.get(conf);
if (fs.exists(new Path(outputpath))) {
fs.delete(new Path(outputpath), true);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTestMR.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTestMR.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTestMR.java
index 4ca2449..4092e5f 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTestMR.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTestMR.java
@@ -36,7 +36,8 @@ public class NonSortTestMR {
private final Text word = new Text();
@Override
-public void map(Object key, Text value, Context context) throws
IOException, InterruptedException {
+public void map(Object key, Text value, Context context)
+ throws IOException, InterruptedException {
final String line = value.toString();
final StringTokenizer tokenizer = new StringTokenizer(line);
while (tokenizer.hasMoreTokens()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/serde/TestKVSerializer.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/serde/TestKVSerializer.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/serde/TestKVSerializer.java
index 004e8b8..fd5b100 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/serde/TestKVSerializer.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-clie
[52/52] [abbrv] git commit: Merge remote-tracking branch 'origin/trunk' into HDFS-6584
Merge remote-tracking branch 'origin/trunk' into HDFS-6584 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ece3ecab Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ece3ecab Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ece3ecab Branch: refs/heads/HDFS-6584 Commit: ece3ecabdf348dd9f6876f2819e65f72c28cc25f Parents: 2689b6c 24d920b Author: Jing Zhao Authored: Mon Sep 15 10:45:45 2014 -0700 Committer: Jing Zhao Committed: Mon Sep 15 10:45:45 2014 -0700 -- LICENSE.txt |33 + .../assemblies/hadoop-mapreduce-dist.xml| 7 + hadoop-common-project/hadoop-common/CHANGES.txt | 6 + .../src/main/conf/log4j.properties | 5 + .../src/main/resources/core-default.xml |86 + .../apache/hadoop/crypto/key/TestKeyShell.java | 4 +- .../hadoop/security/alias/TestCredShell.java|22 +- .../crypto/key/kms/server/KMSConfiguration.java |15 +- .../hadoop/crypto/key/kms/server/MiniKMS.java | 3 +- .../hadoop/crypto/key/kms/server/TestKMS.java | 3 +- hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 8 + hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../apache/hadoop/hdfs/TestEncryptionZones.java |10 + hadoop-mapreduce-project/CHANGES.txt|65 + .../mapreduce/v2/app/ControlledClock.java |43 - .../v2/app/job/impl/TestTaskAttempt.java| 3 +- .../v2/app/rm/TestRMContainerAllocator.java | 3 +- .../mapreduce/v2/hs/TestHistoryFileManager.java | 3 +- .../hadoop-mapreduce-client-jobclient/pom.xml | 6 + .../v2/TestSpeculativeExecutionWithMRApp.java | 3 +- .../hadoop-mapreduce-client-nativetask/pom.xml | 216 + .../src/CMakeLists.txt | 276 + .../src/JNIFlags.cmake | 118 + .../src/config.h.cmake |23 + .../hadoop/mapred/nativetask/Command.java |57 + .../mapred/nativetask/CommandDispatcher.java|33 + .../hadoop/mapred/nativetask/Constants.java |48 + .../hadoop/mapred/nativetask/DataChannel.java |41 + .../hadoop/mapred/nativetask/DataReceiver.java |37 + .../mapred/nativetask/HadoopPlatform.java |84 + .../mapred/nativetask/ICombineHandler.java |44 + .../mapred/nativetask/INativeComparable.java|54 + .../mapred/nativetask/INativeHandler.java |53 + .../mapred/nativetask/NativeBatchProcessor.java | 279 + .../mapred/nativetask/NativeDataSource.java |47 + .../mapred/nativetask/NativeDataTarget.java |47 + .../NativeMapOutputCollectorDelegator.java | 171 + .../hadoop/mapred/nativetask/NativeRuntime.java | 197 + .../hadoop/mapred/nativetask/Platform.java | 100 + .../hadoop/mapred/nativetask/Platforms.java |79 + .../mapred/nativetask/StatusReportChecker.java |99 + .../hadoop/mapred/nativetask/TaskContext.java |94 + .../mapred/nativetask/buffer/BufferType.java|27 + .../nativetask/buffer/ByteBufferDataReader.java | 148 + .../nativetask/buffer/ByteBufferDataWriter.java | 169 + .../nativetask/buffer/DataInputStream.java |27 + .../nativetask/buffer/DataOutputStream.java |39 + .../mapred/nativetask/buffer/InputBuffer.java | 136 + .../mapred/nativetask/buffer/OutputBuffer.java |76 + .../nativetask/handlers/BufferPullee.java | 121 + .../nativetask/handlers/BufferPuller.java | 203 + .../nativetask/handlers/BufferPushee.java | 151 + .../nativetask/handlers/BufferPusher.java |91 + .../nativetask/handlers/CombinerHandler.java| 145 + .../mapred/nativetask/handlers/IDataLoader.java |37 + .../handlers/NativeCollectorOnlyHandler.java| 171 + .../serde/BoolWritableSerializer.java |35 + .../serde/ByteWritableSerializer.java |35 + .../serde/BytesWritableSerializer.java |48 + .../nativetask/serde/DefaultSerializer.java |71 + .../serde/DoubleWritableSerializer.java |35 + .../serde/FloatWritableSerializer.java |36 + .../mapred/nativetask/serde/IKVSerializer.java |48 + .../nativetask/serde/INativeSerializer.java |49 + .../nativetask/serde/IntWritableSerializer.java |35 + .../mapred/nativetask/serde/KVSerializer.java | 115 + .../serde/LongWritableSerializer.java |34 + .../nativetask/serde/NativeSerialization.java |91 + .../serde/NullWritableSerializer.java |35 + .../serde/SerializationFramework.java |35 + .../mapred/nativetask/serde/TextSerializer.java |49 + .../serde/VIntWritableSerializer.java |27 + .../serde/VLongWritableSerializer.java |27 + .../mapred/nativetask/util/BytesUtil.java | 172 +
[28/52] [abbrv] git commit: Merge remote-tracking branch 'origin/trunk' into MR-2841
Merge remote-tracking branch 'origin/trunk' into MR-2841 Conflicts: hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cce7d1e2 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cce7d1e2 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cce7d1e2 Branch: refs/heads/HDFS-6584 Commit: cce7d1e2f96ff75fe19ab2879ddd90a898ca5b18 Parents: 7ade9b0 727331b Author: Todd Lipcon Authored: Tue Sep 2 15:55:54 2014 -0700 Committer: Todd Lipcon Committed: Tue Sep 2 15:55:54 2014 -0700 -- .gitignore |1 + BUILDING.txt| 22 + dev-support/test-patch.sh | 26 +- .../main/resources/assemblies/hadoop-dist.xml |7 + .../dev-support/findbugsExcludeFile.xml | 38 + hadoop-common-project/hadoop-auth/pom.xml | 26 + .../authentication/client/AuthenticatedURL.java | 36 +- .../server/AuthenticationFilter.java| 109 +- .../server/KerberosAuthenticationHandler.java | 21 +- .../server/PseudoAuthenticationHandler.java | 21 +- .../util/RandomSignerSecretProvider.java| 49 + .../util/RolloverSignerSecretProvider.java | 139 ++ .../security/authentication/util/Signer.java| 46 +- .../util/SignerSecretProvider.java | 62 + .../util/StringSignerSecretProvider.java| 49 + .../client/AuthenticatorTestCase.java | 137 +- .../client/TestAuthenticatedURL.java| 38 +- .../client/TestKerberosAuthenticator.java | 58 +- .../server/TestAuthenticationFilter.java| 118 +- .../util/TestRandomSignerSecretProvider.java| 63 + .../util/TestRolloverSignerSecretProvider.java | 79 + .../authentication/util/TestSigner.java | 85 +- .../util/TestStringSignerSecretProvider.java| 33 + hadoop-common-project/hadoop-common/CHANGES.txt | 502 +-- .../dev-support/findbugsExcludeFile.xml |5 + hadoop-common-project/hadoop-common/pom.xml | 34 +- .../hadoop-common/src/CMakeLists.txt| 34 + .../hadoop-common/src/JNIFlags.cmake|6 + .../hadoop-common/src/config.h.cmake|1 + .../src/contrib/bash-tab-completion/hadoop.sh | 28 +- .../hadoop-common/src/main/bin/hadoop | 229 +-- .../src/main/bin/hadoop-config.cmd | 10 +- .../hadoop-common/src/main/bin/hadoop-config.sh | 409 ++--- .../hadoop-common/src/main/bin/hadoop-daemon.sh | 214 +-- .../src/main/bin/hadoop-daemons.sh | 37 +- .../src/main/bin/hadoop-functions.sh| 1066 ++ .../src/main/bin/hadoop-layout.sh.example | 93 ++ .../hadoop-common/src/main/bin/hadoop.cmd | 33 +- .../hadoop-common/src/main/bin/rcc | 52 +- .../hadoop-common/src/main/bin/slaves.sh| 51 +- .../hadoop-common/src/main/bin/start-all.sh | 38 +- .../hadoop-common/src/main/bin/stop-all.sh | 36 +- .../hadoop-common/src/main/conf/hadoop-env.sh | 432 +- .../org/apache/hadoop/conf/Configuration.java | 78 +- .../hadoop/conf/ReconfigurationServlet.java |2 + .../apache/hadoop/crypto/AesCtrCryptoCodec.java | 67 + .../org/apache/hadoop/crypto/CipherSuite.java | 115 ++ .../org/apache/hadoop/crypto/CryptoCodec.java | 179 +++ .../apache/hadoop/crypto/CryptoInputStream.java | 680 + .../hadoop/crypto/CryptoOutputStream.java | 280 .../apache/hadoop/crypto/CryptoStreamUtils.java | 70 + .../org/apache/hadoop/crypto/Decryptor.java | 72 + .../org/apache/hadoop/crypto/Encryptor.java | 71 + .../hadoop/crypto/JceAesCtrCryptoCodec.java | 165 +++ .../hadoop/crypto/OpensslAesCtrCryptoCodec.java | 164 +++ .../org/apache/hadoop/crypto/OpensslCipher.java | 289 .../hadoop/crypto/key/JavaKeyStoreProvider.java | 248 +++- .../apache/hadoop/crypto/key/KeyProvider.java | 32 +- .../crypto/key/KeyProviderCryptoExtension.java | 91 +- .../KeyProviderDelegationTokenExtension.java|8 +- .../hadoop/crypto/key/KeyProviderExtension.java |1 + .../org/apache/hadoop/crypto/key/KeyShell.java | 89 +- .../apache/hadoop/crypto/key/UserProvider.java |5 +- .../crypto/key/kms/KMSClientProvider.java | 117 +- .../hadoop/crypto/key/kms/KMSRESTConstants.java |2 +- .../crypto/random/OpensslSecureRandom.java | 122 ++ .../hadoop/crypto/random/OsSecureRandom.java| 115 ++ .../apache/hadoop/fs/AbstractFileSystem.java| 103 +- .../apache/hadoop/fs/ChecksumFileSystem.java|8 +- .../java/org/apache/hadoop/fs/ChecksumFs.java |8 +- .../hadoop/fs/CommonConfigurationKeys.java |4 +- .../fs/CommonC
[34/52] [abbrv] MAPREDUCE-6069. Lint/style fixes and removal of unused code. Contributed by Todd Lipcon.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/NativeTask.h
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/NativeTask.h
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/NativeTask.h
index 60bb6f5..ba026f5 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/NativeTask.h
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/NativeTask.h
@@ -38,13 +38,6 @@ using std::pair;
enum NativeObjectType {
UnknownObjectType = 0,
BatchHandlerType = 1,
- MapperType = 2,
- ReducerType = 3,
- PartitionerType = 4,
- CombinerType = 5,
- FolderType = 6,
- RecordReaderType = 7,
- RecordWriterType = 8
};
/**
@@ -69,7 +62,6 @@ enum Endium {
#define INPUT_LINE_KV_SEPERATOR
"mapreduce.input.keyvaluelinerecordreader.key.value.separator"
#define MAPRED_TEXTOUTPUT_FORMAT_SEPERATOR
"mapreduce.output.textoutputformat.separator"
#define MAPRED_WORK_OUT_DIR "mapreduce.task.output.dir"
-#define NATIVE_OUTPUT_FILE_NAME "native.output.file.name"
#define MAPRED_COMPRESS_OUTPUT "mapreduce.output.fileoutputformat.compress"
#define MAPRED_OUTPUT_COMPRESSION_CODEC
"mapreduce.output.fileoutputformat.compress.codec"
#define TOTAL_ORDER_PARTITIONER_PATH "total.order.partitioner.path"
@@ -386,32 +378,6 @@ public:
virtual bool next(Buffer & key, Buffer & value) = 0;
};
-class RecordReader : public KVIterator, public Configurable, public Progress {
-public:
- virtual NativeObjectType type() {
-return RecordReaderType;
- }
-
- virtual bool next(Buffer & key, Buffer & value) = 0;
-
- virtual float getProgress() = 0;
-
- virtual void close() = 0;
-};
-
-class RecordWriter : public Collector, public Configurable {
-public:
- virtual NativeObjectType type() {
-return RecordWriterType;
- }
-
- virtual void collect(const void * key, uint32_t keyLen, const void * value,
uint32_t valueLen) {
- }
-
- virtual void close() {
- }
-
-};
class ProcessorBase : public Configurable {
protected:
@@ -444,36 +410,6 @@ public:
}
};
-class Mapper : public ProcessorBase {
-public:
- virtual NativeObjectType type() {
-return MapperType;
- }
-
- /**
- * Map interface, default IdenticalMapper
- */
- virtual void map(const char * key, uint32_t keyLen, const char * value,
uint32_t valueLen) {
-collect(key, keyLen, value, valueLen);
- }
-};
-
-class Partitioner : public Configurable {
-public:
- virtual NativeObjectType type() {
-return PartitionerType;
- }
-
- /**
- * Partition interface
- * @param key key buffer
- * @param keyLen key length, can be modified to smaller value
- * to truncate key
- * @return partition number
- */
- virtual uint32_t getPartition(const char * key, uint32_t & keyLen, uint32_t
numPartition);
-};
-
enum KeyGroupIterState {
SAME_KEY,
NEW_KEY,
@@ -502,80 +438,7 @@ public:
virtual const char * nextValue(uint32_t & len) = 0;
};
-class Reducer : public ProcessorBase {
-public:
- virtual NativeObjectType type() {
-return ReducerType;
- }
-
- /**
- * Reduce interface, default IdenticalReducer
- */
- virtual void reduce(KeyGroupIterator & input) {
-const char * key;
-const char * value;
-uint32_t keyLen;
-uint32_t valueLen;
-key = input.getKey(keyLen);
-while (NULL != (value = input.nextValue(valueLen))) {
- collect(key, keyLen, value, valueLen);
-}
- }
-};
-
-/**
- * Folder API used for hashtable based aggregation
- * Folder will be used in this way:
- * on(key, value):
- * state = hashtable.get(key)
- * if state == None:
- * size = size()
- * if size == -1:
- * state = init(null, -1)
- * elif size > 0:
- * state = fixallocator.get(key)
- * init(state, size)
- * folder(state, value, value.len)
- *
- * final():
- * for k,state in hashtable:
- * final(key, key.len, state)
- */
-class Folder : public ProcessorBase {
-public:
- virtual NativeObjectType type() {
-return FolderType;
- }
-
- /**
- * Get aggregator state size
- * @return state storage size
- * -1 size not fixed or unknown, default
- *e.g. list map tree
- * 0 don't need to store state
- * >0 fixed sized state
- *e.g. int32 int64 float.
- */
- virtual int32_t size() {
-return -1;
- }
- /**
- * Create and/or init new state
- */
- virtual void * init(const char * key, uint32_t keyLen) {
-return NULL;
- }
-
- /**
- * Aggregation function
- */
- virtual void folder(void * dest, const char * value, uint32_t valueLen) {
- }
-
- virtual void final(const char * key, u
[20/52] [abbrv] MAPREDUCE-5977. Fix or suppress native-task gcc warnings. Contributed by Manu Zhang.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/NativeTask.h
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/NativeTask.h
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/NativeTask.h
index 11712ab..60bb6f5 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/NativeTask.h
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/NativeTask.h
@@ -622,7 +622,6 @@ typedef void (*ANY_FUNC_PTR)();
#define DEFINE_NATIVE_LIBRARY(Library) \
static std::map
Library##ClassMap__; \
extern "C" void * Library##GetFunctionGetter(const std::string & name) { \
- void * ret = NULL; \
std::map::iterator itr =
Library##ClassMap__.find(name); \
if (itr != Library##ClassMap__.end()) { \
return (void *)(itr->second); \
@@ -630,7 +629,6 @@ typedef void (*ANY_FUNC_PTR)();
return NULL; \
} \
extern "C" NativeTask::ObjectCreatorFunc Library##GetObjectCreator(const
std::string & name) { \
-NativeObject * ret = NULL; \
std::map::iterator itr =
Library##ClassMap__.find(name); \
if (itr != Library##ClassMap__.end()) { \
return itr->second; \
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/BlockCodec.cc
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/BlockCodec.cc
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/BlockCodec.cc
index a59b9d8..59cd8fb 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/BlockCodec.cc
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/BlockCodec.cc
@@ -23,7 +23,7 @@
namespace NativeTask {
BlockCompressStream::BlockCompressStream(OutputStream * stream, uint32_t
bufferSizeHint)
-: CompressStream(stream), _compressedBytesWritten(0), _tempBufferSize(0),
_tempBuffer(NULL) {
+: CompressStream(stream), _tempBuffer(NULL), _tempBufferSize(0),
_compressedBytesWritten(0) {
_hint = bufferSizeHint;
_blockMax = bufferSizeHint / 2 * 3;
}
@@ -68,7 +68,7 @@ uint64_t BlockCompressStream::compressedBytesWritten() {
//
BlockDecompressStream::BlockDecompressStream(InputStream * stream, uint32_t
bufferSizeHint)
-: DecompressStream(stream), _tempBufferSize(0), _tempBuffer(NULL) {
+: DecompressStream(stream), _tempBuffer(NULL), _tempBufferSize(0) {
_hint = bufferSizeHint;
_blockMax = bufferSizeHint / 2 * 3;
_tempDecompressBuffer = NULL;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/BatchHandler.h
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/BatchHandler.h
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/BatchHandler.h
index bc022a8..15ce2ea 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/BatchHandler.h
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/BatchHandler.h
@@ -103,10 +103,10 @@ protected:
*/
inline void output(const char * buff, uint32_t length) {
while (length > 0) {
- if (length > _out.remain()) {
+ uint32_t remain = _out.remain();
+ if (length > remain) {
flushOutput();
}
- uint32_t remain = _out.remain();
uint32_t cp = length < remain ? length : remain;
simple_memcpy(_out.current(), buff, cp);
buff += cp;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad4524c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/CombineHandler.cc
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/CombineHandler.cc
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/nat
[42/52] [abbrv] git commit: YARN-2525. yarn logs command gives error on trunk (Akira AJISAKA via aw)
YARN-2525. yarn logs command gives error on trunk (Akira AJISAKA via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/40364dc4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/40364dc4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/40364dc4
Branch: refs/heads/HDFS-6584
Commit: 40364dc47c03efa295ae03fe8aa8467017fb6f26
Parents: 957414d
Author: Allen Wittenauer
Authored: Fri Sep 12 10:35:41 2014 -0700
Committer: Allen Wittenauer
Committed: Fri Sep 12 14:56:41 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 2 ++
hadoop-yarn-project/hadoop-yarn/bin/yarn | 2 +-
2 files changed, 3 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40364dc4/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 3993f5e..9002e6a 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -23,6 +23,8 @@ Trunk - Unreleased
YARN-2436. [post-HADOOP-9902] yarn application help doesn't work (aw)
+YARN-2525. yarn logs command gives error on trunk (Akira AJISAKA via aw)
+
Release 2.6.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40364dc4/hadoop-yarn-project/hadoop-yarn/bin/yarn
--
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn
b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index 371d23d..12f7bb5 100644
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -99,7 +99,7 @@ case "${COMMAND}" in
CLASS='org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer'
;;
logs)
-CLASS=org.apache.hadoop.yarn.logaggregation.LogDumper
+CLASS=org.apache.hadoop.yarn.client.cli.LogsCLI
hadoop_debug "Append YARN_CLIENT_OPTS onto YARN_OPTS"
YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}"
;;
[48/52] [abbrv] git commit: YARN-611. Added an API to let apps specify an interval beyond which AM failures should be ignored towards counting max-attempts. Contributed by Xuan Gong.
YARN-611. Added an API to let apps specify an interval beyond which AM failures
should be ignored towards counting max-attempts. Contributed by Xuan Gong.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/14e2639f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/14e2639f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/14e2639f
Branch: refs/heads/HDFS-6584
Commit: 14e2639fd0d53f7e0b58f2f4744af44983d4e867
Parents: 98588cf
Author: Vinod Kumar Vavilapalli
Authored: Sat Sep 13 18:04:05 2014 -0700
Committer: Vinod Kumar Vavilapalli
Committed: Sat Sep 13 18:04:05 2014 -0700
--
.../mapreduce/v2/app/ControlledClock.java | 43 ---
.../v2/app/job/impl/TestTaskAttempt.java| 3 +-
.../v2/app/rm/TestRMContainerAllocator.java | 3 +-
.../mapreduce/v2/hs/TestHistoryFileManager.java | 3 +-
.../hadoop-mapreduce-client-jobclient/pom.xml | 6 +
.../v2/TestSpeculativeExecutionWithMRApp.java | 3 +-
hadoop-yarn-project/CHANGES.txt | 6 +-
.../records/ApplicationSubmissionContext.java | 43 +++
.../src/main/proto/yarn_protos.proto| 1 +
.../pb/ApplicationSubmissionContextPBImpl.java | 13 ++
.../hadoop/yarn/util/ControlledClock.java | 42 ++
.../recovery/FileSystemRMStateStore.java| 1 +
.../recovery/MemoryRMStateStore.java| 1 +
.../resourcemanager/recovery/RMStateStore.java | 10 +-
.../recovery/ZKRMStateStore.java| 2 +
.../records/ApplicationAttemptStateData.java| 15 ++-
.../pb/ApplicationAttemptStateDataPBImpl.java | 12 ++
.../server/resourcemanager/rmapp/RMAppImpl.java | 44 ++-
.../rmapp/attempt/RMAppAttempt.java | 6 +
.../rmapp/attempt/RMAppAttemptImpl.java | 27 +++-
.../yarn_server_resourcemanager_recovery.proto | 1 +
.../yarn/server/resourcemanager/MockRM.java | 23 +++-
.../applicationsmanager/TestAMRestart.java | 127 +++
.../recovery/RMStateStoreTestBase.java | 6 +-
24 files changed, 372 insertions(+), 69 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14e2639f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/ControlledClock.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/ControlledClock.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/ControlledClock.java
deleted file mode 100644
index 198117b..000
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/ControlledClock.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-package org.apache.hadoop.mapreduce.v2.app;
-
-import org.apache.hadoop.yarn.util.Clock;
-
-public class ControlledClock implements Clock {
- private long time = -1;
- private final Clock actualClock;
- public ControlledClock(Clock actualClock) {
-this.actualClock = actualClock;
- }
- public synchronized void setTime(long time) {
-this.time = time;
- }
- public synchronized void reset() {
-time = -1;
- }
-
- @Override
- public synchronized long getTime() {
-if (time != -1) {
- return time;
-}
-return actualClock.getTime();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14e2639f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
b/hadoop-mapreduce-project/hadoop-ma
[01/52] [abbrv] MAPREDUCE-5996. native-task: Rename system tests into standard directory layout. Contributed by Todd Lipcon.
Repository: hadoop
Updated Branches:
refs/heads/HDFS-6584 2689b6ca7 -> ece3ecabd
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/ResultVerifier.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/ResultVerifier.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/ResultVerifier.java
deleted file mode 100644
index b665971..000
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/ResultVerifier.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.testutil;
-
-import java.io.IOException;
-import java.util.zip.CRC32;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.Path;
-
-public class ResultVerifier {
- /**
- * verify the result
- *
- * @param sample
- * :nativetask output
- * @param source
- * :yuanwenjian
- * @throws Exception
- */
- public static boolean verify(String sample, String source) throws Exception {
-FSDataInputStream sourcein = null;
-FSDataInputStream samplein = null;
-
-final Configuration conf = new Configuration();
-final FileSystem fs = FileSystem.get(conf);
-final Path hdfssource = new Path(source);
-final Path[] sourcepaths = FileUtil.stat2Paths(fs.listStatus(hdfssource));
-
-final Path hdfssample = new Path(sample);
-final Path[] samplepaths = FileUtil.stat2Paths(fs.listStatus(hdfssample));
-if (sourcepaths == null) {
- throw new Exception("source file can not be found");
-}
-if (samplepaths == null) {
- throw new Exception("sample file can not be found");
-}
-if (sourcepaths.length != samplepaths.length) {
- return false;
-}
-for (int i = 0; i < sourcepaths.length; i++) {
- final Path sourcepath = sourcepaths[i];
- // op result file start with "part-r" like part-r-0
-
- if (!sourcepath.getName().startsWith("part-r")) {
-continue;
- }
- Path samplepath = null;
- for (int j = 0; j < samplepaths.length; j++) {
-if (samplepaths[i].getName().equals(sourcepath.getName())) {
- samplepath = samplepaths[i];
- break;
-}
- }
- if (samplepath == null) {
-throw new Exception("cound not found file " +
samplepaths[0].getParent() + "/" + sourcepath.getName()
-+ " , as sourcepaths has such file");
- }
-
- // compare
- try {
-if (fs.exists(sourcepath) && fs.exists(samplepath)) {
- sourcein = fs.open(sourcepath);
- samplein = fs.open(samplepath);
-} else {
- System.err.println("result file not found:" + sourcepath + " or " +
samplepath);
- return false;
-}
-
-CRC32 sourcecrc, samplecrc;
-samplecrc = new CRC32();
-sourcecrc = new CRC32();
-final byte[] bufin = new byte[1 << 16];
-int readnum = 0;
-int totalRead = 0;
-while (samplein.available() > 0) {
- readnum = samplein.read(bufin);
- totalRead += readnum;
- samplecrc.update(bufin, 0, readnum);
-}
-
-if (0 == totalRead) {
- throw new Exception("source " + sample + " is empty file");
-}
-
-totalRead = 0;
-while (sourcein.available() > 0) {
- readnum = sourcein.read(bufin);
- totalRead += readnum;
- sourcecrc.update(bufin, 0, readnum);
-}
-if (0 == totalRead) {
- throw new Exception("source " + sample +
[37/52] [abbrv] git commit: MAPREDUCE-6074. native-task: Fix release audit warnings
MAPREDUCE-6074. native-task: Fix release audit warnings Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7c91f9b1 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7c91f9b1 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7c91f9b1 Branch: refs/heads/HDFS-6584 Commit: 7c91f9b1484d487e792dca051fbd418697049422 Parents: 4b3f1e2 Author: Todd Lipcon Authored: Fri Sep 5 13:41:18 2014 -0700 Committer: Todd Lipcon Committed: Fri Sep 5 19:58:41 2014 -0700 -- LICENSE.txt | 33 +++ .../CHANGES.MAPREDUCE-2841.txt | 1 + .../hadoop-mapreduce-client-nativetask/LICENSE | 266 --- .../hadoop-mapreduce-client-nativetask/pom.xml | 12 +- .../mapred/nativetask/INativeComparable.java| 27 +- .../src/main/native/COPYING | 87 -- ...oop_mapred_nativetask_NativeBatchProcessor.h | 54 ...che_hadoop_mapred_nativetask_NativeRuntime.h | 66 - .../src/main/native/test.sh | 11 + .../nativetask/buffer/TestBufferPushPull.java | 13 +- .../buffer/TestByteBufferReadWrite.java | 16 +- .../nativetask/combinertest/CombinerTest.java | 4 +- .../combinertest/OldAPICombinerTest.java| 5 +- .../nativetask/combinertest/WordCount.java | 3 +- .../nativetask/compresstest/CompressMapper.java | 4 +- .../nativetask/compresstest/CompressTest.java | 2 +- .../nativetask/handlers/TestCombineHandler.java | 13 +- .../TestNativeCollectorOnlyHandler.java | 17 +- .../hadoop/mapred/nativetask/kvtest/KVJob.java | 3 +- .../mapred/nativetask/kvtest/TestInputFile.java | 7 +- .../nativetask/nonsorttest/NonSortTest.java | 2 +- .../serde/TestNativeSerialization.java | 10 +- .../nativetask/testutil/BytesFactory.java | 2 +- .../mapred/nativetask/utils/TestBytesUtil.java | 20 +- hadoop-mapreduce-project/pom.xml| 2 +- 25 files changed, 143 insertions(+), 537 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c91f9b1/LICENSE.txt -- diff --git a/LICENSE.txt b/LICENSE.txt index 946a6df..99989f1 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -288,3 +288,36 @@ lz4_encoder.h,lz4hc.h,lz4hc.c,lz4hc_encoder.h}, - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html - LZ4 source repository : http://code.google.com/p/lz4/ */ + + +For hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/gtest +- +Copyright 2008, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. +* Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c91f9b1/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt -- diff --git a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt index ea17907..539e7be 100644 --- a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt +++ b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt @@ -22,3 +22,4 @@ MAPREDUCE-6056. native-task: move system test working dir to target dir and clea MAPREDUCE-6055. native-task: findbugs, interface annotations, and other misc cleanup (tod
[06/52] [abbrv] git commit: Merge trunk into branch
Merge trunk into branch git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/MR-2841@1613007 13f79535-47bb-0310-9956-ffa450edef68 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5149a8a6 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5149a8a6 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5149a8a6 Branch: refs/heads/HDFS-6584 Commit: 5149a8a6f1d2fc673a7d011d7e0e1a5407ceac4c Parents: b2cba48 2054453 Author: Todd Lipcon Authored: Thu Jul 24 06:23:41 2014 + Committer: Todd Lipcon Committed: Thu Jul 24 06:23:41 2014 + -- hadoop-common-project/hadoop-common/CHANGES.txt | 14 + .../apache/hadoop/fs/ChecksumFileSystem.java| 4 +- .../java/org/apache/hadoop/fs/FileContext.java | 29 ++ .../java/org/apache/hadoop/fs/FileSystem.java | 2 +- .../org/apache/hadoop/fs/viewfs/ChRootedFs.java | 34 ++ .../apache/hadoop/fs/viewfs/ViewFileSystem.java | 34 ++ .../org/apache/hadoop/fs/viewfs/ViewFs.java | 80 + .../security/alias/JavaKeyStoreProvider.java| 5 +- .../apache/hadoop/util/DirectBufferPool.java| 109 ++ .../org/apache/hadoop/fs/TestDFVariations.java | 21 +- .../apache/hadoop/fs/shell/TestPathData.java| 12 +- .../fs/viewfs/ViewFileSystemBaseTest.java | 30 ++ .../apache/hadoop/fs/viewfs/ViewFsBaseTest.java | 30 ++ .../hadoop/util/TestDirectBufferPool.java | 97 + .../java/org/apache/hadoop/nfs/NfsExports.java | 7 +- .../hadoop/hdfs/nfs/mount/RpcProgramMountd.java | 19 +- .../hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java| 5 +- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 10 + .../apache/hadoop/hdfs/BlockReaderLocal.java| 2 +- .../hadoop/hdfs/BlockReaderLocalLegacy.java | 2 +- .../hadoop/hdfs/protocol/ClientProtocol.java| 2 +- .../protocol/datatransfer/PacketReceiver.java | 2 +- .../server/datanode/BlockPoolSliceScanner.java | 54 ++- .../hadoop/hdfs/server/namenode/FSEditLog.java | 3 +- .../hdfs/server/namenode/FSEditLogLoader.java | 4 + .../hdfs/server/namenode/FSEditLogOp.java | 5 + .../hdfs/server/namenode/FSNamesystem.java | 62 +++- .../server/namenode/XAttrPermissionFilter.java | 17 +- .../hadoop/hdfs/util/DirectBufferPool.java | 107 -- .../hdfs/web/resources/XAttrNameParam.java | 4 +- .../org/apache/hadoop/hdfs/TestDFSShell.java| 69 .../hdfs/server/namenode/FSXAttrBaseTest.java | 351 +-- .../server/namenode/TestNamenodeRetryCache.java | 4 +- .../namenode/ha/TestRetryCacheWithHA.java | 54 ++- .../hadoop/hdfs/util/TestDirectBufferPool.java | 97 - .../hadoop/hdfs/web/resources/TestParam.java| 6 - .../hadoop-hdfs/src/test/resources/editsStored | Bin 4970 -> 4992 bytes .../src/test/resources/editsStored.xml | 2 + hadoop-yarn-project/CHANGES.txt | 12 + .../dev-support/findbugs-exclude.xml| 6 + .../BaseContainerTokenSecretManager.java| 2 +- .../yarn/server/nodemanager/NodeManager.java| 6 +- .../recovery/NMLeveldbStateStoreService.java| 117 ++- .../recovery/NMNullStateStoreService.java | 30 +- .../recovery/NMStateStoreService.java | 40 ++- .../security/NMContainerTokenSecretManager.java | 105 +- .../security/NMTokenSecretManagerInNM.java | 5 +- .../recovery/NMMemoryStateStoreService.java | 55 ++- .../TestNMLeveldbStateStoreService.java | 96 - .../TestNMContainerTokenSecretManager.java | 144 .../security/TestNMTokenSecretManagerInNM.java | 8 +- .../scheduler/fair/FairScheduler.java | 15 +- .../fair/FairSchedulerConfiguration.java| 9 + .../security/DelegationTokenRenewer.java| 6 +- .../fair/TestFairSchedulerPreemption.java | 2 +- .../security/TestDelegationTokenRenewer.java| 36 +- .../src/site/apt/FairScheduler.apt.vm | 6 + .../src/site/apt/ResourceManagerRest.apt.vm | 105 +++--- 58 files changed, 1789 insertions(+), 405 deletions(-) --
[19/52] [abbrv] git commit: MAPREDUCE-6025: distribute nativetask.so to right folder (Manu Zhang)
MAPREDUCE-6025: distribute nativetask.so to right folder (Manu Zhang)
git-svn-id:
https://svn.apache.org/repos/asf/hadoop/common/branches/MR-2841@1618120
13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6d39367f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6d39367f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6d39367f
Branch: refs/heads/HDFS-6584
Commit: 6d39367f6a95383b634d79b6df35a278e0d89fe0
Parents: eace4dd
Author: Sean Zhong
Authored: Fri Aug 15 08:55:49 2014 +
Committer: Sean Zhong
Committed: Fri Aug 15 08:55:49 2014 +
--
hadoop-dist/pom.xml | 1 -
hadoop-mapreduce-project/pom.xml | 28
2 files changed, 28 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d39367f/hadoop-dist/pom.xml
--
diff --git a/hadoop-dist/pom.xml b/hadoop-dist/pom.xml
index 62a27f4..149f404 100644
--- a/hadoop-dist/pom.xml
+++ b/hadoop-dist/pom.xml
@@ -122,7 +122,6 @@
run cp -r
$ROOT/hadoop-hdfs-project/hadoop-hdfs-nfs/target/hadoop-hdfs-nfs-${project.version}/*
.
run cp -r
$ROOT/hadoop-yarn-project/target/hadoop-yarn-project-${project.version}/* .
run cp -r
$ROOT/hadoop-mapreduce-project/target/hadoop-mapreduce-${project.version}/* .
- run cp -r
$ROOT/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/target/hadoop-mapreduce-client-nativetask-${project.version}/*
.
run cp -r
$ROOT/hadoop-tools/hadoop-tools-dist/target/hadoop-tools-dist-${project.version}/*
.
echo
echo "Hadoop dist layout available at:
${project.build.directory}/hadoop-${project.version}"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d39367f/hadoop-mapreduce-project/pom.xml
--
diff --git a/hadoop-mapreduce-project/pom.xml b/hadoop-mapreduce-project/pom.xml
index 772a2a3..9110942 100644
--- a/hadoop-mapreduce-project/pom.xml
+++ b/hadoop-mapreduce-project/pom.xml
@@ -236,6 +236,34 @@
org.apache.maven.plugins
+maven-antrun-plugin
+
+
+dist
+prepare-package
+
+ run
+
+
+
+
+
LIB_DIR="${mr.basedir}/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/target/native/target/usr/local/lib"
+ if [ -d $${LIB_DIR} ] ; then
+
TARGET_DIR="${project.build.directory}/${project.artifactId}-${project.version}/lib/native"
+mkdir -p $${TARGET_DIR}
+cp -R $${LIB_DIR}/lib* $${TARGET_DIR}
+ fi
+
+
+
+
+
+
+
+
+
+
+org.apache.maven.plugins
maven-assembly-plugin
[45/52] [abbrv] git commit: MAPREDUCE-2841. Merge remote-tracking branch 'apache/MR-2841' into trunk
MAPREDUCE-2841. Merge remote-tracking branch 'apache/MR-2841' into trunk Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8a3a327c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8a3a327c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8a3a327c Branch: refs/heads/HDFS-6584 Commit: 8a3a327cd3c7fd1049b521325e485c59137855e8 Parents: a0ad975 b160707 Author: Todd Lipcon Authored: Fri Sep 12 18:31:53 2014 -0700 Committer: Todd Lipcon Committed: Fri Sep 12 18:31:53 2014 -0700 -- LICENSE.txt |33 + .../assemblies/hadoop-mapreduce-dist.xml| 7 + .../CHANGES.MAPREDUCE-2841.txt |26 + .../hadoop-mapreduce-client-nativetask/pom.xml | 216 + .../src/CMakeLists.txt | 276 + .../src/JNIFlags.cmake | 118 + .../src/config.h.cmake |23 + .../hadoop/mapred/nativetask/Command.java |57 + .../mapred/nativetask/CommandDispatcher.java|33 + .../hadoop/mapred/nativetask/Constants.java |48 + .../hadoop/mapred/nativetask/DataChannel.java |41 + .../hadoop/mapred/nativetask/DataReceiver.java |37 + .../mapred/nativetask/HadoopPlatform.java |84 + .../mapred/nativetask/ICombineHandler.java |44 + .../mapred/nativetask/INativeComparable.java|54 + .../mapred/nativetask/INativeHandler.java |53 + .../mapred/nativetask/NativeBatchProcessor.java | 279 + .../mapred/nativetask/NativeDataSource.java |47 + .../mapred/nativetask/NativeDataTarget.java |47 + .../NativeMapOutputCollectorDelegator.java | 171 + .../hadoop/mapred/nativetask/NativeRuntime.java | 197 + .../hadoop/mapred/nativetask/Platform.java | 100 + .../hadoop/mapred/nativetask/Platforms.java |79 + .../mapred/nativetask/StatusReportChecker.java |99 + .../hadoop/mapred/nativetask/TaskContext.java |94 + .../mapred/nativetask/buffer/BufferType.java|27 + .../nativetask/buffer/ByteBufferDataReader.java | 148 + .../nativetask/buffer/ByteBufferDataWriter.java | 169 + .../nativetask/buffer/DataInputStream.java |27 + .../nativetask/buffer/DataOutputStream.java |39 + .../mapred/nativetask/buffer/InputBuffer.java | 136 + .../mapred/nativetask/buffer/OutputBuffer.java |76 + .../nativetask/handlers/BufferPullee.java | 121 + .../nativetask/handlers/BufferPuller.java | 203 + .../nativetask/handlers/BufferPushee.java | 151 + .../nativetask/handlers/BufferPusher.java |91 + .../nativetask/handlers/CombinerHandler.java| 145 + .../mapred/nativetask/handlers/IDataLoader.java |37 + .../handlers/NativeCollectorOnlyHandler.java| 171 + .../serde/BoolWritableSerializer.java |35 + .../serde/ByteWritableSerializer.java |35 + .../serde/BytesWritableSerializer.java |48 + .../nativetask/serde/DefaultSerializer.java |71 + .../serde/DoubleWritableSerializer.java |35 + .../serde/FloatWritableSerializer.java |36 + .../mapred/nativetask/serde/IKVSerializer.java |48 + .../nativetask/serde/INativeSerializer.java |49 + .../nativetask/serde/IntWritableSerializer.java |35 + .../mapred/nativetask/serde/KVSerializer.java | 115 + .../serde/LongWritableSerializer.java |34 + .../nativetask/serde/NativeSerialization.java |91 + .../serde/NullWritableSerializer.java |35 + .../serde/SerializationFramework.java |35 + .../mapred/nativetask/serde/TextSerializer.java |49 + .../serde/VIntWritableSerializer.java |27 + .../serde/VLongWritableSerializer.java |27 + .../mapred/nativetask/util/BytesUtil.java | 172 + .../mapred/nativetask/util/ConfigUtil.java |47 + .../nativetask/util/LocalJobOutputFiles.java| 159 + .../nativetask/util/NativeTaskOutput.java | 106 + .../nativetask/util/NativeTaskOutputFiles.java | 170 + .../mapred/nativetask/util/OutputUtil.java |47 + .../mapred/nativetask/util/ReadWriteBuffer.java | 159 + .../mapred/nativetask/util/SizedWritable.java |55 + .../src/main/native/gtest/gtest-all.cc | 9120 .../src/main/native/gtest/gtest_main.cc |39 + .../src/main/native/gtest/include/gtest/gtest.h | 19541 + .../src/main/native/src/NativeTask.h| 507 + .../src/main/native/src/codec/BlockCodec.cc | 179 + .../src/main/native/src/codec/BlockCodec.h |98 + .../src/main/native/src/codec/GzipCodec.cc | 192 + .../src/main/native/src/codec/GzipCodec.h |82 + .../src/main/native/src/codec/Lz4Codec.cc |88 + .../src/main/native/src/cod
[18/52] [abbrv] git commit: MAPREDUCE-6035. native-task: sources/test-sources jar distribution. Contributed by Manu Zhang.
MAPREDUCE-6035. native-task: sources/test-sources jar distribution. Contributed
by Manu Zhang.
git-svn-id:
https://svn.apache.org/repos/asf/hadoop/common/branches/MR-2841@1617879
13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eace4dd5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eace4dd5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eace4dd5
Branch: refs/heads/HDFS-6584
Commit: eace4dd50b1214a8842e47c522da258e1746386a
Parents: 808bf8b
Author: Todd Lipcon
Authored: Thu Aug 14 04:56:43 2014 +
Committer: Todd Lipcon
Committed: Thu Aug 14 04:56:43 2014 +
--
.../src/main/resources/assemblies/hadoop-mapreduce-dist.xml | 7 +++
hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt | 1 +
2 files changed, 8 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eace4dd5/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml
--
diff --git
a/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml
b/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml
index 0cb01ea..749e16a 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml
@@ -136,6 +136,13 @@
+
hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/target
+
/share/hadoop/${hadoop.component}/sources
+
+*-sources.jar
+
+
+
hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/target
/share/hadoop/${hadoop.component}/sources
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eace4dd5/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
index 123eb6d..95f7858 100644
--- a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
+++ b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
@@ -14,3 +14,4 @@ MAPREDUCE-5976. native-task: should not fail to build if
snappy is missing (Manu
MAPREDUCE-5978. native-task: remove test case for not supported codec
Bzip2Codec and DefaultCodec (Manu Zhang)
MAPREDUCE-6006. native-task: add native tests to maven and fix bug in pom.xml
(Binglin Chang via todd)
MAPREDUCE-6026. native-task: fix logging (Manu Zhang via todd)
+MAPREDUCE-6035. native-task: sources/test-sources jar distribution (Manu Zhang
via todd)
[43/52] [abbrv] git commit: YARN-2456. Possible livelock in CapacityScheduler when RM is recovering apps. Contributed by Jian He
YARN-2456. Possible livelock in CapacityScheduler when RM is recovering
apps. Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e65ae575
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e65ae575
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e65ae575
Branch: refs/heads/HDFS-6584
Commit: e65ae575a059a426c4c38fdabe22a31eabbb349e
Parents: 40364dc
Author: XuanGong
Authored: Fri Sep 12 15:21:46 2014 -0700
Committer: XuanGong
Committed: Fri Sep 12 15:21:46 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 ++
.../resourcemanager/recovery/RMStateStore.java | 3 +-
.../server/resourcemanager/TestRMRestart.java | 43
3 files changed, 48 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e65ae575/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 9002e6a..efc3e09 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -345,6 +345,9 @@ Release 2.6.0 - UNRELEASED
YARN-2484. FileSystemRMStateStore#readFile/writeFile should close
FSData(In|Out)putStream in final block (Tsuyoshi OZAWA via jlowe)
+YARN-2456. Possible livelock in CapacityScheduler when RM is recovering
apps.
+(Jian He via xgong)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e65ae575/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
index ac51a17..df4f3a9 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
@@ -22,6 +22,7 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
+import java.util.TreeMap;
import javax.crypto.SecretKey;
@@ -421,7 +422,7 @@ public abstract class RMStateStore extends AbstractService {
*/
public static class RMState {
Map appState =
-new HashMap();
+new TreeMap();
RMDTSecretManagerState rmSecretManagerState = new RMDTSecretManagerState();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e65ae575/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
index 7d511db..caa5647 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
@@ -19,9 +19,11 @@
package org.apache.hadoop.yarn.server.resourcemanager;
import static org.mockito.Matchers.isA;
+import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
import java.io.File;
import java.io.FileOutputStream;
@@ -1656,6 +1658,47 @@ public class TestRMRestart {
rm2.waitForState(app1.getApplicationId(), RMAppState.FAILED);
}
+ @Test (timeout = 2)
+ public void testAppRecoveredInOrderOnRMRestart() throws Exception {
+MemoryRMStateStore memStore = new MemoryRMStateStore();
+memStore.init(conf);
+
+for (int i = 10; i > 0; i--) {
+ ApplicationState appState = mock(ApplicationState.class);
[17/52] [abbrv] git commit: MAPREDUCE-6026. native-task: fix logging. Contributed by Manu Zhang.
MAPREDUCE-6026. native-task: fix logging. Contributed by Manu Zhang.
git-svn-id:
https://svn.apache.org/repos/asf/hadoop/common/branches/MR-2841@1617878
13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/808bf8ba
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/808bf8ba
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/808bf8ba
Branch: refs/heads/HDFS-6584
Commit: 808bf8bac1d4bdf83fb3bdf6e096b4038c9567ce
Parents: 886338f
Author: Todd Lipcon
Authored: Thu Aug 14 04:53:36 2014 +
Committer: Todd Lipcon
Committed: Thu Aug 14 04:53:36 2014 +
--
.../CHANGES.MAPREDUCE-2841.txt | 1 +
.../hadoop/mapred/nativetask/HadoopPlatform.java | 5 +++--
.../hadoop/mapred/nativetask/Platforms.java | 5 +++--
.../combinertest/LargeKVCombinerTest.java| 5 -
.../hadoop/mapred/nativetask/kvtest/KVTest.java | 8 ++--
.../mapred/nativetask/kvtest/LargeKVTest.java| 7 +--
.../mapred/nativetask/kvtest/TestInputFile.java | 9 ++---
.../EnforceNativeOutputCollectorDelegator.java | 6 --
.../src/test/resources/log4j.properties | 19 +++
9 files changed, 51 insertions(+), 14 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/808bf8ba/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
index c3c6ab9..123eb6d 100644
--- a/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
+++ b/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt
@@ -13,3 +13,4 @@ MAPREDUCE-5984. native-task: Reuse lz4 sources in
hadoop-common (Binglin Chang)
MAPREDUCE-5976. native-task: should not fail to build if snappy is missing
(Manu Zhang)
MAPREDUCE-5978. native-task: remove test case for not supported codec
Bzip2Codec and DefaultCodec (Manu Zhang)
MAPREDUCE-6006. native-task: add native tests to maven and fix bug in pom.xml
(Binglin Chang via todd)
+MAPREDUCE-6026. native-task: fix logging (Manu Zhang via todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/808bf8ba/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/HadoopPlatform.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/HadoopPlatform.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/HadoopPlatform.java
index 1c4ede5..7599bb8 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/HadoopPlatform.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/HadoopPlatform.java
@@ -19,6 +19,8 @@ package org.apache.hadoop.mapred.nativetask;
import java.io.IOException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.io.BooleanWritable;
import org.apache.hadoop.io.ByteWritable;
import org.apache.hadoop.io.BytesWritable;
@@ -33,10 +35,9 @@ import org.apache.hadoop.io.VLongWritable;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.nativetask.serde.*;
-import org.apache.log4j.Logger;
public class HadoopPlatform extends Platform {
- private static final Logger LOG = Logger.getLogger(HadoopPlatform.class);
+ private static final Log LOG = LogFactory.getLog(HadoopPlatform.class);
public HadoopPlatform() throws IOException {
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/808bf8ba/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Platforms.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Platforms.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Platforms.java
index 154bbc8..d0a8496 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Platforms.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nat
[51/52] [abbrv] git commit: HADOOP-10400. Incorporate new S3A FileSystem implementation. Contributed by Jordan Mendelson and Dave Wang.
HADOOP-10400. Incorporate new S3A FileSystem implementation. Contributed by
Jordan Mendelson and Dave Wang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/24d920b8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/24d920b8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/24d920b8
Branch: refs/heads/HDFS-6584
Commit: 24d920b80eb3626073925a1d0b6dcf148add8cc0
Parents: fc741b5
Author: Aaron T. Myers
Authored: Mon Sep 15 08:27:07 2014 -0700
Committer: Aaron T. Myers
Committed: Mon Sep 15 08:27:07 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt |3 +
.../src/main/conf/log4j.properties |5 +
.../src/main/resources/core-default.xml | 86 ++
hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml |8 +
hadoop-project/pom.xml | 26 +-
hadoop-tools/hadoop-aws/pom.xml | 10 +
.../fs/s3a/AnonymousAWSCredentialsProvider.java | 37 +
.../fs/s3a/BasicAWSCredentialsProvider.java | 51 +
.../org/apache/hadoop/fs/s3a/Constants.java | 90 ++
.../org/apache/hadoop/fs/s3a/S3AFileStatus.java | 62 ++
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 1019 ++
.../apache/hadoop/fs/s3a/S3AInputStream.java| 207
.../apache/hadoop/fs/s3a/S3AOutputStream.java | 208
.../services/org.apache.hadoop.fs.FileSystem|1 +
.../hadoop/fs/contract/s3a/S3AContract.java | 43 +
.../fs/contract/s3a/TestS3AContractCreate.java | 38 +
.../fs/contract/s3a/TestS3AContractDelete.java | 31 +
.../fs/contract/s3a/TestS3AContractMkdir.java | 34 +
.../fs/contract/s3a/TestS3AContractOpen.java| 31 +
.../fs/contract/s3a/TestS3AContractRename.java | 64 ++
.../fs/contract/s3a/TestS3AContractRootDir.java | 35 +
.../fs/contract/s3a/TestS3AContractSeek.java| 31 +
.../fs/s3a/S3AFileSystemContractBaseTest.java | 327 ++
.../src/test/resources/contract/s3a.xml | 105 ++
.../src/test/resources/contract/s3n.xml |7 +-
hadoop-tools/hadoop-azure/pom.xml | 10 +-
26 files changed, 2552 insertions(+), 17 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/24d920b8/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 051eac1..c2ae5ed 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -342,6 +342,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-10893. isolated classloader on the client side (Sangjin Lee via
jlowe)
+HADOOP-10400. Incorporate new S3A FileSystem implementation. (Jordan
+Mendelson and Dave Wang via atm)
+
IMPROVEMENTS
HADOOP-10808. Remove unused native code for munlock. (cnauroth)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/24d920b8/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
--
diff --git a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
index ef9acbf..5fa21fa 100644
--- a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
+++ b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
@@ -174,6 +174,11 @@
log4j.appender.MRAUDIT.MaxBackupIndex=${mapred.audit.log.maxbackupindex}
# Jets3t library
log4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR
+# AWS SDK & S3A FileSystem
+log4j.logger.com.amazonaws=ERROR
+log4j.logger.com.amazonaws.http.AmazonHttpClient=ERROR
+log4j.logger.org.apache.hadoop.fs.s3a.S3AFileSystem=WARN
+
#
# Event Counter Appender
# Sends counts of logging messages at different severity levels to Hadoop
Metrics.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/24d920b8/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 3cc7545..828dec2 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -690,6 +690,92 @@ for ldap providers in the same way as above does.
+ fs.s3a.access.key
+ AWS access key ID. Omit for Role-based
authentication.
+
+
+
+ fs.s3a.secret.key
+ AWS secret key. Omit for Role-based
authentication.
+
+
+
+ fs.s3a.connection.maximum
+ 15
+ Controls the maximum number of simultaneous
git commit: YARN-2549. TestContainerLaunch fails due to classpath problem with hamcrest classes. Contributed by Chris Nauroth.
Repository: hadoop Updated Branches: refs/heads/trunk 43b030300 -> 9d4ec97c9 YARN-2549. TestContainerLaunch fails due to classpath problem with hamcrest classes. Contributed by Chris Nauroth. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9d4ec97c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9d4ec97c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9d4ec97c Branch: refs/heads/trunk Commit: 9d4ec97c95b51c6f99112030da91950551b22b9c Parents: 43b0303 Author: cnauroth Authored: Mon Sep 15 11:53:57 2014 -0700 Committer: cnauroth Committed: Mon Sep 15 11:53:57 2014 -0700 -- hadoop-yarn-project/CHANGES.txt | 3 +++ .../hadoop-yarn-server-nodemanager/pom.xml| 14 +- 2 files changed, 12 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d4ec97c/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 9c1abf7..5cfc460 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -359,6 +359,9 @@ Release 2.6.0 - UNRELEASED header and made it accept multiple origins in CrossOriginFilter. (Jonathan Eagles via zjshen) +YARN-2549. TestContainerLaunch fails due to classpath problem with hamcrest +classes. (cnauroth) + Release 2.5.1 - 2014-09-05 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d4ec97c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml index 370cc36..b1efa5f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml @@ -102,11 +102,6 @@ org.apache.hadoop hadoop-annotations - - org.mockito - mockito-all - test - org.apache.hadoop @@ -122,12 +117,21 @@ com.google.protobuf protobuf-java + junit junit test + org.mockito + mockito-all + test + + com.google.inject guice
git commit: YARN-2549. TestContainerLaunch fails due to classpath problem with hamcrest classes. Contributed by Chris Nauroth.
Repository: hadoop Updated Branches: refs/heads/branch-2 d59555785 -> aaef460e7 YARN-2549. TestContainerLaunch fails due to classpath problem with hamcrest classes. Contributed by Chris Nauroth. (cherry picked from commit 9d4ec97c95b51c6f99112030da91950551b22b9c) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aaef460e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aaef460e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aaef460e Branch: refs/heads/branch-2 Commit: aaef460e7a4a75e723557ea027ba9df329350ec9 Parents: d595557 Author: cnauroth Authored: Mon Sep 15 11:53:57 2014 -0700 Committer: cnauroth Committed: Mon Sep 15 11:54:41 2014 -0700 -- hadoop-yarn-project/CHANGES.txt | 3 +++ .../hadoop-yarn-server-nodemanager/pom.xml| 14 +- 2 files changed, 12 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaef460e/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index c675e75..6b29f2f 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -331,6 +331,9 @@ Release 2.6.0 - UNRELEASED header and made it accept multiple origins in CrossOriginFilter. (Jonathan Eagles via zjshen) +YARN-2549. TestContainerLaunch fails due to classpath problem with hamcrest +classes. (cnauroth) + Release 2.5.1 - 2014-09-05 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaef460e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml index 24bc784..e72d489 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml @@ -120,11 +120,6 @@ org.apache.hadoop hadoop-annotations - - org.mockito - mockito-all - test - org.apache.hadoop @@ -140,12 +135,21 @@ com.google.protobuf protobuf-java + junit junit test + org.mockito + mockito-all + test + + com.google.inject guice
git commit: YARN-2529. Generic history service RPC interface doesn't work when service authorization is enabled. Contributed by Zhijie Shen
Repository: hadoop
Updated Branches:
refs/heads/trunk 9d4ec97c9 -> 88e329fdf
YARN-2529. Generic history service RPC interface doesn't work when service
authorization is enabled. Contributed by Zhijie Shen
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/88e329fd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/88e329fd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/88e329fd
Branch: refs/heads/trunk
Commit: 88e329fdfb27c11f1964c0345188bcf328c95fd6
Parents: 9d4ec97
Author: Jian He
Authored: Mon Sep 15 13:13:54 2014 -0700
Committer: Jian He
Committed: Mon Sep 15 13:13:54 2014 -0700
--
.../src/main/conf/hadoop-policy.xml | 9
hadoop-yarn-project/CHANGES.txt | 3 ++
.../hadoop/yarn/conf/YarnConfiguration.java | 4 ++
.../ApplicationHistoryClientService.java| 15 ++-
.../authorize/TimelinePolicyProvider.java | 44
5 files changed, 74 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88e329fd/hadoop-common-project/hadoop-common/src/main/conf/hadoop-policy.xml
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-policy.xml
b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-policy.xml
index 39e4f4d..2bf5c02 100644
--- a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-policy.xml
+++ b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-policy.xml
@@ -214,4 +214,13 @@
A special value of "*" means all users are allowed.
+
+security.applicationhistory.protocol.acl
+*
+ACL for ApplicationHistoryProtocol, used by the timeline
+server and the generic history service client to communicate with each
other.
+The ACL is a comma-separated list of user and group names. The user and
+group list is separated by a blank. For e.g. "alice,bob users,wheel".
+A special value of "*" means all users are allowed.
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88e329fd/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 5cfc460..629d3b3 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -362,6 +362,9 @@ Release 2.6.0 - UNRELEASED
YARN-2549. TestContainerLaunch fails due to classpath problem with hamcrest
classes. (cnauroth)
+YARN-2529. Generic history service RPC interface doesn't work when service
+authorization is enabled. (Zhijie Shen via jianhe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88e329fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 43f510d..a92b358 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -991,6 +991,10 @@ public class YarnConfiguration extends Configuration {
YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCE_LOCALIZER =
"security.resourcelocalizer.protocol.acl";
+ public static final String
+ YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONHISTORY_PROTOCOL =
+ "security.applicationhistory.protocol.acl";
+
/** No. of milliseconds to wait between sending a SIGTERM and SIGKILL
* to a running container */
public static final String NM_SLEEP_DELAY_BEFORE_SIGKILL_MS =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88e329fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java
index 6372056..2334fde 100644
---
a/hadoop-yarn-pro
git commit: YARN-2529. Generic history service RPC interface doesn't work when service authorization is enabled. Contributed by Zhijie Shen (cherry picked from commit 88e329fdfb27c11f1964c0345188bcf32
Repository: hadoop
Updated Branches:
refs/heads/branch-2 aaef460e7 -> 1c3f83783
YARN-2529. Generic history service RPC interface doesn't work when service
authorization is enabled. Contributed by Zhijie Shen
(cherry picked from commit 88e329fdfb27c11f1964c0345188bcf328c95fd6)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1c3f8378
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1c3f8378
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1c3f8378
Branch: refs/heads/branch-2
Commit: 1c3f8378358b01e6ea7312f7f673a9bd37ddf49a
Parents: aaef460
Author: Jian He
Authored: Mon Sep 15 13:13:54 2014 -0700
Committer: Jian He
Committed: Mon Sep 15 13:15:07 2014 -0700
--
.../src/main/conf/hadoop-policy.xml | 9
hadoop-yarn-project/CHANGES.txt | 3 ++
.../hadoop/yarn/conf/YarnConfiguration.java | 4 ++
.../ApplicationHistoryClientService.java| 15 ++-
.../authorize/TimelinePolicyProvider.java | 44
5 files changed, 74 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c3f8378/hadoop-common-project/hadoop-common/src/main/conf/hadoop-policy.xml
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-policy.xml
b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-policy.xml
index 39e4f4d..2bf5c02 100644
--- a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-policy.xml
+++ b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-policy.xml
@@ -214,4 +214,13 @@
A special value of "*" means all users are allowed.
+
+security.applicationhistory.protocol.acl
+*
+ACL for ApplicationHistoryProtocol, used by the timeline
+server and the generic history service client to communicate with each
other.
+The ACL is a comma-separated list of user and group names. The user and
+group list is separated by a blank. For e.g. "alice,bob users,wheel".
+A special value of "*" means all users are allowed.
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c3f8378/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 6b29f2f..da90f27 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -334,6 +334,9 @@ Release 2.6.0 - UNRELEASED
YARN-2549. TestContainerLaunch fails due to classpath problem with hamcrest
classes. (cnauroth)
+YARN-2529. Generic history service RPC interface doesn't work when service
+authorization is enabled. (Zhijie Shen via jianhe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c3f8378/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index f3104e2..9e88c97 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -991,6 +991,10 @@ public class YarnConfiguration extends Configuration {
YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCE_LOCALIZER =
"security.resourcelocalizer.protocol.acl";
+ public static final String
+ YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONHISTORY_PROTOCOL =
+ "security.applicationhistory.protocol.acl";
+
/** No. of milliseconds to wait between sending a SIGTERM and SIGKILL
* to a running container */
public static final String NM_SLEEP_DELAY_BEFORE_SIGKILL_MS =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c3f8378/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHis
git commit: YARN-2438. yarn-env.sh cleanup (aw)
Repository: hadoop
Updated Branches:
refs/heads/trunk 88e329fdf -> 0c8aec43f
YARN-2438. yarn-env.sh cleanup (aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0c8aec43
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0c8aec43
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0c8aec43
Branch: refs/heads/trunk
Commit: 0c8aec43f0acd4f896df8cd605d97808f9a09887
Parents: 88e329f
Author: Allen Wittenauer
Authored: Mon Sep 15 14:09:45 2014 -0700
Committer: Allen Wittenauer
Committed: Mon Sep 15 14:09:50 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 2 +
.../hadoop-yarn/conf/yarn-env.sh| 52 +---
2 files changed, 37 insertions(+), 17 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c8aec43/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 629d3b3..b41ad82 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -8,6 +8,8 @@ Trunk - Unreleased
IMPROVEMENTS
+YARN-2438. yarn-env.sh cleanup (aw)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c8aec43/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
--
diff --git a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
index 755cfd8..0f5e9b7 100644
--- a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
+++ b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
@@ -1,3 +1,4 @@
+#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -25,27 +26,21 @@
##
###
-# Generic settings for YARN
+# YARN-specific overrides for generic settings
###
-# User for YARN daemons
-export HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn}
-
-#
-# By default, YARN will use HADOOP_CONF_DIR. Specify a custom
-# YARN_CONF_DIR here
-# export YARN_CONF_DIR="${YARN_CONF_DIR:-$HADOOP_YARN_HOME/conf}"
-#
-
-# Override Hadoop's log directory & file
-# export YARN_LOG_DIR="$HADOOP_YARN_HOME/logs"
-# export YARN_LOGFILE='yarn.log'
+# By default, YARN will use HADOOP_LOG_DIR for YARN logging. Specify a custom
+# log directory for YARN things here:
+# export YARN_LOG_DIR="${HADOOP_LOG_DIR}"
-# Need a custom-to-YARN service-level authorization policy file?
-# export YARN_POLICYFILE="yarn-policy.xml"
+# By default, YARN will use the value of HADOOP_LOGFILE as the 'fallback' log
+# file # when log4j settings are not defined. Specify a custom YARN log file
+# here:
+# export YARN_LOGFILE=${HADOOP_LOGFILE}
-#Override the log4j settings for all YARN apps
-# export YARN_ROOT_LOGGER="INFO,console"
+#Override the log4j settings for all YARN apps By default, YARN will use
+# HADOOP_ROOT_LOGGER.
+# export YARN_ROOT_LOGGER=${HADOOP_ROOT_LOGGER}
###
# Resource Manager specific parameters
@@ -125,3 +120,26 @@ export HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn}
#
#export YARN_TIMELINESERVER_OPTS=
+###
+# Web App Proxy Server specifc parameters
+###
+
+# Specify the max Heapsize for the proxy server using a numerical value
+# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
+# the value to 1000.
+# This value will be overridden by an Xmx setting specified in either
YARN_OPTS,
+# HADOOP_OPTS, and/or YARN_PROXYSERVER_OPTS.
+# If not specified, the default value will be picked from either YARN_HEAPMAX
+# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
+#
+#export YARN_PROXYSERVER_HEAPSIZE=1000
+
+# Specify the JVM options to be used when starting the proxy server.
+# These options will be appended to the options specified as YARN_OPTS
+# and therefore may override any similar flags set in YARN_OPTS
+#
+# See ResourceManager for some examples
+#
+#export YARN_PROXYSERVER_OPTS=
+
+
git commit: HDFS-7059. HAadmin transtionToActive with forceActive option can show confusing message. Contributed by Rushabh Shah.
Repository: hadoop
Updated Branches:
refs/heads/trunk 0c8aec43f -> 2f847b335
HDFS-7059. HAadmin transtionToActive with forceActive option can show
confusing message. Contributed by Rushabh Shah.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2f847b33
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2f847b33
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2f847b33
Branch: refs/heads/trunk
Commit: 2f847b3357448184aaec935052f34d116ad3a42c
Parents: 0c8aec4
Author: Kihwal Lee
Authored: Mon Sep 15 16:27:22 2014 -0500
Committer: Kihwal Lee
Committed: Mon Sep 15 16:27:22 2014 -0500
--
.../main/java/org/apache/hadoop/ha/HAAdmin.java | 6 --
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hdfs/tools/TestDFSHAAdminMiniCluster.java | 20
3 files changed, 7 insertions(+), 22 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f847b33/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
index dfa03e8..bd6366c 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
@@ -143,8 +143,10 @@ public abstract class HAAdmin extends Configured
implements Tool {
}
/* returns true if other target node is active or some exception occurred
and forceActive was not set */
-if(isOtherTargetNodeActive(argv[0], cmd.hasOption(FORCEACTIVE))) {
- return -1;
+if(!cmd.hasOption(FORCEACTIVE)) {
+ if(isOtherTargetNodeActive(argv[0], cmd.hasOption(FORCEACTIVE))) {
+return -1;
+ }
}
HAServiceTarget target = resolveTarget(argv[0]);
if (!checkManualStateManagementOK(target)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f847b33/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 209c7c1..f596265 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -458,6 +458,9 @@ Release 2.6.0 - UNRELEASED
HDFS-7061. Add test to verify encryption zone creation after NameNode
restart without saving namespace. (Stephen Chu via wang)
+HDFS-7059. HAadmin transtionToActive with forceActive option can show
+confusing message.
+
OPTIMIZATIONS
HDFS-6690. Deduplicate xattr names in memory. (wang)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f847b33/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java
index 288bcd0..ee1c184 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java
@@ -232,26 +232,6 @@ public class TestDFSHAAdminMiniCluster {
assertFalse("Both namenodes cannot be active", nn1.isActiveState()
&& nn2.isActiveState());
-/* This test case doesn't allow nn2 to transition to Active even with
-forceActive switch since nn1 is already active */
-if(nn1.getState() != null && !nn1.getState().
-equals(HAServiceState.STANDBY.name()) ) {
- cluster.transitionToStandby(0);
-}
-if(nn2.getState() != null && !nn2.getState().
-equals(HAServiceState.STANDBY.name()) ) {
- cluster.transitionToStandby(1);
-}
-//Making sure both the namenode are in standby state
-assertTrue(nn1.isStandbyState());
-assertTrue(nn2.isStandbyState());
-
-runTool("-transitionToActive", "nn1");
-runTool("-transitionToActive", "nn2","--forceactive");
-
-assertFalse("Both namenodes cannot be active even though with forceActive",
-nn1.isActiveState() && nn2.isActiveState());
-
/* In this test case, we have deliberately shut down nn1 and this will
cause HAAAdmin#isOtherTargetNodeActive to throw an Exception
and transitionToActive for nn2 with forceActive switch will succeed
git commit: HDFS-7059. HAadmin transtionToActive with forceActive option can show confusing message. Contributed by Rushabh Shah. (cherry picked from commit 2f847b3357448184aaec935052f34d116ad3a42c)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 1c3f83783 -> e6e31fbbb
HDFS-7059. HAadmin transtionToActive with forceActive option can show
confusing message. Contributed by Rushabh Shah.
(cherry picked from commit 2f847b3357448184aaec935052f34d116ad3a42c)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e6e31fbb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e6e31fbb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e6e31fbb
Branch: refs/heads/branch-2
Commit: e6e31fbbba1f06c2e7caf0b0d1abbe1a20acaf91
Parents: 1c3f837
Author: Kihwal Lee
Authored: Mon Sep 15 16:30:22 2014 -0500
Committer: Kihwal Lee
Committed: Mon Sep 15 16:30:22 2014 -0500
--
.../main/java/org/apache/hadoop/ha/HAAdmin.java | 6 --
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hdfs/tools/TestDFSHAAdminMiniCluster.java | 20
3 files changed, 7 insertions(+), 22 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6e31fbb/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
index dfa03e8..bd6366c 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
@@ -143,8 +143,10 @@ public abstract class HAAdmin extends Configured
implements Tool {
}
/* returns true if other target node is active or some exception occurred
and forceActive was not set */
-if(isOtherTargetNodeActive(argv[0], cmd.hasOption(FORCEACTIVE))) {
- return -1;
+if(!cmd.hasOption(FORCEACTIVE)) {
+ if(isOtherTargetNodeActive(argv[0], cmd.hasOption(FORCEACTIVE))) {
+return -1;
+ }
}
HAServiceTarget target = resolveTarget(argv[0]);
if (!checkManualStateManagementOK(target)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6e31fbb/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 9afe524..080f3b1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -200,6 +200,9 @@ Release 2.6.0 - UNRELEASED
HDFS-7061. Add test to verify encryption zone creation after NameNode
restart without saving namespace. (Stephen Chu via wang)
+HDFS-7059. HAadmin transtionToActive with forceActive option can show
+confusing message.
+
OPTIMIZATIONS
HDFS-6690. Deduplicate xattr names in memory. (wang)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6e31fbb/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java
index 288bcd0..ee1c184 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java
@@ -232,26 +232,6 @@ public class TestDFSHAAdminMiniCluster {
assertFalse("Both namenodes cannot be active", nn1.isActiveState()
&& nn2.isActiveState());
-/* This test case doesn't allow nn2 to transition to Active even with
-forceActive switch since nn1 is already active */
-if(nn1.getState() != null && !nn1.getState().
-equals(HAServiceState.STANDBY.name()) ) {
- cluster.transitionToStandby(0);
-}
-if(nn2.getState() != null && !nn2.getState().
-equals(HAServiceState.STANDBY.name()) ) {
- cluster.transitionToStandby(1);
-}
-//Making sure both the namenode are in standby state
-assertTrue(nn1.isStandbyState());
-assertTrue(nn2.isStandbyState());
-
-runTool("-transitionToActive", "nn1");
-runTool("-transitionToActive", "nn2","--forceactive");
-
-assertFalse("Both namenodes cannot be active even though with forceActive",
-nn1.isActiveState() && nn2.isActiveState());
-
/* In this test case, we have deliberately shut down nn1 and this will
cause HAAAdmin#isOtherTargetNodeActive to throw an Exception
and transitionToActive for nn
git commit: HDFS-6912. SharedFileDescriptorFactory should not allocate sparse files (cmccabe)
Repository: hadoop
Updated Branches:
refs/heads/trunk 2f847b335 -> 8008f0e81
HDFS-6912. SharedFileDescriptorFactory should not allocate sparse files
(cmccabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8008f0e8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8008f0e8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8008f0e8
Branch: refs/heads/trunk
Commit: 8008f0e8191b1c7adbed96ed4c380208e3a37692
Parents: 2f847b3
Author: Colin Patrick Mccabe
Authored: Mon Sep 15 14:47:27 2014 -0700
Committer: Colin Patrick Mccabe
Committed: Mon Sep 15 14:47:27 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++
.../io/nativeio/SharedFileDescriptorFactory.c | 32 ++--
2 files changed, 33 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8008f0e8/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index c2ae5ed..6f80eb2 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -708,6 +708,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11056. OsSecureRandom.setConf() might leak file descriptors (yzhang
via cmccabe)
+HDFS-6912. SharedFileDescriptorFactory should not allocate sparse files
+(cmccabe)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HADOOP-10734. Implement high-performance secure random number sources.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8008f0e8/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/SharedFileDescriptorFactory.c
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/SharedFileDescriptorFactory.c
b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/SharedFileDescriptorFactory.c
index 8368402..3a8540c 100644
---
a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/SharedFileDescriptorFactory.c
+++
b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/SharedFileDescriptorFactory.c
@@ -37,6 +37,8 @@
#include
#include
+#define ZERO_FULLY_BUF_SIZE 8192
+
static pthread_mutex_t g_rand_lock = PTHREAD_MUTEX_INITIALIZER;
JNIEXPORT void JNICALL
@@ -83,6 +85,24 @@ done:
}
}
+static int zero_fully(int fd, jint length)
+{
+ char buf[ZERO_FULLY_BUF_SIZE];
+ int res;
+
+ memset(buf, 0, sizeof(buf));
+ while (length > 0) {
+res = write(fd, buf,
+ (length > ZERO_FULLY_BUF_SIZE) ? ZERO_FULLY_BUF_SIZE : length);
+if (res < 0) {
+ if (errno == EINTR) continue;
+ return errno;
+}
+length -= res;
+ }
+ return 0;
+}
+
JNIEXPORT jobject JNICALL
Java_org_apache_hadoop_io_nativeio_SharedFileDescriptorFactory_createDescriptor0(
JNIEnv *env, jclass clazz, jstring jprefix, jstring jpath, jint length)
@@ -136,12 +156,20 @@
Java_org_apache_hadoop_io_nativeio_SharedFileDescriptorFactory_createDescriptor0
(*env)->Throw(env, jthr);
goto done;
}
- if (ftruncate(fd, length) < 0) {
-jthr = newIOException(env, "ftruncate(%s, %d) failed: error %d (%s)",
+ ret = zero_fully(fd, length);
+ if (ret) {
+jthr = newIOException(env, "zero_fully(%s, %d) failed: error %d (%s)",
path, length, ret, terror(ret));
(*env)->Throw(env, jthr);
goto done;
}
+ if (lseek(fd, 0, SEEK_SET) < 0) {
+ret = errno;
+jthr = newIOException(env, "lseek(%s, 0, SEEK_SET) failed: error %d (%s)",
+ path, ret, terror(ret));
+(*env)->Throw(env, jthr);
+goto done;
+ }
jret = fd_create(env, fd); // throws exception on error.
done:
git commit: HDFS-6912. SharedFileDescriptorFactory should not allocate sparse files (cmccabe) (cherry picked from commit 8008f0e8191b1c7adbed96ed4c380208e3a37692)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 e6e31fbbb -> b4d9aca06
HDFS-6912. SharedFileDescriptorFactory should not allocate sparse files
(cmccabe)
(cherry picked from commit 8008f0e8191b1c7adbed96ed4c380208e3a37692)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b4d9aca0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b4d9aca0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b4d9aca0
Branch: refs/heads/branch-2
Commit: b4d9aca066a09f3536d2df2de21744e02cbed0dd
Parents: e6e31fb
Author: Colin Patrick Mccabe
Authored: Mon Sep 15 14:47:27 2014 -0700
Committer: Colin Patrick Mccabe
Committed: Mon Sep 15 14:51:59 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++
.../io/nativeio/SharedFileDescriptorFactory.c | 32 ++--
2 files changed, 33 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b4d9aca0/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index e3dd7d1..d2deee6 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -378,6 +378,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11056. OsSecureRandom.setConf() might leak file descriptors. (yzhang
via cmccabe)
+HDFS-6912. SharedFileDescriptorFactory should not allocate sparse files
+(cmccabe)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HADOOP-10734. Implement high-performance secure random number sources.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b4d9aca0/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/SharedFileDescriptorFactory.c
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/SharedFileDescriptorFactory.c
b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/SharedFileDescriptorFactory.c
index 8368402..3a8540c 100644
---
a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/SharedFileDescriptorFactory.c
+++
b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/SharedFileDescriptorFactory.c
@@ -37,6 +37,8 @@
#include
#include
+#define ZERO_FULLY_BUF_SIZE 8192
+
static pthread_mutex_t g_rand_lock = PTHREAD_MUTEX_INITIALIZER;
JNIEXPORT void JNICALL
@@ -83,6 +85,24 @@ done:
}
}
+static int zero_fully(int fd, jint length)
+{
+ char buf[ZERO_FULLY_BUF_SIZE];
+ int res;
+
+ memset(buf, 0, sizeof(buf));
+ while (length > 0) {
+res = write(fd, buf,
+ (length > ZERO_FULLY_BUF_SIZE) ? ZERO_FULLY_BUF_SIZE : length);
+if (res < 0) {
+ if (errno == EINTR) continue;
+ return errno;
+}
+length -= res;
+ }
+ return 0;
+}
+
JNIEXPORT jobject JNICALL
Java_org_apache_hadoop_io_nativeio_SharedFileDescriptorFactory_createDescriptor0(
JNIEnv *env, jclass clazz, jstring jprefix, jstring jpath, jint length)
@@ -136,12 +156,20 @@
Java_org_apache_hadoop_io_nativeio_SharedFileDescriptorFactory_createDescriptor0
(*env)->Throw(env, jthr);
goto done;
}
- if (ftruncate(fd, length) < 0) {
-jthr = newIOException(env, "ftruncate(%s, %d) failed: error %d (%s)",
+ ret = zero_fully(fd, length);
+ if (ret) {
+jthr = newIOException(env, "zero_fully(%s, %d) failed: error %d (%s)",
path, length, ret, terror(ret));
(*env)->Throw(env, jthr);
goto done;
}
+ if (lseek(fd, 0, SEEK_SET) < 0) {
+ret = errno;
+jthr = newIOException(env, "lseek(%s, 0, SEEK_SET) failed: error %d (%s)",
+ path, ret, terror(ret));
+(*env)->Throw(env, jthr);
+goto done;
+ }
jret = fd_create(env, fd); // throws exception on error.
done:
git commit: HDFS-7066. LazyWriter#evictBlocks misses a null check for replicaState. (Contributed by Xiaoyu Yao)
Repository: hadoop
Updated Branches:
refs/heads/HDFS-6581 fdf0542d8 -> a4dcbaa33
HDFS-7066. LazyWriter#evictBlocks misses a null check for replicaState.
(Contributed by Xiaoyu Yao)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a4dcbaa3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a4dcbaa3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a4dcbaa3
Branch: refs/heads/HDFS-6581
Commit: a4dcbaa33255cd1dd8d6c54763f55486c9e4317c
Parents: fdf0542
Author: arp
Authored: Mon Sep 15 15:28:17 2014 -0700
Committer: arp
Committed: Mon Sep 15 15:28:17 2014 -0700
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt| 3 +++
.../hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java | 4
2 files changed, 7 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4dcbaa3/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
index 346f912..0877636 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
@@ -44,3 +44,6 @@
HDFS-6978. Directory scanner should correctly reconcile blocks on RAM
disk. (Arpit Agarwal)
+HDFS-7066. LazyWriter#evictBlocks misses a null check for replicaState.
+(Xiaoyu Yao via Arpit Agarwal)
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4dcbaa3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 6366a4f..23753bd 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -2331,6 +2331,10 @@ class FsDatasetImpl implements
FsDatasetSpi {
LazyWriteReplicaTracker.ReplicaState replicaState =
lazyWriteReplicaTracker.getNextCandidateForEviction();
+if (replicaState == null) {
+ break;
+}
+
if (LOG.isDebugEnabled()) {
LOG.debug("Evicting block " + replicaState);
}
[05/10] git commit: YARN-2549. TestContainerLaunch fails due to classpath problem with hamcrest classes. Contributed by Chris Nauroth.
YARN-2549. TestContainerLaunch fails due to classpath problem with hamcrest classes. Contributed by Chris Nauroth. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9d4ec97c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9d4ec97c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9d4ec97c Branch: refs/heads/HDFS-6581 Commit: 9d4ec97c95b51c6f99112030da91950551b22b9c Parents: 43b0303 Author: cnauroth Authored: Mon Sep 15 11:53:57 2014 -0700 Committer: cnauroth Committed: Mon Sep 15 11:53:57 2014 -0700 -- hadoop-yarn-project/CHANGES.txt | 3 +++ .../hadoop-yarn-server-nodemanager/pom.xml| 14 +- 2 files changed, 12 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d4ec97c/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 9c1abf7..5cfc460 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -359,6 +359,9 @@ Release 2.6.0 - UNRELEASED header and made it accept multiple origins in CrossOriginFilter. (Jonathan Eagles via zjshen) +YARN-2549. TestContainerLaunch fails due to classpath problem with hamcrest +classes. (cnauroth) + Release 2.5.1 - 2014-09-05 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d4ec97c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml index 370cc36..b1efa5f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml @@ -102,11 +102,6 @@ org.apache.hadoop hadoop-annotations - - org.mockito - mockito-all - test - org.apache.hadoop @@ -122,12 +117,21 @@ com.google.protobuf protobuf-java + junit junit test + org.mockito + mockito-all + test + + com.google.inject guice
[10/10] git commit: Merge branch 'trunk' into HDFS-6581
Merge branch 'trunk' into HDFS-6581 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a7814e1a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a7814e1a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a7814e1a Branch: refs/heads/HDFS-6581 Commit: a7814e1aa440066ee20fde4c4992173d0b1514c1 Parents: a4dcbaa 8008f0e Author: arp Authored: Mon Sep 15 15:30:19 2014 -0700 Committer: arp Committed: Mon Sep 15 15:30:19 2014 -0700 -- hadoop-common-project/hadoop-common/CHANGES.txt |6 + .../src/main/conf/hadoop-policy.xml |9 + .../src/main/conf/log4j.properties |5 + .../main/java/org/apache/hadoop/ha/HAAdmin.java |6 +- .../io/nativeio/SharedFileDescriptorFactory.c | 32 +- .../src/main/resources/core-default.xml | 86 ++ hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml |8 + hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt |9 + .../web/resources/DatanodeWebHdfsMethods.java | 13 +- .../apache/hadoop/hdfs/TestEncryptionZones.java | 62 ++ .../hdfs/tools/TestDFSHAAdminMiniCluster.java | 20 - hadoop-project/pom.xml | 26 +- hadoop-tools/hadoop-aws/pom.xml | 10 + .../fs/s3a/AnonymousAWSCredentialsProvider.java | 37 + .../fs/s3a/BasicAWSCredentialsProvider.java | 51 + .../org/apache/hadoop/fs/s3a/Constants.java | 90 ++ .../org/apache/hadoop/fs/s3a/S3AFileStatus.java | 62 ++ .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 1019 ++ .../apache/hadoop/fs/s3a/S3AInputStream.java| 207 .../apache/hadoop/fs/s3a/S3AOutputStream.java | 208 .../services/org.apache.hadoop.fs.FileSystem|1 + .../hadoop/fs/contract/s3a/S3AContract.java | 43 + .../fs/contract/s3a/TestS3AContractCreate.java | 38 + .../fs/contract/s3a/TestS3AContractDelete.java | 31 + .../fs/contract/s3a/TestS3AContractMkdir.java | 34 + .../fs/contract/s3a/TestS3AContractOpen.java| 31 + .../fs/contract/s3a/TestS3AContractRename.java | 64 ++ .../fs/contract/s3a/TestS3AContractRootDir.java | 35 + .../fs/contract/s3a/TestS3AContractSeek.java| 31 + .../fs/s3a/S3AFileSystemContractBaseTest.java | 327 ++ .../src/test/resources/contract/s3a.xml | 105 ++ .../src/test/resources/contract/s3n.xml |7 +- hadoop-tools/hadoop-azure/pom.xml | 10 +- hadoop-yarn-project/CHANGES.txt |8 + .../hadoop-yarn/conf/yarn-env.sh| 52 +- .../hadoop/yarn/conf/YarnConfiguration.java |4 + .../ApplicationHistoryClientService.java| 15 +- .../authorize/TimelinePolicyProvider.java | 44 + .../hadoop-yarn-server-nodemanager/pom.xml | 14 +- 39 files changed, 2791 insertions(+), 69 deletions(-) --
[02/10] HADOOP-10400. Incorporate new S3A FileSystem implementation. Contributed by Jordan Mendelson and Dave Wang.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/24d920b8/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractSeek.java
--
diff --git
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractSeek.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractSeek.java
new file mode 100644
index 000..d677ec4
--- /dev/null
+++
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractSeek.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3a;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractSeekTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+public class TestS3AContractSeek extends AbstractContractSeekTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+return new S3AContract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/24d920b8/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3AFileSystemContractBaseTest.java
--
diff --git
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3AFileSystemContractBaseTest.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3AFileSystemContractBaseTest.java
new file mode 100644
index 000..8455233
--- /dev/null
+++
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3AFileSystemContractBaseTest.java
@@ -0,0 +1,327 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import static org.junit.Assume.*;
+
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystemContractBaseTest;
+import org.apache.hadoop.fs.Path;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.util.UUID;
+
+/**
+ * Tests a live S3 system. If you keys and bucket aren't specified, all tests
+ * are marked as passed
+ *
+ * This uses BlockJUnit4ClassRunner because FileSystemContractBaseTest from
+ * TestCase which uses the old Junit3 runner that doesn't ignore assumptions
+ * properly making it impossible to skip the tests if we don't have a valid
+ * bucket.
+ **/
+public class S3AFileSystemContractBaseTest extends FileSystemContractBaseTest {
+ private static final int TEST_BUFFER_SIZE = 128;
+ private static final int MODULUS = 128;
+
+ protected static final Logger LOG =
LoggerFactory.getLogger(S3AFileSystemContractBaseTest.class);
+
+ @Override
+ public void setUp() throws Exception {
+Configuration conf = new Configuration();
+
+URI testURI = URI.create(conf.get("test.fs.s3a.name"));
+
+boolean liveTest = testURI != null && !testURI.equals("s3a:///");
+
+// This doesn't work with our JUnit 3 style test cases, so instead we'll
+// make this whole class not run by default
+assumeTrue(liveTest);
+
+fs = new S3AFileSystem();
+fs.initialize(testURI, conf);
+super.setUp();
+ }
+
+ @Override
+ protected void tearDown
[09/10] git commit: HDFS-6912. SharedFileDescriptorFactory should not allocate sparse files (cmccabe)
HDFS-6912. SharedFileDescriptorFactory should not allocate sparse files
(cmccabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8008f0e8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8008f0e8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8008f0e8
Branch: refs/heads/HDFS-6581
Commit: 8008f0e8191b1c7adbed96ed4c380208e3a37692
Parents: 2f847b3
Author: Colin Patrick Mccabe
Authored: Mon Sep 15 14:47:27 2014 -0700
Committer: Colin Patrick Mccabe
Committed: Mon Sep 15 14:47:27 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++
.../io/nativeio/SharedFileDescriptorFactory.c | 32 ++--
2 files changed, 33 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8008f0e8/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index c2ae5ed..6f80eb2 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -708,6 +708,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11056. OsSecureRandom.setConf() might leak file descriptors (yzhang
via cmccabe)
+HDFS-6912. SharedFileDescriptorFactory should not allocate sparse files
+(cmccabe)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HADOOP-10734. Implement high-performance secure random number sources.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8008f0e8/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/SharedFileDescriptorFactory.c
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/SharedFileDescriptorFactory.c
b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/SharedFileDescriptorFactory.c
index 8368402..3a8540c 100644
---
a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/SharedFileDescriptorFactory.c
+++
b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/SharedFileDescriptorFactory.c
@@ -37,6 +37,8 @@
#include
#include
+#define ZERO_FULLY_BUF_SIZE 8192
+
static pthread_mutex_t g_rand_lock = PTHREAD_MUTEX_INITIALIZER;
JNIEXPORT void JNICALL
@@ -83,6 +85,24 @@ done:
}
}
+static int zero_fully(int fd, jint length)
+{
+ char buf[ZERO_FULLY_BUF_SIZE];
+ int res;
+
+ memset(buf, 0, sizeof(buf));
+ while (length > 0) {
+res = write(fd, buf,
+ (length > ZERO_FULLY_BUF_SIZE) ? ZERO_FULLY_BUF_SIZE : length);
+if (res < 0) {
+ if (errno == EINTR) continue;
+ return errno;
+}
+length -= res;
+ }
+ return 0;
+}
+
JNIEXPORT jobject JNICALL
Java_org_apache_hadoop_io_nativeio_SharedFileDescriptorFactory_createDescriptor0(
JNIEnv *env, jclass clazz, jstring jprefix, jstring jpath, jint length)
@@ -136,12 +156,20 @@
Java_org_apache_hadoop_io_nativeio_SharedFileDescriptorFactory_createDescriptor0
(*env)->Throw(env, jthr);
goto done;
}
- if (ftruncate(fd, length) < 0) {
-jthr = newIOException(env, "ftruncate(%s, %d) failed: error %d (%s)",
+ ret = zero_fully(fd, length);
+ if (ret) {
+jthr = newIOException(env, "zero_fully(%s, %d) failed: error %d (%s)",
path, length, ret, terror(ret));
(*env)->Throw(env, jthr);
goto done;
}
+ if (lseek(fd, 0, SEEK_SET) < 0) {
+ret = errno;
+jthr = newIOException(env, "lseek(%s, 0, SEEK_SET) failed: error %d (%s)",
+ path, ret, terror(ret));
+(*env)->Throw(env, jthr);
+goto done;
+ }
jret = fd_create(env, fd); // throws exception on error.
done:
[01/10] git commit: HDFS-7061. Add test to verify encryption zone creation after NameNode restart without saving namespace. Contributed by Stephen Chu.
Repository: hadoop
Updated Branches:
refs/heads/HDFS-6581 a4dcbaa33 -> a7814e1aa
HDFS-7061. Add test to verify encryption zone creation after NameNode restart
without saving namespace. Contributed by Stephen Chu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fc741b5d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fc741b5d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fc741b5d
Branch: refs/heads/HDFS-6581
Commit: fc741b5d78e7e006355e17b1b5839f502e37261b
Parents: 14e2639
Author: Andrew Wang
Authored: Sun Sep 14 23:48:24 2014 -0700
Committer: Andrew Wang
Committed: Sun Sep 14 23:49:01 2014 -0700
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/hdfs/TestEncryptionZones.java | 10 ++
2 files changed, 13 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc741b5d/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index bddf303..57a4a0f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -455,6 +455,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6482. Use block ID-based block layout on datanodes (James Thomas via
Colin Patrick McCabe)
+HDFS-7061. Add test to verify encryption zone creation after NameNode
+restart without saving namespace. (Stephen Chu via wang)
+
OPTIMIZATIONS
HDFS-6690. Deduplicate xattr names in memory. (wang)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc741b5d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
index db3c085..b3bf5d9 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
@@ -338,6 +338,16 @@ public class TestEncryptionZones {
cluster.restartNameNode(true);
assertNumZones(numZones);
assertZonePresent(null, zone1.toString());
+
+// Verify newly added ez is present after restarting the NameNode
+// without persisting the namespace.
+Path nonpersistZone = new Path("/nonpersistZone");
+fsWrapper.mkdir(nonpersistZone, FsPermission.getDirDefault(), false);
+dfsAdmin.createEncryptionZone(nonpersistZone, TEST_KEY);
+numZones++;
+cluster.restartNameNode(true);
+assertNumZones(numZones);
+assertZonePresent(null, nonpersistZone.toString());
}
/**
[06/10] git commit: YARN-2529. Generic history service RPC interface doesn't work when service authorization is enabled. Contributed by Zhijie Shen
YARN-2529. Generic history service RPC interface doesn't work when service
authorization is enabled. Contributed by Zhijie Shen
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/88e329fd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/88e329fd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/88e329fd
Branch: refs/heads/HDFS-6581
Commit: 88e329fdfb27c11f1964c0345188bcf328c95fd6
Parents: 9d4ec97
Author: Jian He
Authored: Mon Sep 15 13:13:54 2014 -0700
Committer: Jian He
Committed: Mon Sep 15 13:13:54 2014 -0700
--
.../src/main/conf/hadoop-policy.xml | 9
hadoop-yarn-project/CHANGES.txt | 3 ++
.../hadoop/yarn/conf/YarnConfiguration.java | 4 ++
.../ApplicationHistoryClientService.java| 15 ++-
.../authorize/TimelinePolicyProvider.java | 44
5 files changed, 74 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88e329fd/hadoop-common-project/hadoop-common/src/main/conf/hadoop-policy.xml
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-policy.xml
b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-policy.xml
index 39e4f4d..2bf5c02 100644
--- a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-policy.xml
+++ b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-policy.xml
@@ -214,4 +214,13 @@
A special value of "*" means all users are allowed.
+
+security.applicationhistory.protocol.acl
+*
+ACL for ApplicationHistoryProtocol, used by the timeline
+server and the generic history service client to communicate with each
other.
+The ACL is a comma-separated list of user and group names. The user and
+group list is separated by a blank. For e.g. "alice,bob users,wheel".
+A special value of "*" means all users are allowed.
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88e329fd/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 5cfc460..629d3b3 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -362,6 +362,9 @@ Release 2.6.0 - UNRELEASED
YARN-2549. TestContainerLaunch fails due to classpath problem with hamcrest
classes. (cnauroth)
+YARN-2529. Generic history service RPC interface doesn't work when service
+authorization is enabled. (Zhijie Shen via jianhe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88e329fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 43f510d..a92b358 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -991,6 +991,10 @@ public class YarnConfiguration extends Configuration {
YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCE_LOCALIZER =
"security.resourcelocalizer.protocol.acl";
+ public static final String
+ YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONHISTORY_PROTOCOL =
+ "security.applicationhistory.protocol.acl";
+
/** No. of milliseconds to wait between sending a SIGTERM and SIGKILL
* to a running container */
public static final String NM_SLEEP_DELAY_BEFORE_SIGKILL_MS =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88e329fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java
index 6372056..2334fde 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryserv
[07/10] git commit: YARN-2438. yarn-env.sh cleanup (aw)
YARN-2438. yarn-env.sh cleanup (aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0c8aec43
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0c8aec43
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0c8aec43
Branch: refs/heads/HDFS-6581
Commit: 0c8aec43f0acd4f896df8cd605d97808f9a09887
Parents: 88e329f
Author: Allen Wittenauer
Authored: Mon Sep 15 14:09:45 2014 -0700
Committer: Allen Wittenauer
Committed: Mon Sep 15 14:09:50 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 2 +
.../hadoop-yarn/conf/yarn-env.sh| 52 +---
2 files changed, 37 insertions(+), 17 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c8aec43/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 629d3b3..b41ad82 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -8,6 +8,8 @@ Trunk - Unreleased
IMPROVEMENTS
+YARN-2438. yarn-env.sh cleanup (aw)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c8aec43/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
--
diff --git a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
index 755cfd8..0f5e9b7 100644
--- a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
+++ b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
@@ -1,3 +1,4 @@
+#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -25,27 +26,21 @@
##
###
-# Generic settings for YARN
+# YARN-specific overrides for generic settings
###
-# User for YARN daemons
-export HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn}
-
-#
-# By default, YARN will use HADOOP_CONF_DIR. Specify a custom
-# YARN_CONF_DIR here
-# export YARN_CONF_DIR="${YARN_CONF_DIR:-$HADOOP_YARN_HOME/conf}"
-#
-
-# Override Hadoop's log directory & file
-# export YARN_LOG_DIR="$HADOOP_YARN_HOME/logs"
-# export YARN_LOGFILE='yarn.log'
+# By default, YARN will use HADOOP_LOG_DIR for YARN logging. Specify a custom
+# log directory for YARN things here:
+# export YARN_LOG_DIR="${HADOOP_LOG_DIR}"
-# Need a custom-to-YARN service-level authorization policy file?
-# export YARN_POLICYFILE="yarn-policy.xml"
+# By default, YARN will use the value of HADOOP_LOGFILE as the 'fallback' log
+# file # when log4j settings are not defined. Specify a custom YARN log file
+# here:
+# export YARN_LOGFILE=${HADOOP_LOGFILE}
-#Override the log4j settings for all YARN apps
-# export YARN_ROOT_LOGGER="INFO,console"
+#Override the log4j settings for all YARN apps By default, YARN will use
+# HADOOP_ROOT_LOGGER.
+# export YARN_ROOT_LOGGER=${HADOOP_ROOT_LOGGER}
###
# Resource Manager specific parameters
@@ -125,3 +120,26 @@ export HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn}
#
#export YARN_TIMELINESERVER_OPTS=
+###
+# Web App Proxy Server specifc parameters
+###
+
+# Specify the max Heapsize for the proxy server using a numerical value
+# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
+# the value to 1000.
+# This value will be overridden by an Xmx setting specified in either
YARN_OPTS,
+# HADOOP_OPTS, and/or YARN_PROXYSERVER_OPTS.
+# If not specified, the default value will be picked from either YARN_HEAPMAX
+# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
+#
+#export YARN_PROXYSERVER_HEAPSIZE=1000
+
+# Specify the JVM options to be used when starting the proxy server.
+# These options will be appended to the options specified as YARN_OPTS
+# and therefore may override any similar flags set in YARN_OPTS
+#
+# See ResourceManager for some examples
+#
+#export YARN_PROXYSERVER_OPTS=
+
+
[08/10] git commit: HDFS-7059. HAadmin transtionToActive with forceActive option can show confusing message. Contributed by Rushabh Shah.
HDFS-7059. HAadmin transtionToActive with forceActive option can show
confusing message. Contributed by Rushabh Shah.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2f847b33
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2f847b33
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2f847b33
Branch: refs/heads/HDFS-6581
Commit: 2f847b3357448184aaec935052f34d116ad3a42c
Parents: 0c8aec4
Author: Kihwal Lee
Authored: Mon Sep 15 16:27:22 2014 -0500
Committer: Kihwal Lee
Committed: Mon Sep 15 16:27:22 2014 -0500
--
.../main/java/org/apache/hadoop/ha/HAAdmin.java | 6 --
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hdfs/tools/TestDFSHAAdminMiniCluster.java | 20
3 files changed, 7 insertions(+), 22 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f847b33/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
index dfa03e8..bd6366c 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
@@ -143,8 +143,10 @@ public abstract class HAAdmin extends Configured
implements Tool {
}
/* returns true if other target node is active or some exception occurred
and forceActive was not set */
-if(isOtherTargetNodeActive(argv[0], cmd.hasOption(FORCEACTIVE))) {
- return -1;
+if(!cmd.hasOption(FORCEACTIVE)) {
+ if(isOtherTargetNodeActive(argv[0], cmd.hasOption(FORCEACTIVE))) {
+return -1;
+ }
}
HAServiceTarget target = resolveTarget(argv[0]);
if (!checkManualStateManagementOK(target)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f847b33/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 209c7c1..f596265 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -458,6 +458,9 @@ Release 2.6.0 - UNRELEASED
HDFS-7061. Add test to verify encryption zone creation after NameNode
restart without saving namespace. (Stephen Chu via wang)
+HDFS-7059. HAadmin transtionToActive with forceActive option can show
+confusing message.
+
OPTIMIZATIONS
HDFS-6690. Deduplicate xattr names in memory. (wang)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f847b33/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java
index 288bcd0..ee1c184 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java
@@ -232,26 +232,6 @@ public class TestDFSHAAdminMiniCluster {
assertFalse("Both namenodes cannot be active", nn1.isActiveState()
&& nn2.isActiveState());
-/* This test case doesn't allow nn2 to transition to Active even with
-forceActive switch since nn1 is already active */
-if(nn1.getState() != null && !nn1.getState().
-equals(HAServiceState.STANDBY.name()) ) {
- cluster.transitionToStandby(0);
-}
-if(nn2.getState() != null && !nn2.getState().
-equals(HAServiceState.STANDBY.name()) ) {
- cluster.transitionToStandby(1);
-}
-//Making sure both the namenode are in standby state
-assertTrue(nn1.isStandbyState());
-assertTrue(nn2.isStandbyState());
-
-runTool("-transitionToActive", "nn1");
-runTool("-transitionToActive", "nn2","--forceactive");
-
-assertFalse("Both namenodes cannot be active even though with forceActive",
-nn1.isActiveState() && nn2.isActiveState());
-
/* In this test case, we have deliberately shut down nn1 and this will
cause HAAAdmin#isOtherTargetNodeActive to throw an Exception
and transitionToActive for nn2 with forceActive switch will succeed
[03/10] git commit: HADOOP-10400. Incorporate new S3A FileSystem implementation. Contributed by Jordan Mendelson and Dave Wang.
HADOOP-10400. Incorporate new S3A FileSystem implementation. Contributed by
Jordan Mendelson and Dave Wang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/24d920b8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/24d920b8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/24d920b8
Branch: refs/heads/HDFS-6581
Commit: 24d920b80eb3626073925a1d0b6dcf148add8cc0
Parents: fc741b5
Author: Aaron T. Myers
Authored: Mon Sep 15 08:27:07 2014 -0700
Committer: Aaron T. Myers
Committed: Mon Sep 15 08:27:07 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt |3 +
.../src/main/conf/log4j.properties |5 +
.../src/main/resources/core-default.xml | 86 ++
hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml |8 +
hadoop-project/pom.xml | 26 +-
hadoop-tools/hadoop-aws/pom.xml | 10 +
.../fs/s3a/AnonymousAWSCredentialsProvider.java | 37 +
.../fs/s3a/BasicAWSCredentialsProvider.java | 51 +
.../org/apache/hadoop/fs/s3a/Constants.java | 90 ++
.../org/apache/hadoop/fs/s3a/S3AFileStatus.java | 62 ++
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 1019 ++
.../apache/hadoop/fs/s3a/S3AInputStream.java| 207
.../apache/hadoop/fs/s3a/S3AOutputStream.java | 208
.../services/org.apache.hadoop.fs.FileSystem|1 +
.../hadoop/fs/contract/s3a/S3AContract.java | 43 +
.../fs/contract/s3a/TestS3AContractCreate.java | 38 +
.../fs/contract/s3a/TestS3AContractDelete.java | 31 +
.../fs/contract/s3a/TestS3AContractMkdir.java | 34 +
.../fs/contract/s3a/TestS3AContractOpen.java| 31 +
.../fs/contract/s3a/TestS3AContractRename.java | 64 ++
.../fs/contract/s3a/TestS3AContractRootDir.java | 35 +
.../fs/contract/s3a/TestS3AContractSeek.java| 31 +
.../fs/s3a/S3AFileSystemContractBaseTest.java | 327 ++
.../src/test/resources/contract/s3a.xml | 105 ++
.../src/test/resources/contract/s3n.xml |7 +-
hadoop-tools/hadoop-azure/pom.xml | 10 +-
26 files changed, 2552 insertions(+), 17 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/24d920b8/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 051eac1..c2ae5ed 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -342,6 +342,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-10893. isolated classloader on the client side (Sangjin Lee via
jlowe)
+HADOOP-10400. Incorporate new S3A FileSystem implementation. (Jordan
+Mendelson and Dave Wang via atm)
+
IMPROVEMENTS
HADOOP-10808. Remove unused native code for munlock. (cnauroth)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/24d920b8/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
--
diff --git a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
index ef9acbf..5fa21fa 100644
--- a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
+++ b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
@@ -174,6 +174,11 @@
log4j.appender.MRAUDIT.MaxBackupIndex=${mapred.audit.log.maxbackupindex}
# Jets3t library
log4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR
+# AWS SDK & S3A FileSystem
+log4j.logger.com.amazonaws=ERROR
+log4j.logger.com.amazonaws.http.AmazonHttpClient=ERROR
+log4j.logger.org.apache.hadoop.fs.s3a.S3AFileSystem=WARN
+
#
# Event Counter Appender
# Sends counts of logging messages at different severity levels to Hadoop
Metrics.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/24d920b8/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 3cc7545..828dec2 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -690,6 +690,92 @@ for ldap providers in the same way as above does.
+ fs.s3a.access.key
+ AWS access key ID. Omit for Role-based
authentication.
+
+
+
+ fs.s3a.secret.key
+ AWS secret key. Omit for Role-based
authentication.
+
+
+
+ fs.s3a.connection.maximum
+ 15
+ Controls the maximum number of simultaneous
[04/10] git commit: HDFS-7032. Add WebHDFS support for reading and writing to encryption zones. Contributed by Charles Lamb.
HDFS-7032. Add WebHDFS support for reading and writing to encryption zones.
Contributed by Charles Lamb.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/43b03030
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/43b03030
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/43b03030
Branch: refs/heads/HDFS-6581
Commit: 43b03030084839db041d0337013806aaeef12aaa
Parents: 24d920b
Author: Andrew Wang
Authored: Mon Sep 15 10:23:57 2014 -0700
Committer: Andrew Wang
Committed: Mon Sep 15 10:23:57 2014 -0700
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../web/resources/DatanodeWebHdfsMethods.java | 13 +++--
.../apache/hadoop/hdfs/TestEncryptionZones.java | 52
3 files changed, 63 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/43b03030/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 57a4a0f..209c7c1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -647,6 +647,9 @@ Release 2.6.0 - UNRELEASED
HDFS-7045. Fix NameNode deadlock when opening file under /.reserved path.
(Yi Liu via wang)
+HDFS-7032. Add WebHDFS support for reading and writing to encryption zones.
+(clamb via wang)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HDFS-6387. HDFS CLI admin tool for creating & deleting an
http://git-wip-us.apache.org/repos/asf/hadoop/blob/43b03030/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java
index 51731c8..0f0f3be 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java
@@ -231,11 +231,13 @@ public class DatanodeWebHdfsMethods {
DFSClient dfsclient = newDfsClient(nnId, conf);
FSDataOutputStream out = null;
try {
-out = new FSDataOutputStream(dfsclient.create(
+out = dfsclient.createWrappedOutputStream(dfsclient.create(
fullpath, permission.getFsPermission(),
-overwrite.getValue() ? EnumSet.of(CreateFlag.CREATE,
CreateFlag.OVERWRITE)
-: EnumSet.of(CreateFlag.CREATE),
-replication.getValue(conf), blockSize.getValue(conf), null, b,
null), null);
+overwrite.getValue() ?
+EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE) :
+EnumSet.of(CreateFlag.CREATE),
+replication.getValue(conf), blockSize.getValue(conf), null,
+b, null), null);
IOUtils.copyBytes(in, out, b);
out.close();
out = null;
@@ -418,7 +420,8 @@ public class DatanodeWebHdfsMethods {
final DFSClient dfsclient = newDfsClient(nnId, conf);
HdfsDataInputStream in = null;
try {
-in = new HdfsDataInputStream(dfsclient.open(fullpath, b, true));
+in = dfsclient.createWrappedInputStream(
+dfsclient.open(fullpath, b, true));
in.seek(offset.getValue());
} catch(IOException ioe) {
IOUtils.cleanup(LOG, in);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/43b03030/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
index b3bf5d9..68fc850 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.crypto.key.JavaKeyStoreProvider;
import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
import org.apache.hadoop.crypto.key.KeyProviderFactory;
+import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FSTestWrapper;
import org.
git commit: HADOOP-11091. Eliminate old configuration parameter names from s3a (dsw via cmccabe) (cherry picked from commit 0ac760a58d96b36ab30e9d60679bbea6365ef120)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 b4d9aca06 -> f80b10e1e
HADOOP-11091. Eliminate old configuration parameter names from s3a (dsw via
cmccabe)
(cherry picked from commit 0ac760a58d96b36ab30e9d60679bbea6365ef120)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f80b10e1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f80b10e1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f80b10e1
Branch: refs/heads/branch-2
Commit: f80b10e1e7d47550996ba3c8b0f23f4a97a8c16a
Parents: b4d9aca
Author: Colin Patrick Mccabe
Authored: Mon Sep 15 16:49:15 2014 -0700
Committer: Colin Patrick Mccabe
Committed: Mon Sep 15 16:50:10 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++
.../org/apache/hadoop/fs/s3a/Constants.java | 36 +--
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 46 +---
.../apache/hadoop/fs/s3a/S3AOutputStream.java | 6 +--
4 files changed, 38 insertions(+), 53 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f80b10e1/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index d2deee6..fddd86d 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -186,6 +186,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11074. Move s3-related FS connector code to hadoop-aws (David S.
Wang via Colin Patrick McCabe)
+HADOOP-11091. Eliminate old configuration parameter names from s3a (David
+S. Wang via Colin Patrick McCabe)
+
OPTIMIZATIONS
HADOOP-10838. Byte array native checksumming. (James Thomas via todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f80b10e1/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
--
diff --git
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
index 9723b82..26b7ddd 100644
---
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
+++
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
@@ -21,46 +21,37 @@ package org.apache.hadoop.fs.s3a;
public class Constants {
// s3 access key
- public static final String OLD_ACCESS_KEY = "fs.s3a.awsAccessKeyId";
- public static final String NEW_ACCESS_KEY = "fs.s3a.access.key";
+ public static final String ACCESS_KEY = "fs.s3a.access.key";
// s3 secret key
- public static final String OLD_SECRET_KEY = "fs.s3a.awsSecretAccessKey";
- public static final String NEW_SECRET_KEY = "fs.s3a.secret.key";
+ public static final String SECRET_KEY = "fs.s3a.secret.key";
// number of simultaneous connections to s3
- public static final String OLD_MAXIMUM_CONNECTIONS = "fs.s3a.maxConnections";
- public static final String NEW_MAXIMUM_CONNECTIONS =
"fs.s3a.connection.maximum";
+ public static final String MAXIMUM_CONNECTIONS = "fs.s3a.connection.maximum";
public static final int DEFAULT_MAXIMUM_CONNECTIONS = 15;
// connect to s3 over ssl?
- public static final String OLD_SECURE_CONNECTIONS =
"fs.s3a.secureConnections";
- public static final String NEW_SECURE_CONNECTIONS =
"fs.s3a.connection.ssl.enabled";
+ public static final String SECURE_CONNECTIONS =
"fs.s3a.connection.ssl.enabled";
public static final boolean DEFAULT_SECURE_CONNECTIONS = true;
// number of times we should retry errors
- public static final String OLD_MAX_ERROR_RETRIES = "fs.s3a.maxErrorRetries";
- public static final String NEW_MAX_ERROR_RETRIES = "fs.s3a.attempts.maximum";
+ public static final String MAX_ERROR_RETRIES = "fs.s3a.attempts.maximum";
public static final int DEFAULT_MAX_ERROR_RETRIES = 10;
// seconds until we give up on a connection to s3
- public static final String OLD_SOCKET_TIMEOUT = "fs.s3a.socketTimeout";
- public static final String NEW_SOCKET_TIMEOUT = "fs.s3a.connection.timeout";
+ public static final String SOCKET_TIMEOUT = "fs.s3a.connection.timeout";
public static final int DEFAULT_SOCKET_TIMEOUT = 5;
// number of records to get while paging through a directory listing
- public static final String OLD_MAX_PAGING_KEYS = "fs.s3a.maxPagingKeys";
- public static final String NEW_MAX_PAGING_KEYS = "fs.s3a.paging.maximum";
+ public static final String MAX_PAGING_KEYS = "fs.s3a.paging.maximum";
public static final int DEFAULT_MAX_PAGING_KEYS = 5000;
// size of each of or multipart pieces in bytes
- public static final String OLD_MULTIPART_SIZE =
git commit: HADOOP-11091. Eliminate old configuration parameter names from s3a (dsw via cmccabe)
Repository: hadoop
Updated Branches:
refs/heads/trunk 8008f0e81 -> 0ac760a58
HADOOP-11091. Eliminate old configuration parameter names from s3a (dsw via
cmccabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0ac760a5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0ac760a5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0ac760a5
Branch: refs/heads/trunk
Commit: 0ac760a58d96b36ab30e9d60679bbea6365ef120
Parents: 8008f0e
Author: Colin Patrick Mccabe
Authored: Mon Sep 15 16:49:15 2014 -0700
Committer: Colin Patrick Mccabe
Committed: Mon Sep 15 16:49:15 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++
.../org/apache/hadoop/fs/s3a/Constants.java | 36 +--
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 46 +---
.../apache/hadoop/fs/s3a/S3AOutputStream.java | 6 +--
4 files changed, 38 insertions(+), 53 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ac760a5/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 6f80eb2..89bce4d 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -517,6 +517,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11074. Move s3-related FS connector code to hadoop-aws (David S.
Wang via Colin Patrick McCabe)
+HADOOP-11091. Eliminate old configuration parameter names from s3a (David
+S. Wang via Colin Patrick McCabe)
+
OPTIMIZATIONS
HADOOP-10838. Byte array native checksumming. (James Thomas via todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ac760a5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
--
diff --git
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
index 9723b82..26b7ddd 100644
---
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
+++
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
@@ -21,46 +21,37 @@ package org.apache.hadoop.fs.s3a;
public class Constants {
// s3 access key
- public static final String OLD_ACCESS_KEY = "fs.s3a.awsAccessKeyId";
- public static final String NEW_ACCESS_KEY = "fs.s3a.access.key";
+ public static final String ACCESS_KEY = "fs.s3a.access.key";
// s3 secret key
- public static final String OLD_SECRET_KEY = "fs.s3a.awsSecretAccessKey";
- public static final String NEW_SECRET_KEY = "fs.s3a.secret.key";
+ public static final String SECRET_KEY = "fs.s3a.secret.key";
// number of simultaneous connections to s3
- public static final String OLD_MAXIMUM_CONNECTIONS = "fs.s3a.maxConnections";
- public static final String NEW_MAXIMUM_CONNECTIONS =
"fs.s3a.connection.maximum";
+ public static final String MAXIMUM_CONNECTIONS = "fs.s3a.connection.maximum";
public static final int DEFAULT_MAXIMUM_CONNECTIONS = 15;
// connect to s3 over ssl?
- public static final String OLD_SECURE_CONNECTIONS =
"fs.s3a.secureConnections";
- public static final String NEW_SECURE_CONNECTIONS =
"fs.s3a.connection.ssl.enabled";
+ public static final String SECURE_CONNECTIONS =
"fs.s3a.connection.ssl.enabled";
public static final boolean DEFAULT_SECURE_CONNECTIONS = true;
// number of times we should retry errors
- public static final String OLD_MAX_ERROR_RETRIES = "fs.s3a.maxErrorRetries";
- public static final String NEW_MAX_ERROR_RETRIES = "fs.s3a.attempts.maximum";
+ public static final String MAX_ERROR_RETRIES = "fs.s3a.attempts.maximum";
public static final int DEFAULT_MAX_ERROR_RETRIES = 10;
// seconds until we give up on a connection to s3
- public static final String OLD_SOCKET_TIMEOUT = "fs.s3a.socketTimeout";
- public static final String NEW_SOCKET_TIMEOUT = "fs.s3a.connection.timeout";
+ public static final String SOCKET_TIMEOUT = "fs.s3a.connection.timeout";
public static final int DEFAULT_SOCKET_TIMEOUT = 5;
// number of records to get while paging through a directory listing
- public static final String OLD_MAX_PAGING_KEYS = "fs.s3a.maxPagingKeys";
- public static final String NEW_MAX_PAGING_KEYS = "fs.s3a.paging.maximum";
+ public static final String MAX_PAGING_KEYS = "fs.s3a.paging.maximum";
public static final int DEFAULT_MAX_PAGING_KEYS = 5000;
// size of each of or multipart pieces in bytes
- public static final String OLD_MULTIPART_SIZE = "fs.s3a.multipartSize";
- public static final String NEW_MULTIPART_SIZE =
git commit: YARN-1710. Logic to find allocations within a Plan that satisfy user ReservationRequest(s). Contributed by Carlo Curino and Subru Krishnan.
Repository: hadoop
Updated Branches:
refs/heads/YARN-1051 ca9be6fda -> 7800b53c4
YARN-1710. Logic to find allocations within a Plan that satisfy user
ReservationRequest(s). Contributed by Carlo Curino and Subru Krishnan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7800b53c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7800b53c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7800b53c
Branch: refs/heads/YARN-1051
Commit: 7800b53c4d1de918f2a2f970e838f151403504a3
Parents: ca9be6f
Author: carlo curino
Authored: Mon Sep 15 16:56:28 2014 -0700
Committer: carlo curino
Committed: Mon Sep 15 16:56:28 2014 -0700
--
YARN-1051-CHANGES.txt | 12 +-
.../reservation/GreedyReservationAgent.java | 367
.../reservation/ReservationAgent.java | 55 ++
.../exceptions/ContractValidationException.java | 12 +
.../reservation/TestGreedyReservationAgent.java | 588 +++
5 files changed, 1031 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7800b53c/YARN-1051-CHANGES.txt
--
diff --git a/YARN-1051-CHANGES.txt b/YARN-1051-CHANGES.txt
index 410d974..deece7c 100644
--- a/YARN-1051-CHANGES.txt
+++ b/YARN-1051-CHANGES.txt
@@ -4,7 +4,13 @@ CapacityScheduler. (Carlo Curino and Subru Krishnan via curino)
YARN-2475. Logic for responding to capacity drops for the
ReservationSystem. (Carlo Curino and Subru Krishnan via curino)
-YARN-1708. Public YARN APIs for creating/updating/deleting reservations.
(subru)
+YARN-1708. Public YARN APIs for creating/updating/deleting
+reservations. (Carlo Curino and Subru Krishnan via subru)
-YARN-1709. In-memory data structures used to track resources over time to
-enable reservations. (subru)
+YARN-1709. In-memory data structures used to track resources over
+time to enable reservations. (Carlo Curino and Subru Krishnan via
+subru)
+
+YARN-1710. Logic to find allocations within a Plan that satisfy
+user ReservationRequest(s). (Carlo Curino and Subru Krishnan via
+curino)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7800b53c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/GreedyReservationAgent.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/GreedyReservationAgent.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/GreedyReservationAgent.java
new file mode 100644
index 000..3214f93
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/GreedyReservationAgent.java
@@ -0,0 +1,367 @@
+package org.apache.hadoop.yarn.server.resourcemanager.reservation;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.yarn.api.records.ReservationDefinition;
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.api.records.ReservationRequest;
+import org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter;
+import org.apache.hadoop.yarn.api.records.Resource;
+import
org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.ContractValidationException;
+import
org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This Agent employs a simple greedy placement strategy, placing the various
+ * stages of a {@link ReservationRequest} from the deadline moving backward
+ * towards the arrival. This allows jobs with earlier deadline to be scheduled
+ * greedily as well. Combined with an opportunistic anticipation of work if the
+ * cluster is not fully utilized also seems to provide good latency for
+ * best-effort jobs (i.e., jobs running without a reservation).
+ *
+ * This agent does not account for locality and only consider container
+ * granularity for validation purposes (i.e., you can't exceed max-container
+ * size).
+ */
+public class GreedyReservationAgent implements ReservationAgent {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(GreedyReservationAgent.class);
+
+ @Override
+ public boolean createReservation(
[2/2] git commit: HADOOP-10868. AuthenticationFilter should support externalizing the secret for signing and provide rotation support. (rkanter via tucu)
HADOOP-10868. AuthenticationFilter should support externalizing the secret for
signing and provide rotation support. (rkanter via tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/932ae036
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/932ae036
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/932ae036
Branch: refs/heads/trunk
Commit: 932ae036acb96634c5dd435d57ba02ce4d5e8918
Parents: 0ac760a
Author: Alejandro Abdelnur
Authored: Mon Sep 15 17:05:42 2014 -0700
Committer: Alejandro Abdelnur
Committed: Mon Sep 15 17:05:42 2014 -0700
--
hadoop-common-project/hadoop-auth/pom.xml | 13 +
.../server/AuthenticationFilter.java| 152 --
.../util/RandomSignerSecretProvider.java| 4 +-
.../util/RolloverSignerSecretProvider.java | 7 +-
.../util/SignerSecretProvider.java | 9 +-
.../util/StringSignerSecretProvider.java| 15 +-
.../util/ZKSignerSecretProvider.java| 503 +++
.../src/site/apt/Configuration.apt.vm | 148 +-
.../hadoop-auth/src/site/apt/index.apt.vm | 5 +
.../server/TestAuthenticationFilter.java| 117 -
.../util/TestJaasConfiguration.java | 55 ++
.../util/TestRandomSignerSecretProvider.java| 2 +-
.../util/TestRolloverSignerSecretProvider.java | 2 +-
.../authentication/util/TestSigner.java | 23 +-
.../util/TestStringSignerSecretProvider.java| 9 +-
.../util/TestZKSignerSecretProvider.java| 270 ++
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../hadoop/fs/http/server/TestHttpFSServer.java | 8 +-
hadoop-project/pom.xml | 11 +
19 files changed, 1259 insertions(+), 97 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/932ae036/hadoop-common-project/hadoop-auth/pom.xml
--
diff --git a/hadoop-common-project/hadoop-auth/pom.xml
b/hadoop-common-project/hadoop-auth/pom.xml
index 564518c..5f7d774 100644
--- a/hadoop-common-project/hadoop-auth/pom.xml
+++ b/hadoop-common-project/hadoop-auth/pom.xml
@@ -130,6 +130,19 @@
+
+ org.apache.zookeeper
+ zookeeper
+
+
+ org.apache.curator
+ curator-framework
+
+
+ org.apache.curator
+ curator-test
+ test
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/932ae036/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
--
diff --git
a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
index 9330444..47cf54c 100644
---
a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
+++
b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
@@ -22,6 +22,7 @@ import
org.apache.hadoop.security.authentication.util.SignerException;
import
org.apache.hadoop.security.authentication.util.RandomSignerSecretProvider;
import org.apache.hadoop.security.authentication.util.SignerSecretProvider;
import
org.apache.hadoop.security.authentication.util.StringSignerSecretProvider;
+import org.apache.hadoop.security.authentication.util.ZKSignerSecretProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -42,7 +43,7 @@ import java.util.*;
/**
* The {@link AuthenticationFilter} enables protecting web application
resources with different (pluggable)
- * authentication mechanisms.
+ * authentication mechanisms and signer secret providers.
*
* Out of the box it provides 2 authentication mechanisms: Pseudo and Kerberos
SPNEGO.
*
@@ -60,10 +61,13 @@ import java.util.*;
* [#PREFIX#.]type: simple|kerberos|#CLASS#, 'simple' is short for the
* {@link PseudoAuthenticationHandler}, 'kerberos' is short for {@link
KerberosAuthenticationHandler}, otherwise
* the full class name of the {@link AuthenticationHandler} must be
specified.
- * [#PREFIX#.]signature.secret: the secret used to sign the HTTP cookie
value. The default value is a random
- * value. Unless multiple webapp instances need to share the secret the random
value is adequate.
- * [#PREFIX#.]token.validity: time -in seconds- that the generated token
is valid before a
- * new authentication is triggered, default value is 3600
seconds.
+ * [#PREFIX#.]signature.secret: when signer.secret.provider is set to
+
[1/2] HADOOP-10868. AuthenticationFilter should support externalizing the secret for signing and provide rotation support. (rkanter via tucu)
Repository: hadoop
Updated Branches:
refs/heads/trunk 0ac760a58 -> 932ae036a
http://git-wip-us.apache.org/repos/asf/hadoop/blob/932ae036/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestZKSignerSecretProvider.java
--
diff --git
a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestZKSignerSecretProvider.java
b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestZKSignerSecretProvider.java
new file mode 100644
index 000..d7b6e17
--- /dev/null
+++
b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestZKSignerSecretProvider.java
@@ -0,0 +1,270 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. See accompanying LICENSE file.
+ */
+package org.apache.hadoop.security.authentication.util;
+
+import java.util.Arrays;
+import java.util.Properties;
+import java.util.Random;
+import javax.servlet.ServletContext;
+import org.apache.curator.test.TestingServer;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+public class TestZKSignerSecretProvider {
+
+ private TestingServer zkServer;
+
+ @Before
+ public void setup() throws Exception {
+zkServer = new TestingServer();
+ }
+
+ @After
+ public void teardown() throws Exception {
+if (zkServer != null) {
+ zkServer.stop();
+ zkServer.close();
+}
+ }
+
+ @Test
+ // Test just one ZKSignerSecretProvider to verify that it works in the
+ // simplest case
+ public void testOne() throws Exception {
+long rolloverFrequency = 15 * 1000; // rollover every 15 sec
+// use the same seed so we can predict the RNG
+long seed = System.currentTimeMillis();
+Random rand = new Random(seed);
+byte[] secret2 = Long.toString(rand.nextLong()).getBytes();
+byte[] secret1 = Long.toString(rand.nextLong()).getBytes();
+byte[] secret3 = Long.toString(rand.nextLong()).getBytes();
+ZKSignerSecretProvider secretProvider = new ZKSignerSecretProvider(seed);
+Properties config = new Properties();
+config.setProperty(
+ZKSignerSecretProvider.ZOOKEEPER_CONNECTION_STRING,
+zkServer.getConnectString());
+config.setProperty(ZKSignerSecretProvider.ZOOKEEPER_PATH,
+"/secret");
+try {
+ secretProvider.init(config, getDummyServletContext(), rolloverFrequency);
+
+ byte[] currentSecret = secretProvider.getCurrentSecret();
+ byte[][] allSecrets = secretProvider.getAllSecrets();
+ Assert.assertArrayEquals(secret1, currentSecret);
+ Assert.assertEquals(2, allSecrets.length);
+ Assert.assertArrayEquals(secret1, allSecrets[0]);
+ Assert.assertNull(allSecrets[1]);
+ Thread.sleep((rolloverFrequency + 2000));
+
+ currentSecret = secretProvider.getCurrentSecret();
+ allSecrets = secretProvider.getAllSecrets();
+ Assert.assertArrayEquals(secret2, currentSecret);
+ Assert.assertEquals(2, allSecrets.length);
+ Assert.assertArrayEquals(secret2, allSecrets[0]);
+ Assert.assertArrayEquals(secret1, allSecrets[1]);
+ Thread.sleep((rolloverFrequency + 2000));
+
+ currentSecret = secretProvider.getCurrentSecret();
+ allSecrets = secretProvider.getAllSecrets();
+ Assert.assertArrayEquals(secret3, currentSecret);
+ Assert.assertEquals(2, allSecrets.length);
+ Assert.assertArrayEquals(secret3, allSecrets[0]);
+ Assert.assertArrayEquals(secret2, allSecrets[1]);
+ Thread.sleep((rolloverFrequency + 2000));
+} finally {
+ secretProvider.destroy();
+}
+ }
+
+ @Test
+ public void testMultipleInit() throws Exception {
+long rolloverFrequency = 15 * 1000; // rollover every 15 sec
+// use the same seed so we can predict the RNG
+long seedA = System.currentTimeMillis();
+Random rand = new Random(seedA);
+byte[] secretA2 = Long.toString(rand.nextLong()).getBytes();
+byte[] secretA1 = Long.toString(rand.nextLong()).getBytes();
+// use the same seed so we can predict the RNG
+long seedB = System.currentTimeMillis() + rand.nextLong();
+rand = new Random(seedB);
+byte[] secretB2 = Long.toString(rand.nextLong()).getBytes();
+byte[] secretB1 = Long.toString(rand.nextLong()).getBytes();
+// use the same seed so we can pre
[1/2] HADOOP-10868. AuthenticationFilter should support externalizing the secret for signing and provide rotation support. (rkanter via tucu)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 f80b10e1e -> e59f6771e
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e59f6771/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestZKSignerSecretProvider.java
--
diff --git
a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestZKSignerSecretProvider.java
b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestZKSignerSecretProvider.java
new file mode 100644
index 000..d7b6e17
--- /dev/null
+++
b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestZKSignerSecretProvider.java
@@ -0,0 +1,270 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. See accompanying LICENSE file.
+ */
+package org.apache.hadoop.security.authentication.util;
+
+import java.util.Arrays;
+import java.util.Properties;
+import java.util.Random;
+import javax.servlet.ServletContext;
+import org.apache.curator.test.TestingServer;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+public class TestZKSignerSecretProvider {
+
+ private TestingServer zkServer;
+
+ @Before
+ public void setup() throws Exception {
+zkServer = new TestingServer();
+ }
+
+ @After
+ public void teardown() throws Exception {
+if (zkServer != null) {
+ zkServer.stop();
+ zkServer.close();
+}
+ }
+
+ @Test
+ // Test just one ZKSignerSecretProvider to verify that it works in the
+ // simplest case
+ public void testOne() throws Exception {
+long rolloverFrequency = 15 * 1000; // rollover every 15 sec
+// use the same seed so we can predict the RNG
+long seed = System.currentTimeMillis();
+Random rand = new Random(seed);
+byte[] secret2 = Long.toString(rand.nextLong()).getBytes();
+byte[] secret1 = Long.toString(rand.nextLong()).getBytes();
+byte[] secret3 = Long.toString(rand.nextLong()).getBytes();
+ZKSignerSecretProvider secretProvider = new ZKSignerSecretProvider(seed);
+Properties config = new Properties();
+config.setProperty(
+ZKSignerSecretProvider.ZOOKEEPER_CONNECTION_STRING,
+zkServer.getConnectString());
+config.setProperty(ZKSignerSecretProvider.ZOOKEEPER_PATH,
+"/secret");
+try {
+ secretProvider.init(config, getDummyServletContext(), rolloverFrequency);
+
+ byte[] currentSecret = secretProvider.getCurrentSecret();
+ byte[][] allSecrets = secretProvider.getAllSecrets();
+ Assert.assertArrayEquals(secret1, currentSecret);
+ Assert.assertEquals(2, allSecrets.length);
+ Assert.assertArrayEquals(secret1, allSecrets[0]);
+ Assert.assertNull(allSecrets[1]);
+ Thread.sleep((rolloverFrequency + 2000));
+
+ currentSecret = secretProvider.getCurrentSecret();
+ allSecrets = secretProvider.getAllSecrets();
+ Assert.assertArrayEquals(secret2, currentSecret);
+ Assert.assertEquals(2, allSecrets.length);
+ Assert.assertArrayEquals(secret2, allSecrets[0]);
+ Assert.assertArrayEquals(secret1, allSecrets[1]);
+ Thread.sleep((rolloverFrequency + 2000));
+
+ currentSecret = secretProvider.getCurrentSecret();
+ allSecrets = secretProvider.getAllSecrets();
+ Assert.assertArrayEquals(secret3, currentSecret);
+ Assert.assertEquals(2, allSecrets.length);
+ Assert.assertArrayEquals(secret3, allSecrets[0]);
+ Assert.assertArrayEquals(secret2, allSecrets[1]);
+ Thread.sleep((rolloverFrequency + 2000));
+} finally {
+ secretProvider.destroy();
+}
+ }
+
+ @Test
+ public void testMultipleInit() throws Exception {
+long rolloverFrequency = 15 * 1000; // rollover every 15 sec
+// use the same seed so we can predict the RNG
+long seedA = System.currentTimeMillis();
+Random rand = new Random(seedA);
+byte[] secretA2 = Long.toString(rand.nextLong()).getBytes();
+byte[] secretA1 = Long.toString(rand.nextLong()).getBytes();
+// use the same seed so we can predict the RNG
+long seedB = System.currentTimeMillis() + rand.nextLong();
+rand = new Random(seedB);
+byte[] secretB2 = Long.toString(rand.nextLong()).getBytes();
+byte[] secretB1 = Long.toString(rand.nextLong()).getBytes();
+// use the same seed so we can
[2/2] git commit: HADOOP-10868. AuthenticationFilter should support externalizing the secret for signing and provide rotation support. (rkanter via tucu)
HADOOP-10868. AuthenticationFilter should support externalizing the secret for
signing and provide rotation support. (rkanter via tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e59f6771
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e59f6771
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e59f6771
Branch: refs/heads/branch-2
Commit: e59f6771e89ded737cc91698763a02f6ebf23c61
Parents: f80b10e
Author: Alejandro Abdelnur
Authored: Mon Sep 15 17:10:43 2014 -0700
Committer: Alejandro Abdelnur
Committed: Mon Sep 15 17:10:43 2014 -0700
--
hadoop-common-project/hadoop-auth/pom.xml | 13 +
.../server/AuthenticationFilter.java| 152 --
.../util/RandomSignerSecretProvider.java| 4 +-
.../util/RolloverSignerSecretProvider.java | 7 +-
.../util/SignerSecretProvider.java | 9 +-
.../util/StringSignerSecretProvider.java| 15 +-
.../util/ZKSignerSecretProvider.java| 503 +++
.../src/site/apt/Configuration.apt.vm | 148 +-
.../hadoop-auth/src/site/apt/index.apt.vm | 5 +
.../server/TestAuthenticationFilter.java| 117 -
.../util/TestJaasConfiguration.java | 55 ++
.../util/TestRandomSignerSecretProvider.java| 2 +-
.../util/TestRolloverSignerSecretProvider.java | 2 +-
.../authentication/util/TestSigner.java | 23 +-
.../util/TestStringSignerSecretProvider.java| 9 +-
.../util/TestZKSignerSecretProvider.java| 270 ++
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../hadoop/fs/http/server/TestHttpFSServer.java | 8 +-
hadoop-project/pom.xml | 11 +
19 files changed, 1259 insertions(+), 97 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e59f6771/hadoop-common-project/hadoop-auth/pom.xml
--
diff --git a/hadoop-common-project/hadoop-auth/pom.xml
b/hadoop-common-project/hadoop-auth/pom.xml
index 20304e1..1da98dc 100644
--- a/hadoop-common-project/hadoop-auth/pom.xml
+++ b/hadoop-common-project/hadoop-auth/pom.xml
@@ -135,6 +135,19 @@
+
+ org.apache.zookeeper
+ zookeeper
+
+
+ org.apache.curator
+ curator-framework
+
+
+ org.apache.curator
+ curator-test
+ test
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e59f6771/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
--
diff --git
a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
index 9330444..47cf54c 100644
---
a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
+++
b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
@@ -22,6 +22,7 @@ import
org.apache.hadoop.security.authentication.util.SignerException;
import
org.apache.hadoop.security.authentication.util.RandomSignerSecretProvider;
import org.apache.hadoop.security.authentication.util.SignerSecretProvider;
import
org.apache.hadoop.security.authentication.util.StringSignerSecretProvider;
+import org.apache.hadoop.security.authentication.util.ZKSignerSecretProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -42,7 +43,7 @@ import java.util.*;
/**
* The {@link AuthenticationFilter} enables protecting web application
resources with different (pluggable)
- * authentication mechanisms.
+ * authentication mechanisms and signer secret providers.
*
* Out of the box it provides 2 authentication mechanisms: Pseudo and Kerberos
SPNEGO.
*
@@ -60,10 +61,13 @@ import java.util.*;
* [#PREFIX#.]type: simple|kerberos|#CLASS#, 'simple' is short for the
* {@link PseudoAuthenticationHandler}, 'kerberos' is short for {@link
KerberosAuthenticationHandler}, otherwise
* the full class name of the {@link AuthenticationHandler} must be
specified.
- * [#PREFIX#.]signature.secret: the secret used to sign the HTTP cookie
value. The default value is a random
- * value. Unless multiple webapp instances need to share the secret the random
value is adequate.
- * [#PREFIX#.]token.validity: time -in seconds- that the generated token
is valid before a
- * new authentication is triggered, default value is 3600
seconds.
+ * [#PREFIX#.]signature.secret: when signer.secret.provider is set to
git commit: HADOOP-10868. Addendum
Repository: hadoop
Updated Branches:
refs/heads/trunk 932ae036a -> 7e08c0f23
HADOOP-10868. Addendum
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7e08c0f2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7e08c0f2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7e08c0f2
Branch: refs/heads/trunk
Commit: 7e08c0f23f58aa143f0997f2472e8051175142e9
Parents: 932ae03
Author: Alejandro Abdelnur
Authored: Mon Sep 15 19:39:27 2014 -0700
Committer: Alejandro Abdelnur
Committed: Mon Sep 15 19:39:27 2014 -0700
--
.../security/authentication/util/ZKSignerSecretProvider.java | 7 +--
1 file changed, 5 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e08c0f2/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java
--
diff --git
a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java
b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java
index 45d4d65..a17b6d4 100644
---
a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java
+++
b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java
@@ -139,6 +139,9 @@ public class ZKSignerSecretProvider extends
RolloverSignerSecretProvider {
ZOOKEEPER_SIGNER_SECRET_PROVIDER_CURATOR_CLIENT_ATTRIBUTE =
CONFIG_PREFIX + "curator.client";
+ private static final String JAAS_LOGIN_ENTRY_NAME =
+ "ZKSignerSecretProviderClient";
+
private static Logger LOG = LoggerFactory.getLogger(
ZKSignerSecretProvider.class);
private String path;
@@ -384,7 +387,7 @@ public class ZKSignerSecretProvider extends
RolloverSignerSecretProvider {
+ "and using 'sasl' ACLs");
String principal = setJaasConfiguration(config);
System.setProperty(ZooKeeperSaslClient.LOGIN_CONTEXT_NAME_KEY,
- "ZKSignerSecretProviderClient");
+ JAAS_LOGIN_ENTRY_NAME);
System.setProperty("zookeeper.authProvider.1",
"org.apache.zookeeper.server.auth.SASLAuthenticationProvider");
aclProvider = new SASLOwnerACLProvider(principal);
@@ -417,7 +420,7 @@ public class ZKSignerSecretProvider extends
RolloverSignerSecretProvider {
// This is equivalent to writing a jaas.conf file and setting the system
// property, "java.security.auth.login.config", to point to it
JaasConfiguration jConf =
-new JaasConfiguration("Client", principal, keytabFile);
+new JaasConfiguration(JAAS_LOGIN_ENTRY_NAME, principal,
keytabFile);
Configuration.setConfiguration(jConf);
return principal.split("[/@]")[0];
}
git commit: HADOOP-10868. Addendum
Repository: hadoop
Updated Branches:
refs/heads/branch-2 e59f6771e -> 1023196ce
HADOOP-10868. Addendum
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1023196c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1023196c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1023196c
Branch: refs/heads/branch-2
Commit: 1023196ceaa600f92f328cfe67a8bccac3445a64
Parents: e59f677
Author: Alejandro Abdelnur
Authored: Mon Sep 15 19:39:12 2014 -0700
Committer: Alejandro Abdelnur
Committed: Mon Sep 15 19:39:12 2014 -0700
--
.../security/authentication/util/ZKSignerSecretProvider.java | 7 +--
1 file changed, 5 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1023196c/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java
--
diff --git
a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java
b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java
index 45d4d65..a17b6d4 100644
---
a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java
+++
b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java
@@ -139,6 +139,9 @@ public class ZKSignerSecretProvider extends
RolloverSignerSecretProvider {
ZOOKEEPER_SIGNER_SECRET_PROVIDER_CURATOR_CLIENT_ATTRIBUTE =
CONFIG_PREFIX + "curator.client";
+ private static final String JAAS_LOGIN_ENTRY_NAME =
+ "ZKSignerSecretProviderClient";
+
private static Logger LOG = LoggerFactory.getLogger(
ZKSignerSecretProvider.class);
private String path;
@@ -384,7 +387,7 @@ public class ZKSignerSecretProvider extends
RolloverSignerSecretProvider {
+ "and using 'sasl' ACLs");
String principal = setJaasConfiguration(config);
System.setProperty(ZooKeeperSaslClient.LOGIN_CONTEXT_NAME_KEY,
- "ZKSignerSecretProviderClient");
+ JAAS_LOGIN_ENTRY_NAME);
System.setProperty("zookeeper.authProvider.1",
"org.apache.zookeeper.server.auth.SASLAuthenticationProvider");
aclProvider = new SASLOwnerACLProvider(principal);
@@ -417,7 +420,7 @@ public class ZKSignerSecretProvider extends
RolloverSignerSecretProvider {
// This is equivalent to writing a jaas.conf file and setting the system
// property, "java.security.auth.login.config", to point to it
JaasConfiguration jConf =
-new JaasConfiguration("Client", principal, keytabFile);
+new JaasConfiguration(JAAS_LOGIN_ENTRY_NAME, principal,
keytabFile);
Configuration.setConfiguration(jConf);
return principal.split("[/@]")[0];
}
