hadoop git commit: HADOOP-15480 AbstractS3GuardToolTestBase.testDiffCommand fails when using dynamo (Gabor Bota)
Repository: hadoop
Updated Branches:
refs/heads/trunk 135941e00 -> 5f6769f79
HADOOP-15480 AbstractS3GuardToolTestBase.testDiffCommand fails when using
dynamo (Gabor Bota)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5f6769f7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5f6769f7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5f6769f7
Branch: refs/heads/trunk
Commit: 5f6769f7964ff002b6c04a95893b5baeb424b6db
Parents: 135941e
Author: Aaron Fabbri
Authored: Tue May 29 19:20:22 2018 -0700
Committer: Aaron Fabbri
Committed: Tue May 29 19:20:22 2018 -0700
--
.../s3guard/AbstractS3GuardToolTestBase.java| 37 +---
.../s3a/s3guard/ITestS3GuardToolDynamoDB.java | 5 ---
.../fs/s3a/s3guard/ITestS3GuardToolLocal.java | 5 ---
3 files changed, 25 insertions(+), 22 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f6769f7/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
--
diff --git
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
index 2b43810..7d75f52 100644
---
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
+++
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
@@ -25,6 +25,7 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintStream;
+import java.net.URI;
import java.util.Collection;
import java.util.HashSet;
import java.util.Set;
@@ -32,6 +33,8 @@ import java.util.concurrent.Callable;
import java.util.concurrent.TimeUnit;
import org.apache.hadoop.util.StopWatch;
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.fs.FileSystem;
import org.junit.Assume;
import org.junit.Test;
@@ -48,6 +51,8 @@ import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.util.StringUtils;
+import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_METASTORE_NULL;
+import static org.apache.hadoop.fs.s3a.Constants.S3_METADATA_STORE_IMPL;
import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.E_BAD_STATE;
import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.SUCCESS;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
@@ -65,6 +70,7 @@ public abstract class AbstractS3GuardToolTestBase extends
AbstractS3ATestBase {
private static final int PRUNE_MAX_AGE_SECS = 2;
private MetadataStore ms;
+ private S3AFileSystem rawFs;
protected static void expectResult(int expected,
String message,
@@ -129,28 +135,34 @@ public abstract class AbstractS3GuardToolTestBase extends
AbstractS3ATestBase {
return ms;
}
- protected abstract MetadataStore newMetadataStore();
-
@Override
public void setup() throws Exception {
super.setup();
S3ATestUtils.assumeS3GuardState(true, getConfiguration());
-ms = newMetadataStore();
-ms.initialize(getFileSystem());
+ms = getFileSystem().getMetadataStore();
+
+// Also create a "raw" fs without any MetadataStore configured
+Configuration conf = new Configuration(getConfiguration());
+conf.set(S3_METADATA_STORE_IMPL, S3GUARD_METASTORE_NULL);
+URI fsUri = getFileSystem().getUri();
+rawFs = (S3AFileSystem) FileSystem.newInstance(fsUri, conf);
}
@Override
public void teardown() throws Exception {
super.teardown();
IOUtils.cleanupWithLogger(LOG, ms);
+IOUtils.closeStream(rawFs);
}
protected void mkdirs(Path path, boolean onS3, boolean onMetadataStore)
throws IOException {
+Preconditions.checkArgument(onS3 || onMetadataStore);
+// getFileSystem() returns an fs with MetadataStore configured
+S3AFileSystem fs = onMetadataStore ? getFileSystem() : rawFs;
if (onS3) {
- getFileSystem().mkdirs(path);
-}
-if (onMetadataStore) {
+ fs.mkdirs(path);
+} else if (onMetadataStore) {
S3AFileStatus status = new S3AFileStatus(true, path, OWNER);
ms.put(new PathMetadata(status));
}
@@ -178,13 +190,14 @@ public abstract class AbstractS3GuardToolTestBase extends
AbstractS3ATestBase {
*/
protected void createFile(Path path, boolean onS3, boolean onMetadataStore)
throws IOException {
+Preconditions.checkArgument(onS3 || onMetadataStore);
+// getFileSystem() returns an fs with MetadataStore configured
+S3AFileSystem fs = onMetadataStore ? getFileSystem() : rawFs;
if (onS3) {
-
hadoop git commit: YARN-8362. Bugfix logic in container retries in node manager. Contributed by Chandni Singh
Repository: hadoop
Updated Branches:
refs/heads/branch-3.1 11a425d11 -> 03209e896
YARN-8362. Bugfix logic in container retries in node manager.
Contributed by Chandni Singh
(cherry picked from commit 135941e00d762a417c3b4cc524cdc59b0d1810b1)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/03209e89
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/03209e89
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/03209e89
Branch: refs/heads/branch-3.1
Commit: 03209e8966a80500f700a2f3a3c12e5975dc127a
Parents: 11a425d
Author: Eric Yang
Authored: Tue May 29 16:56:58 2018 -0400
Committer: Eric Yang
Committed: Tue May 29 17:04:01 2018 -0400
--
.../container/ContainerImpl.java| 4 +-
.../container/SlidingWindowRetryPolicy.java | 62 +++-
.../container/TestSlidingWindowRetryPolicy.java | 6 ++
3 files changed, 44 insertions(+), 28 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/03209e89/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
index c09c7f1..5527ac4 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
@@ -1602,8 +1602,10 @@ public class ContainerImpl implements Container {
}
container.addDiagnostics(exitEvent.getDiagnosticInfo() + "\n");
}
-
if (container.shouldRetry(container.exitCode)) {
+// Updates to the retry context should be protected from concurrent
+// writes. It should only be called from this transition.
+container.retryPolicy.updateRetryContext(container.windowRetryContext);
container.storeRetryContext();
doRelaunch(container,
container.windowRetryContext.getRemainingRetries(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/03209e89/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/SlidingWindowRetryPolicy.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/SlidingWindowRetryPolicy.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/SlidingWindowRetryPolicy.java
index 0208879..36a8b91 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/SlidingWindowRetryPolicy.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/SlidingWindowRetryPolicy.java
@@ -42,49 +42,40 @@ public class SlidingWindowRetryPolicy {
public boolean shouldRetry(RetryContext retryContext,
int errorCode) {
-ContainerRetryContext containerRC = retryContext
-.containerRetryContext;
+ContainerRetryContext containerRC = retryContext.containerRetryContext;
Preconditions.checkNotNull(containerRC, "container retry context null");
ContainerRetryPolicy retryPolicy = containerRC.getRetryPolicy();
if (retryPolicy == ContainerRetryPolicy.RETRY_ON_ALL_ERRORS
|| (retryPolicy == ContainerRetryPolicy.RETRY_ON_SPECIFIC_ERROR_CODES
&& containerRC.getErrorCodes() != null
&& containerRC.getErrorCodes().contains(errorCode))) {
- if (containerRC.getMaxRetries() == ContainerRetryContext.RETRY_FOREVER) {
-return true;
- }
- int pendingRetries = calculatePendingRetries(retryContext);
- updateRetryContext(retryContext, pendingRetries);
- return pendingRetries > 0;
+ return
hadoop git commit: YARN-8362. Bugfix logic in container retries in node manager. Contributed by Chandni Singh
Repository: hadoop
Updated Branches:
refs/heads/trunk 24169062e -> 135941e00
YARN-8362. Bugfix logic in container retries in node manager.
Contributed by Chandni Singh
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/135941e0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/135941e0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/135941e0
Branch: refs/heads/trunk
Commit: 135941e00d762a417c3b4cc524cdc59b0d1810b1
Parents: 2416906
Author: Eric Yang
Authored: Tue May 29 16:56:58 2018 -0400
Committer: Eric Yang
Committed: Tue May 29 16:56:58 2018 -0400
--
.../container/ContainerImpl.java| 4 +-
.../container/SlidingWindowRetryPolicy.java | 62 +++-
.../container/TestSlidingWindowRetryPolicy.java | 6 ++
3 files changed, 44 insertions(+), 28 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/135941e0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
index c09c7f1..5527ac4 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
@@ -1602,8 +1602,10 @@ public class ContainerImpl implements Container {
}
container.addDiagnostics(exitEvent.getDiagnosticInfo() + "\n");
}
-
if (container.shouldRetry(container.exitCode)) {
+// Updates to the retry context should be protected from concurrent
+// writes. It should only be called from this transition.
+container.retryPolicy.updateRetryContext(container.windowRetryContext);
container.storeRetryContext();
doRelaunch(container,
container.windowRetryContext.getRemainingRetries(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/135941e0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/SlidingWindowRetryPolicy.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/SlidingWindowRetryPolicy.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/SlidingWindowRetryPolicy.java
index 0208879..36a8b91 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/SlidingWindowRetryPolicy.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/SlidingWindowRetryPolicy.java
@@ -42,49 +42,40 @@ public class SlidingWindowRetryPolicy {
public boolean shouldRetry(RetryContext retryContext,
int errorCode) {
-ContainerRetryContext containerRC = retryContext
-.containerRetryContext;
+ContainerRetryContext containerRC = retryContext.containerRetryContext;
Preconditions.checkNotNull(containerRC, "container retry context null");
ContainerRetryPolicy retryPolicy = containerRC.getRetryPolicy();
if (retryPolicy == ContainerRetryPolicy.RETRY_ON_ALL_ERRORS
|| (retryPolicy == ContainerRetryPolicy.RETRY_ON_SPECIFIC_ERROR_CODES
&& containerRC.getErrorCodes() != null
&& containerRC.getErrorCodes().contains(errorCode))) {
- if (containerRC.getMaxRetries() == ContainerRetryContext.RETRY_FOREVER) {
-return true;
- }
- int pendingRetries = calculatePendingRetries(retryContext);
- updateRetryContext(retryContext, pendingRetries);
- return pendingRetries > 0;
+ return containerRC.getMaxRetries() == ContainerRetryContext.RETRY_FOREVER
+ ||
hadoop git commit: HDDS-114. Ozone Datanode mbean registration fails for StorageLocation. Contributed by Elek, Marton.
Repository: hadoop
Updated Branches:
refs/heads/trunk 30284d020 -> 24169062e
HDDS-114. Ozone Datanode mbean registration fails for StorageLocation.
Contributed by Elek, Marton.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/24169062
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/24169062
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/24169062
Branch: refs/heads/trunk
Commit: 24169062e5f4e7798a47c5e6e3e94504cba73092
Parents: 30284d0
Author: Anu Engineer
Authored: Tue May 29 13:23:58 2018 -0700
Committer: Anu Engineer
Committed: Tue May 29 13:48:55 2018 -0700
--
.../common/impl/StorageLocationReport.java | 52 +++-
.../ContainerLocationManagerMXBean.java | 4 +-
.../interfaces/StorageLocationReportMXBean.java | 40 +++
3 files changed, 71 insertions(+), 25 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/24169062/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java
--
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java
index 87b9656..061d09b 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java
@@ -23,6 +23,8 @@ import org.apache.hadoop.hdds.protocol.proto.
StorageContainerDatanodeProtocolProtos.StorageReportProto;
import org.apache.hadoop.hdds.protocol.proto.
StorageContainerDatanodeProtocolProtos.StorageTypeProto;
+import org.apache.hadoop.ozone.container.common.interfaces
+.StorageLocationReportMXBean;
import java.io.IOException;
@@ -30,7 +32,8 @@ import java.io.IOException;
* Storage location stats of datanodes that provide back store for containers.
*
*/
-public class StorageLocationReport {
+public final class StorageLocationReport implements
+StorageLocationReportMXBean {
private final String id;
private final boolean failed;
@@ -76,6 +79,11 @@ public class StorageLocationReport {
return storageLocation;
}
+ @Override
+ public String getStorageTypeName() {
+return storageType.name();
+ }
+
public StorageType getStorageType() {
return storageType;
}
@@ -204,76 +212,76 @@ public class StorageLocationReport {
/**
* Sets the storageId.
*
- * @param id storageId
+ * @param idValue storageId
* @return StorageLocationReport.Builder
*/
-public Builder setId(String id) {
- this.id = id;
+public Builder setId(String idValue) {
+ this.id = idValue;
return this;
}
/**
* Sets whether the volume failed or not.
*
- * @param failed whether volume failed or not
+ * @param failedValue whether volume failed or not
* @return StorageLocationReport.Builder
*/
-public Builder setFailed(boolean failed) {
- this.failed = failed;
+public Builder setFailed(boolean failedValue) {
+ this.failed = failedValue;
return this;
}
/**
* Sets the capacity of volume.
*
- * @param capacity capacity
+ * @param capacityValue capacity
* @return StorageLocationReport.Builder
*/
-public Builder setCapacity(long capacity) {
- this.capacity = capacity;
+public Builder setCapacity(long capacityValue) {
+ this.capacity = capacityValue;
return this;
}
/**
* Sets the scmUsed Value.
*
- * @param scmUsed storage space used by scm
+ * @param scmUsedValue storage space used by scm
* @return StorageLocationReport.Builder
*/
-public Builder setScmUsed(long scmUsed) {
- this.scmUsed = scmUsed;
+public Builder setScmUsed(long scmUsedValue) {
+ this.scmUsed = scmUsedValue;
return this;
}
/**
* Sets the remaining free space value.
*
- * @param remaining remaining free space
+ * @param remainingValue remaining free space
* @return StorageLocationReport.Builder
*/
-public Builder setRemaining(long remaining) {
- this.remaining = remaining;
+public Builder setRemaining(long remainingValue) {
+ this.remaining = remainingValue;
return this;
}
/**
* Sets the storageType.
*
- * @param storageType type of the storage used
+ * @param storageTypeValue type of the storage used
* @return StorageLocationReport.Builder
hadoop git commit: HADOOP-14946 S3Guard testPruneCommandCLI can fail. Contributed by Gabor Bota.
Repository: hadoop
Updated Branches:
refs/heads/trunk 201440b98 -> 30284d020
HADOOP-14946 S3Guard testPruneCommandCLI can fail. Contributed by Gabor Bota.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/30284d02
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/30284d02
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/30284d02
Branch: refs/heads/trunk
Commit: 30284d020d36c502dad5bdbae61ec48e9dfe9f8c
Parents: 201440b
Author: Aaron Fabbri
Authored: Tue May 29 13:38:15 2018 -0700
Committer: Aaron Fabbri
Committed: Tue May 29 13:38:15 2018 -0700
--
.../s3guard/AbstractS3GuardToolTestBase.java| 52 +---
1 file changed, 44 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/30284d02/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
--
diff --git
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
index 4381749..2b43810 100644
---
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
+++
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
@@ -31,6 +31,7 @@ import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.TimeUnit;
+import org.apache.hadoop.util.StopWatch;
import org.junit.Assume;
import org.junit.Test;
@@ -61,6 +62,8 @@ public abstract class AbstractS3GuardToolTestBase extends
AbstractS3ATestBase {
protected static final String S3A_THIS_BUCKET_DOES_NOT_EXIST
= "s3a://this-bucket-does-not-exist-000";
+ private static final int PRUNE_MAX_AGE_SECS = 2;
+
private MetadataStore ms;
protected static void expectResult(int expected,
@@ -186,24 +189,57 @@ public abstract class AbstractS3GuardToolTestBase extends
AbstractS3ATestBase {
}
}
+ /**
+ * Attempt to test prune() with sleep() without having flaky tests
+ * when things run slowly. Test is basically:
+ * 1. Set max path age to X seconds
+ * 2. Create some files (which writes entries to MetadataStore)
+ * 3. Sleep X+2 seconds (all files from above are now "stale")
+ * 4. Create some other files (these are "fresh").
+ * 5. Run prune on MetadataStore.
+ * 6. Assert that only files that were created before the sleep() were
pruned.
+ *
+ * Problem is: #6 can fail if X seconds elapse between steps 4 and 5, since
+ * the newer files also become stale and get pruned. This is easy to
+ * reproduce by running all integration tests in parallel with a ton of
+ * threads, or anything else that slows down execution a lot.
+ *
+ * Solution: Keep track of time elapsed between #4 and #5, and if it
+ * exceeds X, just print a warn() message instead of failing.
+ *
+ * @param cmdConf configuration for command
+ * @param parent path
+ * @param args command args
+ * @throws Exception
+ */
private void testPruneCommand(Configuration cmdConf, Path parent,
String...args) throws Exception {
Path keepParent = path("prune-cli-keep");
+StopWatch timer = new StopWatch();
try {
- getFileSystem().mkdirs(parent);
- getFileSystem().mkdirs(keepParent);
-
S3GuardTool.Prune cmd = new S3GuardTool.Prune(cmdConf);
cmd.setMetadataStore(ms);
+ getFileSystem().mkdirs(parent);
+ getFileSystem().mkdirs(keepParent);
createFile(new Path(parent, "stale"), true, true);
createFile(new Path(keepParent, "stale-to-keep"), true, true);
- Thread.sleep(TimeUnit.SECONDS.toMillis(2));
+
+ Thread.sleep(TimeUnit.SECONDS.toMillis(PRUNE_MAX_AGE_SECS + 2));
+
+ timer.start();
createFile(new Path(parent, "fresh"), true, true);
assertMetastoreListingCount(parent, "Children count before pruning", 2);
exec(cmd, args);
- assertMetastoreListingCount(parent, "Pruned children count", 1);
+ long msecElapsed = timer.now(TimeUnit.MILLISECONDS);
+ if (msecElapsed >= PRUNE_MAX_AGE_SECS * 1000) {
+LOG.warn("Skipping an assertion: Test running too slowly ({} msec)",
+msecElapsed);
+ } else {
+assertMetastoreListingCount(parent, "Pruned children count remaining",
+1);
+ }
assertMetastoreListingCount(keepParent,
"This child should have been kept (prefix restriction).", 1);
} finally {
@@ -224,13 +260,14 @@ public abstract class AbstractS3GuardToolTestBase extends
AbstractS3ATestBase {
public void testPruneCommandCLI() throws Exception {
hadoop git commit: Additional check when unpacking archives. Contributed by Wilfred Spiegelenburg.
Repository: hadoop
Updated Branches:
refs/heads/branch-3.1 a1fd04c4f -> 11a425d11
Additional check when unpacking archives. Contributed by Wilfred Spiegelenburg.
(cherry picked from commit e3236a9680709de7a95ffbc11b20e1bdc95a8605)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/11a425d1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/11a425d1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/11a425d1
Branch: refs/heads/branch-3.1
Commit: 11a425d11a329010d0ff8255ecbcd1eb51b642e3
Parents: a1fd04c
Author: Kihwal Lee
Authored: Tue May 29 15:02:33 2018 -0500
Committer: Kihwal Lee
Committed: Tue May 29 15:02:33 2018 -0500
--
.../java/org/apache/hadoop/util/RunJar.java | 10 +
.../java/org/apache/hadoop/util/TestRunJar.java | 46 +++-
2 files changed, 55 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/11a425d1/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
index 9dd770c..239d464 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
@@ -113,12 +113,17 @@ public class RunJar {
throws IOException {
try (JarInputStream jar = new JarInputStream(inputStream)) {
int numOfFailedLastModifiedSet = 0;
+ String targetDirPath = toDir.getCanonicalPath() + File.separator;
for (JarEntry entry = jar.getNextJarEntry();
entry != null;
entry = jar.getNextJarEntry()) {
if (!entry.isDirectory() &&
unpackRegex.matcher(entry.getName()).matches()) {
File file = new File(toDir, entry.getName());
+ if (!file.getCanonicalPath().startsWith(targetDirPath)) {
+throw new IOException("expanding " + entry.getName()
++ " would create file outside of " + toDir);
+ }
ensureDirectory(file.getParentFile());
try (OutputStream out = new FileOutputStream(file)) {
IOUtils.copyBytes(jar, out, BUFFER_SIZE);
@@ -178,6 +183,7 @@ public class RunJar {
throws IOException {
try (JarFile jar = new JarFile(jarFile)) {
int numOfFailedLastModifiedSet = 0;
+ String targetDirPath = toDir.getCanonicalPath() + File.separator;
Enumeration entries = jar.entries();
while (entries.hasMoreElements()) {
final JarEntry entry = entries.nextElement();
@@ -185,6 +191,10 @@ public class RunJar {
unpackRegex.matcher(entry.getName()).matches()) {
try (InputStream in = jar.getInputStream(entry)) {
File file = new File(toDir, entry.getName());
+if (!file.getCanonicalPath().startsWith(targetDirPath)) {
+ throw new IOException("expanding " + entry.getName()
+ + " would create file outside of " + toDir);
+}
ensureDirectory(file.getParentFile());
try (OutputStream out = new FileOutputStream(file)) {
IOUtils.copyBytes(in, out, BUFFER_SIZE);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/11a425d1/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java
index 19485d6..237751c 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java
@@ -17,9 +17,12 @@
*/
package org.apache.hadoop.util;
+import static org.apache.hadoop.util.RunJar.MATCH_ANY;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
@@ -28,6 +31,7 @@ import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
import java.util.Random;
import java.util.jar.JarEntry;
import java.util.jar.JarOutputStream;
@@ -222,4 +226,44 @@ public class TestRunJar {
runJar.run(args);
// it
hadoop git commit: YARN-8329. Docker client configuration can still be set incorrectly. Contributed by Shane Kumpf
Repository: hadoop
Updated Branches:
refs/heads/branch-3.1 3eb1cb18c -> a1fd04c4f
YARN-8329. Docker client configuration can still be set incorrectly.
Contributed by Shane Kumpf
(cherry picked from commit 4827e9a9085b306bc379cb6e0b1fe4b92326edcd)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a1fd04c4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a1fd04c4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a1fd04c4
Branch: refs/heads/branch-3.1
Commit: a1fd04c4f472fdc0835f491a719220684ee1f255
Parents: 3eb1cb1
Author: Jason Lowe
Authored: Tue May 29 14:43:17 2018 -0500
Committer: Jason Lowe
Committed: Tue May 29 14:48:01 2018 -0500
--
.../yarn/util/DockerClientConfigHandler.java| 23 +++-
.../security/TestDockerClientConfigHandler.java | 4 ++--
.../runtime/DockerLinuxContainerRuntime.java| 7 +++---
3 files changed, 19 insertions(+), 15 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1fd04c4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/DockerClientConfigHandler.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/DockerClientConfigHandler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/DockerClientConfigHandler.java
index 5522cf4..8ec4deb 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/DockerClientConfigHandler.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/DockerClientConfigHandler.java
@@ -154,14 +154,15 @@ public final class DockerClientConfigHandler {
* @param outConfigFile the File to write the Docker client configuration to.
* @param credentials the populated Credentials object.
* @throws IOException if the write fails.
+ * @return true if a Docker credential is found in the supplied credentials.
*/
- public static void writeDockerCredentialsToPath(File outConfigFile,
+ public static boolean writeDockerCredentialsToPath(File outConfigFile,
Credentials credentials) throws IOException {
-ObjectMapper mapper = new ObjectMapper();
-ObjectNode rootNode = mapper.createObjectNode();
-ObjectNode registryUrlNode = mapper.createObjectNode();
boolean foundDockerCred = false;
if (credentials.numberOfTokens() > 0) {
+ ObjectMapper mapper = new ObjectMapper();
+ ObjectNode rootNode = mapper.createObjectNode();
+ ObjectNode registryUrlNode = mapper.createObjectNode();
for (Token tk : credentials.getAllTokens()) {
if (tk.getKind().equals(DockerCredentialTokenIdentifier.KIND)) {
foundDockerCred = true;
@@ -176,12 +177,14 @@ public final class DockerClientConfigHandler {
}
}
}
+ if (foundDockerCred) {
+rootNode.put(CONFIG_AUTHS_KEY, registryUrlNode);
+String json = mapper.writerWithDefaultPrettyPrinter()
+.writeValueAsString(rootNode);
+FileUtils.writeStringToFile(
+outConfigFile, json, StandardCharsets.UTF_8);
+ }
}
-if (foundDockerCred) {
- rootNode.put(CONFIG_AUTHS_KEY, registryUrlNode);
- String json =
- mapper.writerWithDefaultPrettyPrinter().writeValueAsString(rootNode);
- FileUtils.writeStringToFile(outConfigFile, json, StandardCharsets.UTF_8);
-}
+return foundDockerCred;
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1fd04c4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/security/TestDockerClientConfigHandler.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/security/TestDockerClientConfigHandler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/security/TestDockerClientConfigHandler.java
index c4cbe45..cfe5a45 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/security/TestDockerClientConfigHandler.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/security/TestDockerClientConfigHandler.java
@@ -116,8 +116,8 @@ public class TestDockerClientConfigHandler {
Credentials credentials =
DockerClientConfigHandler.readCredentialsFromConfigFile(
new Path(file.toURI()), conf, APPLICATION_ID);
-DockerClientConfigHandler.writeDockerCredentialsToPath(outFile,
-credentials);
+
hadoop git commit: Additional check when unpacking archives. Contributed by Wilfred Spiegelenburg.
Repository: hadoop
Updated Branches:
refs/heads/branch-2.7 5b57f9cae -> eaa2b8035
Additional check when unpacking archives. Contributed by Wilfred Spiegelenburg.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eaa2b803
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eaa2b803
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eaa2b803
Branch: refs/heads/branch-2.7
Commit: eaa2b8035b584dfcf7c79a33484eb2dffd3fdb11
Parents: 5b57f9c
Author: Kihwal Lee
Authored: Tue May 29 14:47:55 2018 -0500
Committer: Kihwal Lee
Committed: Tue May 29 14:48:46 2018 -0500
--
.../java/org/apache/hadoop/util/RunJar.java | 5 +++
.../java/org/apache/hadoop/util/TestRunJar.java | 39 +++-
2 files changed, 43 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eaa2b803/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
index 4b26b76..a3b5b0b 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
@@ -93,6 +93,7 @@ public class RunJar {
throws IOException {
JarFile jar = new JarFile(jarFile);
try {
+ String targetDirPath = toDir.getCanonicalPath() + File.separator;
Enumeration entries = jar.entries();
while (entries.hasMoreElements()) {
final JarEntry entry = entries.nextElement();
@@ -102,6 +103,10 @@ public class RunJar {
try {
File file = new File(toDir, entry.getName());
ensureDirectory(file.getParentFile());
+if (!file.getCanonicalPath().startsWith(targetDirPath)) {
+ throw new IOException("expanding " + entry.getName()
+ + " would create file outside of " + toDir);
+}
OutputStream out = new FileOutputStream(file);
try {
IOUtils.copyBytes(in, out, 8192);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eaa2b803/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java
index f592d04..b2a6537 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.util;
+import static org.junit.Assert.fail;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
@@ -25,6 +26,8 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.jar.JarEntry;
import java.util.jar.JarOutputStream;
import java.util.regex.Pattern;
import java.util.zip.ZipEntry;
@@ -32,6 +35,7 @@ import java.util.zip.ZipEntry;
import junit.framework.TestCase;
import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.test.GenericTestUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -169,4 +173,37 @@ public class TestRunJar extends TestCase {
return jarFile;
}
-}
\ No newline at end of file
+
+ @Test
+ public void testUnJar2() throws IOException {
+// make a simple zip
+File jarFile = new File(TEST_ROOT_DIR, TEST_JAR_NAME);
+JarOutputStream jstream =
+new JarOutputStream(new FileOutputStream(jarFile));
+JarEntry je = new JarEntry("META-INF/MANIFEST.MF");
+byte[] data = "Manifest-Version: 1.0\nCreated-By: 1.8.0_1 (Manual)"
+.getBytes(StandardCharsets.UTF_8);
+je.setSize(data.length);
+jstream.putNextEntry(je);
+jstream.write(data);
+jstream.closeEntry();
+je = new JarEntry("../outside.path");
+data = "any data here".getBytes(StandardCharsets.UTF_8);
+je.setSize(data.length);
+jstream.putNextEntry(je);
+jstream.write(data);
+jstream.closeEntry();
+jstream.close();
+
+File unjarDir = new File(TEST_ROOT_DIR, "unjar-path");
+
+// Unjar everything
+try {
+ RunJar.unJar(jarFile, unjarDir);
+ fail("unJar should throw IOException.");
+} catch (IOException e) {
+ GenericTestUtils.assertExceptionContains(
+
[1/3] hadoop git commit: HDDS-81. Moving ContainerReport inside Datanode heartbeat. Contributed by Nanda Kumar.
Repository: hadoop
Updated Branches:
refs/heads/trunk 4827e9a90 -> 201440b98
http://git-wip-us.apache.org/repos/asf/hadoop/blob/201440b9/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationDatanodeStateManager.java
--
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationDatanodeStateManager.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationDatanodeStateManager.java
deleted file mode 100644
index 50fd18f..000
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationDatanodeStateManager.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations
under
- * the License.
- */
-package org.apache.hadoop.ozone.container.testutils;
-
-import com.google.common.primitives.Longs;
-import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.hadoop.hdds.scm.node.NodeManager;
-import org.apache.hadoop.hdds.scm.node.NodePoolManager;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto
-.StorageContainerDatanodeProtocolProtos.ContainerInfo;
-import org.apache.hadoop.hdds.protocol.proto
-.StorageContainerDatanodeProtocolProtos.ContainerReportsRequestProto;
-
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Random;
-
-import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState
-.HEALTHY;
-
-/**
- * This class manages the state of datanode
- * in conjunction with the node pool and node managers.
- */
-public class ReplicationDatanodeStateManager {
- private final NodeManager nodeManager;
- private final NodePoolManager poolManager;
- private final Random r;
-
- /**
- * The datanode state Manager.
- *
- * @param nodeManager
- * @param poolManager
- */
- public ReplicationDatanodeStateManager(NodeManager nodeManager,
- NodePoolManager poolManager) {
-this.nodeManager = nodeManager;
-this.poolManager = poolManager;
-r = new Random();
- }
-
- /**
- * Get Container Report as if it is from a datanode in the cluster.
- * @param containerID - Container ID.
- * @param poolName - Pool Name.
- * @param dataNodeCount - Datanode Count.
- * @return List of Container Reports.
- */
- public List getContainerReport(
- long containerID, String poolName, int dataNodeCount) {
-List containerList = new LinkedList<>();
-List nodesInPool = poolManager.getNodes(poolName);
-
-if (nodesInPool == null) {
- return containerList;
-}
-
-if (nodesInPool.size() < dataNodeCount) {
- throw new IllegalStateException("Not enough datanodes to create " +
- "required container reports");
-}
-
-while (containerList.size() < dataNodeCount && nodesInPool.size() > 0) {
- DatanodeDetails id = nodesInPool.get(r.nextInt(nodesInPool.size()));
- nodesInPool.remove(id);
- containerID++;
- // We return container reports only for nodes that are healthy.
- if (nodeManager.getNodeState(id) == HEALTHY) {
-ContainerInfo info = ContainerInfo.newBuilder()
-.setContainerID(containerID)
-.setFinalhash(DigestUtils.sha256Hex(
-Longs.toByteArray(containerID)))
-.setContainerID(containerID)
-.build();
-ContainerReportsRequestProto containerReport =
-ContainerReportsRequestProto.newBuilder().addReports(info)
-.setDatanodeDetails(id.getProtoBufMessage())
-.setType(ContainerReportsRequestProto.reportType.fullReport)
-.build();
-containerList.add(containerReport);
- }
-}
-return containerList;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/201440b9/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
--
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
[2/3] hadoop git commit: HDDS-81. Moving ContainerReport inside Datanode heartbeat. Contributed by Nanda Kumar.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/201440b9/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
--
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
index 2d88621..f5fe46a 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
@@ -20,6 +20,7 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.primitives.Longs;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.container.closer.ContainerCloser;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
@@ -33,7 +34,7 @@ import
org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos;
import org.apache.hadoop.hdds.protocol.proto
-.StorageContainerDatanodeProtocolProtos.ContainerReportsRequestProto;
+.StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.lease.Lease;
import org.apache.hadoop.ozone.lease.LeaseException;
@@ -368,11 +369,12 @@ public class ContainerMapping implements Mapping {
* @param reports Container report
*/
@Override
- public void processContainerReports(ContainerReportsRequestProto reports)
+ public void processContainerReports(DatanodeDetails datanodeDetails,
+ ContainerReportsProto reports)
throws IOException {
List
containerInfos = reports.getReportsList();
-containerSupervisor.handleContainerReport(reports);
+containerSupervisor.handleContainerReport(datanodeDetails, reports);
for (StorageContainerDatanodeProtocolProtos.ContainerInfo datanodeState :
containerInfos) {
byte[] dbKey = Longs.toByteArray(datanodeState.getContainerID());
@@ -402,7 +404,7 @@ public class ContainerMapping implements Mapping {
// Container not found in our container db.
LOG.error("Error while processing container report from datanode :" +
" {}, for container: {}, reason: container doesn't exist in"
+
- "container database.", reports.getDatanodeDetails(),
+ "container database.", datanodeDetails,
datanodeState.getContainerID());
}
} finally {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/201440b9/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/Mapping.java
--
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/Mapping.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/Mapping.java
index f560174..ee8e344 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/Mapping.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/Mapping.java
@@ -16,10 +16,11 @@
*/
package org.apache.hadoop.hdds.scm.container;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto
-.StorageContainerDatanodeProtocolProtos.ContainerReportsRequestProto;
+.StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
import java.io.Closeable;
import java.io.IOException;
@@ -98,7 +99,8 @@ public interface Mapping extends Closeable {
*
* @param reports Container report
*/
- void processContainerReports(ContainerReportsRequestProto reports)
+ void processContainerReports(DatanodeDetails datanodeDetails,
+ ContainerReportsProto reports)
throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/201440b9/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerSupervisor.java
--
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerSupervisor.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerSupervisor.java
index c14303f..5bd0574 100644
---
hadoop git commit: YARN-8329. Docker client configuration can still be set incorrectly. Contributed by Shane Kumpf
Repository: hadoop
Updated Branches:
refs/heads/trunk e3236a968 -> 4827e9a90
YARN-8329. Docker client configuration can still be set incorrectly.
Contributed by Shane Kumpf
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4827e9a9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4827e9a9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4827e9a9
Branch: refs/heads/trunk
Commit: 4827e9a9085b306bc379cb6e0b1fe4b92326edcd
Parents: e3236a9
Author: Jason Lowe
Authored: Tue May 29 14:43:17 2018 -0500
Committer: Jason Lowe
Committed: Tue May 29 14:43:17 2018 -0500
--
.../yarn/util/DockerClientConfigHandler.java| 23 +++-
.../security/TestDockerClientConfigHandler.java | 4 ++--
.../runtime/DockerLinuxContainerRuntime.java| 7 +++---
3 files changed, 19 insertions(+), 15 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4827e9a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/DockerClientConfigHandler.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/DockerClientConfigHandler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/DockerClientConfigHandler.java
index 5522cf4..8ec4deb 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/DockerClientConfigHandler.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/DockerClientConfigHandler.java
@@ -154,14 +154,15 @@ public final class DockerClientConfigHandler {
* @param outConfigFile the File to write the Docker client configuration to.
* @param credentials the populated Credentials object.
* @throws IOException if the write fails.
+ * @return true if a Docker credential is found in the supplied credentials.
*/
- public static void writeDockerCredentialsToPath(File outConfigFile,
+ public static boolean writeDockerCredentialsToPath(File outConfigFile,
Credentials credentials) throws IOException {
-ObjectMapper mapper = new ObjectMapper();
-ObjectNode rootNode = mapper.createObjectNode();
-ObjectNode registryUrlNode = mapper.createObjectNode();
boolean foundDockerCred = false;
if (credentials.numberOfTokens() > 0) {
+ ObjectMapper mapper = new ObjectMapper();
+ ObjectNode rootNode = mapper.createObjectNode();
+ ObjectNode registryUrlNode = mapper.createObjectNode();
for (Token tk : credentials.getAllTokens()) {
if (tk.getKind().equals(DockerCredentialTokenIdentifier.KIND)) {
foundDockerCred = true;
@@ -176,12 +177,14 @@ public final class DockerClientConfigHandler {
}
}
}
+ if (foundDockerCred) {
+rootNode.put(CONFIG_AUTHS_KEY, registryUrlNode);
+String json = mapper.writerWithDefaultPrettyPrinter()
+.writeValueAsString(rootNode);
+FileUtils.writeStringToFile(
+outConfigFile, json, StandardCharsets.UTF_8);
+ }
}
-if (foundDockerCred) {
- rootNode.put(CONFIG_AUTHS_KEY, registryUrlNode);
- String json =
- mapper.writerWithDefaultPrettyPrinter().writeValueAsString(rootNode);
- FileUtils.writeStringToFile(outConfigFile, json, StandardCharsets.UTF_8);
-}
+return foundDockerCred;
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4827e9a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/security/TestDockerClientConfigHandler.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/security/TestDockerClientConfigHandler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/security/TestDockerClientConfigHandler.java
index c4cbe45..cfe5a45 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/security/TestDockerClientConfigHandler.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/security/TestDockerClientConfigHandler.java
@@ -116,8 +116,8 @@ public class TestDockerClientConfigHandler {
Credentials credentials =
DockerClientConfigHandler.readCredentialsFromConfigFile(
new Path(file.toURI()), conf, APPLICATION_ID);
-DockerClientConfigHandler.writeDockerCredentialsToPath(outFile,
-credentials);
+assertTrue(DockerClientConfigHandler.writeDockerCredentialsToPath(outFile,
+
hadoop git commit: dditional check when unpacking archives. Contributed by Wilfred Spiegelenburg.
Repository: hadoop
Updated Branches:
refs/heads/branch-2.8 74c7024cc -> 3808e5d62
dditional check when unpacking archives. Contributed by Wilfred Spiegelenburg.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3808e5d6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3808e5d6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3808e5d6
Branch: refs/heads/branch-2.8
Commit: 3808e5d62aa21d7393d98fbc9d54b9ad1e79ab99
Parents: 74c7024
Author: Kihwal Lee
Authored: Tue May 29 14:39:53 2018 -0500
Committer: Kihwal Lee
Committed: Tue May 29 14:39:53 2018 -0500
--
.../java/org/apache/hadoop/util/RunJar.java | 5 +++
.../java/org/apache/hadoop/util/TestRunJar.java | 38 +++-
2 files changed, 42 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3808e5d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
index 52cf05c..a56f6ea 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
@@ -98,6 +98,7 @@ public class RunJar {
JarFile jar = new JarFile(jarFile);
try {
int numOfFailedLastModifiedSet = 0;
+ String targetDirPath = toDir.getCanonicalPath() + File.separator;
Enumeration entries = jar.entries();
while (entries.hasMoreElements()) {
final JarEntry entry = entries.nextElement();
@@ -107,6 +108,10 @@ public class RunJar {
try {
File file = new File(toDir, entry.getName());
ensureDirectory(file.getParentFile());
+if (!file.getCanonicalPath().startsWith(targetDirPath)) {
+ throw new IOException("expanding " + entry.getName()
+ + " would create file outside of " + toDir);
+}
OutputStream out = new FileOutputStream(file);
try {
IOUtils.copyBytes(in, out, 8192);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3808e5d6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java
index 7262534..20650c0 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.util;
+import static org.junit.Assert.fail;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
@@ -25,6 +26,8 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.jar.JarEntry;
import java.util.jar.JarOutputStream;
import java.util.regex.Pattern;
import java.util.zip.ZipEntry;
@@ -186,4 +189,37 @@ public class TestRunJar extends TestCase {
return jarFile;
}
-}
\ No newline at end of file
+
+ @Test
+ public void testUnJar2() throws IOException {
+// make a simple zip
+File jarFile = new File(TEST_ROOT_DIR, TEST_JAR_NAME);
+JarOutputStream jstream =
+new JarOutputStream(new FileOutputStream(jarFile));
+JarEntry je = new JarEntry("META-INF/MANIFEST.MF");
+byte[] data = "Manifest-Version: 1.0\nCreated-By: 1.8.0_1 (Manual)"
+.getBytes(StandardCharsets.UTF_8);
+je.setSize(data.length);
+jstream.putNextEntry(je);
+jstream.write(data);
+jstream.closeEntry();
+je = new JarEntry("../outside.path");
+data = "any data here".getBytes(StandardCharsets.UTF_8);
+je.setSize(data.length);
+jstream.putNextEntry(je);
+jstream.write(data);
+jstream.closeEntry();
+jstream.close();
+
+File unjarDir = new File(TEST_ROOT_DIR, "unjar-path");
+
+// Unjar everything
+try {
+ RunJar.unJar(jarFile, unjarDir);
+ fail("unJar should throw IOException.");
+} catch (IOException e) {
+ GenericTestUtils.assertExceptionContains(
+ "would create file outside of", e);
+}
+ }
+}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail:
hadoop git commit: Additional check when unpacking archives. Contributed by Wilfred Spiegelenburg.
Repository: hadoop
Updated Branches:
refs/heads/branch-2.9 c3dce2620 -> 6a4ae6f6e
Additional check when unpacking archives. Contributed by Wilfred Spiegelenburg.
(cherry picked from commit 65e55097da2bb3f2fbdf9ba1946da25fe58bec98)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6a4ae6f6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6a4ae6f6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6a4ae6f6
Branch: refs/heads/branch-2.9
Commit: 6a4ae6f6eeed1392a4828a5721fa1499f65bdde4
Parents: c3dce26
Author: Kihwal Lee
Authored: Tue May 29 14:35:28 2018 -0500
Committer: Kihwal Lee
Committed: Tue May 29 14:35:28 2018 -0500
--
.../java/org/apache/hadoop/util/RunJar.java | 5 +++
.../java/org/apache/hadoop/util/TestRunJar.java | 36
2 files changed, 41 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a4ae6f6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
index 19b51ad..678e459 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
@@ -109,6 +109,7 @@ public class RunJar {
throws IOException {
try (JarFile jar = new JarFile(jarFile)) {
int numOfFailedLastModifiedSet = 0;
+ String targetDirPath = toDir.getCanonicalPath() + File.separator;
Enumeration entries = jar.entries();
while (entries.hasMoreElements()) {
final JarEntry entry = entries.nextElement();
@@ -117,6 +118,10 @@ public class RunJar {
try (InputStream in = jar.getInputStream(entry)) {
File file = new File(toDir, entry.getName());
ensureDirectory(file.getParentFile());
+if (!file.getCanonicalPath().startsWith(targetDirPath)) {
+ throw new IOException("expanding " + entry.getName()
+ + " would create file outside of " + toDir);
+}
try (OutputStream out = new FileOutputStream(file)) {
IOUtils.copyBytes(in, out, BUFFER_SIZE);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a4ae6f6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java
index 7b61b32..cb2cfa8 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java
@@ -20,12 +20,15 @@ package org.apache.hadoop.util;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.jar.JarEntry;
import java.util.jar.JarOutputStream;
import java.util.regex.Pattern;
import java.util.zip.ZipEntry;
@@ -165,4 +168,37 @@ public class TestRunJar {
runJar.run(args);
// it should not throw an exception
}
+
+ @Test
+ public void testUnJar2() throws IOException {
+// make a simple zip
+File jarFile = new File(TEST_ROOT_DIR, TEST_JAR_NAME);
+JarOutputStream jstream =
+new JarOutputStream(new FileOutputStream(jarFile));
+JarEntry je = new JarEntry("META-INF/MANIFEST.MF");
+byte[] data = "Manifest-Version: 1.0\nCreated-By: 1.8.0_1 (Manual)"
+.getBytes(StandardCharsets.UTF_8);
+je.setSize(data.length);
+jstream.putNextEntry(je);
+jstream.write(data);
+jstream.closeEntry();
+je = new JarEntry("../outside.path");
+data = "any data here".getBytes(StandardCharsets.UTF_8);
+je.setSize(data.length);
+jstream.putNextEntry(je);
+jstream.write(data);
+jstream.closeEntry();
+jstream.close();
+
+File unjarDir = getUnjarDir("unjar-path");
+
+// Unjar everything
+try {
+ RunJar.unJar(jarFile, unjarDir);
+ fail("unJar should throw IOException.");
+} catch (IOException e) {
+ GenericTestUtils.assertExceptionContains(
+ "would create file outside of", e);
hadoop git commit: Additional check when unpacking archives. Contributed by Wilfred Spiegelenburg.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 09fbbff69 -> 6d7d192e4
Additional check when unpacking archives. Contributed by Wilfred Spiegelenburg.
(cherry picked from commit 65e55097da2bb3f2fbdf9ba1946da25fe58bec98)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6d7d192e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6d7d192e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6d7d192e
Branch: refs/heads/branch-2
Commit: 6d7d192e4799b51931e55217e02baec14d49607b
Parents: 09fbbff
Author: Kihwal Lee
Authored: Tue May 29 14:32:58 2018 -0500
Committer: Kihwal Lee
Committed: Tue May 29 14:33:31 2018 -0500
--
.../java/org/apache/hadoop/util/RunJar.java | 5 +++
.../java/org/apache/hadoop/util/TestRunJar.java | 36
2 files changed, 41 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d7d192e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
index 19b51ad..678e459 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
@@ -109,6 +109,7 @@ public class RunJar {
throws IOException {
try (JarFile jar = new JarFile(jarFile)) {
int numOfFailedLastModifiedSet = 0;
+ String targetDirPath = toDir.getCanonicalPath() + File.separator;
Enumeration entries = jar.entries();
while (entries.hasMoreElements()) {
final JarEntry entry = entries.nextElement();
@@ -117,6 +118,10 @@ public class RunJar {
try (InputStream in = jar.getInputStream(entry)) {
File file = new File(toDir, entry.getName());
ensureDirectory(file.getParentFile());
+if (!file.getCanonicalPath().startsWith(targetDirPath)) {
+ throw new IOException("expanding " + entry.getName()
+ + " would create file outside of " + toDir);
+}
try (OutputStream out = new FileOutputStream(file)) {
IOUtils.copyBytes(in, out, BUFFER_SIZE);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d7d192e/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java
index 7b61b32..cb2cfa8 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java
@@ -20,12 +20,15 @@ package org.apache.hadoop.util;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.jar.JarEntry;
import java.util.jar.JarOutputStream;
import java.util.regex.Pattern;
import java.util.zip.ZipEntry;
@@ -165,4 +168,37 @@ public class TestRunJar {
runJar.run(args);
// it should not throw an exception
}
+
+ @Test
+ public void testUnJar2() throws IOException {
+// make a simple zip
+File jarFile = new File(TEST_ROOT_DIR, TEST_JAR_NAME);
+JarOutputStream jstream =
+new JarOutputStream(new FileOutputStream(jarFile));
+JarEntry je = new JarEntry("META-INF/MANIFEST.MF");
+byte[] data = "Manifest-Version: 1.0\nCreated-By: 1.8.0_1 (Manual)"
+.getBytes(StandardCharsets.UTF_8);
+je.setSize(data.length);
+jstream.putNextEntry(je);
+jstream.write(data);
+jstream.closeEntry();
+je = new JarEntry("../outside.path");
+data = "any data here".getBytes(StandardCharsets.UTF_8);
+je.setSize(data.length);
+jstream.putNextEntry(je);
+jstream.write(data);
+jstream.closeEntry();
+jstream.close();
+
+File unjarDir = getUnjarDir("unjar-path");
+
+// Unjar everything
+try {
+ RunJar.unJar(jarFile, unjarDir);
+ fail("unJar should throw IOException.");
+} catch (IOException e) {
+ GenericTestUtils.assertExceptionContains(
+ "would create file outside of", e);
+
hadoop git commit: Additional check when unpacking archives. Contributed by Wilfred Spiegelenburg.
Repository: hadoop
Updated Branches:
refs/heads/branch-3.0 d5708bbcd -> 65e55097d
Additional check when unpacking archives. Contributed by Wilfred Spiegelenburg.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/65e55097
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/65e55097
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/65e55097
Branch: refs/heads/branch-3.0
Commit: 65e55097da2bb3f2fbdf9ba1946da25fe58bec98
Parents: d5708bb
Author: Kihwal Lee
Authored: Tue May 29 14:30:29 2018 -0500
Committer: Kihwal Lee
Committed: Tue May 29 14:30:29 2018 -0500
--
.../java/org/apache/hadoop/util/RunJar.java | 5 +++
.../java/org/apache/hadoop/util/TestRunJar.java | 36
2 files changed, 41 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/65e55097/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
index 19b51ad..678e459 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
@@ -109,6 +109,7 @@ public class RunJar {
throws IOException {
try (JarFile jar = new JarFile(jarFile)) {
int numOfFailedLastModifiedSet = 0;
+ String targetDirPath = toDir.getCanonicalPath() + File.separator;
Enumeration entries = jar.entries();
while (entries.hasMoreElements()) {
final JarEntry entry = entries.nextElement();
@@ -117,6 +118,10 @@ public class RunJar {
try (InputStream in = jar.getInputStream(entry)) {
File file = new File(toDir, entry.getName());
ensureDirectory(file.getParentFile());
+if (!file.getCanonicalPath().startsWith(targetDirPath)) {
+ throw new IOException("expanding " + entry.getName()
+ + " would create file outside of " + toDir);
+}
try (OutputStream out = new FileOutputStream(file)) {
IOUtils.copyBytes(in, out, BUFFER_SIZE);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/65e55097/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java
index 7b61b32..cb2cfa8 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java
@@ -20,12 +20,15 @@ package org.apache.hadoop.util;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.jar.JarEntry;
import java.util.jar.JarOutputStream;
import java.util.regex.Pattern;
import java.util.zip.ZipEntry;
@@ -165,4 +168,37 @@ public class TestRunJar {
runJar.run(args);
// it should not throw an exception
}
+
+ @Test
+ public void testUnJar2() throws IOException {
+// make a simple zip
+File jarFile = new File(TEST_ROOT_DIR, TEST_JAR_NAME);
+JarOutputStream jstream =
+new JarOutputStream(new FileOutputStream(jarFile));
+JarEntry je = new JarEntry("META-INF/MANIFEST.MF");
+byte[] data = "Manifest-Version: 1.0\nCreated-By: 1.8.0_1 (Manual)"
+.getBytes(StandardCharsets.UTF_8);
+je.setSize(data.length);
+jstream.putNextEntry(je);
+jstream.write(data);
+jstream.closeEntry();
+je = new JarEntry("../outside.path");
+data = "any data here".getBytes(StandardCharsets.UTF_8);
+je.setSize(data.length);
+jstream.putNextEntry(je);
+jstream.write(data);
+jstream.closeEntry();
+jstream.close();
+
+File unjarDir = getUnjarDir("unjar-path");
+
+// Unjar everything
+try {
+ RunJar.unJar(jarFile, unjarDir);
+ fail("unJar should throw IOException.");
+} catch (IOException e) {
+ GenericTestUtils.assertExceptionContains(
+ "would create file outside of", e);
+}
+ }
}
\ No newline at end of file
hadoop git commit: Additional check when unpacking archives. Contributed by Wilfred Spiegelenburg.
Repository: hadoop
Updated Branches:
refs/heads/trunk 9502b47bd -> e3236a968
Additional check when unpacking archives. Contributed by Wilfred Spiegelenburg.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e3236a96
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e3236a96
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e3236a96
Branch: refs/heads/trunk
Commit: e3236a9680709de7a95ffbc11b20e1bdc95a8605
Parents: 9502b47
Author: Kihwal Lee
Authored: Tue May 29 14:15:12 2018 -0500
Committer: Kihwal Lee
Committed: Tue May 29 14:15:12 2018 -0500
--
.../java/org/apache/hadoop/util/RunJar.java | 10 +
.../java/org/apache/hadoop/util/TestRunJar.java | 42
2 files changed, 52 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3236a96/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
index f1b643c..4c94dbc 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
@@ -117,12 +117,17 @@ public class RunJar {
throws IOException {
try (JarInputStream jar = new JarInputStream(inputStream)) {
int numOfFailedLastModifiedSet = 0;
+ String targetDirPath = toDir.getCanonicalPath() + File.separator;
for (JarEntry entry = jar.getNextJarEntry();
entry != null;
entry = jar.getNextJarEntry()) {
if (!entry.isDirectory() &&
unpackRegex.matcher(entry.getName()).matches()) {
File file = new File(toDir, entry.getName());
+ if (!file.getCanonicalPath().startsWith(targetDirPath)) {
+throw new IOException("expanding " + entry.getName()
++ " would create file outside of " + toDir);
+ }
ensureDirectory(file.getParentFile());
try (OutputStream out = new FileOutputStream(file)) {
IOUtils.copyBytes(jar, out, BUFFER_SIZE);
@@ -182,6 +187,7 @@ public class RunJar {
throws IOException {
try (JarFile jar = new JarFile(jarFile)) {
int numOfFailedLastModifiedSet = 0;
+ String targetDirPath = toDir.getCanonicalPath() + File.separator;
Enumeration entries = jar.entries();
while (entries.hasMoreElements()) {
final JarEntry entry = entries.nextElement();
@@ -189,6 +195,10 @@ public class RunJar {
unpackRegex.matcher(entry.getName()).matches()) {
try (InputStream in = jar.getInputStream(entry)) {
File file = new File(toDir, entry.getName());
+if (!file.getCanonicalPath().startsWith(targetDirPath)) {
+ throw new IOException("expanding " + entry.getName()
+ + " would create file outside of " + toDir);
+}
ensureDirectory(file.getParentFile());
try (OutputStream out = new FileOutputStream(file)) {
IOUtils.copyBytes(in, out, BUFFER_SIZE);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3236a96/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java
index ea07b97..a8c27d4 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java
@@ -21,6 +21,7 @@ import static org.apache.hadoop.util.RunJar.MATCH_ANY;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
@@ -32,6 +33,7 @@ import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
import java.util.Random;
import java.util.jar.JarEntry;
import java.util.jar.JarOutputStream;
@@ -255,4 +257,44 @@ public class TestRunJar {
// it should not throw an exception
verify(runJar, times(0)).unJar(any(File.class), any(File.class));
}
+
+ @Test
+ public void testUnJar2() throws IOException
[22/50] [abbrv] hadoop git commit: HADOOP-15494. TestRawLocalFileSystemContract fails on Windows. Contributed by Anbang Hu.
HADOOP-15494. TestRawLocalFileSystemContract fails on Windows.
Contributed by Anbang Hu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bddfe796
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bddfe796
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bddfe796
Branch: refs/heads/YARN-7402
Commit: bddfe796f2f992fc1dcc8a1dd44d64ff2b3c9cf4
Parents: 86bc642
Author: Steve Loughran
Authored: Fri May 25 11:12:47 2018 +0100
Committer: Steve Loughran
Committed: Fri May 25 11:12:47 2018 +0100
--
.../java/org/apache/hadoop/fs/TestRawLocalFileSystemContract.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bddfe796/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestRawLocalFileSystemContract.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestRawLocalFileSystemContract.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestRawLocalFileSystemContract.java
index ebf9ea7..908e330 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestRawLocalFileSystemContract.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestRawLocalFileSystemContract.java
@@ -42,7 +42,7 @@ public class TestRawLocalFileSystemContract extends
FileSystemContractBaseTest {
private static final Logger LOG =
LoggerFactory.getLogger(TestRawLocalFileSystemContract.class);
private final static Path TEST_BASE_DIR =
- new Path(GenericTestUtils.getTempPath(""));
+ new Path(GenericTestUtils.getRandomizedTestDir().getAbsolutePath());
@Before
public void setUp() throws Exception {
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[01/50] [abbrv] hadoop git commit: HADOOP-15457. Add Security-Related HTTP Response Header in WEBUIs. (kanwaljeets via rkanter) [Forced Update!]
Repository: hadoop
Updated Branches:
refs/heads/YARN-7402 db183f2ea -> c5bf22dc1 (forced update)
HADOOP-15457. Add Security-Related HTTP Response Header in WEBUIs. (kanwaljeets
via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aa23d49f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aa23d49f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aa23d49f
Branch: refs/heads/YARN-7402
Commit: aa23d49fc8b9c2537529dbdc13512000e2ab295a
Parents: bc6d9d4
Author: Robert Kanter
Authored: Wed May 23 10:23:17 2018 -0700
Committer: Robert Kanter
Committed: Wed May 23 10:24:09 2018 -0700
--
.../org/apache/hadoop/http/HttpServer2.java | 79 +++-
.../org/apache/hadoop/http/TestHttpServer.java | 61 +++
2 files changed, 121 insertions(+), 19 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa23d49f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
index 47ca841..c273c78 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
@@ -34,6 +34,8 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import javax.servlet.Filter;
import javax.servlet.FilterChain;
@@ -172,10 +174,16 @@ public final class HttpServer2 implements FilterContainer
{
private final SignerSecretProvider secretProvider;
private XFrameOption xFrameOption;
private boolean xFrameOptionIsEnabled;
- private static final String X_FRAME_VALUE = "xFrameOption";
- private static final String X_FRAME_ENABLED = "X_FRAME_ENABLED";
-
-
+ public static final String HTTP_HEADER_PREFIX = "hadoop.http.header.";
+ private static final String HTTP_HEADER_REGEX =
+ "hadoop\\.http\\.header\\.([a-zA-Z\\-_]+)";
+ static final String X_XSS_PROTECTION =
+ "X-XSS-Protection:1; mode=block";
+ static final String X_CONTENT_TYPE_OPTIONS =
+ "X-Content-Type-Options:nosniff";
+ private static final String X_FRAME_OPTIONS = "X-FRAME-OPTIONS";
+ private static final Pattern PATTERN_HTTP_HEADER_REGEX =
+ Pattern.compile(HTTP_HEADER_REGEX);
/**
* Class to construct instances of HTTP server with specific options.
*/
@@ -574,10 +582,7 @@ public final class HttpServer2 implements FilterContainer {
addDefaultApps(contexts, appDir, conf);
webServer.setHandler(handlers);
-Map xFrameParams = new HashMap<>();
-xFrameParams.put(X_FRAME_ENABLED,
-String.valueOf(this.xFrameOptionIsEnabled));
-xFrameParams.put(X_FRAME_VALUE, this.xFrameOption.toString());
+Map xFrameParams = setHeaders(conf);
addGlobalFilter("safety", QuotingInputFilter.class.getName(),
xFrameParams);
final FilterInitializer[] initializers = getFilterInitializers(conf);
if (initializers != null) {
@@ -1475,9 +1480,11 @@ public final class HttpServer2 implements
FilterContainer {
public static class QuotingInputFilter implements Filter {
private FilterConfig config;
+private Map headerMap;
public static class RequestQuoter extends HttpServletRequestWrapper {
private final HttpServletRequest rawRequest;
+
public RequestQuoter(HttpServletRequest rawRequest) {
super(rawRequest);
this.rawRequest = rawRequest;
@@ -1566,6 +1573,7 @@ public final class HttpServer2 implements FilterContainer
{
@Override
public void init(FilterConfig config) throws ServletException {
this.config = config;
+ initHttpHeaderMap();
}
@Override
@@ -1593,11 +1601,7 @@ public final class HttpServer2 implements
FilterContainer {
} else if (mime.startsWith("application/xml")) {
httpResponse.setContentType("text/xml; charset=utf-8");
}
-
- if(Boolean.valueOf(this.config.getInitParameter(X_FRAME_ENABLED))) {
-httpResponse.addHeader("X-FRAME-OPTIONS",
-this.config.getInitParameter(X_FRAME_VALUE));
- }
+ headerMap.forEach((k, v) -> httpResponse.addHeader(k, v));
chain.doFilter(quoted, httpResponse);
}
@@ -1613,14 +1617,25 @@ public final class HttpServer2 implements
FilterContainer {
return (mime == null) ? null : mime;
}
+private void initHttpHeaderMap() {
+ Enumeration params = this.config.getInitParameterNames();
+
[42/50] [abbrv] hadoop git commit: HADOOP-15497. TestTrash should use proper test path to avoid failing on Windows. Contributed by Anbang Hu.
HADOOP-15497. TestTrash should use proper test path to avoid failing on
Windows. Contributed by Anbang Hu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3c75f8e4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3c75f8e4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3c75f8e4
Branch: refs/heads/YARN-7402
Commit: 3c75f8e4933221fa60a87e86a3db5e4727530b6f
Parents: 31ab960
Author: Inigo Goiri
Authored: Tue May 29 09:11:08 2018 -0700
Committer: Inigo Goiri
Committed: Tue May 29 09:11:08 2018 -0700
--
.../src/test/java/org/apache/hadoop/fs/TestTrash.java | 10 ++
1 file changed, 6 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c75f8e4/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
index 12aed29..fa2d21f 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
@@ -49,9 +49,11 @@ import org.apache.hadoop.util.Time;
*/
public class TestTrash {
- private final static Path TEST_DIR = new Path(GenericTestUtils.getTempPath(
+ private final static File BASE_PATH = new File(GenericTestUtils.getTempPath(
"testTrash"));
+ private final static Path TEST_DIR = new Path(BASE_PATH.getAbsolutePath());
+
@Before
public void setUp() throws IOException {
// ensure each test initiates a FileSystem instance,
@@ -682,7 +684,7 @@ public class TestTrash {
static class TestLFS extends LocalFileSystem {
Path home;
TestLFS() {
- this(new Path(TEST_DIR, "user/test"));
+ this(TEST_DIR);
}
TestLFS(final Path home) {
super(new RawLocalFileSystem() {
@@ -809,8 +811,8 @@ public class TestTrash {
*/
public static void verifyTrashPermission(FileSystem fs, Configuration conf)
throws IOException {
-Path caseRoot = new Path(
-GenericTestUtils.getTempPath("testTrashPermission"));
+Path caseRoot = new Path(BASE_PATH.getPath(),
+"testTrashPermission");
try (FileSystem fileSystem = fs){
Trash trash = new Trash(fileSystem, conf);
FileSystemTestWrapper wrapper =
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[24/50] [abbrv] hadoop git commit: HADOOP-15473. Configure serialFilter in KeyProvider to avoid UnrecoverableKeyException caused by JDK-8189997. Contributed by Gabor Bota.
HADOOP-15473. Configure serialFilter in KeyProvider to avoid
UnrecoverableKeyException caused by JDK-8189997. Contributed by Gabor Bota.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/02322de3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/02322de3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/02322de3
Branch: refs/heads/YARN-7402
Commit: 02322de3f95ba78a22c057037ef61aa3ab1d3824
Parents: 8d5509c
Author: Xiao Chen
Authored: Fri May 25 09:08:15 2018 -0700
Committer: Xiao Chen
Committed: Fri May 25 09:10:51 2018 -0700
--
.../apache/hadoop/crypto/key/KeyProvider.java | 18 +++
.../fs/CommonConfigurationKeysPublic.java | 7 ++
.../src/main/resources/core-default.xml | 23
3 files changed, 48 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/02322de3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
index 5d670e5..050540b 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
@@ -42,6 +42,8 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import javax.crypto.KeyGenerator;
+import static
org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_JCEKS_KEY_SERIALFILTER;
+
/**
* A provider of secret key material for Hadoop applications. Provides an
* abstraction to separate key storage from users of encryption. It
@@ -61,6 +63,14 @@ public abstract class KeyProvider {
CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_DEFAULT_BITLENGTH_KEY;
public static final int DEFAULT_BITLENGTH = CommonConfigurationKeysPublic.
HADOOP_SECURITY_KEY_DEFAULT_BITLENGTH_DEFAULT;
+ public static final String JCEKS_KEY_SERIALFILTER_DEFAULT =
+ "java.lang.Enum;"
+ + "java.security.KeyRep;"
+ + "java.security.KeyRep$Type;"
+ + "javax.crypto.spec.SecretKeySpec;"
+ + "org.apache.hadoop.crypto.key.JavaKeyStoreProvider$KeyMetadata;"
+ + "!*";
+ public static final String JCEKS_KEY_SERIAL_FILTER =
"jceks.key.serialFilter";
private final Configuration conf;
@@ -394,6 +404,14 @@ public abstract class KeyProvider {
*/
public KeyProvider(Configuration conf) {
this.conf = new Configuration(conf);
+// Added for HADOOP-15473. Configured serialFilter property fixes
+// java.security.UnrecoverableKeyException in JDK 8u171.
+if(System.getProperty(JCEKS_KEY_SERIAL_FILTER) == null) {
+ String serialFilter =
+ conf.get(HADOOP_SECURITY_CRYPTO_JCEKS_KEY_SERIALFILTER,
+ JCEKS_KEY_SERIALFILTER_DEFAULT);
+ System.setProperty(JCEKS_KEY_SERIAL_FILTER, serialFilter);
+}
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/02322de3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
index 8837cfb..9e0ba20 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
@@ -662,6 +662,13 @@ public class CommonConfigurationKeysPublic {
*
* core-default.xml
*/
+ public static final String HADOOP_SECURITY_CRYPTO_JCEKS_KEY_SERIALFILTER =
+ "hadoop.security.crypto.jceks.key.serialfilter";
+ /**
+ * @see
+ *
+ * core-default.xml
+ */
public static final String HADOOP_SECURITY_CRYPTO_BUFFER_SIZE_KEY =
"hadoop.security.crypto.buffer.size";
/** Defalt value for HADOOP_SECURITY_CRYPTO_BUFFER_SIZE_KEY */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/02322de3/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index fad2985..9564587 100644
---
[13/50] [abbrv] hadoop git commit: YARN-8319. More YARN pages need to honor yarn.resourcemanager.display.per-user-apps. Contributed by Sunil G.
YARN-8319. More YARN pages need to honor
yarn.resourcemanager.display.per-user-apps. Contributed by Sunil G.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c05b5d42
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c05b5d42
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c05b5d42
Branch: refs/heads/YARN-7402
Commit: c05b5d424b000bab766f57e88a07f2b4e9a56647
Parents: 4cc0c9b
Author: Rohith Sharma K S
Authored: Thu May 24 14:19:46 2018 +0530
Committer: Rohith Sharma K S
Committed: Thu May 24 14:19:46 2018 +0530
--
.../hadoop/yarn/conf/YarnConfiguration.java | 11 +++-
.../yarn/conf/TestYarnConfigurationFields.java | 2 +
.../src/main/resources/yarn-default.xml | 2 +-
.../nodemanager/webapp/NMWebServices.java | 63 +-
.../webapp/TestNMWebServicesApps.java | 68 +---
.../server/resourcemanager/ClientRMService.java | 10 +--
.../resourcemanager/webapp/RMWebServices.java | 8 +--
.../reader/TimelineReaderWebServices.java | 33 ++
8 files changed, 175 insertions(+), 22 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c05b5d42/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 6d08831..004a59f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -121,6 +121,10 @@ public class YarnConfiguration extends Configuration {
new DeprecationDelta(RM_ZK_RETRY_INTERVAL_MS,
CommonConfigurationKeys.ZK_RETRY_INTERVAL_MS),
});
+Configuration.addDeprecations(new DeprecationDelta[] {
+new DeprecationDelta("yarn.resourcemanager.display.per-user-apps",
+FILTER_ENTITY_LIST_BY_USER)
+});
}
//Configurations
@@ -3569,11 +3573,16 @@ public class YarnConfiguration extends Configuration {
public static final String NM_SCRIPT_BASED_NODE_LABELS_PROVIDER_SCRIPT_OPTS =
NM_SCRIPT_BASED_NODE_LABELS_PROVIDER_PREFIX + "opts";
- /*
+ /**
* Support to view apps for given user in secure cluster.
+ * @deprecated This field is deprecated for {@link
#FILTER_ENTITY_LIST_BY_USER}
*/
+ @Deprecated
public static final String DISPLAY_APPS_FOR_LOGGED_IN_USER =
RM_PREFIX + "display.per-user-apps";
+
+ public static final String FILTER_ENTITY_LIST_BY_USER =
+ "yarn.webapp.filter-entity-list-by-user";
public static final boolean DEFAULT_DISPLAY_APPS_FOR_LOGGED_IN_USER =
false;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c05b5d42/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
index f4d1ac0..b9ba543 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
@@ -182,6 +182,8 @@ public class TestYarnConfigurationFields extends
TestConfigurationFieldsBase {
// Ignore deprecated properties
configurationPrefixToSkipCompare
.add(YarnConfiguration.YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS);
+configurationPrefixToSkipCompare
+.add(YarnConfiguration.DISPLAY_APPS_FOR_LOGGED_IN_USER);
// Allocate for usage
xmlPropsToSkipCompare = new HashSet();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c05b5d42/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index da44ccb..c82474c 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++
[39/50] [abbrv] hadoop git commit: HADOOP-15498. TestHadoopArchiveLogs (#testGenerateScript, #testPrepareWorkingDir) fails on Windows. Contributed by Anbang Hu.
HADOOP-15498. TestHadoopArchiveLogs (#testGenerateScript,
#testPrepareWorkingDir) fails on Windows. Contributed by Anbang Hu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8fdc993a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8fdc993a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8fdc993a
Branch: refs/heads/YARN-7402
Commit: 8fdc993a993728c65084d7dc3ac469059cb1f603
Parents: 9dbf4f0
Author: Inigo Goiri
Authored: Mon May 28 16:45:42 2018 -0700
Committer: Inigo Goiri
Committed: Mon May 28 16:45:42 2018 -0700
--
.../org/apache/hadoop/tools/TestHadoopArchiveLogs.java | 12
1 file changed, 8 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8fdc993a/hadoop-tools/hadoop-archive-logs/src/test/java/org/apache/hadoop/tools/TestHadoopArchiveLogs.java
--
diff --git
a/hadoop-tools/hadoop-archive-logs/src/test/java/org/apache/hadoop/tools/TestHadoopArchiveLogs.java
b/hadoop-tools/hadoop-archive-logs/src/test/java/org/apache/hadoop/tools/TestHadoopArchiveLogs.java
index 2ddd4c5..a1b662c 100644
---
a/hadoop-tools/hadoop-archive-logs/src/test/java/org/apache/hadoop/tools/TestHadoopArchiveLogs.java
+++
b/hadoop-tools/hadoop-archive-logs/src/test/java/org/apache/hadoop/tools/TestHadoopArchiveLogs.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
@@ -278,7 +279,7 @@ public class TestHadoopArchiveLogs {
hal.generateScript(localScript);
Assert.assertTrue(localScript.exists());
String script = IOUtils.toString(localScript.toURI());
-String[] lines = script.split(System.lineSeparator());
+String[] lines = script.split("\n");
Assert.assertEquals(22, lines.length);
Assert.assertEquals("#!/bin/bash", lines[0]);
Assert.assertEquals("set -e", lines[1]);
@@ -368,7 +369,8 @@ public class TestHadoopArchiveLogs {
Assert.assertTrue(dirPrepared);
Assert.assertTrue(fs.exists(workingDir));
Assert.assertEquals(
-new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL, true),
+new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL,
+!Shell.WINDOWS),
fs.getFileStatus(workingDir).getPermission());
// Throw a file in the dir
Path dummyFile = new Path(workingDir, "dummy.txt");
@@ -381,7 +383,8 @@ public class TestHadoopArchiveLogs {
Assert.assertTrue(fs.exists(workingDir));
Assert.assertTrue(fs.exists(dummyFile));
Assert.assertEquals(
-new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL, true),
+new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL,
+!Shell.WINDOWS),
fs.getFileStatus(workingDir).getPermission());
// -force is true and the dir exists, so it will recreate it and the dummy
// won't exist anymore
@@ -390,7 +393,8 @@ public class TestHadoopArchiveLogs {
Assert.assertTrue(dirPrepared);
Assert.assertTrue(fs.exists(workingDir));
Assert.assertEquals(
-new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL, true),
+new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL,
+!Shell.WINDOWS),
fs.getFileStatus(workingDir).getPermission());
Assert.assertFalse(fs.exists(dummyFile));
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[20/50] [abbrv] hadoop git commit: YARN-8357. Fixed NPE when YARN service is saved and not deployed. Contributed by Chandni Singh
YARN-8357. Fixed NPE when YARN service is saved and not deployed.
Contributed by Chandni Singh
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d9852eb5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d9852eb5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d9852eb5
Branch: refs/heads/YARN-7402
Commit: d9852eb5897a25323ab0302c2c0decb61d310e5e
Parents: 7ff5a40
Author: Eric Yang
Authored: Thu May 24 16:32:13 2018 -0400
Committer: Eric Yang
Committed: Thu May 24 16:32:13 2018 -0400
--
.../java/org/apache/hadoop/yarn/service/client/ServiceClient.java | 1 +
1 file changed, 1 insertion(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9852eb5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
index 93a74e3..0ab3322 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -1198,6 +1198,7 @@ public class ServiceClient extends AppAdminClient
implements SliderExitCodes,
ServiceApiUtil.validateNameFormat(serviceName, getConfig());
Service appSpec = new Service();
appSpec.setName(serviceName);
+appSpec.setState(ServiceState.STOPPED);
ApplicationId currentAppId = getAppId(serviceName);
if (currentAppId == null) {
LOG.info("Service {} does not have an application ID", serviceName);
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[30/50] [abbrv] hadoop git commit: YARN-8213. Add Capacity Scheduler performance metrics. (Weiwei Yang via wangda)
YARN-8213. Add Capacity Scheduler performance metrics. (Weiwei Yang via wangda)
Change-Id: Ieea6f3eeb83c90cd74233fea896f0fcd0f325d5f
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f24c842d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f24c842d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f24c842d
Branch: refs/heads/YARN-7402
Commit: f24c842d52e166e8566337ef93c96438f1c870d8
Parents: 8605a38
Author: Wangda Tan
Authored: Fri May 25 21:53:20 2018 -0700
Committer: Wangda Tan
Committed: Fri May 25 21:53:20 2018 -0700
--
.../server/resourcemanager/ResourceManager.java | 1 +
.../scheduler/AbstractYarnScheduler.java| 5 +
.../scheduler/ResourceScheduler.java| 5 +
.../scheduler/capacity/CapacityScheduler.java | 31 -
.../capacity/CapacitySchedulerMetrics.java | 119 +++
.../TestCapacitySchedulerMetrics.java | 110 +
6 files changed, 269 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f24c842d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index 05745ec..c533111 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -1216,6 +1216,7 @@ public class ResourceManager extends CompositeService
implements Recoverable {
void reinitialize(boolean initialize) {
ClusterMetrics.destroy();
QueueMetrics.clearQueueMetrics();
+getResourceScheduler().resetSchedulerMetrics();
if (initialize) {
resetRMContext();
createAndInitActiveServices(true);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f24c842d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index b2747f7..18c7b4e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -1464,4 +1464,9 @@ public abstract class AbstractYarnScheduler
SchedulingRequest schedulingRequest, SchedulerNode schedulerNode) {
return false;
}
+
+ @Override
+ public void resetSchedulerMetrics() {
+// reset scheduler metrics
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f24c842d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java
index 5a56ac7..dcb6edd 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java
+++
[03/50] [abbrv] hadoop git commit: HDFS-13587. TestQuorumJournalManager fails on Windows. Contributed by Anbang Hu.
HDFS-13587. TestQuorumJournalManager fails on Windows. Contributed by Anbang Hu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c13dea87
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c13dea87
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c13dea87
Branch: refs/heads/YARN-7402
Commit: c13dea87d9de7a9872fc8b0c939b41b1666a61e5
Parents: 51ce02b
Author: Inigo Goiri
Authored: Wed May 23 11:36:03 2018 -0700
Committer: Inigo Goiri
Committed: Wed May 23 11:36:03 2018 -0700
--
.../org/apache/hadoop/hdfs/qjournal/MiniJournalCluster.java | 5 +
.../hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java | 3 ++-
2 files changed, 7 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c13dea87/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniJournalCluster.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniJournalCluster.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniJournalCluster.java
index 2314e22..f936d75 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniJournalCluster.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniJournalCluster.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.qjournal.client.QuorumJournalManager;
import org.apache.hadoop.hdfs.qjournal.server.JournalNode;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.net.NetUtils;
import com.google.common.base.Joiner;
@@ -50,6 +51,10 @@ public class MiniJournalCluster {
private int numJournalNodes = 3;
private boolean format = true;
private final Configuration conf;
+
+static {
+ DefaultMetricsSystem.setMiniClusterMode(true);
+}
public Builder(Configuration conf) {
this.conf = conf;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c13dea87/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java
index 34a0348..69856ae 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java
@@ -93,7 +93,8 @@ public class TestQuorumJournalManager {
conf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY,
0);
cluster = new MiniJournalCluster.Builder(conf)
- .build();
+.baseDir(GenericTestUtils.getRandomizedTestDir().getAbsolutePath())
+.build();
cluster.waitActive();
qjm = createSpyingQJM();
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[28/50] [abbrv] hadoop git commit: HDDS-113. Rest and Rpc Client should verify resource name using HddsClientUtils. Contributed by Lokesh Jain.
HDDS-113. Rest and Rpc Client should verify resource name using HddsClientUtils.
Contributed by Lokesh Jain.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2a9652e6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2a9652e6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2a9652e6
Branch: refs/heads/YARN-7402
Commit: 2a9652e69650973f6158b60ff131215827738db6
Parents: 13d2528
Author: Anu Engineer
Authored: Fri May 25 15:40:46 2018 -0700
Committer: Anu Engineer
Committed: Fri May 25 15:45:50 2018 -0700
--
.../hadoop/hdds/scm/client/HddsClientUtils.java | 23 +
.../apache/hadoop/ozone/client/ObjectStore.java | 9
.../apache/hadoop/ozone/client/OzoneBucket.java | 24 +
.../apache/hadoop/ozone/client/OzoneVolume.java | 18 +--
.../hadoop/ozone/client/rest/RestClient.java| 52
.../hadoop/ozone/client/rpc/RpcClient.java | 46 +++--
6 files changed, 64 insertions(+), 108 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a9652e6/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/HddsClientUtils.java
--
diff --git
a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/HddsClientUtils.java
b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/HddsClientUtils.java
index bc5f8d6..a6813eb 100644
---
a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/HddsClientUtils.java
+++
b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/HddsClientUtils.java
@@ -170,6 +170,29 @@ public final class HddsClientUtils {
}
/**
+ * verifies that bucket / volume name is a valid DNS name.
+ *
+ * @param resourceNames Array of bucket / volume names to be verified.
+ */
+ public static void verifyResourceName(String... resourceNames) {
+for (String resourceName : resourceNames) {
+ HddsClientUtils.verifyResourceName(resourceName);
+}
+ }
+
+ /**
+ * Checks that object parameters passed as reference is not null.
+ *
+ * @param references Array of object references to be checked.
+ * @param
+ */
+ public static void checkNotNull(T... references) {
+for (T ref: references) {
+ Preconditions.checkNotNull(ref);
+}
+ }
+
+ /**
* Returns the cache value to be used for list calls.
* @param conf Configuration object
* @return list cache size
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a9652e6/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
--
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
index d8b3011..c5f0689 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
@@ -63,8 +63,6 @@ public class ObjectStore {
* @throws IOException
*/
public void createVolume(String volumeName) throws IOException {
-Preconditions.checkNotNull(volumeName);
-HddsClientUtils.verifyResourceName(volumeName);
proxy.createVolume(volumeName);
}
@@ -76,9 +74,6 @@ public class ObjectStore {
*/
public void createVolume(String volumeName, VolumeArgs volumeArgs)
throws IOException {
-Preconditions.checkNotNull(volumeName);
-Preconditions.checkNotNull(volumeArgs);
-HddsClientUtils.verifyResourceName(volumeName);
proxy.createVolume(volumeName, volumeArgs);
}
@@ -89,8 +84,6 @@ public class ObjectStore {
* @throws IOException
*/
public OzoneVolume getVolume(String volumeName) throws IOException {
-Preconditions.checkNotNull(volumeName);
-HddsClientUtils.verifyResourceName(volumeName);
OzoneVolume volume = proxy.getVolumeDetails(volumeName);
return volume;
}
@@ -150,8 +143,6 @@ public class ObjectStore {
* @throws IOException
*/
public void deleteVolume(String volumeName) throws IOException {
-Preconditions.checkNotNull(volumeName);
-HddsClientUtils.verifyResourceName(volumeName);
proxy.deleteVolume(volumeName);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a9652e6/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
--
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
index 5df0254..2f3cff6 100644
---
[12/50] [abbrv] hadoop git commit: YARN-8346. Upgrading to 3.1 kills running containers with error 'Opportunistic container queue is full'. Contributed by Jason Lowe.
YARN-8346. Upgrading to 3.1 kills running containers with error 'Opportunistic
container queue is full'. Contributed by Jason Lowe.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4cc0c9b0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4cc0c9b0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4cc0c9b0
Branch: refs/heads/YARN-7402
Commit: 4cc0c9b0baa93f5a1c0623eee353874e858a7caa
Parents: 7a87add
Author: Rohith Sharma K S
Authored: Thu May 24 12:23:47 2018 +0530
Committer: Rohith Sharma K S
Committed: Thu May 24 12:23:47 2018 +0530
--
.../yarn/security/ContainerTokenIdentifier.java | 4 ++--
.../yarn/security/TestYARNTokenIdentifier.java | 25
2 files changed, 27 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cc0c9b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
index 37c74b8..8dea65f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
@@ -292,7 +292,7 @@ public class ContainerTokenIdentifier extends
TokenIdentifier {
*/
public ContainerType getContainerType(){
if (!proto.hasContainerType()) {
- return null;
+ return ContainerType.TASK;
}
return convertFromProtoFormat(proto.getContainerType());
}
@@ -303,7 +303,7 @@ public class ContainerTokenIdentifier extends
TokenIdentifier {
*/
public ExecutionType getExecutionType(){
if (!proto.hasExecutionType()) {
- return null;
+ return ExecutionType.GUARANTEED;
}
return convertFromProtoFormat(proto.getExecutionType());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cc0c9b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/security/TestYARNTokenIdentifier.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/security/TestYARNTokenIdentifier.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/security/TestYARNTokenIdentifier.java
index 51fbe9a..8109b5e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/security/TestYARNTokenIdentifier.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/security/TestYARNTokenIdentifier.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.impl.pb.LogAggregationContextPBImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
+import
org.apache.hadoop.yarn.proto.YarnSecurityTokenProtos.ContainerTokenIdentifierProto;
import
org.apache.hadoop.yarn.proto.YarnSecurityTokenProtos.YARNDelegationTokenIdentifierProto;
import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier;
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
@@ -170,6 +171,30 @@ public class TestYARNTokenIdentifier {
}
@Test
+ public void testContainerTokenIdentifierProtoMissingFields()
+ throws IOException {
+ContainerTokenIdentifierProto.Builder builder =
+ContainerTokenIdentifierProto.newBuilder();
+ContainerTokenIdentifierProto proto = builder.build();
+Assert.assertFalse(proto.hasContainerType());
+Assert.assertFalse(proto.hasExecutionType());
+Assert.assertFalse(proto.hasNodeLabelExpression());
+
+byte[] tokenData = proto.toByteArray();
+DataInputBuffer dib = new DataInputBuffer();
+dib.reset(tokenData, tokenData.length);
+ContainerTokenIdentifier tid = new ContainerTokenIdentifier();
+tid.readFields(dib);
+
+Assert.assertEquals("container type",
+ContainerType.TASK, tid.getContainerType());
+Assert.assertEquals("execution type",
+ExecutionType.GUARANTEED, tid.getExecutionType());
+Assert.assertEquals("node label expression",
+CommonNodeLabelsManager.NO_LABEL, tid.getNodeLabelExpression());
+ }
+
+ @Test
public void testContainerTokenIdentifier() throws IOException {
[05/50] [abbrv] hadoop git commit: HDFS-13493. Reduce the HttpServer2 thread count on DataNodes. Contributed by Erik Krogen.
HDFS-13493. Reduce the HttpServer2 thread count on DataNodes. Contributed by
Erik Krogen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cddbbe5f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cddbbe5f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cddbbe5f
Branch: refs/heads/YARN-7402
Commit: cddbbe5f690e4617413f6e986adc6fa900629f03
Parents: e30938a
Author: Inigo Goiri
Authored: Wed May 23 12:12:08 2018 -0700
Committer: Inigo Goiri
Committed: Wed May 23 12:12:08 2018 -0700
--
.../hdfs/server/datanode/web/DatanodeHttpServer.java | 14 +-
1 file changed, 13 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cddbbe5f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java
index 0ce327a..4349c26 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java
@@ -89,6 +89,13 @@ public class DatanodeHttpServer implements Closeable {
private InetSocketAddress httpsAddress;
static final Log LOG = LogFactory.getLog(DatanodeHttpServer.class);
+ // HttpServer threads are only used for the web UI and basic servlets, so
+ // set them to the minimum possible
+ private static final int HTTP_SELECTOR_THREADS = 1;
+ private static final int HTTP_ACCEPTOR_THREADS = 1;
+ private static final int HTTP_MAX_THREADS =
+ HTTP_SELECTOR_THREADS + HTTP_ACCEPTOR_THREADS + 1;
+
public DatanodeHttpServer(final Configuration conf,
final DataNode datanode,
final ServerSocketChannel externalHttpChannel)
@@ -97,7 +104,12 @@ public class DatanodeHttpServer implements Closeable {
this.conf = conf;
Configuration confForInfoServer = new Configuration(conf);
-confForInfoServer.setInt(HttpServer2.HTTP_MAX_THREADS_KEY, 10);
+confForInfoServer.setInt(HttpServer2.HTTP_MAX_THREADS_KEY,
+HTTP_MAX_THREADS);
+confForInfoServer.setInt(HttpServer2.HTTP_SELECTOR_COUNT_KEY,
+HTTP_SELECTOR_THREADS);
+confForInfoServer.setInt(HttpServer2.HTTP_ACCEPTOR_COUNT_KEY,
+HTTP_ACCEPTOR_THREADS);
int proxyPort =
confForInfoServer.getInt(DFS_DATANODE_HTTP_INTERNAL_PROXY_PORT, 0);
HttpServer2.Builder builder = new HttpServer2.Builder()
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[35/50] [abbrv] hadoop git commit: HADOOP-15449. Increase default timeout of ZK session to avoid frequent NameNode failover
HADOOP-15449. Increase default timeout of ZK session to avoid frequent NameNode
failover
Signed-off-by: Akira Ajisaka
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/61df174e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/61df174e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/61df174e
Branch: refs/heads/YARN-7402
Commit: 61df174e8b3d582183306cabfa2347c8b96322ff
Parents: 04757e5
Author: Karthik Palanisamy
Authored: Mon May 28 19:41:07 2018 +0900
Committer: Akira Ajisaka
Committed: Mon May 28 19:41:07 2018 +0900
--
.../src/main/java/org/apache/hadoop/ha/ZKFailoverController.java | 2 +-
.../hadoop-common/src/main/resources/core-default.xml | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61df174e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java
index a8c19ab..9295288 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java
@@ -63,7 +63,7 @@ public abstract class ZKFailoverController {
public static final String ZK_QUORUM_KEY = "ha.zookeeper.quorum";
private static final String ZK_SESSION_TIMEOUT_KEY =
"ha.zookeeper.session-timeout.ms";
- private static final int ZK_SESSION_TIMEOUT_DEFAULT = 5*1000;
+ private static final int ZK_SESSION_TIMEOUT_DEFAULT = 10*1000;
private static final String ZK_PARENT_ZNODE_KEY =
"ha.zookeeper.parent-znode";
public static final String ZK_ACL_KEY = "ha.zookeeper.acl";
private static final String ZK_ACL_DEFAULT = "world:anyone:rwcda";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61df174e/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 9564587..75acf48 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -2168,7 +2168,7 @@
ha.zookeeper.session-timeout.ms
- 5000
+ 1
The session timeout to use when the ZKFC connects to ZooKeeper.
Setting this value to a lower value implies that server crashes
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[49/50] [abbrv] hadoop git commit: YARN-7707. [GPG] Policy generator framework. Contributed by Young Chen
YARN-7707. [GPG] Policy generator framework. Contributed by Young Chen
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f5da8ca6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f5da8ca6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f5da8ca6
Branch: refs/heads/YARN-7402
Commit: f5da8ca6f04b7db40fccfd00cc4ff8ca1b2da74b
Parents: 46a4a94
Author: Botong Huang
Authored: Fri Mar 23 17:07:10 2018 -0700
Committer: Botong Huang
Committed: Tue May 29 10:48:40 2018 -0700
--
.../hadoop/yarn/conf/YarnConfiguration.java | 36 +-
.../src/main/resources/yarn-default.xml | 40 +++
.../utils/FederationStateStoreFacade.java | 13 +
.../pom.xml | 18 +
.../globalpolicygenerator/GPGContext.java | 4 +
.../globalpolicygenerator/GPGContextImpl.java | 10 +
.../globalpolicygenerator/GPGPolicyFacade.java | 220
.../server/globalpolicygenerator/GPGUtils.java | 80 +
.../GlobalPolicyGenerator.java | 17 +
.../policygenerator/GlobalPolicy.java | 76 +
.../policygenerator/NoOpGlobalPolicy.java | 36 ++
.../policygenerator/PolicyGenerator.java| 261 ++
.../UniformWeightedLocalityGlobalPolicy.java| 71
.../policygenerator/package-info.java | 24 ++
.../TestGPGPolicyFacade.java| 202 +++
.../policygenerator/TestPolicyGenerator.java| 338 +++
.../src/test/resources/schedulerInfo1.json | 134
.../src/test/resources/schedulerInfo2.json | 196 +++
18 files changed, 1775 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f5da8ca6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 7c78e0d..b224818 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -3326,7 +3326,7 @@ public class YarnConfiguration extends Configuration {
public static final boolean DEFAULT_ROUTER_WEBAPP_PARTIAL_RESULTS_ENABLED =
false;
- private static final String FEDERATION_GPG_PREFIX =
+ public static final String FEDERATION_GPG_PREFIX =
FEDERATION_PREFIX + "gpg.";
// The number of threads to use for the GPG scheduled executor service
@@ -3344,6 +3344,40 @@ public class YarnConfiguration extends Configuration {
FEDERATION_GPG_PREFIX + "subcluster.heartbeat.expiration-ms";
public static final long DEFAULT_GPG_SUBCLUSTER_EXPIRATION_MS = 180;
+ public static final String FEDERATION_GPG_POLICY_PREFIX =
+ FEDERATION_GPG_PREFIX + "policy.generator.";
+
+ /** The interval at which the policy generator runs, default is one hour. */
+ public static final String GPG_POLICY_GENERATOR_INTERVAL_MS =
+ FEDERATION_GPG_POLICY_PREFIX + "interval-ms";
+ public static final long DEFAULT_GPG_POLICY_GENERATOR_INTERVAL_MS = -1;
+
+ /**
+ * The configured policy generator class, runs NoOpGlobalPolicy by
+ * default.
+ */
+ public static final String GPG_GLOBAL_POLICY_CLASS =
+ FEDERATION_GPG_POLICY_PREFIX + "class";
+ public static final String DEFAULT_GPG_GLOBAL_POLICY_CLASS =
+ "org.apache.hadoop.yarn.server.globalpolicygenerator.policygenerator."
+ + "NoOpGlobalPolicy";
+
+ /**
+ * Whether or not the policy generator is running in read only (won't modify
+ * policies), default is false.
+ */
+ public static final String GPG_POLICY_GENERATOR_READONLY =
+ FEDERATION_GPG_POLICY_PREFIX + "readonly";
+ public static final boolean DEFAULT_GPG_POLICY_GENERATOR_READONLY =
+ false;
+
+ /**
+ * Which sub-clusters the policy generator should blacklist.
+ */
+ public static final String GPG_POLICY_GENERATOR_BLACKLIST =
+ FEDERATION_GPG_POLICY_PREFIX + "blacklist";
+
+
// Other Configs
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f5da8ca6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
[38/50] [abbrv] hadoop git commit: HDFS-13591. TestDFSShell#testSetrepLow fails on Windows. Contributed by Anbang Hu.
HDFS-13591. TestDFSShell#testSetrepLow fails on Windows. Contributed by Anbang
Hu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9dbf4f01
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9dbf4f01
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9dbf4f01
Branch: refs/heads/YARN-7402
Commit: 9dbf4f01665d5480a70395a24519cbab5d4db0c5
Parents: 91d7c74
Author: Inigo Goiri
Authored: Mon May 28 16:34:02 2018 -0700
Committer: Inigo Goiri
Committed: Mon May 28 16:34:02 2018 -0700
--
.../test/java/org/apache/hadoop/hdfs/TestDFSShell.java| 10 +-
1 file changed, 5 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9dbf4f01/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
index e82863a..c352dc9 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
@@ -2829,11 +2829,11 @@ public class TestDFSShell {
System.setErr(origErr);
}
- assertEquals("Error message is not the expected error message",
- "setrep: Requested replication factor of 1 is less than "
- + "the required minimum of 2 for /tmp/TestDFSShell-"
- + "testSetrepLow/testFileForSetrepLow\n",
- bao.toString());
+ assertTrue("Error message is not the expected error message"
+ + bao.toString(), bao.toString().startsWith(
+ "setrep: Requested replication factor of 1 is less than "
+ + "the required minimum of 2 for /tmp/TestDFSShell-"
+ + "testSetrepLow/testFileForSetrepLow"));
} finally {
shell.close();
cluster.shutdown();
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[18/50] [abbrv] hadoop git commit: HDDS-80. Remove SendContainerCommand from SCM. Contributed by Nanda Kumar.
HDDS-80. Remove SendContainerCommand from SCM. Contributed by Nanda Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2d19e7d0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2d19e7d0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2d19e7d0
Branch: refs/heads/YARN-7402
Commit: 2d19e7d08f031341078a36fee74860c58de02993
Parents: c9b63de
Author: Xiaoyu Yao
Authored: Thu May 24 11:10:30 2018 -0700
Committer: Xiaoyu Yao
Committed: Thu May 24 11:10:30 2018 -0700
--
.../statemachine/DatanodeStateMachine.java | 3 -
.../commandhandler/ContainerReportHandler.java | 114 ---
.../states/endpoint/HeartbeatEndpointTask.java | 5 -
.../protocol/commands/SendContainerCommand.java | 80 -
.../StorageContainerDatanodeProtocol.proto | 16 ++-
.../container/replication/InProgressPool.java | 57 --
.../scm/server/SCMDatanodeProtocolServer.java | 7 --
7 files changed, 7 insertions(+), 275 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d19e7d0/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
--
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
index a16bfdc..a8fe494 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
@@ -26,8 +26,6 @@ import
org.apache.hadoop.ozone.container.common.statemachine.commandhandler
import org.apache.hadoop.ozone.container.common.statemachine.commandhandler
.CommandDispatcher;
import org.apache.hadoop.ozone.container.common.statemachine.commandhandler
-.ContainerReportHandler;
-import org.apache.hadoop.ozone.container.common.statemachine.commandhandler
.DeleteBlocksCommandHandler;
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
@@ -88,7 +86,6 @@ public class DatanodeStateMachine implements Closeable {
// When we add new handlers just adding a new handler here should do the
// trick.
commandDispatcher = CommandDispatcher.newBuilder()
-.addHandler(new ContainerReportHandler())
.addHandler(new CloseContainerHandler())
.addHandler(new DeleteBlocksCommandHandler(
container.getContainerManager(), conf))
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d19e7d0/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ContainerReportHandler.java
--
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ContainerReportHandler.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ContainerReportHandler.java
deleted file mode 100644
index fbea290..000
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ContainerReportHandler.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations
under
- * the License.
- */
-package org.apache.hadoop.ozone.container.common.statemachine.commandhandler;
-
-import org.apache.hadoop.hdds.protocol.proto
-.StorageContainerDatanodeProtocolProtos.ContainerReportsRequestProto;
-import org.apache.hadoop.hdds.protocol.proto
-.StorageContainerDatanodeProtocolProtos.SCMCmdType;
-import org.apache.hadoop.ozone.container.common.statemachine
-
[43/50] [abbrv] hadoop git commit: YARN-8339. Service AM should localize static/archive resource types to container working directory instead of 'resources'. (Suma Shivaprasad via wangda)
YARN-8339. Service AM should localize static/archive resource types to
container working directory instead of 'resources'. (Suma Shivaprasad via
wangda)
Change-Id: I9f8e8f621650347f6c2f9e3420edee9eb2f356a4
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3061bfcd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3061bfcd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3061bfcd
Branch: refs/heads/YARN-7402
Commit: 3061bfcde53210d2032df3814243498b27a997b7
Parents: 3c75f8e
Author: Wangda Tan
Authored: Tue May 29 09:23:11 2018 -0700
Committer: Wangda Tan
Committed: Tue May 29 09:23:11 2018 -0700
--
.../org/apache/hadoop/yarn/service/provider/ProviderUtils.java | 3 +--
.../apache/hadoop/yarn/service/provider/TestProviderUtils.java | 6 +++---
2 files changed, 4 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3061bfcd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
index 1ad5fd8..ac90992 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
@@ -298,8 +298,7 @@ public class ProviderUtils implements YarnServiceConstants {
destFile = new Path(staticFile.getDestFile());
}
- String symlink = APP_RESOURCES_DIR + "/" + destFile.getName();
- addLocalResource(launcher, symlink, localResource, destFile);
+ addLocalResource(launcher, destFile.getName(), localResource, destFile);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3061bfcd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestProviderUtils.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestProviderUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestProviderUtils.java
index 6e8bc43..5d794d2 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestProviderUtils.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestProviderUtils.java
@@ -154,11 +154,11 @@ public class TestProviderUtils {
ProviderUtils.handleStaticFilesForLocalization(launcher, sfs,
compLaunchCtx);
-
Mockito.verify(launcher).addLocalResource(Mockito.eq("resources/destFile1"),
+Mockito.verify(launcher).addLocalResource(Mockito.eq("destFile1"),
any(LocalResource.class));
Mockito.verify(launcher).addLocalResource(
-Mockito.eq("resources/destFile_2"), any(LocalResource.class));
+Mockito.eq("destFile_2"), any(LocalResource.class));
Mockito.verify(launcher).addLocalResource(
-Mockito.eq("resources/sourceFile4"), any(LocalResource.class));
+Mockito.eq("sourceFile4"), any(LocalResource.class));
}
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[04/50] [abbrv] hadoop git commit: YARN-8336. Fix potential connection leak in SchedConfCLI and YarnWebServiceUtils. Contributed by Giovanni Matteo Fumarola.
YARN-8336. Fix potential connection leak in SchedConfCLI and
YarnWebServiceUtils. Contributed by Giovanni Matteo Fumarola.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e30938af
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e30938af
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e30938af
Branch: refs/heads/YARN-7402
Commit: e30938af1270e079587e7bc06b755f9e93e660a5
Parents: c13dea8
Author: Inigo Goiri
Authored: Wed May 23 11:55:31 2018 -0700
Committer: Inigo Goiri
Committed: Wed May 23 11:55:31 2018 -0700
--
.../hadoop/yarn/client/cli/SchedConfCLI.java| 42
.../yarn/webapp/util/YarnWebServiceUtils.java | 17 +---
2 files changed, 38 insertions(+), 21 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e30938af/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java
index 11bfdd7..a5f3b80 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java
@@ -132,25 +132,35 @@ public class SchedConfCLI extends Configured implements
Tool {
}
Client webServiceClient = Client.create();
-WebResource webResource = webServiceClient.resource(WebAppUtils.
-getRMWebAppURLWithScheme(getConf()));
-ClientResponse response = webResource.path("ws").path("v1").path("cluster")
-.path("scheduler-conf").accept(MediaType.APPLICATION_JSON)
-.entity(YarnWebServiceUtils.toJson(updateInfo,
-SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
-.put(ClientResponse.class);
-if (response != null) {
- if (response.getStatus() == Status.OK.getStatusCode()) {
-System.out.println("Configuration changed successfully.");
-return 0;
+WebResource webResource = webServiceClient
+.resource(WebAppUtils.getRMWebAppURLWithScheme(getConf()));
+ClientResponse response = null;
+
+try {
+ response =
+ webResource.path("ws").path("v1").path("cluster")
+ .path("scheduler-conf").accept(MediaType.APPLICATION_JSON)
+ .entity(YarnWebServiceUtils.toJson(updateInfo,
+ SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+ if (response != null) {
+if (response.getStatus() == Status.OK.getStatusCode()) {
+ System.out.println("Configuration changed successfully.");
+ return 0;
+} else {
+ System.err.println("Configuration change unsuccessful: "
+ + response.getEntity(String.class));
+}
} else {
-System.err.println("Configuration change unsuccessful: "
-+ response.getEntity(String.class));
+System.err.println("Configuration change unsuccessful: null response");
}
-} else {
- System.err.println("Configuration change unsuccessful: null response");
+ return -1;
+} finally {
+ if (response != null) {
+response.close();
+ }
+ webServiceClient.destroy();
}
-return -1;
}
@VisibleForTesting
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e30938af/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/YarnWebServiceUtils.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/YarnWebServiceUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/YarnWebServiceUtils.java
index 1cf1e97..e7bca2c 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/YarnWebServiceUtils.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/YarnWebServiceUtils.java
@@ -58,11 +58,18 @@ public final class YarnWebServiceUtils {
WebResource webResource = webServiceClient.resource(webAppAddress);
-ClientResponse response = webResource.path("ws").path("v1")
-.path("cluster").path("nodes")
-.path(nodeId).accept(MediaType.APPLICATION_JSON)
-.get(ClientResponse.class);
-return
[06/50] [abbrv] hadoop git commit: YARN-8344. Missing nm.stop() in TestNodeManagerResync to fix testKillContainersOnResync. Contributed by Giovanni Matteo Fumarola.
YARN-8344. Missing nm.stop() in TestNodeManagerResync to fix
testKillContainersOnResync. Contributed by Giovanni Matteo Fumarola.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e99e5bf1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e99e5bf1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e99e5bf1
Branch: refs/heads/YARN-7402
Commit: e99e5bf104e9664bc1b43a2639d87355d47a77e2
Parents: cddbbe5
Author: Inigo Goiri
Authored: Wed May 23 14:15:26 2018 -0700
Committer: Inigo Goiri
Committed: Wed May 23 14:15:26 2018 -0700
--
.../nodemanager/TestNodeManagerResync.java | 87 +++-
1 file changed, 48 insertions(+), 39 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e99e5bf1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java
index 97e9922..cf33775 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java
@@ -150,7 +150,6 @@ public class TestNodeManagerResync {
testContainerPreservationOnResyncImpl(nm, true);
}
- @SuppressWarnings("unchecked")
protected void testContainerPreservationOnResyncImpl(TestNodeManager1 nm,
boolean isWorkPreservingRestartEnabled)
throws IOException, YarnException, InterruptedException {
@@ -186,32 +185,35 @@ public class TestNodeManagerResync {
}
}
- @SuppressWarnings("unchecked")
+ @SuppressWarnings("resource")
@Test(timeout=1)
public void testNMshutdownWhenResyncThrowException() throws IOException,
InterruptedException, YarnException {
NodeManager nm = new TestNodeManager3();
YarnConfiguration conf = createNMConfig();
-nm.init(conf);
-nm.start();
-Assert.assertEquals(1, ((TestNodeManager3) nm).getNMRegistrationCount());
-nm.getNMDispatcher().getEventHandler()
-.handle(new NodeManagerEvent(NodeManagerEventType.RESYNC));
-
-synchronized (isNMShutdownCalled) {
- while (isNMShutdownCalled.get() == false) {
-try {
- isNMShutdownCalled.wait();
-} catch (InterruptedException e) {
+try {
+ nm.init(conf);
+ nm.start();
+ Assert.assertEquals(1, ((TestNodeManager3) nm).getNMRegistrationCount());
+ nm.getNMDispatcher().getEventHandler()
+ .handle(new NodeManagerEvent(NodeManagerEventType.RESYNC));
+
+ synchronized (isNMShutdownCalled) {
+while (!isNMShutdownCalled.get()) {
+ try {
+isNMShutdownCalled.wait();
+ } catch (InterruptedException e) {
+ }
}
}
-}
-Assert.assertTrue("NM shutdown not called.",isNMShutdownCalled.get());
-nm.stop();
+ Assert.assertTrue("NM shutdown not called.", isNMShutdownCalled.get());
+} finally {
+ nm.stop();
+}
}
- @SuppressWarnings("unchecked")
+ @SuppressWarnings("resource")
@Test(timeout=6)
public void testContainerResourceIncreaseIsSynchronizedWithRMResync()
throws IOException, InterruptedException, YarnException {
@@ -219,28 +221,32 @@ public class TestNodeManagerResync {
YarnConfiguration conf = createNMConfig();
conf.setBoolean(
YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED, true);
-nm.init(conf);
-nm.start();
-// Start a container and make sure it is in RUNNING state
-((TestNodeManager4)nm).startContainer();
-// Simulate a container resource increase in a separate thread
-((TestNodeManager4)nm).updateContainerResource();
-// Simulate RM restart by sending a RESYNC event
-LOG.info("Sending out RESYNC event");
-nm.getNMDispatcher().getEventHandler().handle(
-new NodeManagerEvent(NodeManagerEventType.RESYNC));
try {
- syncBarrier.await();
-} catch (BrokenBarrierException e) {
- e.printStackTrace();
+ nm.init(conf);
+ nm.start();
+ // Start a container and make sure it is in RUNNING state
+ ((TestNodeManager4) nm).startContainer();
+ // Simulate a container resource increase in a separate thread
+
[29/50] [abbrv] hadoop git commit: HDFS-13620. Randomize the test directory path for TestHDFSFileSystemContract. Contributed by Anbang Hu.
HDFS-13620. Randomize the test directory path for TestHDFSFileSystemContract.
Contributed by Anbang Hu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8605a385
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8605a385
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8605a385
Branch: refs/heads/YARN-7402
Commit: 8605a38514b4f7a2a549c7ecf8e1421e61bb4d67
Parents: 2a9652e
Author: Inigo Goiri
Authored: Fri May 25 19:43:33 2018 -0700
Committer: Inigo Goiri
Committed: Fri May 25 19:43:33 2018 -0700
--
.../org/apache/hadoop/hdfs/TestHDFSFileSystemContract.java | 6 +-
1 file changed, 5 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8605a385/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSFileSystemContract.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSFileSystemContract.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSFileSystemContract.java
index 50d1e75..6da46de 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSFileSystemContract.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSFileSystemContract.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs;
+import java.io.File;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
@@ -25,6 +26,7 @@ import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FileSystemContractBaseTest;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.test.GenericTestUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -39,7 +41,9 @@ public class TestHDFSFileSystemContract extends
FileSystemContractBaseTest {
Configuration conf = new HdfsConfiguration();
conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY,
FileSystemContractBaseTest.TEST_UMASK);
-cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
+File basedir = GenericTestUtils.getRandomizedTestDir();
+cluster = new MiniDFSCluster.Builder(conf, basedir).numDataNodes(2)
+.build();
fs = cluster.getFileSystem();
defaultWorkingDirectory = "/user/" +
UserGroupInformation.getCurrentUser().getShortUserName();
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[27/50] [abbrv] hadoop git commit: HDFS-13619. TestAuditLoggerWithCommands fails on Windows. Contributed by Anbang Hu.
HDFS-13619. TestAuditLoggerWithCommands fails on Windows. Contributed by Anbang
Hu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/13d25289
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/13d25289
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/13d25289
Branch: refs/heads/YARN-7402
Commit: 13d25289076b39daf481fb1ee15939dbfe4a6b23
Parents: 8733012
Author: Inigo Goiri
Authored: Fri May 25 13:32:34 2018 -0700
Committer: Inigo Goiri
Committed: Fri May 25 13:32:34 2018 -0700
--
.../hdfs/server/namenode/TestAuditLoggerWithCommands.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d25289/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLoggerWithCommands.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLoggerWithCommands.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLoggerWithCommands.java
index 41ee03f..222a1de 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLoggerWithCommands.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLoggerWithCommands.java
@@ -1264,8 +1264,9 @@ public class TestAuditLoggerWithCommands {
}
private int verifyAuditLogs(String pattern) {
-int length = auditlog.getOutput().split("\n").length;
-String lastAudit = auditlog.getOutput().split("\n")[length - 1];
+int length = auditlog.getOutput().split(System.lineSeparator()).length;
+String lastAudit = auditlog.getOutput()
+.split(System.lineSeparator())[length - 1];
assertTrue("Unexpected log!", lastAudit.matches(pattern));
return length;
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[47/50] [abbrv] hadoop git commit: YARN-7402. [GPG] Fix potential connection leak in GPGUtils. Contributed by Giovanni Matteo Fumarola.
YARN-7402. [GPG] Fix potential connection leak in GPGUtils. Contributed by
Giovanni Matteo Fumarola.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c5bf22dc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c5bf22dc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c5bf22dc
Branch: refs/heads/YARN-7402
Commit: c5bf22dc13b5bbe57b45fe81dd2d912af3b87602
Parents: f5da8ca
Author: Botong Huang
Authored: Wed May 23 12:45:32 2018 -0700
Committer: Botong Huang
Committed: Tue May 29 10:48:40 2018 -0700
--
.../server/globalpolicygenerator/GPGUtils.java | 31 +---
1 file changed, 20 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c5bf22dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGUtils.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGUtils.java
index 429bec4..31cee1c 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGUtils.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGUtils.java
@@ -18,21 +18,22 @@
package org.apache.hadoop.yarn.server.globalpolicygenerator;
+import static javax.servlet.http.HttpServletResponse.SC_OK;
+
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
-import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.core.MediaType;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.WebResource;
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
/**
* GPGUtils contains utility functions for the GPG.
@@ -53,15 +54,23 @@ public final class GPGUtils {
T obj = null;
WebResource webResource = client.resource(webAddr);
-ClientResponse response = webResource.path("ws/v1/cluster").path(path)
-.accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
-if (response.getStatus() == HttpServletResponse.SC_OK) {
- obj = response.getEntity(returnType);
-} else {
- throw new YarnRuntimeException("Bad response from remote web service: "
- + response.getStatus());
+ClientResponse response = null;
+try {
+ response = webResource.path("ws/v1/cluster").path(path)
+ .accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
+ if (response.getStatus() == SC_OK) {
+obj = response.getEntity(returnType);
+ } else {
+throw new YarnRuntimeException(
+"Bad response from remote web service: " + response.getStatus());
+ }
+ return obj;
+} finally {
+ if (response != null) {
+response.close();
+ }
+ client.destroy();
}
-return obj;
}
/**
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[45/50] [abbrv] hadoop git commit: HDDS-125. Cleanup HDDS CheckStyle issues. Contributed by Anu Engineer.
HDDS-125. Cleanup HDDS CheckStyle issues.
Contributed by Anu Engineer.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9502b47b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9502b47b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9502b47b
Branch: refs/heads/YARN-7402
Commit: 9502b47bd2a3cf32edae635293169883c2914475
Parents: 17aa40f
Author: Anu Engineer
Authored: Tue May 29 09:54:06 2018 -0700
Committer: Anu Engineer
Committed: Tue May 29 09:54:06 2018 -0700
--
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 1 -
.../hdds/scm/block/DeletedBlockLogImpl.java | 2 +-
.../hdds/scm/container/ContainerMapping.java| 6 +-
.../scm/container/ContainerStateManager.java| 24 +++
.../hadoop/hdds/scm/container/Mapping.java | 9 ++-
.../hdds/scm/node/SCMNodeStorageStatMXBean.java | 4 +-
.../hdds/scm/node/SCMNodeStorageStatMap.java| 19 +++---
.../hdds/scm/node/StorageReportResult.java | 8 +--
.../hdds/scm/node/states/Node2ContainerMap.java | 2 +-
.../hdds/scm/pipelines/PipelineSelector.java| 5 +-
.../scm/server/StorageContainerManager.java | 3 +-
.../TestStorageContainerManagerHttpServer.java | 1 -
.../hadoop/hdds/scm/block/package-info.java | 23 +++
.../scm/container/TestContainerMapping.java | 12 ++--
.../hdds/scm/container/closer/package-info.java | 22 +++
.../hadoop/hdds/scm/container/package-info.java | 22 +++
.../hdds/scm/container/states/package-info.java | 22 +++
.../hadoop/hdds/scm/node/TestNodeManager.java | 66 ++--
.../scm/node/TestSCMNodeStorageStatMap.java | 32 +-
.../hadoop/hdds/scm/node/package-info.java | 22 +++
.../ozone/container/common/TestEndPoint.java| 2 -
.../ozone/container/common/package-info.java| 22 +++
.../ozone/container/placement/package-info.java | 22 +++
.../replication/TestContainerSupervisor.java| 7 ++-
24 files changed, 263 insertions(+), 95 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9502b47b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
--
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
index 5a98e85..d17d6c0 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
@@ -41,7 +41,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
-import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9502b47b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java
--
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java
index cabcb46..cedc506 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java
@@ -190,7 +190,7 @@ public class DeletedBlockLogImpl implements DeletedBlockLog
{
try {
for(Long txID : txIDs) {
try {
- byte [] deleteBlockBytes =
+ byte[] deleteBlockBytes =
deletedStore.get(Longs.toByteArray(txID));
if (deleteBlockBytes == null) {
LOG.warn("Delete txID {} not found", txID);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9502b47b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
--
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
index e569874..2d88621 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
@@ -152,7 +152,8 @@ public class ContainerMapping implements Mapping {
ContainerInfo containerInfo;
lock.lock();
try {
- byte[]
[15/50] [abbrv] hadoop git commit: HDDS-45. Removal of old OzoneRestClient. Contributed by Lokesh Jain.
HDDS-45. Removal of old OzoneRestClient. Contributed by Lokesh Jain.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/774daa8d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/774daa8d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/774daa8d
Branch: refs/heads/YARN-7402
Commit: 774daa8d532f91fe8e342a8da2cfa65a8629
Parents: c05b5d4
Author: Mukul Kumar Singh
Authored: Thu May 24 15:53:42 2018 +0530
Committer: Mukul Kumar Singh
Committed: Thu May 24 15:53:42 2018 +0530
--
.../apache/hadoop/hdds/scm/XceiverClient.java | 22 +-
.../hadoop/ozone/web/client/OzoneBucket.java| 646 ---
.../hadoop/ozone/web/client/OzoneKey.java | 44 -
.../ozone/web/client/OzoneRestClient.java | 804 ---
.../hadoop/ozone/web/client/OzoneVolume.java| 584 --
.../hadoop/ozone/web/client/package-info.java | 34 -
.../hadoop/ozone/MiniOzoneClusterImpl.java | 3 +-
.../apache/hadoop/ozone/RatisTestHelper.java| 14 +-
.../ozone/web/TestOzoneRestWithMiniCluster.java | 207 ++---
.../hadoop/ozone/web/client/TestBuckets.java| 193 +++--
.../ozone/web/client/TestBucketsRatis.java | 15 +-
.../hadoop/ozone/web/client/TestKeys.java | 286 ---
.../hadoop/ozone/web/client/TestKeysRatis.java | 29 +-
.../hadoop/ozone/web/client/TestVolume.java | 285 +++
.../ozone/web/client/TestVolumeRatis.java | 29 +-
15 files changed, 548 insertions(+), 2647 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/774daa8d/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClient.java
--
diff --git
a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClient.java
b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClient.java
index 6d33cd4..42e02f9 100644
---
a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClient.java
+++
b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClient.java
@@ -54,6 +54,7 @@ public class XceiverClient extends XceiverClientSpi {
private Bootstrap b;
private EventLoopGroup group;
private final Semaphore semaphore;
+ private boolean closed = false;
/**
* Constructs a client that can communicate with the Container framework on
@@ -74,6 +75,10 @@ public class XceiverClient extends XceiverClientSpi {
@Override
public void connect() throws Exception {
+if (closed) {
+ throw new IOException("This channel is not connected.");
+}
+
if (channel != null && channel.isActive()) {
throw new IOException("This client is already connected to a host.");
}
@@ -97,6 +102,18 @@ public class XceiverClient extends XceiverClientSpi {
channel = b.connect(leader.getHostName(), port).sync().channel();
}
+ public void reconnect() throws IOException {
+try {
+ connect();
+ if (channel == null || !channel.isActive()) {
+throw new IOException("This channel is not connected.");
+ }
+} catch (Exception e) {
+ LOG.error("Error while connecting: ", e);
+ throw new IOException(e);
+}
+ }
+
/**
* Returns if the exceiver client connects to a server.
*
@@ -109,6 +126,7 @@ public class XceiverClient extends XceiverClientSpi {
@Override
public void close() {
+closed = true;
if (group != null) {
group.shutdownGracefully().awaitUninterruptibly();
}
@@ -124,7 +142,7 @@ public class XceiverClient extends XceiverClientSpi {
ContainerProtos.ContainerCommandRequestProto request) throws IOException
{
try {
if ((channel == null) || (!channel.isActive())) {
-throw new IOException("This channel is not connected.");
+reconnect();
}
XceiverClientHandler handler =
channel.pipeline().get(XceiverClientHandler.class);
@@ -160,7 +178,7 @@ public class XceiverClient extends XceiverClientSpi {
sendCommandAsync(ContainerProtos.ContainerCommandRequestProto request)
throws IOException, ExecutionException, InterruptedException {
if ((channel == null) || (!channel.isActive())) {
- throw new IOException("This channel is not connected.");
+ reconnect();
}
XceiverClientHandler handler =
channel.pipeline().get(XceiverClientHandler.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/774daa8d/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/web/client/OzoneBucket.java
--
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/web/client/OzoneBucket.java
[31/50] [abbrv] hadoop git commit: HDDS-78. Add per volume level storage stats in SCM. Contributed by Shashikant Banerjee.
HDDS-78. Add per volume level storage stats in SCM.
Contributed by Shashikant Banerjee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0cf6e87f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0cf6e87f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0cf6e87f
Branch: refs/heads/YARN-7402
Commit: 0cf6e87f9212af10eae39cdcb1fe60e6d8191772
Parents: f24c842
Author: Anu Engineer
Authored: Sat May 26 11:06:22 2018 -0700
Committer: Anu Engineer
Committed: Sat May 26 11:11:14 2018 -0700
--
.../placement/metrics/SCMNodeStat.java | 21 --
.../hdds/scm/node/SCMNodeStorageStatMXBean.java | 8 +
.../hdds/scm/node/SCMNodeStorageStatMap.java| 230 +--
.../hdds/scm/node/StorageReportResult.java | 87 +++
.../scm/node/TestSCMNodeStorageStatMap.java | 141 +---
5 files changed, 356 insertions(+), 131 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0cf6e87f/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/metrics/SCMNodeStat.java
--
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/metrics/SCMNodeStat.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/metrics/SCMNodeStat.java
index 4fe72fc..3c871d3 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/metrics/SCMNodeStat.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/metrics/SCMNodeStat.java
@@ -136,25 +136,4 @@ public class SCMNodeStat implements NodeStat {
public int hashCode() {
return Long.hashCode(capacity.get() ^ scmUsed.get() ^ remaining.get());
}
-
-
- /**
- * Truncate to 4 digits since uncontrolled precision is some times
- * counter intuitive to what users expect.
- * @param value - double.
- * @return double.
- */
- private double truncateDecimals(double value) {
-final int multiplier = 1;
-return (double) ((long) (value * multiplier)) / multiplier;
- }
-
- /**
- * get the scmUsed ratio
- */
- public double getScmUsedratio() {
-double scmUsedRatio =
-truncateDecimals(getScmUsed().get() / (double) getCapacity().get());
-return scmUsedRatio;
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0cf6e87f/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeStorageStatMXBean.java
--
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeStorageStatMXBean.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeStorageStatMXBean.java
index f17a970..d81ff0f 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeStorageStatMXBean.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeStorageStatMXBean.java
@@ -19,7 +19,9 @@
package org.apache.hadoop.hdds.scm.node;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.ozone.container.common.impl.StorageLocationReport;
+import java.util.Set;
import java.util.UUID;
/**
@@ -66,4 +68,10 @@ public interface SCMNodeStorageStatMXBean {
* @return long
*/
long getTotalFreeSpace();
+
+ /**
+ * Returns the set of disks for a given Datanode.
+ * @return set of storage volumes
+ */
+ Set getStorageVolumes(UUID datanodeId);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0cf6e87f/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeStorageStatMap.java
--
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeStorageStatMap.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeStorageStatMap.java
index 25cb357..f8ad2af 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeStorageStatMap.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeStorageStatMap.java
@@ -22,18 +22,18 @@ package org.apache.hadoop.hdds.scm.node;
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
-import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
+import org.apache.hadoop.hdds.protocol.proto.
+StorageContainerDatanodeProtocolProtos.SCMStorageReport;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import
[17/50] [abbrv] hadoop git commit: HDFS-13611. Unsafe use of Text as a ConcurrentHashMap key in PBHelperClient.
HDFS-13611. Unsafe use of Text as a ConcurrentHashMap key in PBHelperClient.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c9b63deb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c9b63deb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c9b63deb
Branch: refs/heads/YARN-7402
Commit: c9b63deb533274ca8ef4939f6cd13f728a067f7b
Parents: 1388de1
Author: Andrew Wang
Authored: Thu May 24 09:56:23 2018 -0700
Committer: Andrew Wang
Committed: Thu May 24 09:56:23 2018 -0700
--
.../java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9b63deb/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
index 579ac43..490ccb4 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
@@ -247,7 +247,7 @@ public class PBHelperClient {
ByteString value = fixedByteStringCache.get(key);
if (value == null) {
value = ByteString.copyFromUtf8(key.toString());
- fixedByteStringCache.put(key, value);
+ fixedByteStringCache.put(new Text(key.copyBytes()), value);
}
return value;
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[46/50] [abbrv] hadoop git commit: YARN-6648. [GPG] Add SubClusterCleaner in Global Policy Generator. (botong)
YARN-6648. [GPG] Add SubClusterCleaner in Global Policy Generator. (botong)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/46a4a945
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/46a4a945
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/46a4a945
Branch: refs/heads/YARN-7402
Commit: 46a4a945732afdefec9828d1c43b77d32609bb8a
Parents: bca8e9b
Author: Botong Huang
Authored: Thu Feb 1 14:43:48 2018 -0800
Committer: Botong Huang
Committed: Tue May 29 10:48:40 2018 -0700
--
.../dev-support/findbugs-exclude.xml| 5 +
.../hadoop/yarn/conf/YarnConfiguration.java | 18 +++
.../src/main/resources/yarn-default.xml | 24
.../store/impl/MemoryFederationStateStore.java | 13 ++
.../utils/FederationStateStoreFacade.java | 41 ++-
.../GlobalPolicyGenerator.java | 92 ++-
.../subclustercleaner/SubClusterCleaner.java| 109 +
.../subclustercleaner/package-info.java | 19 +++
.../TestSubClusterCleaner.java | 118 +++
9 files changed, 409 insertions(+), 30 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46a4a945/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
--
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index 5841361..bf2e376 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -380,6 +380,11 @@
+
+
+
+
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46a4a945/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index f7f82f8..7c78e0d 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -3326,6 +3326,24 @@ public class YarnConfiguration extends Configuration {
public static final boolean DEFAULT_ROUTER_WEBAPP_PARTIAL_RESULTS_ENABLED =
false;
+ private static final String FEDERATION_GPG_PREFIX =
+ FEDERATION_PREFIX + "gpg.";
+
+ // The number of threads to use for the GPG scheduled executor service
+ public static final String GPG_SCHEDULED_EXECUTOR_THREADS =
+ FEDERATION_GPG_PREFIX + "scheduled.executor.threads";
+ public static final int DEFAULT_GPG_SCHEDULED_EXECUTOR_THREADS = 10;
+
+ // The interval at which the subcluster cleaner runs, -1 means disabled
+ public static final String GPG_SUBCLUSTER_CLEANER_INTERVAL_MS =
+ FEDERATION_GPG_PREFIX + "subcluster.cleaner.interval-ms";
+ public static final long DEFAULT_GPG_SUBCLUSTER_CLEANER_INTERVAL_MS = -1;
+
+ // The expiration time for a subcluster heartbeat, default is 30 minutes
+ public static final String GPG_SUBCLUSTER_EXPIRATION_MS =
+ FEDERATION_GPG_PREFIX + "subcluster.heartbeat.expiration-ms";
+ public static final long DEFAULT_GPG_SUBCLUSTER_EXPIRATION_MS = 180;
+
// Other Configs
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46a4a945/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index b0ffc48..8a450d3 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3524,6 +3524,30 @@
+ The number of threads to use for the GPG scheduled executor service.
+
+yarn.federation.gpg.scheduled.executor.threads
+10
+
+
+
+
+ The interval at which the subcluster cleaner runs, -1 means disabled.
+
+yarn.federation.gpg.subcluster.cleaner.interval-ms
+-1
+
+
+
+
+ The expiration time for a subcluster heartbeat, default is 30 minutes.
+
+
[16/50] [abbrv] hadoop git commit: YARN-6919. Add default volume mount list. Contributed by Eric Badger
YARN-6919. Add default volume mount list. Contributed by Eric Badger
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1388de18
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1388de18
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1388de18
Branch: refs/heads/YARN-7402
Commit: 1388de18ad51434569589a8f5b0b05c38fe02ab3
Parents: 774daa8
Author: Shane Kumpf
Authored: Thu May 24 09:30:39 2018 -0600
Committer: Shane Kumpf
Committed: Thu May 24 09:30:39 2018 -0600
--
.../hadoop/yarn/conf/YarnConfiguration.java | 10 ++
.../src/main/resources/yarn-default.xml | 14 ++
.../runtime/DockerLinuxContainerRuntime.java| 38 +
.../runtime/TestDockerContainerRuntime.java | 138 +++
4 files changed, 200 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1388de18/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 004a59f..f7f82f8 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -2002,6 +2002,16 @@ public class YarnConfiguration extends Configuration {
*/
public static final int DEFAULT_NM_DOCKER_STOP_GRACE_PERIOD = 10;
+ /** The default list of read-only mounts to be bind-mounted into all
+ * Docker containers that use DockerContainerRuntime. */
+ public static final String NM_DOCKER_DEFAULT_RO_MOUNTS =
+ DOCKER_CONTAINER_RUNTIME_PREFIX + "default-ro-mounts";
+
+ /** The default list of read-write mounts to be bind-mounted into all
+ * Docker containers that use DockerContainerRuntime. */
+ public static final String NM_DOCKER_DEFAULT_RW_MOUNTS =
+ DOCKER_CONTAINER_RUNTIME_PREFIX + "default-rw-mounts";
+
/** The mode in which the Java Container Sandbox should run detailed by
* the JavaSandboxLinuxContainerRuntime. */
public static final String YARN_CONTAINER_SANDBOX =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1388de18/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index c82474c..b0ffc48 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1811,6 +1811,20 @@
+The default list of read-only mounts to be bind-mounted
+ into all Docker containers that use DockerContainerRuntime.
+yarn.nodemanager.runtime.linux.docker.default-ro-mounts
+
+
+
+
+The default list of read-write mounts to be bind-mounted
+ into all Docker containers that use DockerContainerRuntime.
+yarn.nodemanager.runtime.linux.docker.default-rw-mounts
+
+
+
+
The mode in which the Java Container Sandbox should run
detailed by
the JavaSandboxLinuxContainerRuntime.
yarn.nodemanager.runtime.linux.sandbox-mode
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1388de18/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index e131e9d..5e2233b 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++
[11/50] [abbrv] hadoop git commit: HDFS-13598. Reduce unnecessary byte-to-string transform operation in INodesInPath#toString. Contributed by Gabor Bota.
HDFS-13598. Reduce unnecessary byte-to-string transform operation in
INodesInPath#toString. Contributed by Gabor Bota.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7a87add4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7a87add4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7a87add4
Branch: refs/heads/YARN-7402
Commit: 7a87add4ea4c317aa9377d1fc8e43fb5e7418a46
Parents: d996479
Author: Yiqun Lin
Authored: Thu May 24 10:57:35 2018 +0800
Committer: Yiqun Lin
Committed: Thu May 24 10:57:35 2018 +0800
--
.../java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a87add4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
index 8235bf0..50ead61 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
@@ -484,7 +484,7 @@ public class INodesInPath {
}
final StringBuilder b = new StringBuilder(getClass().getSimpleName())
-.append(": path = ").append(DFSUtil.byteArray2PathString(path))
+.append(": path = ").append(getPath())
.append("\n inodes = ");
if (inodes == null) {
b.append("null");
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[02/50] [abbrv] hadoop git commit: HADOOP-15486. Make NetworkTopology#netLock fair. Contributed by Nanda kumar.
HADOOP-15486. Make NetworkTopology#netLock fair. Contributed by Nanda kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/51ce02bb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/51ce02bb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/51ce02bb
Branch: refs/heads/YARN-7402
Commit: 51ce02bb54d6047a8191624a86d427b0c9445cb1
Parents: aa23d49
Author: Arpit Agarwal
Authored: Wed May 23 10:30:12 2018 -0700
Committer: Arpit Agarwal
Committed: Wed May 23 10:30:12 2018 -0700
--
.../src/main/java/org/apache/hadoop/net/NetworkTopology.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/51ce02bb/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
index 256f07b..1f077a7 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
@@ -105,7 +105,7 @@ public class NetworkTopology {
private boolean clusterEverBeenMultiRack = false;
/** the lock used to manage access */
- protected ReadWriteLock netlock = new ReentrantReadWriteLock();
+ protected ReadWriteLock netlock = new ReentrantReadWriteLock(true);
// keeping the constructor because other components like MR still uses this.
public NetworkTopology() {
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[48/50] [abbrv] hadoop git commit: YARN-7707. [GPG] Policy generator framework. Contributed by Young Chen
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f5da8ca6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/resources/schedulerInfo2.json
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/resources/schedulerInfo2.json
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/resources/schedulerInfo2.json
new file mode 100644
index 000..2ff879e
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/resources/schedulerInfo2.json
@@ -0,0 +1,196 @@
+ {
+ "type": "capacityScheduler",
+ "capacity": 100.0,
+ "usedCapacity": 0.0,
+ "maxCapacity": 100.0,
+ "queueName": "root",
+ "queues": {
+"queue": [
+ {
+"type": "capacitySchedulerLeafQueueInfo",
+"capacity": 100.0,
+"usedCapacity": 0.0,
+"maxCapacity": 100.0,
+"absoluteCapacity": 100.0,
+"absoluteMaxCapacity": 100.0,
+"absoluteUsedCapacity": 0.0,
+"numApplications": 484,
+"queueName": "default",
+"state": "RUNNING",
+"resourcesUsed": {
+ "memory": 0,
+ "vCores": 0
+},
+"hideReservationQueues": false,
+"nodeLabels": [
+ "*"
+],
+"numActiveApplications": 484,
+"numPendingApplications": 0,
+"numContainers": 0,
+"maxApplications": 1,
+"maxApplicationsPerUser": 1,
+"userLimit": 100,
+"users": {
+ "user": [
+{
+ "username": "Default",
+ "resourcesUsed": {
+"memory": 0,
+"vCores": 0
+ },
+ "numPendingApplications": 0,
+ "numActiveApplications": 468,
+ "AMResourceUsed": {
+"memory": 30191616,
+"vCores": 468
+ },
+ "userResourceLimit": {
+"memory": 31490048,
+"vCores": 7612
+ }
+}
+ ]
+},
+"userLimitFactor": 1.0,
+"AMResourceLimit": {
+ "memory": 31490048,
+ "vCores": 7612
+},
+"usedAMResource": {
+ "memory": 30388224,
+ "vCores": 532
+},
+"userAMResourceLimit": {
+ "memory": 31490048,
+ "vCores": 7612
+},
+"preemptionDisabled": true
+ },
+ {
+"type": "capacitySchedulerLeafQueueInfo",
+"capacity": 100.0,
+"usedCapacity": 0.0,
+"maxCapacity": 100.0,
+"absoluteCapacity": 100.0,
+"absoluteMaxCapacity": 100.0,
+"absoluteUsedCapacity": 0.0,
+"numApplications": 484,
+"queueName": "default2",
+"state": "RUNNING",
+"resourcesUsed": {
+ "memory": 0,
+ "vCores": 0
+},
+"hideReservationQueues": false,
+"nodeLabels": [
+ "*"
+],
+"numActiveApplications": 484,
+"numPendingApplications": 0,
+"numContainers": 0,
+"maxApplications": 1,
+"maxApplicationsPerUser": 1,
+"userLimit": 100,
+"users": {
+ "user": [
+{
+ "username": "Default",
+ "resourcesUsed": {
+"memory": 0,
+"vCores": 0
+ },
+ "numPendingApplications": 0,
+ "numActiveApplications": 468,
+ "AMResourceUsed": {
+"memory": 30191616,
+"vCores": 468
+ },
+ "userResourceLimit": {
+"memory": 31490048,
+"vCores": 7612
+ }
+}
+ ]
+},
+"userLimitFactor": 1.0,
+"AMResourceLimit": {
+ "memory": 31490048,
+ "vCores": 7612
+},
+"usedAMResource": {
+ "memory": 30388224,
+ "vCores": 532
+},
+"userAMResourceLimit": {
+ "memory": 31490048,
+ "vCores": 7612
+},
+"preemptionDisabled": true
+ }
+]
+ },
+ "health": {
+"lastrun": 1517951638085,
+"operationsInfo": {
+ "entry": {
+"key":
[21/50] [abbrv] hadoop git commit: YARN-8191. Fair scheduler: queue deletion without RM restart. (Gergo Repas via Haibo Chen)
YARN-8191. Fair scheduler: queue deletion without RM restart. (Gergo Repas via
Haibo Chen)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/86bc6425
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/86bc6425
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/86bc6425
Branch: refs/heads/YARN-7402
Commit: 86bc6425d425913899f1d951498bd040e453b3d0
Parents: d9852eb
Author: Haibo Chen
Authored: Thu May 24 17:07:21 2018 -0700
Committer: Haibo Chen
Committed: Thu May 24 17:12:34 2018 -0700
--
.../fair/AllocationFileLoaderService.java | 16 +-
.../scheduler/fair/FSLeafQueue.java | 31 ++
.../resourcemanager/scheduler/fair/FSQueue.java | 9 +
.../scheduler/fair/FairScheduler.java | 29 +-
.../scheduler/fair/QueueManager.java| 155 +++--
.../fair/TestAllocationFileLoaderService.java | 100 +++---
.../scheduler/fair/TestQueueManager.java| 337 +++
7 files changed, 596 insertions(+), 81 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86bc6425/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
index d8d9051..7a40b6a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
@@ -87,7 +87,7 @@ public class AllocationFileLoaderService extends
AbstractService {
private Path allocFile;
private FileSystem fs;
- private Listener reloadListener;
+ private final Listener reloadListener;
@VisibleForTesting
long reloadIntervalMs = ALLOC_RELOAD_INTERVAL_MS;
@@ -95,15 +95,16 @@ public class AllocationFileLoaderService extends
AbstractService {
private Thread reloadThread;
private volatile boolean running = true;
- public AllocationFileLoaderService() {
-this(SystemClock.getInstance());
+ public AllocationFileLoaderService(Listener reloadListener) {
+this(reloadListener, SystemClock.getInstance());
}
private List defaultPermissions;
- public AllocationFileLoaderService(Clock clock) {
+ public AllocationFileLoaderService(Listener reloadListener, Clock clock) {
super(AllocationFileLoaderService.class.getName());
this.clock = clock;
+this.reloadListener = reloadListener;
}
@Override
@@ -114,6 +115,7 @@ public class AllocationFileLoaderService extends
AbstractService {
reloadThread = new Thread(() -> {
while (running) {
try {
+reloadListener.onCheck();
long time = clock.getTime();
long lastModified =
fs.getFileStatus(allocFile).getModificationTime();
@@ -207,10 +209,6 @@ public class AllocationFileLoaderService extends
AbstractService {
return allocPath;
}
- public synchronized void setReloadListener(Listener reloadListener) {
-this.reloadListener = reloadListener;
- }
-
/**
* Updates the allocation list from the allocation config file. This file is
* expected to be in the XML format specified in the design doc.
@@ -351,5 +349,7 @@ public class AllocationFileLoaderService extends
AbstractService {
public interface Listener {
void onReload(AllocationConfiguration info) throws IOException;
+
+void onCheck();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86bc6425/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
[26/50] [abbrv] hadoop git commit: HDDS-96. Add an option in ozone script to generate a site file with minimally required ozone configs. Contributed by Dinesh Chitlangia.
HDDS-96. Add an option in ozone script to generate a site file with minimally
required ozone configs.
Contributed by Dinesh Chitlangia.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8733012a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8733012a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8733012a
Branch: refs/heads/YARN-7402
Commit: 8733012ae35f2762d704f94975a762885d116795
Parents: 1e0d4b1
Author: Anu Engineer
Authored: Fri May 25 13:06:14 2018 -0700
Committer: Anu Engineer
Committed: Fri May 25 13:06:14 2018 -0700
--
.../hadoop/hdds/conf/OzoneConfiguration.java| 6 +-
hadoop-ozone/common/src/main/bin/ozone | 4 +
...TestGenerateOzoneRequiredConfigurations.java | 100 +++
.../GenerateOzoneRequiredConfigurations.java| 174 +++
.../hadoop/ozone/genconf/package-info.java | 24 +++
5 files changed, 305 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8733012a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java
--
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java
index f07718c..36d953c 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java
@@ -137,7 +137,7 @@ public class OzoneConfiguration extends Configuration {
@Override
public String toString() {
- return this.getName() + " " + this.getValue() + this.getTag();
+ return this.getName() + " " + this.getValue() + " " + this.getTag();
}
@Override
@@ -152,11 +152,11 @@ public class OzoneConfiguration extends Configuration {
}
}
- public static void activate(){
+ public static void activate() {
// adds the default resources
Configuration.addDefaultResource("hdfs-default.xml");
Configuration.addDefaultResource("hdfs-site.xml");
Configuration.addDefaultResource("ozone-default.xml");
Configuration.addDefaultResource("ozone-site.xml");
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8733012a/hadoop-ozone/common/src/main/bin/ozone
--
diff --git a/hadoop-ozone/common/src/main/bin/ozone
b/hadoop-ozone/common/src/main/bin/ozone
index 00261c7..6843bdd 100755
--- a/hadoop-ozone/common/src/main/bin/ozone
+++ b/hadoop-ozone/common/src/main/bin/ozone
@@ -47,6 +47,7 @@ function hadoop_usage
hadoop_add_subcommand "scm" daemon "run the Storage Container Manager
service"
hadoop_add_subcommand "scmcli" client "run the CLI of the Storage Container
Manager "
hadoop_add_subcommand "version" client "print the version"
+ hadoop_add_subcommand "genconf" client "generate minimally required ozone
configs and output to ozone-site.xml in specified path"
hadoop_generate_usage "${HADOOP_SHELL_EXECNAME}" false
}
@@ -118,6 +119,9 @@ function ozonecmd_case
version)
HADOOP_CLASSNAME=org.apache.hadoop.util.VersionInfo
;;
+genconf)
+
HADOOP_CLASSNAME=org.apache.hadoop.ozone.genconf.GenerateOzoneRequiredConfigurations
+;;
*)
HADOOP_CLASSNAME="${subcmd}"
if ! hadoop_validate_classname "${HADOOP_CLASSNAME}"; then
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8733012a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/genconf/TestGenerateOzoneRequiredConfigurations.java
--
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/genconf/TestGenerateOzoneRequiredConfigurations.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/genconf/TestGenerateOzoneRequiredConfigurations.java
new file mode 100644
index 000..82582a6
--- /dev/null
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/genconf/TestGenerateOzoneRequiredConfigurations.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
[25/50] [abbrv] hadoop git commit: HDFS-13618. Fix TestDataNodeFaultInjector test failures on Windows. Contributed by Xiao Liang.
HDFS-13618. Fix TestDataNodeFaultInjector test failures on Windows. Contributed
by Xiao Liang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1e0d4b1c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1e0d4b1c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1e0d4b1c
Branch: refs/heads/YARN-7402
Commit: 1e0d4b1c283fb98a95c60a1723f594befb3c18a9
Parents: 02322de
Author: Inigo Goiri
Authored: Fri May 25 09:10:32 2018 -0700
Committer: Inigo Goiri
Committed: Fri May 25 09:14:28 2018 -0700
--
.../hadoop/hdfs/server/datanode/TestDataNodeFaultInjector.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e0d4b1c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeFaultInjector.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeFaultInjector.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeFaultInjector.java
index 1507844..4afacd9 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeFaultInjector.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeFaultInjector.java
@@ -118,7 +118,7 @@ public class TestDataNodeFaultInjector {
final MetricsDataNodeFaultInjector mdnFaultInjector) throws Exception {
final Path baseDir = new Path(
-PathUtils.getTestDir(getClass()).getAbsolutePath(),
+PathUtils.getTestDir(getClass()).getPath(),
GenericTestUtils.getMethodName());
final DataNodeFaultInjector oldDnInjector = DataNodeFaultInjector.get();
DataNodeFaultInjector.set(mdnFaultInjector);
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[19/50] [abbrv] hadoop git commit: YARN-8316. Improved diagnostic message for ATS unavailability for YARN Service. Contributed by Billie Rinaldi
YARN-8316. Improved diagnostic message for ATS unavailability for YARN Service.
Contributed by Billie Rinaldi
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7ff5a402
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7ff5a402
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7ff5a402
Branch: refs/heads/YARN-7402
Commit: 7ff5a40218241ad2380595175a493794129a7402
Parents: 2d19e7d
Author: Eric Yang
Authored: Thu May 24 16:26:02 2018 -0400
Committer: Eric Yang
Committed: Thu May 24 16:26:02 2018 -0400
--
.../org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java | 2 +-
.../org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ff5a402/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
index 072e606..1ceb462 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
@@ -400,7 +400,7 @@ public class YarnClientImpl extends YarnClient {
+ e.getMessage());
return null;
}
- throw e;
+ throw new IOException(e);
} catch (NoClassDefFoundError e) {
NoClassDefFoundError wrappedError = new NoClassDefFoundError(
e.getMessage() + ". It appears that the timeline client "
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ff5a402/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
index b84b49c..70ff47b 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
@@ -1159,7 +1159,7 @@ public class TestYarnClient extends
ParameterizedSchedulerTestBase {
TimelineClient createTimelineClient() throws IOException, YarnException {
timelineClient = mock(TimelineClient.class);
when(timelineClient.getDelegationToken(any(String.class)))
- .thenThrow(new IOException("Best effort test exception"));
+ .thenThrow(new RuntimeException("Best effort test exception"));
return timelineClient;
}
});
@@ -1175,7 +1175,7 @@ public class TestYarnClient extends
ParameterizedSchedulerTestBase {
client.serviceInit(conf);
client.getTimelineDelegationToken();
Assert.fail("Get delegation token should have thrown an exception");
-} catch (Exception e) {
+} catch (IOException e) {
// Success
}
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[33/50] [abbrv] hadoop git commit: MAPREDUCE-7097. MapReduce JHS should honor yarn.webapp.filter-entity-list-by-user. Contributed by Sunil Govindan.
MAPREDUCE-7097. MapReduce JHS should honor
yarn.webapp.filter-entity-list-by-user. Contributed by Sunil Govindan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/88cbe57c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/88cbe57c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/88cbe57c
Branch: refs/heads/YARN-7402
Commit: 88cbe57c069a1d2dd3bfb32e3ad742566470a10b
Parents: d14e26b
Author: Rohith Sharma K S
Authored: Mon May 28 12:45:07 2018 +0530
Committer: Rohith Sharma K S
Committed: Mon May 28 14:05:49 2018 +0530
--
.../mapreduce/v2/hs/webapp/HsJobBlock.java | 18 ++-
.../mapreduce/v2/hs/webapp/TestHsJobBlock.java | 20 ++--
.../apache/hadoop/yarn/webapp/Controller.java | 4
.../org/apache/hadoop/yarn/webapp/View.java | 24 +---
4 files changed, 55 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88cbe57c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobBlock.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobBlock.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobBlock.java
index 18040f0..9b845cd 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobBlock.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobBlock.java
@@ -27,6 +27,8 @@ import static org.apache.hadoop.yarn.webapp.view.JQueryUI._TH;
import java.util.Date;
import java.util.List;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.JobACL;
import org.apache.hadoop.mapreduce.TaskID;
import org.apache.hadoop.mapreduce.v2.api.records.AMInfo;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
@@ -39,8 +41,10 @@ import org.apache.hadoop.mapreduce.v2.hs.webapp.dao.JobInfo;
import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.mapreduce.v2.util.MRApps.TaskAttemptStateUI;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.mapreduce.v2.util.MRWebAppUtil;
import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.util.Times;
import org.apache.hadoop.yarn.webapp.ResponseInfo;
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;
@@ -56,9 +60,14 @@ import com.google.inject.Inject;
*/
public class HsJobBlock extends HtmlBlock {
final AppContext appContext;
+ private UserGroupInformation ugi;
+ private boolean isFilterAppListByUserEnabled;
- @Inject HsJobBlock(AppContext appctx) {
+ @Inject HsJobBlock(Configuration conf, AppContext appctx, ViewContext ctx) {
+super(ctx);
appContext = appctx;
+isFilterAppListByUserEnabled = conf
+.getBoolean(YarnConfiguration.FILTER_ENTITY_LIST_BY_USER, false);
}
/*
@@ -78,6 +87,13 @@ public class HsJobBlock extends HtmlBlock {
html.p().__("Sorry, ", jid, " not found.").__();
return;
}
+ugi = getCallerUGI();
+if (isFilterAppListByUserEnabled && ugi != null
+&& !j.checkAccess(ugi, JobACL.VIEW_JOB)) {
+ html.p().__("Sorry, ", jid, " could not be viewed for '",
+ ugi.getUserName(), "'.").__();
+ return;
+}
if(j instanceof UnparsedJob) {
final int taskCount = j.getTotalMaps() + j.getTotalReduces();
UnparsedJob oversizedJob = (UnparsedJob) j;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88cbe57c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsJobBlock.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsJobBlock.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsJobBlock.java
index 7fa238e..48e3d3b 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsJobBlock.java
+++
[41/50] [abbrv] hadoop git commit: YARN-8338. TimelineService V1.5 doesn't come up after HADOOP-15406. Contributed by Vinod Kumar Vavilapalli
YARN-8338. TimelineService V1.5 doesn't come up after HADOOP-15406. Contributed by Vinod Kumar Vavilapalli Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/31ab960f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/31ab960f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/31ab960f Branch: refs/heads/YARN-7402 Commit: 31ab960f4f931df273481927b897388895d803ba Parents: 438ef49 Author: Jason Lowe Authored: Tue May 29 11:00:30 2018 -0500 Committer: Jason Lowe Committed: Tue May 29 11:00:30 2018 -0500 -- hadoop-project/pom.xml | 5 + .../hadoop-yarn-server-applicationhistoryservice/pom.xml| 5 + 2 files changed, 10 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ab960f/hadoop-project/pom.xml -- diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 73c3f5b..59a9bd2 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -1144,6 +1144,11 @@ 1.8.5 +org.objenesis +objenesis +1.0 + + org.mock-server mockserver-netty 3.9.2 http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ab960f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml index f310518..0527095 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml @@ -155,6 +155,11 @@ leveldbjni-all + + org.objenesis + objenesis + + org.apache.hadoop - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[23/50] [abbrv] hadoop git commit: YARN-8292: Fix the dominant resource preemption cannot happen when some of the resource vector becomes negative. Contributed by Wangda Tan.
YARN-8292: Fix the dominant resource preemption cannot happen when some of the
resource vector becomes negative. Contributed by Wangda Tan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8d5509c6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8d5509c6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8d5509c6
Branch: refs/heads/YARN-7402
Commit: 8d5509c68156faaa6641f4e747fc9ff80adccf88
Parents: bddfe79
Author: Eric E Payne
Authored: Fri May 25 16:06:09 2018 +
Committer: Eric E Payne
Committed: Fri May 25 16:06:09 2018 +
--
.../resource/DefaultResourceCalculator.java | 15 ++-
.../resource/DominantResourceCalculator.java| 39 ---
.../yarn/util/resource/ResourceCalculator.java | 13 ++-
.../hadoop/yarn/util/resource/Resources.java| 5 -
.../AbstractPreemptableResourceCalculator.java | 58 ---
.../CapacitySchedulerPreemptionUtils.java | 61 +--
.../capacity/FifoCandidatesSelector.java| 8 +-
.../FifoIntraQueuePreemptionPlugin.java | 4 +-
.../capacity/IntraQueueCandidatesSelector.java | 2 +-
.../capacity/PreemptableResourceCalculator.java | 6 +-
.../monitor/capacity/TempQueuePerPartition.java | 8 +-
...alCapacityPreemptionPolicyMockFramework.java | 30 ++
.../TestPreemptionForQueueWithPriorities.java | 103 ---
...pacityPreemptionPolicyInterQueueWithDRF.java | 60 ++-
14 files changed, 312 insertions(+), 100 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d5509c6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
index 6375c4a..ab6d7f5 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
@@ -136,13 +136,18 @@ public class DefaultResourceCalculator extends
ResourceCalculator {
}
@Override
- public boolean isAnyMajorResourceZero(Resource resource) {
-return resource.getMemorySize() == 0f;
- }
-
- @Override
public Resource normalizeDown(Resource r, Resource stepFactor) {
return Resources.createResource(
roundDown((r.getMemorySize()), stepFactor.getMemorySize()));
}
+
+ @Override
+ public boolean isAnyMajorResourceZeroOrNegative(Resource resource) {
+return resource.getMemorySize() <= 0;
+ }
+
+ @Override
+ public boolean isAnyMajorResourceAboveZero(Resource resource) {
+return resource.getMemorySize() > 0;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d5509c6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
index 6fed23b..2e85ebc 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
@@ -577,19 +577,6 @@ public class DominantResourceCalculator extends
ResourceCalculator {
}
@Override
- public boolean isAnyMajorResourceZero(Resource resource) {
-int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
-for (int i = 0; i < maxLength; i++) {
- ResourceInformation resourceInformation = resource
- .getResourceInformation(i);
- if (resourceInformation.getValue() == 0L) {
-return true;
- }
-}
-return false;
- }
-
- @Override
public Resource normalizeDown(Resource r, Resource stepFactor) {
Resource ret = Resource.newInstance(r);
int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
@@ -613,4 +600,30 @@ public class DominantResourceCalculator extends
ResourceCalculator {
}
return ret;
}
+
+ @Override
+ public boolean isAnyMajorResourceZeroOrNegative(Resource resource)
[50/50] [abbrv] hadoop git commit: YARN-3660. [GPG] Federation Global Policy Generator (service hook only). (Contributed by Botong Huang via curino)
YARN-3660. [GPG] Federation Global Policy Generator (service hook only).
(Contributed by Botong Huang via curino)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bca8e9bf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bca8e9bf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bca8e9bf
Branch: refs/heads/YARN-7402
Commit: bca8e9bf9d6c0d99e15d45dfb714ca5677ac4e0a
Parents: 9502b47
Author: Carlo Curino
Authored: Thu Jan 18 17:21:06 2018 -0800
Committer: Botong Huang
Committed: Tue May 29 10:48:40 2018 -0700
--
hadoop-project/pom.xml | 6 +
hadoop-yarn-project/hadoop-yarn/bin/yarn| 5 +
hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd| 55 +---
.../hadoop-yarn/conf/yarn-env.sh| 12 ++
.../pom.xml | 98 +
.../globalpolicygenerator/GPGContext.java | 31 +
.../globalpolicygenerator/GPGContextImpl.java | 41 ++
.../GlobalPolicyGenerator.java | 136 +++
.../globalpolicygenerator/package-info.java | 19 +++
.../TestGlobalPolicyGenerator.java | 38 ++
.../hadoop-yarn/hadoop-yarn-server/pom.xml | 1 +
hadoop-yarn-project/pom.xml | 4 +
12 files changed, 424 insertions(+), 22 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bca8e9bf/hadoop-project/pom.xml
--
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 59a9bd2..2db538e 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -446,6 +446,12 @@
org.apache.hadoop
+hadoop-yarn-server-globalpolicygenerator
+${project.version}
+
+
+
+org.apache.hadoop
hadoop-yarn-services-core
${hadoop.version}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bca8e9bf/hadoop-yarn-project/hadoop-yarn/bin/yarn
--
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn
b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index 69afe6f..8061859 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -39,6 +39,7 @@ function hadoop_usage
hadoop_add_subcommand "container" client "prints container(s) report"
hadoop_add_subcommand "daemonlog" admin "get/set the log level for each
daemon"
hadoop_add_subcommand "envvars" client "display computed Hadoop environment
variables"
+ hadoop_add_subcommand "globalpolicygenerator" daemon "run the Global Policy
Generator"
hadoop_add_subcommand "jar " client "run a jar file"
hadoop_add_subcommand "logs" client "dump container logs"
hadoop_add_subcommand "node" admin "prints node report(s)"
@@ -103,6 +104,10 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
echo "HADOOP_TOOLS_LIB_JARS_DIR='${HADOOP_TOOLS_LIB_JARS_DIR}'"
exit 0
;;
+globalpolicygenerator)
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+
HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.globalpolicygenerator.GlobalPolicyGenerator'
+;;
jar)
HADOOP_CLASSNAME=org.apache.hadoop.util.RunJar
;;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bca8e9bf/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
--
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
index e1ac112..bebfd71 100644
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
@@ -134,6 +134,10 @@ if "%1" == "--loglevel" (
set
CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\yarn-server\yarn-server-router\target\classes
)
+ if exist
%HADOOP_YARN_HOME%\yarn-server\yarn-server-globalpolicygenerator\target\classes
(
+set
CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\yarn-server\yarn-server-globalpolicygenerator\target\classes
+ )
+
if exist %HADOOP_YARN_HOME%\build\test\classes (
set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\build\test\classes
)
@@ -155,7 +159,7 @@ if "%1" == "--loglevel" (
set yarncommands=resourcemanager nodemanager proxyserver rmadmin version jar
^
application applicationattempt container node queue logs daemonlog
historyserver ^
- timelineserver timelinereader router classpath
+ timelineserver timelinereader router globalpolicygenerator classpath
for %%i in ( %yarncommands% ) do (
if %yarn-command% == %%i set yarncommand=true
)
@@ -259,7 +263,13 @@ goto :eof
:router
set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\router-config\log4j.properties
set CLASS=org.apache.hadoop.yarn.server.router.Router
-
[08/50] [abbrv] hadoop git commit: YARN-8327. Fix TestAggregatedLogFormat#testReadAcontainerLogs1 on Windows. Contributed by Giovanni Matteo Fumarola.
YARN-8327. Fix TestAggregatedLogFormat#testReadAcontainerLogs1 on Windows.
Contributed by Giovanni Matteo Fumarola.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f09dc730
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f09dc730
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f09dc730
Branch: refs/heads/YARN-7402
Commit: f09dc73001fd5f3319765fa997f4b0ca9e8f2aff
Parents: d726156
Author: Inigo Goiri
Authored: Wed May 23 15:59:30 2018 -0700
Committer: Inigo Goiri
Committed: Wed May 23 15:59:30 2018 -0700
--
.../logaggregation/TestAggregatedLogFormat.java | 19 ---
1 file changed, 12 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f09dc730/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
index efbaa4c..f85445e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
@@ -254,13 +254,18 @@ public class TestAggregatedLogFormat {
// Since we could not open the fileInputStream for stderr, this file is not
// aggregated.
String s = writer.toString();
-int expectedLength =
-"LogType:stdout".length()
-+ (logUploadedTime ? ("\nLog Upload Time:" + Times.format(System
- .currentTimeMillis())).length() : 0)
-+ ("\nLogLength:" + numChars).length()
-+ "\nLog Contents:\n".length() + numChars + "\n".length()
-+ "\nEnd of LogType:stdout\n".length();
+
+int expectedLength = "LogType:stdout".length()
++ (logUploadedTime
+? (System.lineSeparator() + "Log Upload Time:"
++ Times.format(System.currentTimeMillis())).length()
+: 0)
++ (System.lineSeparator() + "LogLength:" + numChars).length()
++ (System.lineSeparator() + "Log Contents:" + System.lineSeparator())
+.length()
++ numChars + ("\n").length() + ("End of LogType:stdout"
++ System.lineSeparator() + System.lineSeparator()).length();
+
Assert.assertTrue("LogType not matched", s.contains("LogType:stdout"));
Assert.assertTrue("log file:stderr should not be aggregated.",
!s.contains("LogType:stderr"));
Assert.assertTrue("log file:logs should not be aggregated.",
!s.contains("LogType:logs"));
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[37/50] [abbrv] hadoop git commit: HDFS-13627. TestErasureCodingExerciseAPIs fails on Windows. Contributed by Anbang Hu.
HDFS-13627. TestErasureCodingExerciseAPIs fails on Windows. Contributed by
Anbang Hu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/91d7c74e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/91d7c74e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/91d7c74e
Branch: refs/heads/YARN-7402
Commit: 91d7c74e6aa4850922f68bab490b585443e4fccb
Parents: 7c34366
Author: Inigo Goiri
Authored: Mon May 28 10:26:47 2018 -0700
Committer: Inigo Goiri
Committed: Mon May 28 10:26:47 2018 -0700
--
.../org/apache/hadoop/hdfs/TestErasureCodingExerciseAPIs.java | 5 -
1 file changed, 4 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/91d7c74e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingExerciseAPIs.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingExerciseAPIs.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingExerciseAPIs.java
index 4335527..c63ba34 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingExerciseAPIs.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingExerciseAPIs.java
@@ -40,6 +40,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.DataOutputStream;
+import java.io.File;
import java.io.IOException;
import java.nio.file.Paths;
import java.security.NoSuchAlgorithmException;
@@ -91,8 +92,10 @@ public class TestErasureCodingExerciseAPIs {
// Set up java key store
String testRootDir = Paths.get(new FileSystemTestHelper().getTestRootDir())
.toString();
+Path targetFile = new Path(new File(testRootDir).getAbsolutePath(),
+"test.jks");
String keyProviderURI = JavaKeyStoreProvider.SCHEME_NAME + "://file"
-+ new Path(testRootDir, "test.jks").toUri();
++ targetFile.toUri();
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH,
keyProviderURI);
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY,
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[40/50] [abbrv] hadoop git commit: HADOOP-15455. Incorrect debug message in KMSACL#hasAccess. Contributed by Yuen-Kuei Hsueh.
HADOOP-15455. Incorrect debug message in KMSACL#hasAccess. Contributed by
Yuen-Kuei Hsueh.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/438ef495
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/438ef495
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/438ef495
Branch: refs/heads/YARN-7402
Commit: 438ef4951a38171f193eaf2631da31d0f4bc3c62
Parents: 8fdc993
Author: Wei-Chiu Chuang
Authored: Mon May 28 17:32:32 2018 -0700
Committer: Wei-Chiu Chuang
Committed: Mon May 28 17:32:32 2018 -0700
--
.../java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java| 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/438ef495/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java
--
diff --git
a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java
b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java
index b02f34e..17faec2 100644
---
a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java
+++
b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java
@@ -247,9 +247,9 @@ public class KMSACLs implements Runnable, KeyACLs {
if (blacklist == null) {
LOG.debug("No blacklist for {}", type.toString());
} else if (access) {
- LOG.debug("user is in {}" , blacklist.getAclString());
-} else {
LOG.debug("user is not in {}" , blacklist.getAclString());
+} else {
+ LOG.debug("user is in {}" , blacklist.getAclString());
}
}
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[09/50] [abbrv] hadoop git commit: YARN-4599. Set OOM control for memory cgroups. (Miklos Szegedi via Haibo Chen)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9964799/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupElasticMemoryController.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupElasticMemoryController.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupElasticMemoryController.java
new file mode 100644
index 000..118d172
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupElasticMemoryController.java
@@ -0,0 +1,319 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package
org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.junit.Test;
+
+import java.io.File;
+import java.nio.charset.Charset;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * Test for elastic non-strict memory controller based on cgroups.
+ */
+public class TestCGroupElasticMemoryController {
+ private YarnConfiguration conf = new YarnConfiguration();
+ private File script = new File("target/" +
+ TestCGroupElasticMemoryController.class.getName());
+
+ /**
+ * Test that at least one memory type is requested.
+ * @throws YarnException on exception
+ */
+ @Test(expected = YarnException.class)
+ public void testConstructorOff()
+ throws YarnException {
+CGroupElasticMemoryController controller =
+new CGroupElasticMemoryController(
+conf,
+null,
+null,
+false,
+false,
+1
+);
+ }
+
+ /**
+ * Test that the OOM logic is pluggable.
+ * @throws YarnException on exception
+ */
+ @Test
+ public void testConstructorHandler()
+ throws YarnException {
+conf.setClass(YarnConfiguration.NM_ELASTIC_MEMORY_CONTROL_OOM_HANDLER,
+DummyRunnableWithContext.class, Runnable.class);
+CGroupsHandler handler = mock(CGroupsHandler.class);
+when(handler.getPathForCGroup(any(), any())).thenReturn("");
+CGroupElasticMemoryController controller =
+new CGroupElasticMemoryController(
+conf,
+null,
+handler,
+true,
+false,
+1
+);
+ }
+
+ /**
+ * Test that the handler is notified about multiple OOM events.
+ * @throws Exception on exception
+ */
+ @Test
+ public void testMultipleOOMEvents() throws Exception {
+conf.set(YarnConfiguration.NM_ELASTIC_MEMORY_CONTROL_OOM_LISTENER_PATH,
+script.getAbsolutePath());
+try {
+ FileUtils.writeStringToFile(script,
+ "#!/bin/bash\nprintf oomevent;printf oomevent;\n",
+ Charset.defaultCharset(), false);
+ assertTrue("Could not set executable",
+ script.setExecutable(true));
+
+ CGroupsHandler cgroups = mock(CGroupsHandler.class);
+ when(cgroups.getPathForCGroup(any(), any())).thenReturn("");
+ when(cgroups.getCGroupParam(any(), any(), any()))
+ .thenReturn("under_oom 0");
+
+ Runnable handler = mock(Runnable.class);
+ doNothing().when(handler).run();
+
+
[44/50] [abbrv] hadoop git commit: YARN-8369. Javadoc build failed due to 'bad use of >'. (Takanobu Asanuma via wangda)
YARN-8369. Javadoc build failed due to 'bad use of >'. (Takanobu Asanuma via
wangda)
Change-Id: I79a42154e8f86ab1c3cc939b3745024b8eebe5f4
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/17aa40f6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/17aa40f6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/17aa40f6
Branch: refs/heads/YARN-7402
Commit: 17aa40f669f197d43387d67dc00040d14cd00948
Parents: 3061bfc
Author: Wangda Tan
Authored: Tue May 29 09:27:36 2018 -0700
Committer: Wangda Tan
Committed: Tue May 29 09:27:36 2018 -0700
--
.../apache/hadoop/yarn/util/resource/ResourceCalculator.java | 4 ++--
.../monitor/capacity/CapacitySchedulerPreemptionUtils.java | 8
2 files changed, 6 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/17aa40f6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
index 51078cd..27394f7 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
@@ -260,10 +260,10 @@ public abstract class ResourceCalculator {
/**
* Check if resource has any major resource types (which are all NodeManagers
- * included) has a >0 value.
+ * included) has a {@literal >} 0 value.
*
* @param resource resource
- * @return returns true if any resource is >0
+ * @return returns true if any resource is {@literal >} 0
*/
public abstract boolean isAnyMajorResourceAboveZero(Resource resource);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/17aa40f6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionUtils.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionUtils.java
index 5396d61..690eb02 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionUtils.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionUtils.java
@@ -136,12 +136,12 @@ public class CapacitySchedulerPreemptionUtils {
* @param conservativeDRF
* should we do conservativeDRF preemption or not.
* When true:
- *stop preempt container when any major resource type <= 0 for
to-
- *preempt.
+ *stop preempt container when any major resource type
+ *{@literal <=} 0 for to-preempt.
*This is default preemption behavior of intra-queue preemption
* When false:
- *stop preempt container when: all major resource type <= 0 for
- *to-preempt.
+ *stop preempt container when: all major resource type
+ *{@literal <=} 0 for to-preempt.
*This is default preemption behavior of inter-queue preemption
* @return should we preempt rmContainer. If we should, deduct from
* resourceToObtainByPartition
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[07/50] [abbrv] hadoop git commit: YARN-8348. Incorrect and missing AfterClass in HBase-tests to fix NPE failures. Contributed by Giovanni Matteo Fumarola.
YARN-8348. Incorrect and missing AfterClass in HBase-tests to fix NPE failures.
Contributed by Giovanni Matteo Fumarola.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d7261561
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d7261561
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d7261561
Branch: refs/heads/YARN-7402
Commit: d72615611cfa6bd82756270d4b10136ec1e56741
Parents: e99e5bf
Author: Inigo Goiri
Authored: Wed May 23 14:43:59 2018 -0700
Committer: Inigo Goiri
Committed: Wed May 23 14:43:59 2018 -0700
--
.../storage/TestHBaseTimelineStorageApps.java| 4 +++-
.../storage/TestHBaseTimelineStorageDomain.java | 8
.../storage/TestHBaseTimelineStorageEntities.java| 4 +++-
.../storage/TestHBaseTimelineStorageSchema.java | 8
.../storage/flow/TestHBaseStorageFlowActivity.java | 4 +++-
.../storage/flow/TestHBaseStorageFlowRun.java| 4 +++-
.../storage/flow/TestHBaseStorageFlowRunCompaction.java | 4 +++-
7 files changed, 31 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d7261561/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageApps.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageApps.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageApps.java
index bc33427..0dee442 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageApps.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageApps.java
@@ -1936,6 +1936,8 @@ public class TestHBaseTimelineStorageApps {
@AfterClass
public static void tearDownAfterClass() throws Exception {
-util.shutdownMiniCluster();
+if (util != null) {
+ util.shutdownMiniCluster();
+}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d7261561/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageDomain.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageDomain.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageDomain.java
index 2932e0c..1f59088 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageDomain.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageDomain.java
@@ -32,6 +32,7 @@ import
org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnRWHelp
import
org.apache.hadoop.yarn.server.timelineservice.storage.domain.DomainColumn;
import
org.apache.hadoop.yarn.server.timelineservice.storage.domain.DomainRowKey;
import
org.apache.hadoop.yarn.server.timelineservice.storage.domain.DomainTableRW;
+import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -123,4 +124,11 @@ public class TestHBaseTimelineStorageDomain {
assertEquals("user1,user2 group1,group2", readers);
assertEquals("writer1,writer2", writers);
}
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+if (util != null) {
+ util.shutdownMiniCluster();
+}
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d7261561/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageEntities.java
[36/50] [abbrv] hadoop git commit: YARN-4781. Support intra-queue preemption for fairness ordering policy. Contributed by Eric Payne.
YARN-4781. Support intra-queue preemption for fairness ordering policy.
Contributed by Eric Payne.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7c343669
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7c343669
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7c343669
Branch: refs/heads/YARN-7402
Commit: 7c343669baf660df3b70d58987d6e68aec54d6fa
Parents: 61df174
Author: Sunil G
Authored: Mon May 28 16:32:53 2018 +0530
Committer: Sunil G
Committed: Mon May 28 16:32:53 2018 +0530
--
.../FifoIntraQueuePreemptionPlugin.java | 37 ++-
.../capacity/IntraQueueCandidatesSelector.java | 40 +++
.../monitor/capacity/TempAppPerPartition.java | 9 +
.../AbstractComparatorOrderingPolicy.java | 2 -
...alCapacityPreemptionPolicyMockFramework.java | 12 +-
...yPreemptionPolicyIntraQueueFairOrdering.java | 276 +++
6 files changed, 366 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c343669/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoIntraQueuePreemptionPlugin.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoIntraQueuePreemptionPlugin.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoIntraQueuePreemptionPlugin.java
index 40f333f..12c178c 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoIntraQueuePreemptionPlugin.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoIntraQueuePreemptionPlugin.java
@@ -34,6 +34,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Resource;
+import
org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.IntraQueueCandidatesSelector.TAFairOrderingComparator;
import
org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.IntraQueueCandidatesSelector.TAPriorityComparator;
import
org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy.IntraQueuePreemptionOrderPolicy;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
@@ -41,6 +42,8 @@ import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.SchedulingMode;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
+import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FairOrderingPolicy;
+import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.OrderingPolicy;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
@@ -263,8 +266,17 @@ public class FifoIntraQueuePreemptionPlugin
Resource queueReassignableResource,
PriorityQueue orderedByPriority) {
-Comparator reverseComp = Collections
-.reverseOrder(new TAPriorityComparator());
+Comparator reverseComp;
+OrderingPolicy queueOrderingPolicy =
+tq.leafQueue.getOrderingPolicy();
+if (queueOrderingPolicy instanceof FairOrderingPolicy
+&& (context.getIntraQueuePreemptionOrderPolicy()
+== IntraQueuePreemptionOrderPolicy.USERLIMIT_FIRST)) {
+ reverseComp = Collections.reverseOrder(
+ new TAFairOrderingComparator(this.rc, clusterResource));
+} else {
+ reverseComp = Collections.reverseOrder(new TAPriorityComparator());
+}
TreeSet orderedApps = new TreeSet<>(reverseComp);
String partition = tq.partition;
@@ -355,7 +367,16 @@ public class FifoIntraQueuePreemptionPlugin
TempQueuePerPartition tq, Collection apps,
Resource clusterResource,
Map perUserAMUsed) {
-TAPriorityComparator taComparator = new TAPriorityComparator();
+Comparator taComparator;
+OrderingPolicy orderingPolicy =
+tq.leafQueue.getOrderingPolicy();
+if (orderingPolicy instanceof FairOrderingPolicy
+&&
[14/50] [abbrv] hadoop git commit: HDDS-45. Removal of old OzoneRestClient. Contributed by Lokesh Jain.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/774daa8d/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/TestOzoneRestWithMiniCluster.java
--
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/TestOzoneRestWithMiniCluster.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/TestOzoneRestWithMiniCluster.java
index 5b67657..a9b8175 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/TestOzoneRestWithMiniCluster.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/TestOzoneRestWithMiniCluster.java
@@ -23,23 +23,31 @@ import static
org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
import static org.apache.hadoop.ozone.OzoneConsts.CHUNK_SIZE;
import static org.junit.Assert.*;
+import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.RandomStringUtils;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.web.client.OzoneRestClient;
+import org.apache.hadoop.hdds.client.OzoneQuota;
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.ozone.client.VolumeArgs;
+import org.apache.hadoop.ozone.client.io.OzoneInputStream;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.rpc.RpcClient;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
-import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.web.client.OzoneBucket;
-import org.apache.hadoop.ozone.web.client.OzoneVolume;
-import org.apache.hadoop.ozone.web.request.OzoneQuota;
import org.junit.rules.Timeout;
+import java.io.IOException;
+import java.io.InputStream;
+
/**
* End-to-end testing of Ozone REST operations.
*/
@@ -52,7 +60,9 @@ public class TestOzoneRestWithMiniCluster {
private static MiniOzoneCluster cluster;
private static OzoneConfiguration conf;
- private static OzoneRestClient ozoneClient;
+ private static ClientProtocol client;
+ private static ReplicationFactor replicationFactor = ReplicationFactor.ONE;
+ private static ReplicationType replicationType = ReplicationType.STAND_ALONE;
@Rule
public ExpectedException exception = ExpectedException.none();
@@ -62,180 +72,125 @@ public class TestOzoneRestWithMiniCluster {
conf = new OzoneConfiguration();
cluster = MiniOzoneCluster.newBuilder(conf).build();
cluster.waitForClusterToBeReady();
-int port = cluster.getHddsDatanodes().get(0)
-.getDatanodeDetails().getOzoneRestPort();
-ozoneClient = new OzoneRestClient(
-String.format("http://localhost:%d;, port));
-ozoneClient.setUserAuth(OzoneConsts.OZONE_SIMPLE_HDFS_USER);
+client = new RpcClient(conf);
}
@AfterClass
- public static void shutdown() throws InterruptedException {
+ public static void shutdown() throws InterruptedException, IOException {
if (cluster != null) {
cluster.shutdown();
}
-IOUtils.cleanupWithLogger(null, ozoneClient);
+client.close();
}
@Test
public void testCreateAndGetVolume() throws Exception {
-String volumeName = nextId("volume");
-OzoneVolume volume = ozoneClient.createVolume(volumeName, "bilbo",
"100TB");
-assertNotNull(volume);
-assertEquals(volumeName, volume.getVolumeName());
-assertEquals(ozoneClient.getUserAuth(), volume.getCreatedby());
-assertEquals("bilbo", volume.getOwnerName());
-assertNotNull(volume.getQuota());
-assertEquals(OzoneQuota.parseQuota("100TB").sizeInBytes(),
-volume.getQuota().sizeInBytes());
-volume = ozoneClient.getVolume(volumeName);
-assertNotNull(volume);
-assertEquals(volumeName, volume.getVolumeName());
-assertEquals(ozoneClient.getUserAuth(), volume.getCreatedby());
-assertEquals("bilbo", volume.getOwnerName());
-assertNotNull(volume.getQuota());
-assertEquals(OzoneQuota.parseQuota("100TB").sizeInBytes(),
-volume.getQuota().sizeInBytes());
+createAndGetVolume();
}
@Test
public void testCreateAndGetBucket() throws Exception {
-String volumeName = nextId("volume");
-String bucketName = nextId("bucket");
-OzoneVolume volume = ozoneClient.createVolume(volumeName, "bilbo",
"100TB");
-assertNotNull(volume);
-assertEquals(volumeName, volume.getVolumeName());
-assertEquals(ozoneClient.getUserAuth(), volume.getCreatedby());
-assertEquals("bilbo", volume.getOwnerName());
-
hadoop git commit: HDDS-125. Cleanup HDDS CheckStyle issues. Contributed by Anu Engineer.
Repository: hadoop
Updated Branches:
refs/heads/trunk 17aa40f66 -> 9502b47bd
HDDS-125. Cleanup HDDS CheckStyle issues.
Contributed by Anu Engineer.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9502b47b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9502b47b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9502b47b
Branch: refs/heads/trunk
Commit: 9502b47bd2a3cf32edae635293169883c2914475
Parents: 17aa40f
Author: Anu Engineer
Authored: Tue May 29 09:54:06 2018 -0700
Committer: Anu Engineer
Committed: Tue May 29 09:54:06 2018 -0700
--
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 1 -
.../hdds/scm/block/DeletedBlockLogImpl.java | 2 +-
.../hdds/scm/container/ContainerMapping.java| 6 +-
.../scm/container/ContainerStateManager.java| 24 +++
.../hadoop/hdds/scm/container/Mapping.java | 9 ++-
.../hdds/scm/node/SCMNodeStorageStatMXBean.java | 4 +-
.../hdds/scm/node/SCMNodeStorageStatMap.java| 19 +++---
.../hdds/scm/node/StorageReportResult.java | 8 +--
.../hdds/scm/node/states/Node2ContainerMap.java | 2 +-
.../hdds/scm/pipelines/PipelineSelector.java| 5 +-
.../scm/server/StorageContainerManager.java | 3 +-
.../TestStorageContainerManagerHttpServer.java | 1 -
.../hadoop/hdds/scm/block/package-info.java | 23 +++
.../scm/container/TestContainerMapping.java | 12 ++--
.../hdds/scm/container/closer/package-info.java | 22 +++
.../hadoop/hdds/scm/container/package-info.java | 22 +++
.../hdds/scm/container/states/package-info.java | 22 +++
.../hadoop/hdds/scm/node/TestNodeManager.java | 66 ++--
.../scm/node/TestSCMNodeStorageStatMap.java | 32 +-
.../hadoop/hdds/scm/node/package-info.java | 22 +++
.../ozone/container/common/TestEndPoint.java| 2 -
.../ozone/container/common/package-info.java| 22 +++
.../ozone/container/placement/package-info.java | 22 +++
.../replication/TestContainerSupervisor.java| 7 ++-
24 files changed, 263 insertions(+), 95 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9502b47b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
--
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
index 5a98e85..d17d6c0 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
@@ -41,7 +41,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
-import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9502b47b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java
--
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java
index cabcb46..cedc506 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java
@@ -190,7 +190,7 @@ public class DeletedBlockLogImpl implements DeletedBlockLog
{
try {
for(Long txID : txIDs) {
try {
- byte [] deleteBlockBytes =
+ byte[] deleteBlockBytes =
deletedStore.get(Longs.toByteArray(txID));
if (deleteBlockBytes == null) {
LOG.warn("Delete txID {} not found", txID);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9502b47b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
--
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
index e569874..2d88621 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
@@ -152,7 +152,8 @@ public class ContainerMapping implements Mapping {
[1/2] hadoop git commit: YARN-8339. Service AM should localize static/archive resource types to container working directory instead of 'resources'. (Suma Shivaprasad via wangda)
Repository: hadoop
Updated Branches:
refs/heads/branch-3.1 1dd9670dd -> 3eb1cb18c
YARN-8339. Service AM should localize static/archive resource types to
container working directory instead of 'resources'. (Suma Shivaprasad via
wangda)
Change-Id: I9f8e8f621650347f6c2f9e3420edee9eb2f356a4
(cherry picked from commit 3061bfcde53210d2032df3814243498b27a997b7)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b262ea13
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b262ea13
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b262ea13
Branch: refs/heads/branch-3.1
Commit: b262ea13818d74f180f10077e9f47bef30d02a06
Parents: 1dd9670
Author: Wangda Tan
Authored: Tue May 29 09:23:11 2018 -0700
Committer: Wangda Tan
Committed: Tue May 29 09:28:27 2018 -0700
--
.../org/apache/hadoop/yarn/service/provider/ProviderUtils.java | 3 +--
.../apache/hadoop/yarn/service/provider/TestProviderUtils.java | 6 +++---
2 files changed, 4 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b262ea13/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
index 1ad5fd8..ac90992 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
@@ -298,8 +298,7 @@ public class ProviderUtils implements YarnServiceConstants {
destFile = new Path(staticFile.getDestFile());
}
- String symlink = APP_RESOURCES_DIR + "/" + destFile.getName();
- addLocalResource(launcher, symlink, localResource, destFile);
+ addLocalResource(launcher, destFile.getName(), localResource, destFile);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b262ea13/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestProviderUtils.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestProviderUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestProviderUtils.java
index 6e8bc43..5d794d2 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestProviderUtils.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestProviderUtils.java
@@ -154,11 +154,11 @@ public class TestProviderUtils {
ProviderUtils.handleStaticFilesForLocalization(launcher, sfs,
compLaunchCtx);
-
Mockito.verify(launcher).addLocalResource(Mockito.eq("resources/destFile1"),
+Mockito.verify(launcher).addLocalResource(Mockito.eq("destFile1"),
any(LocalResource.class));
Mockito.verify(launcher).addLocalResource(
-Mockito.eq("resources/destFile_2"), any(LocalResource.class));
+Mockito.eq("destFile_2"), any(LocalResource.class));
Mockito.verify(launcher).addLocalResource(
-Mockito.eq("resources/sourceFile4"), any(LocalResource.class));
+Mockito.eq("sourceFile4"), any(LocalResource.class));
}
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[2/2] hadoop git commit: YARN-8369. Javadoc build failed due to 'bad use of >'. (Takanobu Asanuma via wangda)
YARN-8369. Javadoc build failed due to 'bad use of >'. (Takanobu Asanuma via
wangda)
Change-Id: I79a42154e8f86ab1c3cc939b3745024b8eebe5f4
(cherry picked from commit 17aa40f669f197d43387d67dc00040d14cd00948)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3eb1cb18
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3eb1cb18
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3eb1cb18
Branch: refs/heads/branch-3.1
Commit: 3eb1cb18c716dd7b131b4ceb4b7af3892b83187d
Parents: b262ea1
Author: Wangda Tan
Authored: Tue May 29 09:27:36 2018 -0700
Committer: Wangda Tan
Committed: Tue May 29 09:28:34 2018 -0700
--
.../apache/hadoop/yarn/util/resource/ResourceCalculator.java | 4 ++--
.../monitor/capacity/CapacitySchedulerPreemptionUtils.java | 8
2 files changed, 6 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3eb1cb18/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
index 51078cd..27394f7 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
@@ -260,10 +260,10 @@ public abstract class ResourceCalculator {
/**
* Check if resource has any major resource types (which are all NodeManagers
- * included) has a >0 value.
+ * included) has a {@literal >} 0 value.
*
* @param resource resource
- * @return returns true if any resource is >0
+ * @return returns true if any resource is {@literal >} 0
*/
public abstract boolean isAnyMajorResourceAboveZero(Resource resource);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3eb1cb18/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionUtils.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionUtils.java
index 5396d61..690eb02 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionUtils.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionUtils.java
@@ -136,12 +136,12 @@ public class CapacitySchedulerPreemptionUtils {
* @param conservativeDRF
* should we do conservativeDRF preemption or not.
* When true:
- *stop preempt container when any major resource type <= 0 for
to-
- *preempt.
+ *stop preempt container when any major resource type
+ *{@literal <=} 0 for to-preempt.
*This is default preemption behavior of intra-queue preemption
* When false:
- *stop preempt container when: all major resource type <= 0 for
- *to-preempt.
+ *stop preempt container when: all major resource type
+ *{@literal <=} 0 for to-preempt.
*This is default preemption behavior of inter-queue preemption
* @return should we preempt rmContainer. If we should, deduct from
* resourceToObtainByPartition
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[2/2] hadoop git commit: YARN-8369. Javadoc build failed due to 'bad use of >'. (Takanobu Asanuma via wangda)
YARN-8369. Javadoc build failed due to 'bad use of >'. (Takanobu Asanuma via
wangda)
Change-Id: I79a42154e8f86ab1c3cc939b3745024b8eebe5f4
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/17aa40f6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/17aa40f6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/17aa40f6
Branch: refs/heads/trunk
Commit: 17aa40f669f197d43387d67dc00040d14cd00948
Parents: 3061bfc
Author: Wangda Tan
Authored: Tue May 29 09:27:36 2018 -0700
Committer: Wangda Tan
Committed: Tue May 29 09:27:36 2018 -0700
--
.../apache/hadoop/yarn/util/resource/ResourceCalculator.java | 4 ++--
.../monitor/capacity/CapacitySchedulerPreemptionUtils.java | 8
2 files changed, 6 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/17aa40f6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
index 51078cd..27394f7 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
@@ -260,10 +260,10 @@ public abstract class ResourceCalculator {
/**
* Check if resource has any major resource types (which are all NodeManagers
- * included) has a >0 value.
+ * included) has a {@literal >} 0 value.
*
* @param resource resource
- * @return returns true if any resource is >0
+ * @return returns true if any resource is {@literal >} 0
*/
public abstract boolean isAnyMajorResourceAboveZero(Resource resource);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/17aa40f6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionUtils.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionUtils.java
index 5396d61..690eb02 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionUtils.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionUtils.java
@@ -136,12 +136,12 @@ public class CapacitySchedulerPreemptionUtils {
* @param conservativeDRF
* should we do conservativeDRF preemption or not.
* When true:
- *stop preempt container when any major resource type <= 0 for
to-
- *preempt.
+ *stop preempt container when any major resource type
+ *{@literal <=} 0 for to-preempt.
*This is default preemption behavior of intra-queue preemption
* When false:
- *stop preempt container when: all major resource type <= 0 for
- *to-preempt.
+ *stop preempt container when: all major resource type
+ *{@literal <=} 0 for to-preempt.
*This is default preemption behavior of inter-queue preemption
* @return should we preempt rmContainer. If we should, deduct from
* resourceToObtainByPartition
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[1/2] hadoop git commit: YARN-8339. Service AM should localize static/archive resource types to container working directory instead of 'resources'. (Suma Shivaprasad via wangda)
Repository: hadoop
Updated Branches:
refs/heads/trunk 3c75f8e49 -> 17aa40f66
YARN-8339. Service AM should localize static/archive resource types to
container working directory instead of 'resources'. (Suma Shivaprasad via
wangda)
Change-Id: I9f8e8f621650347f6c2f9e3420edee9eb2f356a4
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3061bfcd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3061bfcd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3061bfcd
Branch: refs/heads/trunk
Commit: 3061bfcde53210d2032df3814243498b27a997b7
Parents: 3c75f8e
Author: Wangda Tan
Authored: Tue May 29 09:23:11 2018 -0700
Committer: Wangda Tan
Committed: Tue May 29 09:23:11 2018 -0700
--
.../org/apache/hadoop/yarn/service/provider/ProviderUtils.java | 3 +--
.../apache/hadoop/yarn/service/provider/TestProviderUtils.java | 6 +++---
2 files changed, 4 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3061bfcd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
index 1ad5fd8..ac90992 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
@@ -298,8 +298,7 @@ public class ProviderUtils implements YarnServiceConstants {
destFile = new Path(staticFile.getDestFile());
}
- String symlink = APP_RESOURCES_DIR + "/" + destFile.getName();
- addLocalResource(launcher, symlink, localResource, destFile);
+ addLocalResource(launcher, destFile.getName(), localResource, destFile);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3061bfcd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestProviderUtils.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestProviderUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestProviderUtils.java
index 6e8bc43..5d794d2 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestProviderUtils.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestProviderUtils.java
@@ -154,11 +154,11 @@ public class TestProviderUtils {
ProviderUtils.handleStaticFilesForLocalization(launcher, sfs,
compLaunchCtx);
-
Mockito.verify(launcher).addLocalResource(Mockito.eq("resources/destFile1"),
+Mockito.verify(launcher).addLocalResource(Mockito.eq("destFile1"),
any(LocalResource.class));
Mockito.verify(launcher).addLocalResource(
-Mockito.eq("resources/destFile_2"), any(LocalResource.class));
+Mockito.eq("destFile_2"), any(LocalResource.class));
Mockito.verify(launcher).addLocalResource(
-Mockito.eq("resources/sourceFile4"), any(LocalResource.class));
+Mockito.eq("sourceFile4"), any(LocalResource.class));
}
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: YARN-8338. TimelineService V1.5 doesn't come up after HADOOP-15406. Contributed by Vinod Kumar Vavilapalli
Repository: hadoop Updated Branches: refs/heads/branch-3.0 1f594f31d -> d5708bbcd YARN-8338. TimelineService V1.5 doesn't come up after HADOOP-15406. Contributed by Vinod Kumar Vavilapalli (cherry picked from commit 31ab960f4f931df273481927b897388895d803ba) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d5708bbc Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d5708bbc Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d5708bbc Branch: refs/heads/branch-3.0 Commit: d5708bbcdc86a18cb43f2243629d69dcdae702de Parents: 1f594f3 Author: Jason Lowe Authored: Tue May 29 11:00:30 2018 -0500 Committer: Jason Lowe Committed: Tue May 29 11:15:07 2018 -0500 -- hadoop-project/pom.xml | 5 + .../hadoop-yarn-server-applicationhistoryservice/pom.xml| 5 + 2 files changed, 10 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5708bbc/hadoop-project/pom.xml -- diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index b69bba3..e697f4d 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -979,6 +979,11 @@ 1.8.5 +org.objenesis +objenesis +1.0 + + org.mock-server mockserver-netty 3.9.2 http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5708bbc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml index ddba171..9bdac13 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml @@ -155,6 +155,11 @@ leveldbjni-all + + org.objenesis + objenesis + + org.apache.hadoop - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[4/5] hadoop git commit: HADOOP-15497. TestTrash should use proper test path to avoid failing on Windows. Contributed by Anbang Hu.
HADOOP-15497. TestTrash should use proper test path to avoid failing on
Windows. Contributed by Anbang Hu.
(cherry picked from commit 3c75f8e4933221fa60a87e86a3db5e4727530b6f)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/09fbbff6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/09fbbff6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/09fbbff6
Branch: refs/heads/branch-2
Commit: 09fbbff695f511df7fa82a244483816d037c4898
Parents: 0fc8b43
Author: Inigo Goiri
Authored: Tue May 29 09:11:08 2018 -0700
Committer: Inigo Goiri
Committed: Tue May 29 09:12:47 2018 -0700
--
.../src/test/java/org/apache/hadoop/fs/TestTrash.java | 10 ++
1 file changed, 6 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09fbbff6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
index 7a5b25e..1a6d580 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
@@ -48,9 +48,11 @@ import org.junit.Test;
*/
public class TestTrash extends TestCase {
- private final static Path TEST_DIR = new Path(GenericTestUtils.getTempPath(
+ private final static File BASE_PATH = new File(GenericTestUtils.getTempPath(
"testTrash"));
+ private final static Path TEST_DIR = new Path(BASE_PATH.getAbsolutePath());
+
@Before
public void setUp() throws IOException {
// ensure each test initiates a FileSystem instance,
@@ -680,7 +682,7 @@ public class TestTrash extends TestCase {
static class TestLFS extends LocalFileSystem {
Path home;
TestLFS() {
- this(new Path(TEST_DIR, "user/test"));
+ this(TEST_DIR);
}
TestLFS(final Path home) {
super(new RawLocalFileSystem() {
@@ -807,8 +809,8 @@ public class TestTrash extends TestCase {
*/
public static void verifyTrashPermission(FileSystem fs, Configuration conf)
throws IOException {
-Path caseRoot = new Path(
-GenericTestUtils.getTempPath("testTrashPermission"));
+Path caseRoot = new Path(BASE_PATH.getPath(),
+"testTrashPermission");
try (FileSystem fileSystem = fs){
Trash trash = new Trash(fileSystem, conf);
FileSystemTestWrapper wrapper =
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[2/5] hadoop git commit: HADOOP-15497. TestTrash should use proper test path to avoid failing on Windows. Contributed by Anbang Hu.
HADOOP-15497. TestTrash should use proper test path to avoid failing on
Windows. Contributed by Anbang Hu.
(cherry picked from commit 3c75f8e4933221fa60a87e86a3db5e4727530b6f)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1dd9670d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1dd9670d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1dd9670d
Branch: refs/heads/branch-3.1
Commit: 1dd9670ddd8767f3860c6662d860181aacacf744
Parents: 500b0ee
Author: Inigo Goiri
Authored: Tue May 29 09:11:08 2018 -0700
Committer: Inigo Goiri
Committed: Tue May 29 09:11:38 2018 -0700
--
.../src/test/java/org/apache/hadoop/fs/TestTrash.java | 10 ++
1 file changed, 6 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1dd9670d/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
index 12aed29..fa2d21f 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
@@ -49,9 +49,11 @@ import org.apache.hadoop.util.Time;
*/
public class TestTrash {
- private final static Path TEST_DIR = new Path(GenericTestUtils.getTempPath(
+ private final static File BASE_PATH = new File(GenericTestUtils.getTempPath(
"testTrash"));
+ private final static Path TEST_DIR = new Path(BASE_PATH.getAbsolutePath());
+
@Before
public void setUp() throws IOException {
// ensure each test initiates a FileSystem instance,
@@ -682,7 +684,7 @@ public class TestTrash {
static class TestLFS extends LocalFileSystem {
Path home;
TestLFS() {
- this(new Path(TEST_DIR, "user/test"));
+ this(TEST_DIR);
}
TestLFS(final Path home) {
super(new RawLocalFileSystem() {
@@ -809,8 +811,8 @@ public class TestTrash {
*/
public static void verifyTrashPermission(FileSystem fs, Configuration conf)
throws IOException {
-Path caseRoot = new Path(
-GenericTestUtils.getTempPath("testTrashPermission"));
+Path caseRoot = new Path(BASE_PATH.getPath(),
+"testTrashPermission");
try (FileSystem fileSystem = fs){
Trash trash = new Trash(fileSystem, conf);
FileSystemTestWrapper wrapper =
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[3/5] hadoop git commit: HADOOP-15497. TestTrash should use proper test path to avoid failing on Windows. Contributed by Anbang Hu.
HADOOP-15497. TestTrash should use proper test path to avoid failing on
Windows. Contributed by Anbang Hu.
(cherry picked from commit 3c75f8e4933221fa60a87e86a3db5e4727530b6f)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1f594f31
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1f594f31
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1f594f31
Branch: refs/heads/branch-3.0
Commit: 1f594f31dd11609bfa673848509c5443aabfd958
Parents: 595b44e
Author: Inigo Goiri
Authored: Tue May 29 09:11:08 2018 -0700
Committer: Inigo Goiri
Committed: Tue May 29 09:12:08 2018 -0700
--
.../src/test/java/org/apache/hadoop/fs/TestTrash.java | 10 ++
1 file changed, 6 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f594f31/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
index 12aed29..fa2d21f 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
@@ -49,9 +49,11 @@ import org.apache.hadoop.util.Time;
*/
public class TestTrash {
- private final static Path TEST_DIR = new Path(GenericTestUtils.getTempPath(
+ private final static File BASE_PATH = new File(GenericTestUtils.getTempPath(
"testTrash"));
+ private final static Path TEST_DIR = new Path(BASE_PATH.getAbsolutePath());
+
@Before
public void setUp() throws IOException {
// ensure each test initiates a FileSystem instance,
@@ -682,7 +684,7 @@ public class TestTrash {
static class TestLFS extends LocalFileSystem {
Path home;
TestLFS() {
- this(new Path(TEST_DIR, "user/test"));
+ this(TEST_DIR);
}
TestLFS(final Path home) {
super(new RawLocalFileSystem() {
@@ -809,8 +811,8 @@ public class TestTrash {
*/
public static void verifyTrashPermission(FileSystem fs, Configuration conf)
throws IOException {
-Path caseRoot = new Path(
-GenericTestUtils.getTempPath("testTrashPermission"));
+Path caseRoot = new Path(BASE_PATH.getPath(),
+"testTrashPermission");
try (FileSystem fileSystem = fs){
Trash trash = new Trash(fileSystem, conf);
FileSystemTestWrapper wrapper =
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[5/5] hadoop git commit: HADOOP-15497. TestTrash should use proper test path to avoid failing on Windows. Contributed by Anbang Hu.
HADOOP-15497. TestTrash should use proper test path to avoid failing on
Windows. Contributed by Anbang Hu.
(cherry picked from commit 3c75f8e4933221fa60a87e86a3db5e4727530b6f)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c3dce262
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c3dce262
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c3dce262
Branch: refs/heads/branch-2.9
Commit: c3dce2620e111ce7b4552b570406bb9d18e7acc9
Parents: f8c03a8
Author: Inigo Goiri
Authored: Tue May 29 09:11:08 2018 -0700
Committer: Inigo Goiri
Committed: Tue May 29 09:13:02 2018 -0700
--
.../src/test/java/org/apache/hadoop/fs/TestTrash.java | 10 ++
1 file changed, 6 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3dce262/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
index 7a5b25e..1a6d580 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
@@ -48,9 +48,11 @@ import org.junit.Test;
*/
public class TestTrash extends TestCase {
- private final static Path TEST_DIR = new Path(GenericTestUtils.getTempPath(
+ private final static File BASE_PATH = new File(GenericTestUtils.getTempPath(
"testTrash"));
+ private final static Path TEST_DIR = new Path(BASE_PATH.getAbsolutePath());
+
@Before
public void setUp() throws IOException {
// ensure each test initiates a FileSystem instance,
@@ -680,7 +682,7 @@ public class TestTrash extends TestCase {
static class TestLFS extends LocalFileSystem {
Path home;
TestLFS() {
- this(new Path(TEST_DIR, "user/test"));
+ this(TEST_DIR);
}
TestLFS(final Path home) {
super(new RawLocalFileSystem() {
@@ -807,8 +809,8 @@ public class TestTrash extends TestCase {
*/
public static void verifyTrashPermission(FileSystem fs, Configuration conf)
throws IOException {
-Path caseRoot = new Path(
-GenericTestUtils.getTempPath("testTrashPermission"));
+Path caseRoot = new Path(BASE_PATH.getPath(),
+"testTrashPermission");
try (FileSystem fileSystem = fs){
Trash trash = new Trash(fileSystem, conf);
FileSystemTestWrapper wrapper =
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[1/5] hadoop git commit: HADOOP-15497. TestTrash should use proper test path to avoid failing on Windows. Contributed by Anbang Hu.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 0fc8b43dc -> 09fbbff69
refs/heads/branch-2.9 f8c03a808 -> c3dce2620
refs/heads/branch-3.0 595b44e2d -> 1f594f31d
refs/heads/branch-3.1 500b0ee2c -> 1dd9670dd
refs/heads/trunk 31ab960f4 -> 3c75f8e49
HADOOP-15497. TestTrash should use proper test path to avoid failing on
Windows. Contributed by Anbang Hu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3c75f8e4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3c75f8e4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3c75f8e4
Branch: refs/heads/trunk
Commit: 3c75f8e4933221fa60a87e86a3db5e4727530b6f
Parents: 31ab960
Author: Inigo Goiri
Authored: Tue May 29 09:11:08 2018 -0700
Committer: Inigo Goiri
Committed: Tue May 29 09:11:08 2018 -0700
--
.../src/test/java/org/apache/hadoop/fs/TestTrash.java | 10 ++
1 file changed, 6 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c75f8e4/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
index 12aed29..fa2d21f 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
@@ -49,9 +49,11 @@ import org.apache.hadoop.util.Time;
*/
public class TestTrash {
- private final static Path TEST_DIR = new Path(GenericTestUtils.getTempPath(
+ private final static File BASE_PATH = new File(GenericTestUtils.getTempPath(
"testTrash"));
+ private final static Path TEST_DIR = new Path(BASE_PATH.getAbsolutePath());
+
@Before
public void setUp() throws IOException {
// ensure each test initiates a FileSystem instance,
@@ -682,7 +684,7 @@ public class TestTrash {
static class TestLFS extends LocalFileSystem {
Path home;
TestLFS() {
- this(new Path(TEST_DIR, "user/test"));
+ this(TEST_DIR);
}
TestLFS(final Path home) {
super(new RawLocalFileSystem() {
@@ -809,8 +811,8 @@ public class TestTrash {
*/
public static void verifyTrashPermission(FileSystem fs, Configuration conf)
throws IOException {
-Path caseRoot = new Path(
-GenericTestUtils.getTempPath("testTrashPermission"));
+Path caseRoot = new Path(BASE_PATH.getPath(),
+"testTrashPermission");
try (FileSystem fileSystem = fs){
Trash trash = new Trash(fileSystem, conf);
FileSystemTestWrapper wrapper =
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: YARN-8338. TimelineService V1.5 doesn't come up after HADOOP-15406. Contributed by Vinod Kumar Vavilapalli
Repository: hadoop Updated Branches: refs/heads/branch-3.1 6918d9e9c -> 500b0ee2c YARN-8338. TimelineService V1.5 doesn't come up after HADOOP-15406. Contributed by Vinod Kumar Vavilapalli (cherry picked from commit 31ab960f4f931df273481927b897388895d803ba) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/500b0ee2 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/500b0ee2 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/500b0ee2 Branch: refs/heads/branch-3.1 Commit: 500b0ee2cef5f645e9cd55516df48d3007e3986a Parents: 6918d9e Author: Jason Lowe Authored: Tue May 29 11:00:30 2018 -0500 Committer: Jason Lowe Committed: Tue May 29 11:03:19 2018 -0500 -- hadoop-project/pom.xml | 5 + .../hadoop-yarn-server-applicationhistoryservice/pom.xml| 5 + 2 files changed, 10 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/500b0ee2/hadoop-project/pom.xml -- diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 6caf26f..702a7d3 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -993,6 +993,11 @@ 1.8.5 +org.objenesis +objenesis +1.0 + + org.mock-server mockserver-netty 3.9.2 http://git-wip-us.apache.org/repos/asf/hadoop/blob/500b0ee2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml index 3cab4c6..13a373a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml @@ -155,6 +155,11 @@ leveldbjni-all + + org.objenesis + objenesis + + org.apache.hadoop - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: YARN-8338. TimelineService V1.5 doesn't come up after HADOOP-15406. Contributed by Vinod Kumar Vavilapalli
Repository: hadoop Updated Branches: refs/heads/trunk 438ef4951 -> 31ab960f4 YARN-8338. TimelineService V1.5 doesn't come up after HADOOP-15406. Contributed by Vinod Kumar Vavilapalli Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/31ab960f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/31ab960f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/31ab960f Branch: refs/heads/trunk Commit: 31ab960f4f931df273481927b897388895d803ba Parents: 438ef49 Author: Jason Lowe Authored: Tue May 29 11:00:30 2018 -0500 Committer: Jason Lowe Committed: Tue May 29 11:00:30 2018 -0500 -- hadoop-project/pom.xml | 5 + .../hadoop-yarn-server-applicationhistoryservice/pom.xml| 5 + 2 files changed, 10 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ab960f/hadoop-project/pom.xml -- diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 73c3f5b..59a9bd2 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -1144,6 +1144,11 @@ 1.8.5 +org.objenesis +objenesis +1.0 + + org.mock-server mockserver-netty 3.9.2 http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ab960f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml index f310518..0527095 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml @@ -155,6 +155,11 @@ leveldbjni-all + + org.objenesis + objenesis + + org.apache.hadoop - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
