hadoop git commit: HDFS-13732. ECAdmin should print the policy name when an EC policy is set. Contributed by Zsolt Venczel.
Repository: hadoop
Updated Branches:
refs/heads/trunk d951af22b -> 7dc79a8b5
HDFS-13732. ECAdmin should print the policy name when an EC policy is set.
Contributed by Zsolt Venczel.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7dc79a8b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7dc79a8b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7dc79a8b
Branch: refs/heads/trunk
Commit: 7dc79a8b5b7af0bf37d25a221be8ed446b0edb74
Parents: d951af2
Author: Xiao Chen
Authored: Wed Aug 15 13:51:14 2018 -0700
Committer: Xiao Chen
Committed: Wed Aug 15 13:53:47 2018 -0700
--
.../main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java | 12 ++--
.../src/test/resources/testErasureCodingConf.xml| 6 +++---
2 files changed, 9 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7dc79a8b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
index 56706b2..56d453b 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
@@ -357,16 +357,16 @@ public class ECAdmin extends Configured implements Tool {
final DistributedFileSystem dfs = AdminHelper.getDFS(p.toUri(), conf);
try {
dfs.setErasureCodingPolicy(p, ecPolicyName);
-if (ecPolicyName == null){
- ecPolicyName = "default";
-}
-System.out.println("Set " + ecPolicyName + " erasure coding policy on"
+
-" " + path);
+
+String actualECPolicyName = dfs.getErasureCodingPolicy(p).getName();
+
+System.out.println("Set " + actualECPolicyName +
+" erasure coding policy on "+ path);
RemoteIterator dirIt = dfs.listStatusIterator(p);
if (dirIt.hasNext()) {
System.out.println("Warning: setting erasure coding policy on a " +
"non-empty directory will not automatically convert existing " +
- "files to " + ecPolicyName + " erasure coding policy");
+ "files to " + actualECPolicyName + " erasure coding policy");
}
} catch (Exception e) {
System.err.println(AdminHelper.prettifyException(e));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7dc79a8b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
index 9070367..b47d50f 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
@@ -710,7 +710,7 @@
SubstringComparator
- Set default erasure coding policy on
/ecdir
+ Set RS-6-3-1024k erasure coding policy on
/ecdir
@@ -728,11 +728,11 @@
SubstringComparator
- Set default erasure coding policy on
/ecdir
+ Set RS-6-3-1024k erasure coding policy on
/ecdir
SubstringComparator
- Warning: setting erasure coding policy on a
non-empty directory will not automatically convert existing files to default
erasure coding policy
+ Warning: setting erasure coding policy on a
non-empty directory will not automatically convert existing files to
RS-6-3-1024k erasure coding policy
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: HADOOP-15552. Addendum patch to fix the build break in Ozone File system. Contributed by Anu Engineer.
Repository: hadoop
Updated Branches:
refs/heads/trunk c918d88a9 -> d951af22b
HADOOP-15552. Addendum patch to fix the build break in Ozone File system.
Contributed by Anu Engineer.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d951af22
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d951af22
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d951af22
Branch: refs/heads/trunk
Commit: d951af22b42a22c09cfeecd3c866d5f1cd412120
Parents: c918d88
Author: Anu Engineer
Authored: Wed Aug 15 10:58:29 2018 -0700
Committer: Anu Engineer
Committed: Wed Aug 15 10:58:29 2018 -0700
--
.../hadoop/fs/ozone/contract/ITestOzoneContractGetFileStatus.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d951af22/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractGetFileStatus.java
--
diff --git
a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractGetFileStatus.java
b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractGetFileStatus.java
index 98bbb14..fe442f7 100644
---
a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractGetFileStatus.java
+++
b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractGetFileStatus.java
@@ -50,7 +50,7 @@ public class ITestOzoneContractGetFileStatus
@Override
public void teardown() throws Exception {
-getLog().info("FS details {}", getFileSystem());
+getLogger().info("FS details {}", getFileSystem());
super.teardown();
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: Revert "HDDS-119:Skip Apache license header check for some ozone doc scripts. Contributed by Ajay Kumar"
Repository: hadoop Updated Branches: refs/heads/trunk ef02f9b23 -> c918d88a9 Revert "HDDS-119:Skip Apache license header check for some ozone doc scripts. Contributed by Ajay Kumar" This reverts commit c4bbcd5634d9e32183ae20adbeafca258568bedd. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c918d88a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c918d88a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c918d88a Branch: refs/heads/trunk Commit: c918d88a9b5396a4ef8cf6f9aa323fb70c5c67f4 Parents: ef02f9b Author: Anu Engineer Authored: Wed Aug 15 10:06:17 2018 -0700 Committer: Anu Engineer Committed: Wed Aug 15 10:06:17 2018 -0700 -- .../src/main/resources/checkstyle/suppressions.xml| 1 - hadoop-ozone/pom.xml | 10 -- 2 files changed, 11 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c918d88a/hadoop-build-tools/src/main/resources/checkstyle/suppressions.xml -- diff --git a/hadoop-build-tools/src/main/resources/checkstyle/suppressions.xml b/hadoop-build-tools/src/main/resources/checkstyle/suppressions.xml index 084384d..ccc89c8 100644 --- a/hadoop-build-tools/src/main/resources/checkstyle/suppressions.xml +++ b/hadoop-build-tools/src/main/resources/checkstyle/suppressions.xml @@ -18,5 +18,4 @@ - http://git-wip-us.apache.org/repos/asf/hadoop/blob/c918d88a/hadoop-ozone/pom.xml -- diff --git a/hadoop-ozone/pom.xml b/hadoop-ozone/pom.xml index e82a3d8..20ee71f 100644 --- a/hadoop-ozone/pom.xml +++ b/hadoop-ozone/pom.xml @@ -156,16 +156,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd;> webapps/static/nvd3-1.8.5.min.js.map webapps/static/angular-1.6.4.min.js webapps/static/d3-3.5.17.min.js -static/OzoneOverview.svg -themes/ozonedoc/static/js/jquery.min.js -themes/ozonedoc/static/js/bootstrap.min.js -themes/ozonedoc/static/css/bootstrap.min.css -themes/ozonedoc/static/css/bootstrap.min.css.map - themes/ozonedoc/static/css/bootstrap-theme.min.css - themes/ozonedoc/static/css/bootstrap-theme.min.css.map - themes/ozonedoc/static/fonts/glyphicons-halflings-regular.svg -themes/ozonedoc/layouts/index.html -themes/ozonedoc/theme.toml - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: YARN-7708. [GPG] Load based policy generator. Contributed by Young Chen.
Repository: hadoop
Updated Branches:
refs/heads/YARN-7402 b8e718082 -> 91dd58b76
YARN-7708. [GPG] Load based policy generator. Contributed by Young Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/91dd58b7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/91dd58b7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/91dd58b7
Branch: refs/heads/YARN-7402
Commit: 91dd58b76a14c694145290e511999b847837e945
Parents: b8e7180
Author: Botong Huang
Authored: Wed Aug 15 09:45:50 2018 -0700
Committer: Botong Huang
Committed: Wed Aug 15 09:46:13 2018 -0700
--
.../dev-support/findbugs-exclude.xml| 4 +
.../policygenerator/GlobalPolicy.java | 2 +-
.../policygenerator/LoadBasedGlobalPolicy.java | 255 +++
.../TestLoadBasedGlobalPolicy.java | 211 +++
4 files changed, 471 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/91dd58b7/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
--
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index 9fcafad..6e062c7 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -209,6 +209,10 @@
+
+
+
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/91dd58b7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/GlobalPolicy.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/GlobalPolicy.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/GlobalPolicy.java
index 38d762d..fcd22c9 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/GlobalPolicy.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/GlobalPolicy.java
@@ -51,7 +51,7 @@ public abstract class GlobalPolicy implements Configurable {
* duplicate calls to the same * endpoints as the GlobalPolicy is invoked
* once per queue.
*/
- protected Map registerPaths() {
+ protected Map, String> registerPaths() {
// Default register nothing
return Collections.emptyMap();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/91dd58b7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/LoadBasedGlobalPolicy.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/LoadBasedGlobalPolicy.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/LoadBasedGlobalPolicy.java
new file mode 100644
index 000..03bd48c
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/LoadBasedGlobalPolicy.java
@@ -0,0 +1,255 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions
hadoop git commit: HDFS-13217. Audit log all EC policy names during addErasureCodingPolicies. Contributed by liaoyuxiangqin.
Repository: hadoop
Updated Branches:
refs/heads/trunk 3e3963b03 -> ef02f9b23
HDFS-13217. Audit log all EC policy names during addErasureCodingPolicies.
Contributed by liaoyuxiangqin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ef02f9b2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ef02f9b2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ef02f9b2
Branch: refs/heads/trunk
Commit: ef02f9b233116a10cf9274b3753fe0b8dcbe8d92
Parents: 3e3963b
Author: Xiao Chen
Authored: Wed Aug 15 09:22:24 2018 -0700
Committer: Xiao Chen
Committed: Wed Aug 15 09:23:05 2018 -0700
--
.../apache/hadoop/hdfs/server/namenode/FSNamesystem.java | 10 ++
1 file changed, 6 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef02f9b2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index ecf7fce..cdd7d48 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -7538,9 +7538,10 @@ public class FSNamesystem implements Namesystem,
FSNamesystemMBean,
ErasureCodingPolicy[] policies, final boolean logRetryCache)
throws IOException {
final String operationName = "addErasureCodingPolicies";
-String addECPolicyName = "";
+List addECPolicyNames = new ArrayList<>(policies.length);
checkOperation(OperationCategory.WRITE);
-List responses = new ArrayList<>();
+List responses =
+new ArrayList<>(policies.length);
boolean success = false;
writeLock();
try {
@@ -7551,7 +7552,7 @@ public class FSNamesystem implements Namesystem,
FSNamesystemMBean,
ErasureCodingPolicy newPolicy =
FSDirErasureCodingOp.addErasureCodingPolicy(this, policy,
logRetryCache);
- addECPolicyName = newPolicy.getName();
+ addECPolicyNames.add(newPolicy.getName());
responses.add(new AddErasureCodingPolicyResponse(newPolicy));
} catch (HadoopIllegalArgumentException e) {
responses.add(new AddErasureCodingPolicyResponse(policy, e));
@@ -7564,7 +7565,8 @@ public class FSNamesystem implements Namesystem,
FSNamesystemMBean,
if (success) {
getEditLog().logSync();
}
- logAuditEvent(success, operationName, addECPolicyName, null, null);
+ logAuditEvent(success, operationName, addECPolicyNames.toString(),
+ null, null);
}
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[2/2] hadoop git commit: HADOOP-15552. Move logging APIs over to slf4j in hadoop-tools - Part2. Contributed by Ian Pickering.
HADOOP-15552. Move logging APIs over to slf4j in hadoop-tools - Part2. Contributed by Ian Pickering. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3e3963b0 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3e3963b0 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3e3963b0 Branch: refs/heads/trunk Commit: 3e3963b035911703c61e6d9a2939eb894da5644c Parents: f2315f2 Author: Akira Ajisaka Authored: Thu Aug 16 00:31:59 2018 +0900 Committer: Akira Ajisaka Committed: Thu Aug 16 00:31:59 2018 +0900 -- .../fs/contract/AbstractContractRenameTest.java| 4 ++-- .../fs/contract/AbstractContractSeekTest.java | 2 +- .../fs/contract/AbstractFSContractTestBase.java| 6 +++--- .../namenode/TestNameNodeMetadataConsistency.java | 2 +- .../hadoop/fs/aliyun/oss/AliyunOSSInputStream.java | 6 +++--- .../org/apache/hadoop/tools/HadoopArchiveLogs.java | 6 +++--- .../hadoop/tools/HadoopArchiveLogsRunner.java | 8 .../org/apache/hadoop/tools/HadoopArchives.java| 6 +++--- .../s3a/ITestS3AContractGetFileStatus.java | 2 +- .../s3a/ITestS3AContractGetFileStatusV1List.java | 2 +- .../apache/hadoop/contrib/utils/join/JobBase.java | 6 +++--- .../java/org/apache/hadoop/tools/CopyListing.java | 6 +++--- .../main/java/org/apache/hadoop/tools/DistCp.java | 6 +++--- .../apache/hadoop/tools/GlobbedCopyListing.java| 6 +++--- .../org/apache/hadoop/tools/OptionsParser.java | 6 +++--- .../org/apache/hadoop/tools/RegexCopyFilter.java | 8 .../org/apache/hadoop/tools/SimpleCopyListing.java | 10 +- .../org/apache/hadoop/tools/mapred/CopyMapper.java | 6 +++--- .../tools/mapred/RetriableFileCopyCommand.java | 8 .../tools/mapred/UniformSizeInputFormat.java | 8 .../hadoop/tools/mapred/lib/DynamicInputChunk.java | 8 .../tools/mapred/lib/DynamicInputChunkContext.java | 6 +++--- .../tools/mapred/lib/DynamicInputFormat.java | 6 +++--- .../tools/mapred/lib/DynamicRecordReader.java | 6 +++--- .../org/apache/hadoop/tools/util/DistCpUtils.java | 6 +++--- .../apache/hadoop/tools/util/ProducerConsumer.java | 6 +++--- .../apache/hadoop/tools/util/RetriableCommand.java | 6 +++--- .../org/apache/hadoop/tools/TestCopyListing.java | 6 +++--- .../org/apache/hadoop/tools/TestDistCpSystem.java | 8 .../org/apache/hadoop/tools/TestDistCpViewFs.java | 6 +++--- .../org/apache/hadoop/tools/TestExternalCall.java | 6 +++--- .../hadoop/tools/TestFileBasedCopyListing.java | 6 +++--- .../org/apache/hadoop/tools/TestIntegration.java | 6 +++--- .../tools/contract/AbstractContractDistCpTest.java | 2 +- .../hadoop/tools/mapred/TestCopyCommitter.java | 6 +++--- .../apache/hadoop/tools/mapred/TestCopyMapper.java | 6 +++--- .../hadoop/tools/mapred/TestCopyOutputFormat.java | 6 +++--- .../tools/mapred/lib/TestDynamicInputFormat.java | 6 +++--- .../apache/hadoop/tools/util/TestDistCpUtils.java | 6 +++--- .../tools/util/TestThrottledInputStream.java | 8 .../java/org/apache/hadoop/tools/DistTool.java | 6 +++--- .../hadoop/mapred/gridmix/ClusterSummarizer.java | 6 +++--- .../mapred/gridmix/CompressionEmulationUtil.java | 6 +++--- .../mapred/gridmix/DistributedCacheEmulator.java | 8 .../hadoop/mapred/gridmix/EchoUserResolver.java| 6 +++--- .../hadoop/mapred/gridmix/ExecutionSummarizer.java | 6 +++--- .../org/apache/hadoop/mapred/gridmix/FilePool.java | 6 +++--- .../apache/hadoop/mapred/gridmix/GridmixJob.java | 6 +++--- .../apache/hadoop/mapred/gridmix/InputStriper.java | 6 +++--- .../apache/hadoop/mapred/gridmix/JobFactory.java | 6 +++--- .../apache/hadoop/mapred/gridmix/JobMonitor.java | 6 +++--- .../apache/hadoop/mapred/gridmix/JobSubmitter.java | 6 +++--- .../org/apache/hadoop/mapred/gridmix/LoadJob.java | 6 +++--- .../mapred/gridmix/RandomTextDataGenerator.java| 6 +++--- .../hadoop/mapred/gridmix/ReplayJobFactory.java| 6 +++--- .../mapred/gridmix/RoundRobinUserResolver.java | 6 +++--- .../hadoop/mapred/gridmix/SerialJobFactory.java| 6 +++--- .../org/apache/hadoop/mapred/gridmix/SleepJob.java | 6 +++--- .../apache/hadoop/mapred/gridmix/Statistics.java | 6 +++--- .../hadoop/mapred/gridmix/StressJobFactory.java| 6 +++--- .../mapred/gridmix/SubmitterUserResolver.java | 6 +++--- .../hadoop/mapred/gridmix/CommonJobTest.java | 6 +++--- .../hadoop/mapred/gridmix/DebugJobProducer.java| 6 +++--- .../hadoop/mapred/gridmix/GridmixTestUtils.java| 6 +++--- .../apache/hadoop/mapred/gridmix/TestFilePool.java | 6 +++--- .../hadoop/mapred/gridmix/TestFileQueue.java | 6 +++--- .../hadoop/mapred/gridmix/TestGridMixClasses.java | 6 +++--- .../hadoop/mapred/gridmix/TestGridmixRecord.java
[1/2] hadoop git commit: HADOOP-15552. Move logging APIs over to slf4j in hadoop-tools - Part2. Contributed by Ian Pickering.
Repository: hadoop
Updated Branches:
refs/heads/trunk f2315f2e9 -> 3e3963b03
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e3963b0/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Statistics.java
--
diff --git
a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Statistics.java
b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Statistics.java
index 915788b..bf73f2a 100644
---
a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Statistics.java
+++
b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Statistics.java
@@ -17,8 +17,8 @@
*/
package org.apache.hadoop.mapred.gridmix;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.ClusterStatus;
import org.apache.hadoop.mapred.JobClient;
@@ -50,7 +50,7 @@ import java.util.concurrent.locks.ReentrantLock;
* notified either on every job completion event or some fixed time interval.
*/
public class Statistics implements Component {
- public static final Log LOG = LogFactory.getLog(Statistics.class);
+ public static final Logger LOG = LoggerFactory.getLogger(Statistics.class);
private final StatCollector statistics = new StatCollector();
private JobClient cluster;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e3963b0/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/StressJobFactory.java
--
diff --git
a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/StressJobFactory.java
b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/StressJobFactory.java
index 4dcc1a2..bd15c29 100644
---
a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/StressJobFactory.java
+++
b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/StressJobFactory.java
@@ -17,8 +17,8 @@
*/
package org.apache.hadoop.mapred.gridmix;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
@@ -38,7 +38,7 @@ import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicBoolean;
public class StressJobFactory extends JobFactory {
- public static final Log LOG = LogFactory.getLog(StressJobFactory.class);
+ public static final Logger LOG =
LoggerFactory.getLogger(StressJobFactory.class);
private final LoadStatus loadStatus = new LoadStatus();
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e3963b0/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/SubmitterUserResolver.java
--
diff --git
a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/SubmitterUserResolver.java
b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/SubmitterUserResolver.java
index d0d552a..32ddad9 100644
---
a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/SubmitterUserResolver.java
+++
b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/SubmitterUserResolver.java
@@ -21,14 +21,14 @@ import java.io.IOException;
import java.net.URI;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Resolves all UGIs to the submitting user.
*/
public class SubmitterUserResolver implements UserResolver {
- public static final Log LOG = LogFactory.getLog(SubmitterUserResolver.class);
+ public static final Logger LOG =
LoggerFactory.getLogger(SubmitterUserResolver.class);
private UserGroupInformation ugi = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e3963b0/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/CommonJobTest.java
--
diff --git
a/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/CommonJobTest.java
b/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/CommonJobTest.java
index 6a45ba7..73c03fd 100644
---
a/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/CommonJobTest.java
+++
b/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/CommonJobTest.java
@@ -33,8 +33,8 @@ import
hadoop git commit: HDDS-347. TestCloseContainerByPipeline#testCloseContainerViaStandaAlone sometimes fails. Contributed by Lixin Ge.
Repository: hadoop
Updated Branches:
refs/heads/trunk 8dc07b472 -> f2315f2e9
HDDS-347. TestCloseContainerByPipeline#testCloseContainerViaStandaAlone
sometimes fails. Contributed by Lixin Ge.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f2315f2e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f2315f2e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f2315f2e
Branch: refs/heads/trunk
Commit: f2315f2e9c31879b8c30eb9028f1959dfe48e5f4
Parents: 8dc07b4
Author: Xiaoyu Yao
Authored: Wed Aug 15 08:10:08 2018 -0700
Committer: Xiaoyu Yao
Committed: Wed Aug 15 08:11:17 2018 -0700
--
.../commandhandler/TestCloseContainerByPipeline.java | 10 ++
1 file changed, 6 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f2315f2e/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java
--
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java
index 866bc32..f40e425 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java
@@ -182,9 +182,11 @@ public class TestCloseContainerByPipeline {
new CloseContainerCommand(containerID,
HddsProtos.ReplicationType.STAND_ALONE, pipeline.getId()));
-GenericTestUtils
-.waitFor(() -> isContainerClosed(cluster, containerID,
datanodeDetails),
-500, 5 * 1000);
+// The log will appear after the state changed to closed in standalone,
+// wait for the log to ensure the operation has been done.
+GenericTestUtils.waitFor(() -> logCapturer.getOutput().contains(
+"submitting CloseContainer request over STAND_ALONE server for"
++ " container " + containerID), 500, 5 * 1000);
//double check if it's really closed (waitFor also throws an exception)
Assert.assertTrue(isContainerClosed(cluster, containerID,
datanodeDetails));
@@ -260,7 +262,7 @@ public class TestCloseContainerByPipeline {
containerData =
datanodeService.getDatanodeStateMachine().getContainer()
.getContainerSet().getContainer(containerID).getContainerData();
-return !containerData.isOpen();
+return containerData.isClosed();
}
return false;
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: HADOOP-15456. create base image for running secure ozone cluster. Contributed by Ajay Kumar.
Repository: hadoop
Updated Branches:
refs/heads/docker-hadoop-runner a63c65476 -> 44f08ef7c
HADOOP-15456. create base image for running secure ozone cluster. Contributed
by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/44f08ef7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/44f08ef7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/44f08ef7
Branch: refs/heads/docker-hadoop-runner
Commit: 44f08ef7c28bddfc98d84bb2f5a5c73985caa3c9
Parents: a63c654
Author: Márton Elek
Authored: Wed Aug 15 10:53:15 2018 +0200
Committer: Márton Elek
Committed: Wed Aug 15 10:54:04 2018 +0200
--
Dockerfile| 5 +++--
scripts/krb5.conf | 41 +
scripts/starter.sh| 34 +++---
scripts/transformation.py | 0
4 files changed, 75 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44f08ef7/Dockerfile
--
diff --git a/Dockerfile b/Dockerfile
index 28956097..11e6da8 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -18,11 +18,12 @@ FROM openjdk:8-jdk
RUN apt-get update && apt-get install -y jq curl python sudo && apt-get clean
RUN wget -O /usr/local/bin/dumb-init
https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64
RUN chmod +x /usr/local/bin/dumb-init
-
+RUN mkdir -p /etc/security/keytabs && chmod -R a+wr /etc/security/keytabs
ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
ENV PATH $PATH:/opt/hadoop/bin
-
+RUN apt-get install -y jsvc
+ENV JSVC_HOME=/usr/bin
RUN addgroup --gid 1000 hadoop
RUN adduser --disabled-password --gecos "" --uid 1000 hadoop --gid 100 --home
/opt/hadoop
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44f08ef7/scripts/krb5.conf
--
diff --git a/scripts/krb5.conf b/scripts/krb5.conf
new file mode 100644
index 000..0e03a6f
--- /dev/null
+++ b/scripts/krb5.conf
@@ -0,0 +1,41 @@
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+[logging]
+default = FILE:/var/log/krb5libs.log
+kdc = FILE:/var/log/krb5kdc.log
+admin_server = FILE:/var/log/kadmind.log
+
+[libdefaults]
+ dns_canonicalize_hostname = false
+ dns_lookup_realm = false
+ ticket_lifetime = 24h
+ renew_lifetime = 7d
+ forwardable = true
+ rdns = false
+ default_realm = EXAMPLE.COM
+
+[realms]
+ EXAMPLE.COM = {
+ kdc = SERVER
+ admin_server = SERVER
+ }
+
+[domain_realm]
+ .example.com = EXAMPLE.COM
+ example.com = EXAMPLE.COM
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44f08ef7/scripts/starter.sh
--
diff --git a/scripts/starter.sh b/scripts/starter.sh
index 7c51826..90fdc92 100755
--- a/scripts/starter.sh
+++ b/scripts/starter.sh
@@ -24,12 +24,43 @@ sudo chmod o+rwx /data
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
$DIR/envtoconf.py --destination /opt/hadoop/etc/hadoop
+echo "Setting up environment!"
if [ -n "$SLEEP_SECONDS" ]; then
echo "Sleeping for $SLEEP_SECONDS seconds"
sleep $SLEEP_SECONDS
fi
+if [ -n "$KERBEROS_ENABLED" ]; then
+ echo "Setting up kerberos!!"
+ KERBEROS_SERVER=${KERBEROS_SERVER:-krb5}
+ ISSUER_SERVER=${ISSUER_SERVER:-$KERBEROS_SERVER\:8081}
+ echo "KDC ISSUER_SERVER => $ISSUER_SERVER"
+
+ while true
+ do
+ STATUS=$(curl -s -o /dev/null -w '%{http_code}'
http://$ISSUER_SERVER/keytab/test/test)
+ if [ $STATUS -eq 200 ]; then
+ echo "Got 200, KDC service ready!!"
+ break
+ else
+ echo "Got $STATUS :( KDC service not ready yet..."
+ fi
+ sleep 5
+ done
+
+ export HOST_NAME=`hostname -f`
+ for NAME in ${KERBEROS_KEYTABS}; do
+ echo "Download $NAME/$hostn...@example.com keytab file to
$CONF_DIR/$NAME.keytab"
+ wget http://$ISSUER_SERVER/keytab/$HOST_NAME/$NAME -O
