svn commit: r1503570 - /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
Author: suresh
Date: Tue Jul 16 06:33:40 2013
New Revision: 1503570
URL: http://svn.apache.org/r1503570
Log:
HDFS-4903. Print trash configuration and trash emptier state in namenode log.
Contributed by Arpit Agarwal.
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java?rev=1503570r1=1503569r2=1503570view=diff
==
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
Tue Jul 16 06:33:40 2013
@@ -89,8 +89,11 @@ public class TrashPolicyDefault extends
this.emptierInterval = (long)(conf.getFloat(
FS_TRASH_CHECKPOINT_INTERVAL_KEY, FS_TRASH_CHECKPOINT_INTERVAL_DEFAULT)
* MSECS_PER_MINUTE);
- }
-
+LOG.info(Namenode trash configuration: Deletion interval = +
+ this.deletionInterval + minutes, Emptier interval = +
+ this.emptierInterval + minutes.);
+ }
+
private Path makeTrashRelativePath(Path basePath, Path rmFilePath) {
return Path.mergePaths(basePath, rmFilePath);
}
svn commit: r1503572 - /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
Author: suresh
Date: Tue Jul 16 06:49:05 2013
New Revision: 1503572
URL: http://svn.apache.org/r1503572
Log:
HDFS-4903. Merge 1503570 from trunk
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java?rev=1503572r1=1503571r2=1503572view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
Tue Jul 16 06:49:05 2013
@@ -89,8 +89,11 @@ public class TrashPolicyDefault extends
this.emptierInterval = (long)(conf.getFloat(
FS_TRASH_CHECKPOINT_INTERVAL_KEY, FS_TRASH_CHECKPOINT_INTERVAL_DEFAULT)
* MSECS_PER_MINUTE);
- }
-
+LOG.info(Namenode trash configuration: Deletion interval = +
+ this.deletionInterval + minutes, Emptier interval = +
+ this.emptierInterval + minutes.);
+ }
+
private Path makeTrashRelativePath(Path basePath, Path rmFilePath) {
return Path.mergePaths(basePath, rmFilePath);
}
svn commit: r1503573 - /hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
Author: suresh
Date: Tue Jul 16 06:54:27 2013
New Revision: 1503573
URL: http://svn.apache.org/r1503573
Log:
HDFS-4903. Merge 1503572 from branch-2
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java?rev=1503573r1=1503572r2=1503573view=diff
==
---
hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
(original)
+++
hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
Tue Jul 16 06:54:27 2013
@@ -89,8 +89,11 @@ public class TrashPolicyDefault extends
this.emptierInterval = (long)(conf.getFloat(
FS_TRASH_CHECKPOINT_INTERVAL_KEY, FS_TRASH_CHECKPOINT_INTERVAL_DEFAULT)
* MSECS_PER_MINUTE);
- }
-
+LOG.info(Namenode trash configuration: Deletion interval = +
+ this.deletionInterval + minutes, Emptier interval = +
+ this.emptierInterval + minutes.);
+ }
+
private Path makeTrashRelativePath(Path basePath, Path rmFilePath) {
return Path.mergePaths(basePath, rmFilePath);
}
svn commit: r1503799 - /hadoop/common/branches/YARN-321/
Author: vinodkv Date: Tue Jul 16 17:22:45 2013 New Revision: 1503799 URL: http://svn.apache.org/r1503799 Log: Creating a branch for Generic ApplicationHistoryServer via YARN-321 Added: hadoop/common/branches/YARN-321/ (props changed) - copied from r1503798, hadoop/common/branches/branch-2/ Propchange: hadoop/common/branches/YARN-321/ -- --- svn:ignore (added) +++ svn:ignore Tue Jul 16 17:22:45 2013 @@ -0,0 +1,5 @@ +.classpath +.git +.project +.settings +target Propchange: hadoop/common/branches/YARN-321/ -- --- svn:mergeinfo (added) +++ svn:mergeinfo Tue Jul 16 17:22:45 2013 @@ -0,0 +1,5 @@ +/hadoop/common/branches/HDFS-2802:1360400-1480829 +/hadoop/common/branches/HDFS-3042:1306184-1342109 +/hadoop/common/branches/branch-0.23-PB:1227776-1294021 +/hadoop/common/branches/branch-2.0.4-alpha:1463804 +/hadoop/common/trunk:1161777,1161781,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163490,1163768,1163852,1163858,1163981,1164255,1164301,1164339,1166009,1166402,1167001,1167383,1167662,1170085,1170379,1170459,1171297,1172916,1173402,1176550,1177487,1177531,1177859,1177864,1182189,1182205,1182214,1189613,1189932,1189982,1195575,1196113,1196129,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1212021,1212062,1212073,1212084,1213537,1213586,1213592-1213593,1213954,1214046,1220510,1221348,1225114,1225192,1225456,1225489,1225591,1226211,1226239,1226350,1227091,1227165,1227423,1227964,1229347,1230398,1231569,1231572,1231627,1231640,1233605,1234555,1235135,1235137,1235956,1236456,1239752,1240897,1240928,1243065,1243104,1244766,1245751,1245762,1293419,1295061,1295227,1296556,1298044,1298696,1298700,1299045,1299434,1299963,1301308,1301312,1301820,1301871,1302624,1302704- 1302705,1303474,1304063,1304099,1304112,1304118,1305230,1309625,1310185,1311556,1312029,1329319,1333557,1334216,1342112,1347867,1348207,1349124,1349616,1351445,1351818,1357442,1361813,1373683,1374696,1375450,1375829,1379646,1400347,1430682,1430688,1440245,1445965,1458741,1465121,1476395,1477849,1480838,1480840,1487093
svn commit: r1503806 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java
Author: cmccabe
Date: Tue Jul 16 17:48:24 2013
New Revision: 1503806
URL: http://svn.apache.org/r1503806
Log:
HADOOP-9618. thread which detects GC pauses (Todd Lipcon via Colin Patrick
McCabe)
Added:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1503806r1=1503805r2=1503806view=diff
==
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Tue Jul
16 17:48:24 2013
@@ -310,6 +310,9 @@ Release 2.2.0 - UNRELEASED
HADOOP-9703. org.apache.hadoop.ipc.Client leaks threads on stop.
(Tsuyoshi OZAWA vi Colin Patrick McCabe)
+HADOOP-9618. Add thread which detects GC pauses.
+(Todd Lipcon via Colin Patrick McCabe)
+
OPTIMIZATIONS
BUG FIXES
Added:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java?rev=1503806view=auto
==
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java
(added)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java
Tue Jul 16 17:48:24 2013
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+import java.lang.management.GarbageCollectorMXBean;
+import java.lang.management.ManagementFactory;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Stopwatch;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+
+/**
+ * Class which sets up a simple thread which runs in a loop sleeping
+ * for a short interval of time. If the sleep takes significantly longer
+ * than its target time, it implies that the JVM or host machine has
+ * paused processing, which may cause other problems. If such a pause is
+ * detected, the thread logs a message.
+ */
+@InterfaceAudience.Private
+public class JvmPauseMonitor {
+ private static final Log LOG = LogFactory.getLog(
+ JvmPauseMonitor.class);
+
+ /** The target sleep time */
+ private static final long SLEEP_INTERVAL_MS = 500;
+
+ /** log WARN if we detect a pause longer than this threshold */
+ private final long warnThresholdMs;
+ private static final String WARN_THRESHOLD_KEY =
+ jvm.pause.warn-threshold.ms;
+ private static final long WARN_THRESHOLD_DEFAULT = 1;
+
+ /** log INFO if we detect a pause longer than this threshold */
+ private final long infoThresholdMs;
+ private static final String INFO_THRESHOLD_KEY =
+ jvm.pause.info-threshold.ms;
+ private static final long INFO_THRESHOLD_DEFAULT = 1000;
+
+
+ private Thread monitorThread;
+ private volatile boolean shouldRun = true;
+
+ public JvmPauseMonitor(Configuration conf) {
+this.warnThresholdMs = conf.getLong(WARN_THRESHOLD_KEY,
WARN_THRESHOLD_DEFAULT);
+this.infoThresholdMs = conf.getLong(INFO_THRESHOLD_KEY,
INFO_THRESHOLD_DEFAULT);
+ }
+
+ public void start() {
+Preconditions.checkState(monitorThread == null,
+Already started);
+monitorThread = new Daemon(new Monitor());
+monitorThread.start();
+ }
+
+ public void stop() {
+shouldRun = false;
+
svn commit: r1503810 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java
Author: cmccabe
Date: Tue Jul 16 17:57:44 2013
New Revision: 1503810
URL: http://svn.apache.org/r1503810
Log:
HADOOP-9618. thread which detects GC pauses (Todd Lipcon via Colin Patrick
McCabe)
Added:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1503810r1=1503809r2=1503810view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Tue Jul 16 17:57:44 2013
@@ -36,6 +36,9 @@ Release 2.2.0 - UNRELEASED
HADOOP-9703. org.apache.hadoop.ipc.Client leaks threads on stop.
(Tsuyoshi OZAWA vi Colin Patrick McCabe)
+HADOOP-9618. Add thread which detects GC pauses.
+(Todd Lipcon via Colin Patrick McCabe)
+
OPTIMIZATIONS
BUG FIXES
Added:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java?rev=1503810view=auto
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java
(added)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java
Tue Jul 16 17:57:44 2013
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+import java.lang.management.GarbageCollectorMXBean;
+import java.lang.management.ManagementFactory;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Stopwatch;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+
+/**
+ * Class which sets up a simple thread which runs in a loop sleeping
+ * for a short interval of time. If the sleep takes significantly longer
+ * than its target time, it implies that the JVM or host machine has
+ * paused processing, which may cause other problems. If such a pause is
+ * detected, the thread logs a message.
+ */
+@InterfaceAudience.Private
+public class JvmPauseMonitor {
+ private static final Log LOG = LogFactory.getLog(
+ JvmPauseMonitor.class);
+
+ /** The target sleep time */
+ private static final long SLEEP_INTERVAL_MS = 500;
+
+ /** log WARN if we detect a pause longer than this threshold */
+ private final long warnThresholdMs;
+ private static final String WARN_THRESHOLD_KEY =
+ jvm.pause.warn-threshold.ms;
+ private static final long WARN_THRESHOLD_DEFAULT = 1;
+
+ /** log INFO if we detect a pause longer than this threshold */
+ private final long infoThresholdMs;
+ private static final String INFO_THRESHOLD_KEY =
+ jvm.pause.info-threshold.ms;
+ private static final long INFO_THRESHOLD_DEFAULT = 1000;
+
+
+ private Thread monitorThread;
+ private volatile boolean shouldRun = true;
+
+ public JvmPauseMonitor(Configuration conf) {
+this.warnThresholdMs = conf.getLong(WARN_THRESHOLD_KEY,
WARN_THRESHOLD_DEFAULT);
+this.infoThresholdMs = conf.getLong(INFO_THRESHOLD_KEY,
INFO_THRESHOLD_DEFAULT);
+ }
+
+ public void start() {
+Preconditions.checkState(monitorThread == null,
+Already started);
+monitorThread = new Daemon(new
svn commit: r1503811 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: ./ src/main/java/org/apache/hadoop/ipc/ src/test/java/org/apache/hadoop/ipc/
Author: daryn
Date: Tue Jul 16 17:59:39 2013
New Revision: 1503811
URL: http://svn.apache.org/r1503811
Log:
HADOOP-9683. [RPC v9] Wrap IpcConnectionContext in RPC headers (daryn)
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcConstants.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1503811r1=1503810r2=1503811view=diff
==
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Tue Jul
16 17:59:39 2013
@@ -354,6 +354,8 @@ Release 2.1.0-beta - 2013-07-02
HADOOP-9688. Add globally unique Client ID to RPC requests. (suresh)
+HADOOP-9683. [RPC v9] Wrap IpcConnectionContext in RPC headers (daryn)
+
NEW FEATURES
HADOOP-9283. Add support for running the Hadoop client on AIX. (atm)
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java?rev=1503811r1=1503810r2=1503811view=diff
==
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
Tue Jul 16 17:59:39 2013
@@ -18,6 +18,8 @@
package org.apache.hadoop.ipc;
+import static org.apache.hadoop.ipc.RpcConstants.CONNECTION_CONTEXT_CALL_ID;
+
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.DataInputStream;
@@ -63,7 +65,10 @@ import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.retry.RetryPolicies;
import org.apache.hadoop.io.retry.RetryPolicy;
import org.apache.hadoop.io.retry.RetryPolicy.RetryAction;
+import org.apache.hadoop.ipc.ProtobufRpcEngine.RpcRequestMessageWrapper;
+import org.apache.hadoop.ipc.RPC.RpcKind;
import org.apache.hadoop.ipc.Server.AuthProtocol;
+import
org.apache.hadoop.ipc.protobuf.IpcConnectionContextProtos.IpcConnectionContextProto;
import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcRequestHeaderProto;
import
org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcRequestHeaderProto.OperationProto;
import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcResponseHeaderProto;
@@ -833,17 +838,20 @@ public class Client {
AuthMethod authMethod)
throws IOException {
// Write out the ConnectionHeader
- DataOutputBuffer buf = new DataOutputBuffer();
- ProtoUtil.makeIpcConnectionContext(
+ IpcConnectionContextProto message = ProtoUtil.makeIpcConnectionContext(
RPC.getProtocolName(remoteId.getProtocol()),
remoteId.getTicket(),
- authMethod).writeTo(buf);
+ authMethod);
+ RpcRequestHeaderProto connectionContextHeader =
+ ProtoUtil.makeRpcRequestHeader(RpcKind.RPC_PROTOCOL_BUFFER,
+ OperationProto.RPC_FINAL_PACKET, CONNECTION_CONTEXT_CALL_ID,
+ clientId);
+ RpcRequestMessageWrapper request =
+ new RpcRequestMessageWrapper(connectionContextHeader, message);
// Write out the packet length
- int bufLen = buf.getLength();
-
- out.writeInt(bufLen);
- out.write(buf.getData(), 0, bufLen);
+ out.writeInt(request.getLength());
+ request.write(out);
}
/* wait till someone signals us to start reading RPC response or
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcConstants.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcConstants.java?rev=1503811r1=1503810r2=1503811view=diff
==
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcConstants.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcConstants.java
Tue Jul 16 17:59:39 2013
@@ -32,6 +32,8 @@ public class RpcConstants {
svn commit: r1503830 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: ./ src/main/java/org/apache/hadoop/ipc/ src/test/java/org/apache/hadoop/ipc/
Author: daryn
Date: Tue Jul 16 18:59:29 2013
New Revision: 1503830
URL: http://svn.apache.org/r1503830
Log:
HADOOP-9683. [RPC v9] Wrap IpcConnectionContext in RPC headers (daryn)
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1503830r1=1503829r2=1503830view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Tue Jul 16 18:59:29 2013
@@ -78,6 +78,8 @@ Release 2.1.0-beta - 2013-07-02
HADOOP-9421. [RPC v9] Convert SASL to use ProtoBuf and provide
negotiation capabilities (daryn)
+HADOOP-9683. [RPC v9] Wrap IpcConnectionContext in RPC headers (daryn)
+
NEW FEATURES
HADOOP-9283. Add support for running the Hadoop client on AIX. (atm)
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java?rev=1503830r1=1503829r2=1503830view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
Tue Jul 16 18:59:29 2013
@@ -62,7 +62,10 @@ import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.retry.RetryPolicies;
import org.apache.hadoop.io.retry.RetryPolicy;
import org.apache.hadoop.io.retry.RetryPolicy.RetryAction;
+import org.apache.hadoop.ipc.ProtobufRpcEngine.RpcRequestMessageWrapper;
+import org.apache.hadoop.ipc.RPC.RpcKind;
import org.apache.hadoop.ipc.Server.AuthProtocol;
+import
org.apache.hadoop.ipc.protobuf.IpcConnectionContextProtos.IpcConnectionContextProto;
import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcRequestHeaderProto;
import
org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcRequestHeaderProto.OperationProto;
import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcResponseHeaderProto;
@@ -113,6 +116,8 @@ public class Client {
private final boolean fallbackAllowed;
final static int PING_CALL_ID = -1;
+
+ final static int CONNECTION_CONTEXT_CALL_ID = -3;
/**
* Executor on which IPC calls' parameters are sent.
@@ -832,17 +837,19 @@ public class Client {
AuthMethod authMethod)
throws IOException {
// Write out the ConnectionHeader
- DataOutputBuffer buf = new DataOutputBuffer();
- ProtoUtil.makeIpcConnectionContext(
+ IpcConnectionContextProto message = ProtoUtil.makeIpcConnectionContext(
RPC.getProtocolName(remoteId.getProtocol()),
remoteId.getTicket(),
- authMethod).writeTo(buf);
+ authMethod);
+ RpcRequestHeaderProto connectionContextHeader =
+ ProtoUtil.makeRpcRequestHeader(RpcKind.RPC_PROTOCOL_BUFFER,
+ OperationProto.RPC_FINAL_PACKET, CONNECTION_CONTEXT_CALL_ID);
+ RpcRequestMessageWrapper request =
+ new RpcRequestMessageWrapper(connectionContextHeader, message);
// Write out the packet length
- int bufLen = buf.getLength();
-
- out.writeInt(bufLen);
- out.write(buf.getData(), 0, bufLen);
+ out.writeInt(request.getLength());
+ request.write(out);
}
/* wait till someone signals us to start reading RPC response or
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java?rev=1503830r1=1503829r2=1503830view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
Tue Jul 16
svn commit: r1503832 - in /hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common: ./ src/main/java/org/apache/hadoop/ipc/ src/test/java/org/apache/hadoop/ipc/
Author: daryn
Date: Tue Jul 16 19:04:17 2013
New Revision: 1503832
URL: http://svn.apache.org/r1503832
Log:
merge -c 1503830 FIXES: HADOOP-9683. [RPC v9] Wrap IpcConnectionContext in RPC
headers (daryn)
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1503832r1=1503831r2=1503832view=diff
==
---
hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/CHANGES.txt
Tue Jul 16 19:04:17 2013
@@ -39,6 +39,8 @@ Release 2.1.0-beta - 2013-07-02
HADOOP-9421. [RPC v9] Convert SASL to use ProtoBuf and provide
negotiation capabilities (daryn)
+HADOOP-9683. [RPC v9] Wrap IpcConnectionContext in RPC headers (daryn)
+
NEW FEATURES
HADOOP-9283. Add support for running the Hadoop client on AIX. (atm)
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java?rev=1503832r1=1503831r2=1503832view=diff
==
---
hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
(original)
+++
hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
Tue Jul 16 19:04:17 2013
@@ -62,7 +62,10 @@ import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.retry.RetryPolicies;
import org.apache.hadoop.io.retry.RetryPolicy;
import org.apache.hadoop.io.retry.RetryPolicy.RetryAction;
+import org.apache.hadoop.ipc.ProtobufRpcEngine.RpcRequestMessageWrapper;
+import org.apache.hadoop.ipc.RPC.RpcKind;
import org.apache.hadoop.ipc.Server.AuthProtocol;
+import
org.apache.hadoop.ipc.protobuf.IpcConnectionContextProtos.IpcConnectionContextProto;
import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcRequestHeaderProto;
import
org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcRequestHeaderProto.OperationProto;
import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcResponseHeaderProto;
@@ -113,6 +116,8 @@ public class Client {
private final boolean fallbackAllowed;
final static int PING_CALL_ID = -1;
+
+ final static int CONNECTION_CONTEXT_CALL_ID = -3;
/**
* Executor on which IPC calls' parameters are sent. Deferring
@@ -779,17 +784,19 @@ public class Client {
AuthMethod authMethod)
throws IOException {
// Write out the ConnectionHeader
- DataOutputBuffer buf = new DataOutputBuffer();
- ProtoUtil.makeIpcConnectionContext(
+ IpcConnectionContextProto message = ProtoUtil.makeIpcConnectionContext(
RPC.getProtocolName(remoteId.getProtocol()),
remoteId.getTicket(),
- authMethod).writeTo(buf);
+ authMethod);
+ RpcRequestHeaderProto connectionContextHeader =
+ ProtoUtil.makeRpcRequestHeader(RpcKind.RPC_PROTOCOL_BUFFER,
+ OperationProto.RPC_FINAL_PACKET, CONNECTION_CONTEXT_CALL_ID);
+ RpcRequestMessageWrapper request =
+ new RpcRequestMessageWrapper(connectionContextHeader, message);
// Write out the packet length
- int bufLen = buf.getLength();
-
- out.writeInt(bufLen);
- out.write(buf.getData(), 0, bufLen);
+ out.writeInt(request.getLength());
+ request.write(out);
}
/* wait till someone signals us to start reading RPC response or
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java?rev=1503832r1=1503831r2=1503832view=diff
==
---
hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
svn commit: r1503954 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: ./ dev-support/ src/main/java/org/apache/hadoop/security/protocolPB/ src/main/java/org/apache/hadoop/tools/protocolP
Author: jlowe Date: Wed Jul 17 00:06:48 2013 New Revision: 1503954 URL: http://svn.apache.org/r1503954 Log: HADOOP-9734. Common protobuf definitions for GetUserMappingsProtocol, RefreshAuthorizationPolicyProtocol and RefreshUserMappingsProtocol. Contributed by Jason Lowe Added: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshAuthorizationPolicyProtocolClientSideTranslatorPB.java (with props) hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshAuthorizationPolicyProtocolPB.java (with props) hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshAuthorizationPolicyProtocolServerSideTranslatorPB.java (with props) hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshUserMappingsProtocolClientSideTranslatorPB.java (with props) hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshUserMappingsProtocolPB.java (with props) hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshUserMappingsProtocolServerSideTranslatorPB.java (with props) hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/protocolPB/ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/protocolPB/GetUserMappingsProtocolClientSideTranslatorPB.java (with props) hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/protocolPB/GetUserMappingsProtocolPB.java (with props) hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/protocolPB/GetUserMappingsProtocolServerSideTranslatorPB.java (with props) hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/proto/GetUserMappingsProtocol.proto hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/proto/RefreshAuthorizationPolicyProtocol.proto hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/proto/RefreshUserMappingsProtocol.proto Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/trunk/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml hadoop/common/trunk/hadoop-common-project/hadoop-common/pom.xml Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1503954r1=1503953r2=1503954view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Wed Jul 17 00:06:48 2013 @@ -474,6 +474,9 @@ Release 2.1.0-beta - 2013-07-02 HADOOP-9720. Rename Client#uuid to Client#clientId. (Arpit Agarwal via suresh) +HADOOP-9734. Common protobuf definitions for GetUserMappingsProtocol, +RefreshAuthorizationPolicyProtocol and RefreshUserMappingsProtocol (jlowe) + OPTIMIZATIONS HADOOP-9150. Avoid unnecessary DNS resolution attempts for logical URIs Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml?rev=1503954r1=1503953r2=1503954view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml Wed Jul 17 00:06:48 2013 @@ -19,6 +19,12 @@ Package name=org.apache.hadoop.record.compiler.generated / /Match Match + Package name=org.apache.hadoop.security.proto / + /Match + Match + Package name=org.apache.hadoop.tools.proto / + /Match + Match Bug pattern=EI_EXPOSE_REP / /Match Match Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/pom.xml?rev=1503954r1=1503953r2=1503954view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/pom.xml (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/pom.xml Wed Jul 17 00:06:48 2013 @@ -321,6 +321,9 @@
svn commit: r1503955 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: ./ dev-support/ src/main/java/org/apache/hadoop/security/protocolPB/ src/main/java/org/apache/hadoop/too
Author: jlowe Date: Wed Jul 17 00:12:05 2013 New Revision: 1503955 URL: http://svn.apache.org/r1503955 Log: svn merge -c 1503954 FIXES: HADOOP-9734. Common protobuf definitions for GetUserMappingsProtocol, RefreshAuthorizationPolicyProtocol and RefreshUserMappingsProtocol. Contributed by Jason Lowe Added: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/ - copied from r1503954, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshAuthorizationPolicyProtocolClientSideTranslatorPB.java - copied unchanged from r1503954, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshAuthorizationPolicyProtocolClientSideTranslatorPB.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshAuthorizationPolicyProtocolPB.java - copied unchanged from r1503954, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshAuthorizationPolicyProtocolPB.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshAuthorizationPolicyProtocolServerSideTranslatorPB.java - copied unchanged from r1503954, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshAuthorizationPolicyProtocolServerSideTranslatorPB.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshUserMappingsProtocolClientSideTranslatorPB.java - copied unchanged from r1503954, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshUserMappingsProtocolClientSideTranslatorPB.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshUserMappingsProtocolPB.java - copied unchanged from r1503954, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshUserMappingsProtocolPB.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshUserMappingsProtocolServerSideTranslatorPB.java - copied unchanged from r1503954, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshUserMappingsProtocolServerSideTranslatorPB.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/protocolPB/ - copied from r1503954, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/protocolPB/ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/protocolPB/GetUserMappingsProtocolClientSideTranslatorPB.java - copied unchanged from r1503954, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/protocolPB/GetUserMappingsProtocolClientSideTranslatorPB.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/protocolPB/GetUserMappingsProtocolPB.java - copied unchanged from r1503954, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/protocolPB/GetUserMappingsProtocolPB.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/protocolPB/GetUserMappingsProtocolServerSideTranslatorPB.java - copied unchanged from r1503954, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/protocolPB/GetUserMappingsProtocolServerSideTranslatorPB.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/proto/GetUserMappingsProtocol.proto - copied unchanged from r1503954, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/proto/GetUserMappingsProtocol.proto hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/proto/RefreshAuthorizationPolicyProtocol.proto - copied unchanged from r1503954, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/proto/RefreshAuthorizationPolicyProtocol.proto hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/proto/RefreshUserMappingsProtocol.proto - copied unchanged from r1503954, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/proto/RefreshUserMappingsProtocol.proto Modified:
svn commit: r1503956 - in /hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common: ./ dev-support/ src/main/java/org/apache/hadoop/security/protocolPB/ src/main/java/org/apache/had
Author: jlowe Date: Wed Jul 17 00:15:03 2013 New Revision: 1503956 URL: http://svn.apache.org/r1503956 Log: svn merge -c 1503954 FIXES: HADOOP-9734. Common protobuf definitions for GetUserMappingsProtocol, RefreshAuthorizationPolicyProtocol and RefreshUserMappingsProtocol. Contributed by Jason Lowe Added: hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/ - copied from r1503954, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/ hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshAuthorizationPolicyProtocolClientSideTranslatorPB.java - copied unchanged from r1503954, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshAuthorizationPolicyProtocolClientSideTranslatorPB.java hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshAuthorizationPolicyProtocolPB.java - copied unchanged from r1503954, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshAuthorizationPolicyProtocolPB.java hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshAuthorizationPolicyProtocolServerSideTranslatorPB.java - copied unchanged from r1503954, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshAuthorizationPolicyProtocolServerSideTranslatorPB.java hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshUserMappingsProtocolClientSideTranslatorPB.java - copied unchanged from r1503954, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshUserMappingsProtocolClientSideTranslatorPB.java hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshUserMappingsProtocolPB.java - copied unchanged from r1503954, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshUserMappingsProtocolPB.java hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshUserMappingsProtocolServerSideTranslatorPB.java - copied unchanged from r1503954, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshUserMappingsProtocolServerSideTranslatorPB.java hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/protocolPB/ - copied from r1503954, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/protocolPB/ hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/protocolPB/GetUserMappingsProtocolClientSideTranslatorPB.java - copied unchanged from r1503954, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/protocolPB/GetUserMappingsProtocolClientSideTranslatorPB.java hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/protocolPB/GetUserMappingsProtocolPB.java - copied unchanged from r1503954, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/protocolPB/GetUserMappingsProtocolPB.java hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/protocolPB/GetUserMappingsProtocolServerSideTranslatorPB.java - copied unchanged from r1503954, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/protocolPB/GetUserMappingsProtocolServerSideTranslatorPB.java hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/proto/GetUserMappingsProtocol.proto - copied unchanged from r1503954, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/proto/GetUserMappingsProtocol.proto hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/proto/RefreshAuthorizationPolicyProtocol.proto - copied unchanged from r1503954, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/proto/RefreshAuthorizationPolicyProtocol.proto hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/proto/RefreshUserMappingsProtocol.proto - copied unchanged from r1503954,
svn commit: r1503976 - in /hadoop/common/trunk/hadoop-tools/hadoop-extras/src: main/java/org/apache/hadoop/tools/DistCh.java test/java/org/apache/hadoop/tools/TestDistCh.java
Author: kihwal
Date: Wed Jul 17 02:22:14 2013
New Revision: 1503976
URL: http://svn.apache.org/r1503976
Log:
HADOOP-9738. TestDistCh fails. Contributed by Jing Zhao.
Modified:
hadoop/common/trunk/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCh.java
hadoop/common/trunk/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestDistCh.java
Modified:
hadoop/common/trunk/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCh.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCh.java?rev=1503976r1=1503975r2=1503976view=diff
==
---
hadoop/common/trunk/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCh.java
(original)
+++
hadoop/common/trunk/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCh.java
Wed Jul 17 02:22:14 2013
@@ -24,8 +24,6 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Stack;
-import javax.security.auth.login.LoginException;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -47,10 +45,7 @@ import org.apache.hadoop.mapred.OutputCo
import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.SequenceFileRecordReader;
-import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobSubmissionFiles;
-import org.apache.hadoop.mapreduce.jobhistory.JobSubmittedEvent;
-import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.ToolRunner;
Modified:
hadoop/common/trunk/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestDistCh.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestDistCh.java?rev=1503976r1=1503975r2=1503976view=diff
==
---
hadoop/common/trunk/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestDistCh.java
(original)
+++
hadoop/common/trunk/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestDistCh.java
Wed Jul 17 02:22:14 2013
@@ -113,6 +113,8 @@ public class TestDistCh extends junit.fr
}
static class ChPermissionStatus extends PermissionStatus {
+private final boolean defaultPerm;
+
ChPermissionStatus(FileStatus filestatus) {
this(filestatus, , , );
}
@@ -121,6 +123,7 @@ public class TestDistCh extends junit.fr
super(.equals(owner)? filestatus.getOwner(): owner,
.equals(group)? filestatus.getGroup(): group,
.equals(permission)? filestatus.getPermission(): new
FsPermission(Short.parseShort(permission, 8)));
+ defaultPerm = permission == null || .equals(permission);
}
}
@@ -141,7 +144,7 @@ public class TestDistCh extends junit.fr
runLsr(shell, tree.root, 0);
final String[] args = new String[NUN_SUBS];
- final PermissionStatus[] newstatus = new PermissionStatus[NUN_SUBS];
+ final ChPermissionStatus[] newstatus = new ChPermissionStatus[NUN_SUBS];
args[0]=/test/testDistCh/sub0:sub1::;
@@ -185,13 +188,11 @@ public class TestDistCh extends junit.fr
}
}
-
-
- static void checkFileStatus(PermissionStatus expected, FileStatus actual) {
+ static void checkFileStatus(ChPermissionStatus expected, FileStatus actual) {
assertEquals(expected.getUserName(), actual.getOwner());
assertEquals(expected.getGroupName(), actual.getGroup());
-FsPermission perm = expected.getPermission();
-if (actual.isFile()) {
+FsPermission perm = expected.getPermission();
+if (actual.isFile() expected.defaultPerm) {
perm = perm.applyUMask(UMASK);
}
assertEquals(perm, actual.getPermission());
svn commit: r1503977 - in /hadoop/common/branches/branch-2/hadoop-tools/hadoop-extras/src: main/java/org/apache/hadoop/tools/DistCh.java test/java/org/apache/hadoop/tools/TestDistCh.java
Author: kihwal
Date: Wed Jul 17 02:24:18 2013
New Revision: 1503977
URL: http://svn.apache.org/r1503977
Log:
svn merge -c 1503976 merging to branch-2 to fix HADOOP-9738. TestDistCh fails.
Contributed by Jing Zhao.
Modified:
hadoop/common/branches/branch-2/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCh.java
hadoop/common/branches/branch-2/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestDistCh.java
Modified:
hadoop/common/branches/branch-2/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCh.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCh.java?rev=1503977r1=1503976r2=1503977view=diff
==
---
hadoop/common/branches/branch-2/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCh.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCh.java
Wed Jul 17 02:24:18 2013
@@ -24,8 +24,6 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Stack;
-import javax.security.auth.login.LoginException;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -47,10 +45,7 @@ import org.apache.hadoop.mapred.OutputCo
import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.SequenceFileRecordReader;
-import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobSubmissionFiles;
-import org.apache.hadoop.mapreduce.jobhistory.JobSubmittedEvent;
-import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.ToolRunner;
Modified:
hadoop/common/branches/branch-2/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestDistCh.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestDistCh.java?rev=1503977r1=1503976r2=1503977view=diff
==
---
hadoop/common/branches/branch-2/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestDistCh.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestDistCh.java
Wed Jul 17 02:24:18 2013
@@ -113,6 +113,8 @@ public class TestDistCh extends junit.fr
}
static class ChPermissionStatus extends PermissionStatus {
+private final boolean defaultPerm;
+
ChPermissionStatus(FileStatus filestatus) {
this(filestatus, , , );
}
@@ -121,6 +123,7 @@ public class TestDistCh extends junit.fr
super(.equals(owner)? filestatus.getOwner(): owner,
.equals(group)? filestatus.getGroup(): group,
.equals(permission)? filestatus.getPermission(): new
FsPermission(Short.parseShort(permission, 8)));
+ defaultPerm = permission == null || .equals(permission);
}
}
@@ -141,7 +144,7 @@ public class TestDistCh extends junit.fr
runLsr(shell, tree.root, 0);
final String[] args = new String[NUN_SUBS];
- final PermissionStatus[] newstatus = new PermissionStatus[NUN_SUBS];
+ final ChPermissionStatus[] newstatus = new ChPermissionStatus[NUN_SUBS];
args[0]=/test/testDistCh/sub0:sub1::;
@@ -185,13 +188,11 @@ public class TestDistCh extends junit.fr
}
}
-
-
- static void checkFileStatus(PermissionStatus expected, FileStatus actual) {
+ static void checkFileStatus(ChPermissionStatus expected, FileStatus actual) {
assertEquals(expected.getUserName(), actual.getOwner());
assertEquals(expected.getGroupName(), actual.getGroup());
-FsPermission perm = expected.getPermission();
-if (actual.isFile()) {
+FsPermission perm = expected.getPermission();
+if (actual.isFile() expected.defaultPerm) {
perm = perm.applyUMask(UMASK);
}
assertEquals(perm, actual.getPermission());
svn commit: r1503977 - /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Author: kihwal Date: Wed Jul 17 02:24:18 2013 New Revision: 1503977 URL: http://svn.apache.org/r1503977 Log: svn merge -c 1503976 merging to branch-2 to fix HADOOP-9738. TestDistCh fails. Contributed by Jing Zhao. Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1503977r1=1503976r2=1503977view=diff == --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Wed Jul 17 02:24:18 2013 @@ -347,6 +347,8 @@ Release 2.1.0-beta - 2013-07-02 HADOOP-9707. Fix register lists for crc32c inline assembly. (todd via kihwal) +HADOOP-9738. TestDistCh fails. (jing9 via kihwal) + BREAKDOWN OF HADOOP-8562 SUBTASKS AND RELATED JIRAS HADOOP-8924. Hadoop Common creating package-info.java must not depend on
svn commit: r1503978 - /hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/CHANGES.txt
Author: kihwal Date: Wed Jul 17 02:24:55 2013 New Revision: 1503978 URL: http://svn.apache.org/r1503978 Log: svn merge -c 1503976 merging to branch-2 to fix HADOOP-9738. TestDistCh fails. Contributed by Jing Zhao. Modified: hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/CHANGES.txt Modified: hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1503978r1=1503977r2=1503978view=diff == --- hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/CHANGES.txt Wed Jul 17 02:24:55 2013 @@ -308,6 +308,8 @@ Release 2.1.0-beta - 2013-07-02 HADOOP-9707. Fix register lists for crc32c inline assembly. (todd via kihwal) +HADOOP-9738. TestDistCh fails. (jing9 via kihwal) + BREAKDOWN OF HADOOP-8562 SUBTASKS AND RELATED JIRAS HADOOP-8924. Hadoop Common creating package-info.java must not depend on
svn commit: r1503978 - in /hadoop/common/branches/branch-2.1-beta/hadoop-tools/hadoop-extras/src: main/java/org/apache/hadoop/tools/DistCh.java test/java/org/apache/hadoop/tools/TestDistCh.java
Author: kihwal
Date: Wed Jul 17 02:24:55 2013
New Revision: 1503978
URL: http://svn.apache.org/r1503978
Log:
svn merge -c 1503976 merging to branch-2 to fix HADOOP-9738. TestDistCh fails.
Contributed by Jing Zhao.
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCh.java
hadoop/common/branches/branch-2.1-beta/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestDistCh.java
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCh.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCh.java?rev=1503978r1=1503977r2=1503978view=diff
==
---
hadoop/common/branches/branch-2.1-beta/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCh.java
(original)
+++
hadoop/common/branches/branch-2.1-beta/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCh.java
Wed Jul 17 02:24:55 2013
@@ -24,8 +24,6 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Stack;
-import javax.security.auth.login.LoginException;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -47,10 +45,7 @@ import org.apache.hadoop.mapred.OutputCo
import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.SequenceFileRecordReader;
-import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobSubmissionFiles;
-import org.apache.hadoop.mapreduce.jobhistory.JobSubmittedEvent;
-import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.ToolRunner;
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestDistCh.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestDistCh.java?rev=1503978r1=1503977r2=1503978view=diff
==
---
hadoop/common/branches/branch-2.1-beta/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestDistCh.java
(original)
+++
hadoop/common/branches/branch-2.1-beta/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestDistCh.java
Wed Jul 17 02:24:55 2013
@@ -113,6 +113,8 @@ public class TestDistCh extends junit.fr
}
static class ChPermissionStatus extends PermissionStatus {
+private final boolean defaultPerm;
+
ChPermissionStatus(FileStatus filestatus) {
this(filestatus, , , );
}
@@ -121,6 +123,7 @@ public class TestDistCh extends junit.fr
super(.equals(owner)? filestatus.getOwner(): owner,
.equals(group)? filestatus.getGroup(): group,
.equals(permission)? filestatus.getPermission(): new
FsPermission(Short.parseShort(permission, 8)));
+ defaultPerm = permission == null || .equals(permission);
}
}
@@ -141,7 +144,7 @@ public class TestDistCh extends junit.fr
runLsr(shell, tree.root, 0);
final String[] args = new String[NUN_SUBS];
- final PermissionStatus[] newstatus = new PermissionStatus[NUN_SUBS];
+ final ChPermissionStatus[] newstatus = new ChPermissionStatus[NUN_SUBS];
args[0]=/test/testDistCh/sub0:sub1::;
@@ -185,13 +188,11 @@ public class TestDistCh extends junit.fr
}
}
-
-
- static void checkFileStatus(PermissionStatus expected, FileStatus actual) {
+ static void checkFileStatus(ChPermissionStatus expected, FileStatus actual) {
assertEquals(expected.getUserName(), actual.getOwner());
assertEquals(expected.getGroupName(), actual.getGroup());
-FsPermission perm = expected.getPermission();
-if (actual.isFile()) {
+FsPermission perm = expected.getPermission();
+if (actual.isFile() expected.defaultPerm) {
perm = perm.applyUMask(UMASK);
}
assertEquals(perm, actual.getPermission());
svn commit: r1504007 - in /hadoop/common/branches/branch-1: CHANGES.txt src/core/org/apache/hadoop/fs/FsShell.java src/docs/src/documentation/content/xdocs/file_system_shell.xml src/test/org/apache/ha
Author: suresh
Date: Wed Jul 17 05:53:59 2013
New Revision: 1504007
URL: http://svn.apache.org/r1504007
Log:
HADOOP-8873. Port HADOOP-8175 (Add mkdir -p flag) to branch-1. Contributed by
Akira Ajisaka.
Modified:
hadoop/common/branches/branch-1/CHANGES.txt
hadoop/common/branches/branch-1/src/core/org/apache/hadoop/fs/FsShell.java
hadoop/common/branches/branch-1/src/docs/src/documentation/content/xdocs/file_system_shell.xml
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/TestDFSShell.java
Modified: hadoop/common/branches/branch-1/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1504007r1=1504006r2=1504007view=diff
==
--- hadoop/common/branches/branch-1/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1/CHANGES.txt Wed Jul 17 05:53:59 2013
@@ -18,6 +18,9 @@ Release 1.3.0 - unreleased
support an arbitrary filesystem URI. (Tom White, backported by
Chelsey Chang via ivanmi)
+HADOOP-8873. Port HADOOP-8175 (Add mkdir -p flag) to branch-1.
+(Akira Ajisaka via suresh)
+
BUG FIXES
MAPREDUCE-5047. keep.failed.task.files=true causes job failure on
Modified:
hadoop/common/branches/branch-1/src/core/org/apache/hadoop/fs/FsShell.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/core/org/apache/hadoop/fs/FsShell.java?rev=1504007r1=1504006r2=1504007view=diff
==
--- hadoop/common/branches/branch-1/src/core/org/apache/hadoop/fs/FsShell.java
(original)
+++ hadoop/common/branches/branch-1/src/core/org/apache/hadoop/fs/FsShell.java
Wed Jul 17 05:53:59 2013
@@ -709,26 +709,31 @@ public class FsShell extends Configured
}
/**
- * Create the given dir
+ * Create the given dir.
+ * @param src a path to create dir
+ * @param createParents
+ * if set to true, mkdir will not fail when the given dir already exists
+ * @throws IOException
*/
- void mkdir(String src) throws IOException {
+ void mkdir(String src, boolean createParents) throws IOException {
Path f = new Path(src);
FileSystem srcFs = f.getFileSystem(getConf());
FileStatus fstatus = null;
try {
fstatus = srcFs.getFileStatus(f);
if (fstatus.isDir()) {
-throw new IOException(cannot create directory
-+ src + : File exists);
- }
- else {
+if (!createParents) {
+ throw new IOException(cannot create directory
+ + src + : File exists);
+}
+ } else {
throw new IOException(src + exists but +
is not a directory);
}
-} catch(FileNotFoundException e) {
-if (!srcFs.mkdirs(f)) {
- throw new IOException(failed to create + src);
-}
+} catch (FileNotFoundException e) {
+ if (!srcFs.mkdirs(f)) {
+throw new IOException(failed to create + src);
+ }
}
}
@@ -1294,7 +1299,7 @@ public class FsShell extends Configured
GET_SHORT_USAGE + \n\t +
[-getmerge src localdst [addnl]] [-cat src]\n\t +
[ + COPYTOLOCAL_SHORT_USAGE + ] [-moveToLocal src localdst]\n\t +
- [-mkdir path] [-report] [ + SETREP_SHORT_USAGE + ]\n\t +
+ [-mkdir [-p] path] [-report] [ + SETREP_SHORT_USAGE + ]\n\t +
[-touchz path] [-test -[ezd] path] [-stat [format] path]\n\t +
[-tail [-f] path] [-text path]\n\t +
[ + FsShellPermissions.CHMOD_USAGE + ]\n\t +
@@ -1394,7 +1399,9 @@ public class FsShell extends Configured
String moveToLocal = -moveToLocal src localdst: Not implemented yet
\n;
-String mkdir = -mkdir path: \tCreate a directory in specified location.
\n;
+String mkdir = -mkdir [-p] path:
+ + \tCreate a directory in specified location. \n
+ + \t\t-p Do not fail if the directory already exists.;
String setrep = SETREP_SHORT_USAGE
+ : Set the replication level of a file. \n
@@ -1556,8 +1563,15 @@ public class FsShell extends Configured
private int doall(String cmd, String argv[], int startindex) {
int exitCode = 0;
int i = startindex;
+boolean mkdirCreateParents = false;
boolean rmSkipTrash = false;
+// Check for -p option in mkdir
+if(-mkdir.equals(cmd) -p.equals(argv[i])) {
+ mkdirCreateParents = true;
+ i++;
+}
+
// Check for -skipTrash option in rm/rmr
if((-rm.equals(cmd) || -rmr.equals(cmd))
-skipTrash.equals(argv[i])) {
@@ -1576,7 +1590,7 @@ public class FsShell extends Configured
if (-cat.equals(cmd)) {
cat(argv[i], true);
} else if (-mkdir.equals(cmd)) {
- mkdir(argv[i]);
+ mkdir(argv[i], mkdirCreateParents);
} else if (-rm.equals(cmd)) {
delete(argv[i], false, rmSkipTrash);
} else if (-rmr.equals(cmd)) {
@@ -1644,10 +1658,12 @@ public
