Author: cmccabe
Date: Tue Mar 4 06:36:06 2014
New Revision: 1573886
URL: http://svn.apache.org/r1573886
Log:
HDFS-6040. fix DFSClient issue without libhadoop.so and some other
ShortCircuitShm cleanups (cmccabe)
Modified:
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/ (props changed)
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/ (props
changed)
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
(props changed)
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/ShortCircuitShm.java
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/ShortCircuitCache.java
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/client/TestShortCircuitShm.java
Propchange: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project:r1573883
Propchange: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1573883
Modified:
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1573886&r1=1573885&r2=1573886&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Tue Mar 4 06:36:06 2014
@@ -261,6 +261,9 @@ Release 2.4.0 - UNRELEASED
HDFS-5866. '-maxSize' and '-step' option fail in OfflineImageViewer.
(Akira Ajisaka via wheat9)
+ HDFS-6040. fix DFSClient issue without libhadoop.so and some other
+ ShortCircuitShm cleanups (cmccabe)
+
BREAKDOWN OF HDFS-5698 SUBTASKS AND RELATED JIRAS
HDFS-5717. Save FSImage header in protobuf. (Haohui Mai via jing9)
Modified:
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml?rev=1573886&r1=1573885&r2=1573886&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
(original)
+++
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
Tue Mar 4 06:36:06 2014
@@ -165,13 +165,13 @@
<Bug pattern="DM_STRING_CTOR" />
</Match>
<Match>
- <Class name="org.apache.hadoop.hdfs.client.ClientMmapManager" />
- <Method name="create" />
+ <Class
name="org.apache.hadoop.hdfs.client.DfsClientShmManager$EndpointShmManager" />
+ <Method name="allocSlot" />
<Bug pattern="UL_UNRELEASED_LOCK_EXCEPTION_PATH" />
</Match>
<Match>
- <Class name="org.apache.hadoop.hdfs.client.ClientMmapManager" />
- <Method name="create" />
+ <Class
name="org.apache.hadoop.hdfs.client.DfsClientShmManager$EndpointShmManager" />
+ <Method name="allocSlot" />
<Bug pattern="UL_UNRELEASED_LOCK" />
</Match>
<!-- Manually verified to be okay, we want to throw away the top bit here
-->
Propchange:
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1573883
Modified:
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/ShortCircuitShm.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/ShortCircuitShm.java?rev=1573886&r1=1573885&r2=1573886&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/ShortCircuitShm.java
(original)
+++
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/ShortCircuitShm.java
Tue Mar 4 06:36:06 2014
@@ -30,7 +30,6 @@ import org.apache.commons.lang.builder.H
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.InvalidRequestException;
-import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.nativeio.NativeIO;
import org.apache.hadoop.io.nativeio.NativeIO.POSIX;
import org.apache.hadoop.util.Shell;
@@ -514,7 +513,9 @@ public class ShortCircuitShm {
* @return The base address of the slot.
*/
private final long calculateSlotAddress(int slotIdx) {
- return this.baseAddress + (slotIdx * BYTES_PER_SLOT);
+ long offset = slotIdx;
+ offset *= BYTES_PER_SLOT;
+ return this.baseAddress + offset;
}
/**
@@ -536,7 +537,6 @@ public class ShortCircuitShm {
slot.makeValid();
slots[idx] = slot;
if (LOG.isTraceEnabled()) {
- //LOG.trace(this + ": allocAndRegisterSlot " + idx);
LOG.trace(this + ": allocAndRegisterSlot " + idx + ": allocatedSlots=" +
allocatedSlots +
StringUtils.getStackTrace(Thread.currentThread()));
}
@@ -567,6 +567,14 @@ public class ShortCircuitShm {
*/
synchronized public final Slot registerSlot(int slotIdx,
ExtendedBlockId blockId) throws InvalidRequestException {
+ if (slotIdx < 0) {
+ throw new InvalidRequestException(this + ": invalid negative slot " +
+ "index " + slotIdx);
+ }
+ if (slotIdx >= slots.length) {
+ throw new InvalidRequestException(this + ": invalid slot " +
+ "index " + slotIdx);
+ }
if (allocatedSlots.get(slotIdx)) {
throw new InvalidRequestException(this + ": slot " + slotIdx +
" is already in use.");
@@ -579,7 +587,6 @@ public class ShortCircuitShm {
slots[slotIdx] = slot;
allocatedSlots.set(slotIdx, true);
if (LOG.isTraceEnabled()) {
- //LOG.trace(this + ": registerSlot " + slotIdx);
LOG.trace(this + ": registerSlot " + slotIdx + ": allocatedSlots=" +
allocatedSlots +
StringUtils.getStackTrace(Thread.currentThread()));
}
Modified:
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/ShortCircuitCache.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/ShortCircuitCache.java?rev=1573886&r1=1573885&r2=1573886&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/ShortCircuitCache.java
(original)
+++
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/ShortCircuitCache.java
Tue Mar 4 06:36:06 2014
@@ -53,6 +53,7 @@ import org.apache.hadoop.hdfs.protocolPB
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ipc.RetriableException;
import org.apache.hadoop.net.unix.DomainSocket;
+import org.apache.hadoop.net.unix.DomainSocketWatcher;
import org.apache.hadoop.security.token.SecretManager.InvalidToken;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Time;
@@ -375,7 +376,8 @@ public class ShortCircuitCache implement
this.mmapRetryTimeoutMs = mmapRetryTimeoutMs;
this.staleThresholdMs = staleThresholdMs;
DfsClientShmManager shmManager = null;
- if (shmInterruptCheckMs > 0) {
+ if ((shmInterruptCheckMs > 0) &&
+ (DomainSocketWatcher.getLoadingFailureReason() == null)) {
try {
shmManager = new DfsClientShmManager(shmInterruptCheckMs);
} catch (IOException e) {
Modified:
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/client/TestShortCircuitShm.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/client/TestShortCircuitShm.java?rev=1573886&r1=1573885&r2=1573886&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/client/TestShortCircuitShm.java
(original)
+++
hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/client/TestShortCircuitShm.java
Tue Mar 4 06:36:06 2014
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.client;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.util.ArrayList;
+import java.util.Iterator;
+
+import org.apache.commons.lang.SystemUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.io.nativeio.NativeIO;
+import org.apache.hadoop.io.nativeio.SharedFileDescriptorFactory;
+import org.apache.hadoop.hdfs.ExtendedBlockId;
+import org.apache.hadoop.hdfs.ShortCircuitShm;
+import org.apache.hadoop.hdfs.ShortCircuitShm.ShmId;
+import org.apache.hadoop.hdfs.ShortCircuitShm.Slot;
+import org.junit.Assume;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.Assert;
+
+public class TestShortCircuitShm {
+ public static final Log LOG = LogFactory.getLog(TestShortCircuitShm.class);
+
+ private static final File TEST_BASE =
+ new File(System.getProperty("test.build.data", "/tmp"));
+
+ @Before
+ public void before() {
+ Assume.assumeTrue(NativeIO.isAvailable());
+ Assume.assumeTrue(SystemUtils.IS_OS_UNIX);
+ }
+
+ @Test(timeout=60000)
+ public void testStartupShutdown() throws Exception {
+ File path = new File(TEST_BASE, "testStartupShutdown");
+ path.mkdirs();
+ SharedFileDescriptorFactory factory =
+ new SharedFileDescriptorFactory("shm_", path.getAbsolutePath());
+ FileInputStream stream =
+ factory.createDescriptor("testStartupShutdown", 4096);
+ ShortCircuitShm shm = new ShortCircuitShm(ShmId.createRandom(), stream);
+ shm.free();
+ stream.close();
+ FileUtil.fullyDelete(path);
+ }
+
+ @Test(timeout=60000)
+ public void testAllocateSlots() throws Exception {
+ File path = new File(TEST_BASE, "testAllocateSlots");
+ path.mkdirs();
+ SharedFileDescriptorFactory factory =
+ new SharedFileDescriptorFactory("shm_", path.getAbsolutePath());
+ FileInputStream stream =
+ factory.createDescriptor("testAllocateSlots", 4096);
+ ShortCircuitShm shm = new ShortCircuitShm(ShmId.createRandom(), stream);
+ int numSlots = 0;
+ ArrayList<Slot> slots = new ArrayList<Slot>();
+ while (!shm.isFull()) {
+ Slot slot = shm.allocAndRegisterSlot(new ExtendedBlockId(123L,
"test_bp1"));
+ slots.add(slot);
+ numSlots++;
+ }
+ LOG.info("allocated " + numSlots + " slots before running out.");
+ int slotIdx = 0;
+ for (Iterator<Slot> iter = shm.slotIterator();
+ iter.hasNext(); ) {
+ Assert.assertTrue(slots.contains(iter.next()));
+ }
+ for (Slot slot : slots) {
+ Assert.assertFalse(slot.addAnchor());
+ Assert.assertEquals(slotIdx++, slot.getSlotIdx());
+ }
+ for (Slot slot : slots) {
+ slot.makeAnchorable();
+ }
+ for (Slot slot : slots) {
+ Assert.assertTrue(slot.addAnchor());
+ }
+ for (Slot slot : slots) {
+ slot.removeAnchor();
+ }
+ for (Slot slot : slots) {
+ shm.unregisterSlot(slot.getSlotIdx());
+ slot.makeInvalid();
+ }
+ shm.free();
+ stream.close();
+ FileUtil.fullyDelete(path);
+ }
+}
