Author: cmccabe Date: Tue Mar 4 06:36:06 2014 New Revision: 1573886 URL: http://svn.apache.org/r1573886 Log: HDFS-6040. fix DFSClient issue without libhadoop.so and some other ShortCircuitShm cleanups (cmccabe)
Modified: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/ (props changed) hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/ (props changed) hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ (props changed) hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/ShortCircuitShm.java hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/ShortCircuitCache.java hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/client/TestShortCircuitShm.java Propchange: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project:r1573883 Propchange: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1573883 Modified: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1573886&r1=1573885&r2=1573886&view=diff ============================================================================== --- hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original) +++ hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Tue Mar 4 06:36:06 2014 @@ -261,6 +261,9 @@ Release 2.4.0 - UNRELEASED HDFS-5866. '-maxSize' and '-step' option fail in OfflineImageViewer. (Akira Ajisaka via wheat9) + HDFS-6040. fix DFSClient issue without libhadoop.so and some other + ShortCircuitShm cleanups (cmccabe) + BREAKDOWN OF HDFS-5698 SUBTASKS AND RELATED JIRAS HDFS-5717. Save FSImage header in protobuf. (Haohui Mai via jing9) Modified: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml?rev=1573886&r1=1573885&r2=1573886&view=diff ============================================================================== --- hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml (original) +++ hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml Tue Mar 4 06:36:06 2014 @@ -165,13 +165,13 @@ <Bug pattern="DM_STRING_CTOR" /> </Match> <Match> - <Class name="org.apache.hadoop.hdfs.client.ClientMmapManager" /> - <Method name="create" /> + <Class name="org.apache.hadoop.hdfs.client.DfsClientShmManager$EndpointShmManager" /> + <Method name="allocSlot" /> <Bug pattern="UL_UNRELEASED_LOCK_EXCEPTION_PATH" /> </Match> <Match> - <Class name="org.apache.hadoop.hdfs.client.ClientMmapManager" /> - <Method name="create" /> + <Class name="org.apache.hadoop.hdfs.client.DfsClientShmManager$EndpointShmManager" /> + <Method name="allocSlot" /> <Bug pattern="UL_UNRELEASED_LOCK" /> </Match> <!-- Manually verified to be okay, we want to throw away the top bit here --> Propchange: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1573883 Modified: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/ShortCircuitShm.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/ShortCircuitShm.java?rev=1573886&r1=1573885&r2=1573886&view=diff ============================================================================== --- hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/ShortCircuitShm.java (original) +++ hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/ShortCircuitShm.java Tue Mar 4 06:36:06 2014 @@ -30,7 +30,6 @@ import org.apache.commons.lang.builder.H import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.InvalidRequestException; -import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.nativeio.NativeIO; import org.apache.hadoop.io.nativeio.NativeIO.POSIX; import org.apache.hadoop.util.Shell; @@ -514,7 +513,9 @@ public class ShortCircuitShm { * @return The base address of the slot. */ private final long calculateSlotAddress(int slotIdx) { - return this.baseAddress + (slotIdx * BYTES_PER_SLOT); + long offset = slotIdx; + offset *= BYTES_PER_SLOT; + return this.baseAddress + offset; } /** @@ -536,7 +537,6 @@ public class ShortCircuitShm { slot.makeValid(); slots[idx] = slot; if (LOG.isTraceEnabled()) { - //LOG.trace(this + ": allocAndRegisterSlot " + idx); LOG.trace(this + ": allocAndRegisterSlot " + idx + ": allocatedSlots=" + allocatedSlots + StringUtils.getStackTrace(Thread.currentThread())); } @@ -567,6 +567,14 @@ public class ShortCircuitShm { */ synchronized public final Slot registerSlot(int slotIdx, ExtendedBlockId blockId) throws InvalidRequestException { + if (slotIdx < 0) { + throw new InvalidRequestException(this + ": invalid negative slot " + + "index " + slotIdx); + } + if (slotIdx >= slots.length) { + throw new InvalidRequestException(this + ": invalid slot " + + "index " + slotIdx); + } if (allocatedSlots.get(slotIdx)) { throw new InvalidRequestException(this + ": slot " + slotIdx + " is already in use."); @@ -579,7 +587,6 @@ public class ShortCircuitShm { slots[slotIdx] = slot; allocatedSlots.set(slotIdx, true); if (LOG.isTraceEnabled()) { - //LOG.trace(this + ": registerSlot " + slotIdx); LOG.trace(this + ": registerSlot " + slotIdx + ": allocatedSlots=" + allocatedSlots + StringUtils.getStackTrace(Thread.currentThread())); } Modified: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/ShortCircuitCache.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/ShortCircuitCache.java?rev=1573886&r1=1573885&r2=1573886&view=diff ============================================================================== --- hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/ShortCircuitCache.java (original) +++ hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/ShortCircuitCache.java Tue Mar 4 06:36:06 2014 @@ -53,6 +53,7 @@ import org.apache.hadoop.hdfs.protocolPB import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.ipc.RetriableException; import org.apache.hadoop.net.unix.DomainSocket; +import org.apache.hadoop.net.unix.DomainSocketWatcher; import org.apache.hadoop.security.token.SecretManager.InvalidToken; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Time; @@ -375,7 +376,8 @@ public class ShortCircuitCache implement this.mmapRetryTimeoutMs = mmapRetryTimeoutMs; this.staleThresholdMs = staleThresholdMs; DfsClientShmManager shmManager = null; - if (shmInterruptCheckMs > 0) { + if ((shmInterruptCheckMs > 0) && + (DomainSocketWatcher.getLoadingFailureReason() == null)) { try { shmManager = new DfsClientShmManager(shmInterruptCheckMs); } catch (IOException e) { Modified: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/client/TestShortCircuitShm.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/client/TestShortCircuitShm.java?rev=1573886&r1=1573885&r2=1573886&view=diff ============================================================================== --- hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/client/TestShortCircuitShm.java (original) +++ hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/client/TestShortCircuitShm.java Tue Mar 4 06:36:06 2014 @@ -0,0 +1,109 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.client; + +import java.io.File; +import java.io.FileInputStream; +import java.util.ArrayList; +import java.util.Iterator; + +import org.apache.commons.lang.SystemUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.io.nativeio.NativeIO; +import org.apache.hadoop.io.nativeio.SharedFileDescriptorFactory; +import org.apache.hadoop.hdfs.ExtendedBlockId; +import org.apache.hadoop.hdfs.ShortCircuitShm; +import org.apache.hadoop.hdfs.ShortCircuitShm.ShmId; +import org.apache.hadoop.hdfs.ShortCircuitShm.Slot; +import org.junit.Assume; +import org.junit.Before; +import org.junit.Test; +import org.junit.Assert; + +public class TestShortCircuitShm { + public static final Log LOG = LogFactory.getLog(TestShortCircuitShm.class); + + private static final File TEST_BASE = + new File(System.getProperty("test.build.data", "/tmp")); + + @Before + public void before() { + Assume.assumeTrue(NativeIO.isAvailable()); + Assume.assumeTrue(SystemUtils.IS_OS_UNIX); + } + + @Test(timeout=60000) + public void testStartupShutdown() throws Exception { + File path = new File(TEST_BASE, "testStartupShutdown"); + path.mkdirs(); + SharedFileDescriptorFactory factory = + new SharedFileDescriptorFactory("shm_", path.getAbsolutePath()); + FileInputStream stream = + factory.createDescriptor("testStartupShutdown", 4096); + ShortCircuitShm shm = new ShortCircuitShm(ShmId.createRandom(), stream); + shm.free(); + stream.close(); + FileUtil.fullyDelete(path); + } + + @Test(timeout=60000) + public void testAllocateSlots() throws Exception { + File path = new File(TEST_BASE, "testAllocateSlots"); + path.mkdirs(); + SharedFileDescriptorFactory factory = + new SharedFileDescriptorFactory("shm_", path.getAbsolutePath()); + FileInputStream stream = + factory.createDescriptor("testAllocateSlots", 4096); + ShortCircuitShm shm = new ShortCircuitShm(ShmId.createRandom(), stream); + int numSlots = 0; + ArrayList<Slot> slots = new ArrayList<Slot>(); + while (!shm.isFull()) { + Slot slot = shm.allocAndRegisterSlot(new ExtendedBlockId(123L, "test_bp1")); + slots.add(slot); + numSlots++; + } + LOG.info("allocated " + numSlots + " slots before running out."); + int slotIdx = 0; + for (Iterator<Slot> iter = shm.slotIterator(); + iter.hasNext(); ) { + Assert.assertTrue(slots.contains(iter.next())); + } + for (Slot slot : slots) { + Assert.assertFalse(slot.addAnchor()); + Assert.assertEquals(slotIdx++, slot.getSlotIdx()); + } + for (Slot slot : slots) { + slot.makeAnchorable(); + } + for (Slot slot : slots) { + Assert.assertTrue(slot.addAnchor()); + } + for (Slot slot : slots) { + slot.removeAnchor(); + } + for (Slot slot : slots) { + shm.unregisterSlot(slot.getSlotIdx()); + slot.makeInvalid(); + } + shm.free(); + stream.close(); + FileUtil.fullyDelete(path); + } +}