Author: wang Date: Thu Dec 5 21:09:30 2013 New Revision: 1548309 URL: http://svn.apache.org/r1548309 Log: HDFS-5630. Hook up cache directive and pool usage statistics. (wang)
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirective.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveStats.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CachePoolStats.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CachePool.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCacheAdminConf.xml Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1548309&r1=1548308&r2=1548309&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Thu Dec 5 21:09:30 2013 @@ -231,6 +231,8 @@ Trunk (Unreleased) HDFS-5536. Implement HTTP policy for Namenode and DataNode. (Haohui Mai via jing9) + HDFS-5630. Hook up cache directive and pool usage statistics. (wang) + OPTIMIZATIONS HDFS-5349. DNA_CACHE and DNA_UNCACHE should be by blockId only. (cmccabe) Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirective.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirective.java?rev=1548309&r1=1548308&r2=1548309&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirective.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirective.java Thu Dec 5 21:09:30 2013 @@ -46,7 +46,9 @@ public final class CacheDirective implem private long bytesNeeded; private long bytesCached; - private long filesAffected; + private long filesNeeded; + private long filesCached; + private Element prev; private Element next; @@ -58,9 +60,6 @@ public final class CacheDirective implem Preconditions.checkArgument(replication > 0); this.replication = replication; this.expiryTime = expiryTime; - this.bytesNeeded = 0; - this.bytesCached = 0; - this.filesAffected = 0; } public long getId() { @@ -112,7 +111,8 @@ public final class CacheDirective implem return new CacheDirectiveStats.Builder(). setBytesNeeded(bytesNeeded). setBytesCached(bytesCached). - setFilesAffected(filesAffected). + setFilesNeeded(filesNeeded). + setFilesCached(filesCached). setHasExpired(new Date().getTime() > expiryTime). build(); } @@ -131,7 +131,8 @@ public final class CacheDirective implem append(", expiryTime: ").append(getExpiryTimeString()). append(", bytesNeeded:").append(bytesNeeded). append(", bytesCached:").append(bytesCached). - append(", filesAffected:").append(filesAffected). + append(", filesNeeded:").append(filesNeeded). + append(", filesCached:").append(filesCached). append(" }"); return builder.toString(); } @@ -152,42 +153,60 @@ public final class CacheDirective implem return new HashCodeBuilder().append(id).toHashCode(); } - public long getBytesNeeded() { - return bytesNeeded; + // + // Stats related getters and setters + // + + /** + * Resets the byte and file statistics being tracked by this CacheDirective. + */ + public void resetStatistics() { + bytesNeeded = 0; + bytesCached = 0; + filesNeeded = 0; + filesCached = 0; } - public void clearBytesNeeded() { - this.bytesNeeded = 0; + public long getBytesNeeded() { + return bytesNeeded; } - public void addBytesNeeded(long toAdd) { - this.bytesNeeded += toAdd; + public void addBytesNeeded(long bytes) { + this.bytesNeeded += bytes; + pool.addBytesNeeded(bytes); } public long getBytesCached() { return bytesCached; } - public void clearBytesCached() { - this.bytesCached = 0; + public void addBytesCached(long bytes) { + this.bytesCached += bytes; + pool.addBytesCached(bytes); } - public void addBytesCached(long toAdd) { - this.bytesCached += toAdd; + public long getFilesNeeded() { + return filesNeeded; } - public long getFilesAffected() { - return filesAffected; + public void addFilesNeeded(long files) { + this.filesNeeded += files; + pool.addFilesNeeded(files); } - public void clearFilesAffected() { - this.filesAffected = 0; + public long getFilesCached() { + return filesCached; } - public void incrementFilesAffected() { - this.filesAffected++; + public void addFilesCached(long files) { + this.filesCached += files; + pool.addFilesCached(files); } + // + // IntrusiveCollection.Element implementation + // + @SuppressWarnings("unchecked") @Override // IntrusiveCollection.Element public void insertInternal(IntrusiveCollection<? extends Element> list, Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveStats.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveStats.java?rev=1548309&r1=1548308&r2=1548309&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveStats.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveStats.java Thu Dec 5 21:09:30 2013 @@ -29,7 +29,8 @@ public class CacheDirectiveStats { public static class Builder { private long bytesNeeded; private long bytesCached; - private long filesAffected; + private long filesNeeded; + private long filesCached; private boolean hasExpired; /** @@ -38,8 +39,8 @@ public class CacheDirectiveStats { * @return New CacheDirectiveStats. */ public CacheDirectiveStats build() { - return new CacheDirectiveStats(bytesNeeded, bytesCached, filesAffected, - hasExpired); + return new CacheDirectiveStats(bytesNeeded, bytesCached, filesNeeded, + filesCached, hasExpired); } /** @@ -71,13 +72,23 @@ public class CacheDirectiveStats { } /** - * Sets the files affected by this directive. + * Sets the files needed by this directive. + * @param filesNeeded The number of files needed + * @return This builder, for call chaining. + */ + public Builder setFilesNeeded(long filesNeeded) { + this.filesNeeded = filesNeeded; + return this; + } + + /** + * Sets the files cached by this directive. * - * @param filesAffected The files affected. + * @param filesCached The number of files cached. * @return This builder, for call chaining. */ - public Builder setFilesAffected(long filesAffected) { - this.filesAffected = filesAffected; + public Builder setFilesCached(long filesCached) { + this.filesCached = filesCached; return this; } @@ -95,14 +106,16 @@ public class CacheDirectiveStats { private final long bytesNeeded; private final long bytesCached; - private final long filesAffected; + private final long filesNeeded; + private final long filesCached; private final boolean hasExpired; private CacheDirectiveStats(long bytesNeeded, long bytesCached, - long filesAffected, boolean hasExpired) { + long filesNeeded, long filesCached, boolean hasExpired) { this.bytesNeeded = bytesNeeded; this.bytesCached = bytesCached; - this.filesAffected = filesAffected; + this.filesNeeded = filesNeeded; + this.filesCached = filesCached; this.hasExpired = hasExpired; } @@ -121,10 +134,17 @@ public class CacheDirectiveStats { } /** - * @return The files affected. + * @return The number of files needed. + */ + public long getFilesNeeded() { + return filesNeeded; + } + + /** + * @return The number of files cached. */ - public long getFilesAffected() { - return filesAffected; + public long getFilesCached() { + return filesCached; } /** @@ -140,7 +160,8 @@ public class CacheDirectiveStats { builder.append("{"); builder.append("bytesNeeded: ").append(bytesNeeded); builder.append(", ").append("bytesCached: ").append(bytesCached); - builder.append(", ").append("filesAffected: ").append(filesAffected); + builder.append(", ").append("filesNeeded: ").append(filesNeeded); + builder.append(", ").append("filesCached: ").append(filesCached); builder.append(", ").append("hasExpired: ").append(hasExpired); builder.append("}"); return builder.toString(); Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CachePoolStats.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CachePoolStats.java?rev=1548309&r1=1548308&r2=1548309&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CachePoolStats.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CachePoolStats.java Thu Dec 5 21:09:30 2013 @@ -30,7 +30,8 @@ public class CachePoolStats { public static class Builder { private long bytesNeeded; private long bytesCached; - private long filesAffected; + private long filesNeeded; + private long filesCached; public Builder() { } @@ -45,24 +46,33 @@ public class CachePoolStats { return this; } - public Builder setFilesAffected(long filesAffected) { - this.filesAffected = filesAffected; + public Builder setFilesNeeded(long filesNeeded) { + this.filesNeeded = filesNeeded; + return this; + } + + public Builder setFilesCached(long filesCached) { + this.filesCached = filesCached; return this; } public CachePoolStats build() { - return new CachePoolStats(bytesNeeded, bytesCached, filesAffected); + return new CachePoolStats(bytesNeeded, bytesCached, filesNeeded, + filesCached); } }; private final long bytesNeeded; private final long bytesCached; - private final long filesAffected; + private final long filesNeeded; + private final long filesCached; - private CachePoolStats(long bytesNeeded, long bytesCached, long filesAffected) { + private CachePoolStats(long bytesNeeded, long bytesCached, long filesNeeded, + long filesCached) { this.bytesNeeded = bytesNeeded; this.bytesCached = bytesCached; - this.filesAffected = filesAffected; + this.filesNeeded = filesNeeded; + this.filesCached = filesCached; } public long getBytesNeeded() { @@ -70,18 +80,23 @@ public class CachePoolStats { } public long getBytesCached() { - return bytesNeeded; + return bytesCached; + } + + public long getFilesNeeded() { + return filesNeeded; } - public long getFilesAffected() { - return filesAffected; + public long getFilesCached() { + return filesCached; } public String toString() { return new StringBuilder().append("{"). append("bytesNeeded:").append(bytesNeeded). append(", bytesCached:").append(bytesCached). - append(", filesAffected:").append(filesAffected). + append(", filesNeeded:").append(filesNeeded). + append(", filesCached:").append(filesCached). append("}").toString(); } } Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java?rev=1548309&r1=1548308&r2=1548309&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java Thu Dec 5 21:09:30 2013 @@ -1642,7 +1642,8 @@ public class PBHelper { CacheDirectiveStatsProto.newBuilder(); builder.setBytesNeeded(stats.getBytesNeeded()); builder.setBytesCached(stats.getBytesCached()); - builder.setFilesAffected(stats.getFilesAffected()); + builder.setFilesNeeded(stats.getFilesNeeded()); + builder.setFilesCached(stats.getFilesCached()); builder.setHasExpired(stats.hasExpired()); return builder.build(); } @@ -1651,7 +1652,8 @@ public class PBHelper { CacheDirectiveStats.Builder builder = new CacheDirectiveStats.Builder(); builder.setBytesNeeded(proto.getBytesNeeded()); builder.setBytesCached(proto.getBytesCached()); - builder.setFilesAffected(proto.getFilesAffected()); + builder.setFilesNeeded(proto.getFilesNeeded()); + builder.setFilesCached(proto.getFilesCached()); builder.setHasExpired(proto.getHasExpired()); return builder.build(); } @@ -1711,7 +1713,8 @@ public class PBHelper { CachePoolStatsProto.Builder builder = CachePoolStatsProto.newBuilder(); builder.setBytesNeeded(stats.getBytesNeeded()); builder.setBytesCached(stats.getBytesCached()); - builder.setFilesAffected(stats.getFilesAffected()); + builder.setFilesNeeded(stats.getFilesNeeded()); + builder.setFilesCached(stats.getFilesCached()); return builder.build(); } @@ -1719,7 +1722,8 @@ public class PBHelper { CachePoolStats.Builder builder = new CachePoolStats.Builder(); builder.setBytesNeeded(proto.getBytesNeeded()); builder.setBytesCached(proto.getBytesCached()); - builder.setFilesAffected(proto.getFilesAffected()); + builder.setFilesNeeded(proto.getFilesNeeded()); + builder.setFilesCached(proto.getFilesCached()); return builder.build(); } Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java?rev=1548309&r1=1548308&r2=1548309&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java Thu Dec 5 21:09:30 2013 @@ -37,6 +37,7 @@ import org.apache.hadoop.hdfs.protocol.C import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.CachedBlocksList.Type; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState; import org.apache.hadoop.hdfs.server.namenode.CacheManager; +import org.apache.hadoop.hdfs.server.namenode.CachePool; import org.apache.hadoop.hdfs.server.namenode.CachedBlock; import org.apache.hadoop.hdfs.server.namenode.FSDirectory; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; @@ -198,6 +199,7 @@ public class CacheReplicationMonitor ext scannedBlocks = 0; namesystem.writeLock(); try { + resetStatistics(); rescanCacheDirectives(); rescanCachedBlockMap(); blockManager.getDatanodeManager().resetLastCachingDirectiveSentTime(); @@ -206,6 +208,15 @@ public class CacheReplicationMonitor ext } } + private void resetStatistics() { + for (CachePool pool: cacheManager.getCachePools()) { + pool.resetStatistics(); + } + for (CacheDirective directive: cacheManager.getCacheDirectives()) { + directive.resetStatistics(); + } + } + /** * Scan all CacheDirectives. Use the information to figure out * what cache replication factor each block should have. @@ -213,11 +224,9 @@ public class CacheReplicationMonitor ext private void rescanCacheDirectives() { FSDirectory fsDir = namesystem.getFSDirectory(); final long now = new Date().getTime(); - for (CacheDirective directive : cacheManager.getEntriesById().values()) { - // Reset the directive - directive.clearBytesNeeded(); - directive.clearBytesCached(); - directive.clearFilesAffected(); + for (CacheDirective directive : cacheManager.getCacheDirectives()) { + // Reset the directive's statistics + directive.resetStatistics(); // Skip processing this entry if it has expired LOG.info("Directive expiry is at " + directive.getExpiryTime()); if (directive.getExpiryTime() > 0 && directive.getExpiryTime() <= now) { @@ -262,26 +271,34 @@ public class CacheReplicationMonitor ext /** * Apply a CacheDirective to a file. - * - * @param pce The CacheDirective to apply. - * @param file The file. + * + * @param directive The CacheDirective to apply. + * @param file The file. */ - private void rescanFile(CacheDirective pce, INodeFile file) { - pce.incrementFilesAffected(); + private void rescanFile(CacheDirective directive, INodeFile file) { BlockInfo[] blockInfos = file.getBlocks(); - long cachedTotal = 0; + + // Increment the "needed" statistics + directive.addFilesNeeded(1); long neededTotal = 0; for (BlockInfo blockInfo : blockInfos) { + long neededByBlock = + directive.getReplication() * blockInfo.getNumBytes(); + neededTotal += neededByBlock; + } + directive.addBytesNeeded(neededTotal); + + // TODO: Enforce per-pool quotas + + long cachedTotal = 0; + for (BlockInfo blockInfo : blockInfos) { if (!blockInfo.getBlockUCState().equals(BlockUCState.COMPLETE)) { // We don't try to cache blocks that are under construction. continue; } - long neededByBlock = - pce.getReplication() * blockInfo.getNumBytes(); - neededTotal += neededByBlock; Block block = new Block(blockInfo.getBlockId()); CachedBlock ncblock = new CachedBlock(block.getBlockId(), - pce.getReplication(), mark); + directive.getReplication(), mark); CachedBlock ocblock = cachedBlocks.get(ncblock); if (ocblock == null) { cachedBlocks.put(ncblock); @@ -294,26 +311,30 @@ public class CacheReplicationMonitor ext // both get them added to their bytesCached. List<DatanodeDescriptor> cachedOn = ocblock.getDatanodes(Type.CACHED); - long cachedByBlock = Math.min(cachedOn.size(), pce.getReplication()) * - blockInfo.getNumBytes(); + long cachedByBlock = Math.min(cachedOn.size(), + directive.getReplication()) * blockInfo.getNumBytes(); cachedTotal += cachedByBlock; if (mark != ocblock.getMark()) { // Mark hasn't been set in this scan, so update replication and mark. - ocblock.setReplicationAndMark(pce.getReplication(), mark); + ocblock.setReplicationAndMark(directive.getReplication(), mark); } else { // Mark already set in this scan. Set replication to highest value in // any CacheDirective that covers this file. ocblock.setReplicationAndMark((short)Math.max( - pce.getReplication(), ocblock.getReplication()), mark); + directive.getReplication(), ocblock.getReplication()), mark); } } } - pce.addBytesNeeded(neededTotal); - pce.addBytesCached(cachedTotal); + // Increment the "cached" statistics + directive.addBytesCached(cachedTotal); + if (cachedTotal == neededTotal) { + directive.addFilesCached(1); + } if (LOG.isTraceEnabled()) { - LOG.debug("Directive " + pce.getId() + " is caching " + - file.getFullPathName() + ": " + cachedTotal + "/" + neededTotal); + LOG.trace("Directive " + directive.getId() + " is caching " + + file.getFullPathName() + ": " + cachedTotal + "/" + neededTotal + + " bytes"); } } Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java?rev=1548309&r1=1548308&r2=1548309&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java Thu Dec 5 21:09:30 2013 @@ -31,6 +31,7 @@ import java.io.DataOutput; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.Iterator; import java.util.LinkedList; import java.util.List; @@ -238,9 +239,20 @@ public final class CacheManager { return active; } - public TreeMap<Long, CacheDirective> getEntriesById() { + /** + * @return Unmodifiable view of the collection of CachePools. + */ + public Collection<CachePool> getCachePools() { + assert namesystem.hasReadLock(); + return Collections.unmodifiableCollection(cachePools.values()); + } + + /** + * @return Unmodifiable view of the collection of CacheDirectives. + */ + public Collection<CacheDirective> getCacheDirectives() { assert namesystem.hasReadLock(); - return directivesById; + return Collections.unmodifiableCollection(directivesById.values()); } @VisibleForTesting Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CachePool.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CachePool.java?rev=1548309&r1=1548308&r2=1548309&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CachePool.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CachePool.java Thu Dec 5 21:09:30 2013 @@ -70,9 +70,14 @@ public final class CachePool { */ @Nonnull private FsPermission mode; - + private int weight; + private long bytesNeeded; + private long bytesCached; + private long filesNeeded; + private long filesCached; + public final static class DirectiveList extends IntrusiveCollection<CacheDirective> { private CachePool cachePool; @@ -203,15 +208,58 @@ public final class CachePool { } /** + * Resets statistics related to this CachePool + */ + public void resetStatistics() { + bytesNeeded = 0; + bytesCached = 0; + filesNeeded = 0; + filesCached = 0; + } + + public void addBytesNeeded(long bytes) { + bytesNeeded += bytes; + } + + public void addBytesCached(long bytes) { + bytesCached += bytes; + } + + public void addFilesNeeded(long files) { + filesNeeded += files; + } + + public void addFilesCached(long files) { + filesCached += files; + } + + public long getBytesNeeded() { + return bytesNeeded; + } + + public long getBytesCached() { + return bytesCached; + } + + public long getFilesNeeded() { + return filesNeeded; + } + + public long getFilesCached() { + return filesCached; + } + + /** * Get statistics about this CachePool. * * @return Cache pool statistics. */ private CachePoolStats getStats() { return new CachePoolStats.Builder(). - setBytesNeeded(0). - setBytesCached(0). - setFilesAffected(0). + setBytesNeeded(bytesNeeded). + setBytesCached(bytesCached). + setFilesNeeded(filesNeeded). + setFilesCached(filesCached). build(); } Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java?rev=1548309&r1=1548308&r2=1548309&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java Thu Dec 5 21:09:30 2013 @@ -36,6 +36,7 @@ import org.apache.hadoop.hdfs.protocol.C import org.apache.hadoop.hdfs.protocol.CacheDirectiveStats; import org.apache.hadoop.hdfs.protocol.CachePoolEntry; import org.apache.hadoop.hdfs.protocol.CachePoolInfo; +import org.apache.hadoop.hdfs.protocol.CachePoolStats; import org.apache.hadoop.hdfs.server.namenode.CachePool; import org.apache.hadoop.hdfs.tools.TableListing.Justification; import org.apache.hadoop.ipc.RemoteException; @@ -477,9 +478,10 @@ public class CacheAdmin extends Configur addField("EXPIRY", Justification.LEFT). addField("PATH", Justification.LEFT); if (printStats) { - tableBuilder.addField("NEEDED", Justification.RIGHT). - addField("CACHED", Justification.RIGHT). - addField("FILES", Justification.RIGHT); + tableBuilder.addField("BYTES_NEEDED", Justification.RIGHT). + addField("BYTES_CACHED", Justification.RIGHT). + addField("FILES_NEEDED", Justification.RIGHT). + addField("FILES_CACHED", Justification.RIGHT); } TableListing tableListing = tableBuilder.build(); @@ -507,7 +509,8 @@ public class CacheAdmin extends Configur if (printStats) { row.add("" + stats.getBytesNeeded()); row.add("" + stats.getBytesCached()); - row.add("" + stats.getFilesAffected()); + row.add("" + stats.getFilesNeeded()); + row.add("" + stats.getFilesCached()); } tableListing.addRow(row.toArray(new String[0])); numEntries++; @@ -769,13 +772,14 @@ public class CacheAdmin extends Configur @Override public String getShortUsage() { - return "[" + getName() + " [name]]\n"; + return "[" + getName() + " [-stats] [<name>]]\n"; } @Override public String getLongUsage() { TableListing listing = getOptionDescriptionListing(); - listing.addRow("[name]", "If specified, list only the named cache pool."); + listing.addRow("-stats", "Display additional cache pool statistics."); + listing.addRow("<name>", "If specified, list only the named cache pool."); return getShortUsage() + "\n" + WordUtils.wrap("Display information about one or more cache pools, " + @@ -787,6 +791,7 @@ public class CacheAdmin extends Configur @Override public int run(Configuration conf, List<String> args) throws IOException { String name = StringUtils.popFirstNonOption(args); + final boolean printStats = StringUtils.popOption("-stats", args); if (!args.isEmpty()) { System.err.print("Can't understand arguments: " + Joiner.on(" ").join(args) + "\n"); @@ -794,28 +799,42 @@ public class CacheAdmin extends Configur return 1; } DistributedFileSystem dfs = getDFS(conf); - TableListing listing = new TableListing.Builder(). + TableListing.Builder builder = new TableListing.Builder(). addField("NAME", Justification.LEFT). addField("OWNER", Justification.LEFT). addField("GROUP", Justification.LEFT). addField("MODE", Justification.LEFT). - addField("WEIGHT", Justification.RIGHT). - build(); + addField("WEIGHT", Justification.RIGHT); + if (printStats) { + builder. + addField("BYTES_NEEDED", Justification.RIGHT). + addField("BYTES_CACHED", Justification.RIGHT). + addField("FILES_NEEDED", Justification.RIGHT). + addField("FILES_CACHED", Justification.RIGHT); + } + TableListing listing = builder.build(); int numResults = 0; try { RemoteIterator<CachePoolEntry> iter = dfs.listCachePools(); while (iter.hasNext()) { CachePoolEntry entry = iter.next(); CachePoolInfo info = entry.getInfo(); - String[] row = new String[5]; + LinkedList<String> row = new LinkedList<String>(); if (name == null || info.getPoolName().equals(name)) { - row[0] = info.getPoolName(); - row[1] = info.getOwnerName(); - row[2] = info.getGroupName(); - row[3] = info.getMode() != null ? info.getMode().toString() : null; - row[4] = - info.getWeight() != null ? info.getWeight().toString() : null; - listing.addRow(row); + row.add(info.getPoolName()); + row.add(info.getOwnerName()); + row.add(info.getGroupName()); + row.add(info.getMode() != null ? info.getMode().toString() : null); + row.add( + info.getWeight() != null ? info.getWeight().toString() : null); + if (printStats) { + CachePoolStats stats = entry.getStats(); + row.add(Long.toString(stats.getBytesNeeded())); + row.add(Long.toString(stats.getBytesCached())); + row.add(Long.toString(stats.getFilesNeeded())); + row.add(Long.toString(stats.getFilesCached())); + } + listing.addRow(row.toArray(new String[] {})); ++numResults; if (name != null) { break; Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto?rev=1548309&r1=1548308&r2=1548309&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto Thu Dec 5 21:09:30 2013 @@ -379,8 +379,9 @@ message CacheDirectiveInfoExpirationProt message CacheDirectiveStatsProto { required int64 bytesNeeded = 1; required int64 bytesCached = 2; - required int64 filesAffected = 3; - required bool hasExpired = 4; + required int64 filesNeeded = 3; + required int64 filesCached = 4; + required bool hasExpired = 5; } message AddCacheDirectiveRequestProto { @@ -431,7 +432,8 @@ message CachePoolInfoProto { message CachePoolStatsProto { required int64 bytesNeeded = 1; required int64 bytesCached = 2; - required int64 filesAffected = 3; + required int64 filesNeeded = 3; + required int64 filesCached = 4; } message AddCachePoolRequestProto { Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java?rev=1548309&r1=1548308&r2=1548309&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java Thu Dec 5 21:09:30 2013 @@ -62,6 +62,7 @@ import org.apache.hadoop.hdfs.protocol.C import org.apache.hadoop.hdfs.protocol.CachePoolEntry; import org.apache.hadoop.hdfs.protocol.CachePoolInfo; import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo.Expiration; +import org.apache.hadoop.hdfs.protocol.CachePoolStats; import org.apache.hadoop.hdfs.server.blockmanagement.CacheReplicationMonitor; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.CachedBlocksList.Type; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; @@ -623,45 +624,111 @@ public class TestCacheDirectives { }, 500, 60000); } - private static void waitForCachedStats(final DistributedFileSystem dfs, - final long targetFilesAffected, final long targetBytesNeeded, - final long targetBytesCached, - final CacheDirectiveInfo filter, final String infoString) + private static void waitForCacheDirectiveStats(final DistributedFileSystem dfs, + final long targetBytesNeeded, final long targetBytesCached, + final long targetFilesNeeded, final long targetFilesCached, + final CacheDirectiveInfo filter, final String infoString) throws Exception { - LOG.info("Polling listDirectives{" + - ((filter == null) ? "ALL" : filter.toString()) + - " for " + targetFilesAffected + " targetFilesAffected, " + - targetBytesNeeded + " targetBytesNeeded, " + - targetBytesCached + " targetBytesCached"); - GenericTestUtils.waitFor(new Supplier<Boolean>() { - @Override - public Boolean get() { - RemoteIterator<CacheDirectiveEntry> iter = null; - CacheDirectiveEntry entry = null; + LOG.info("Polling listCacheDirectives " + + ((filter == null) ? "ALL" : filter.toString()) + " for " + + targetBytesNeeded + " targetBytesNeeded, " + + targetBytesCached + " targetBytesCached, " + + targetFilesNeeded + " targetFilesNeeded, " + + targetFilesCached + " targetFilesCached"); + GenericTestUtils.waitFor(new Supplier<Boolean>() { + @Override + public Boolean get() { + RemoteIterator<CacheDirectiveEntry> iter = null; + CacheDirectiveEntry entry = null; + try { + iter = dfs.listCacheDirectives(filter); + entry = iter.next(); + } catch (IOException e) { + fail("got IOException while calling " + + "listCacheDirectives: " + e.getMessage()); + } + Assert.assertNotNull(entry); + CacheDirectiveStats stats = entry.getStats(); + if ((targetBytesNeeded == stats.getBytesNeeded()) && + (targetBytesCached == stats.getBytesCached()) && + (targetFilesNeeded == stats.getFilesNeeded()) && + (targetFilesCached == stats.getFilesCached())) { + return true; + } else { + LOG.info(infoString + ": " + + "filesNeeded: " + + stats.getFilesNeeded() + "/" + targetFilesNeeded + + ", filesCached: " + + stats.getFilesCached() + "/" + targetFilesCached + + ", bytesNeeded: " + + stats.getBytesNeeded() + "/" + targetBytesNeeded + + ", bytesCached: " + + stats.getBytesCached() + "/" + targetBytesCached); + return false; + } + } + }, 500, 60000); + } + + private static void waitForCachePoolStats(final DistributedFileSystem dfs, + final long targetBytesNeeded, final long targetBytesCached, + final long targetFilesNeeded, final long targetFilesCached, + final CachePoolInfo pool, final String infoString) + throws Exception { + LOG.info("Polling listCachePools " + pool.toString() + " for " + + targetBytesNeeded + " targetBytesNeeded, " + + targetBytesCached + " targetBytesCached, " + + targetFilesNeeded + " targetFilesNeeded, " + + targetFilesCached + " targetFilesCached"); + GenericTestUtils.waitFor(new Supplier<Boolean>() { + @Override + public Boolean get() { + RemoteIterator<CachePoolEntry> iter = null; + try { + iter = dfs.listCachePools(); + } catch (IOException e) { + fail("got IOException while calling " + + "listCachePools: " + e.getMessage()); + } + while (true) { + CachePoolEntry entry = null; try { - iter = dfs.listCacheDirectives(filter); + if (!iter.hasNext()) { + break; + } entry = iter.next(); } catch (IOException e) { - fail("got IOException while calling " + - "listCacheDirectives: " + e.getMessage()); + fail("got IOException while iterating through " + + "listCachePools: " + e.getMessage()); + } + if (entry == null) { + break; + } + if (!entry.getInfo().getPoolName().equals(pool.getPoolName())) { + continue; } - Assert.assertNotNull(entry); - CacheDirectiveStats stats = entry.getStats(); - if ((targetFilesAffected == stats.getFilesAffected()) && - (targetBytesNeeded == stats.getBytesNeeded()) && - (targetBytesCached == stats.getBytesCached())) { + CachePoolStats stats = entry.getStats(); + if ((targetBytesNeeded == stats.getBytesNeeded()) && + (targetBytesCached == stats.getBytesCached()) && + (targetFilesNeeded == stats.getFilesNeeded()) && + (targetFilesCached == stats.getFilesCached())) { return true; } else { - LOG.info(infoString + ": filesAffected: " + - stats.getFilesAffected() + "/" + targetFilesAffected + - ", bytesNeeded: " + + LOG.info(infoString + ": " + + "filesNeeded: " + + stats.getFilesNeeded() + "/" + targetFilesNeeded + + ", filesCached: " + + stats.getFilesCached() + "/" + targetFilesCached + + ", bytesNeeded: " + stats.getBytesNeeded() + "/" + targetBytesNeeded + - ", bytesCached: " + + ", bytesCached: " + stats.getBytesCached() + "/" + targetBytesCached); return false; } } - }, 500, 60000); + return false; + } + }, 500, 60000); } private static void checkNumCachedReplicas(final DistributedFileSystem dfs, @@ -837,7 +904,8 @@ public class TestCacheDirectives { NameNode namenode = cluster.getNameNode(); // Create the pool final String pool = "friendlyPool"; - dfs.addCachePool(new CachePoolInfo(pool)); + final CachePoolInfo poolInfo = new CachePoolInfo(pool); + dfs.addCachePool(poolInfo); // Create some test files final List<Path> paths = new LinkedList<Path>(); paths.add(new Path("/foo/bar")); @@ -853,6 +921,7 @@ public class TestCacheDirectives { } waitForCachedBlocks(namenode, 0, 0, "testWaitForCachedReplicasInDirectory:0"); + // cache entire directory long id = dfs.addCacheDirective( new CacheDirectiveInfo.Builder(). @@ -861,14 +930,20 @@ public class TestCacheDirectives { setPool(pool). build()); waitForCachedBlocks(namenode, 4, 8, - "testWaitForCachedReplicasInDirectory:1"); + "testWaitForCachedReplicasInDirectory:1:blocks"); // Verify that listDirectives gives the stats we want. - waitForCachedStats(dfs, 2, - 8 * BLOCK_SIZE, 8 * BLOCK_SIZE, + waitForCacheDirectiveStats(dfs, + 4 * numBlocksPerFile * BLOCK_SIZE, 4 * numBlocksPerFile * BLOCK_SIZE, + 2, 2, new CacheDirectiveInfo.Builder(). setPath(new Path("/foo")). build(), - "testWaitForCachedReplicasInDirectory:2"); + "testWaitForCachedReplicasInDirectory:1:directive"); + waitForCachePoolStats(dfs, + 4 * numBlocksPerFile * BLOCK_SIZE, 4 * numBlocksPerFile * BLOCK_SIZE, + 2, 2, + poolInfo, "testWaitForCachedReplicasInDirectory:1:pool"); + long id2 = dfs.addCacheDirective( new CacheDirectiveInfo.Builder(). setPath(new Path("/foo/bar")). @@ -877,28 +952,42 @@ public class TestCacheDirectives { build()); // wait for an additional 2 cached replicas to come up waitForCachedBlocks(namenode, 4, 10, - "testWaitForCachedReplicasInDirectory:3"); + "testWaitForCachedReplicasInDirectory:2:blocks"); // the directory directive's stats are unchanged - waitForCachedStats(dfs, 2, - 8 * BLOCK_SIZE, 8 * BLOCK_SIZE, + waitForCacheDirectiveStats(dfs, + 4 * numBlocksPerFile * BLOCK_SIZE, 4 * numBlocksPerFile * BLOCK_SIZE, + 2, 2, new CacheDirectiveInfo.Builder(). setPath(new Path("/foo")). build(), - "testWaitForCachedReplicasInDirectory:4"); + "testWaitForCachedReplicasInDirectory:2:directive-1"); // verify /foo/bar's stats - waitForCachedStats(dfs, 1, + waitForCacheDirectiveStats(dfs, 4 * numBlocksPerFile * BLOCK_SIZE, // only 3 because the file only has 3 replicas, not 4 as requested. 3 * numBlocksPerFile * BLOCK_SIZE, + 1, + // only 0 because the file can't be fully cached + 0, new CacheDirectiveInfo.Builder(). setPath(new Path("/foo/bar")). build(), - "testWaitForCachedReplicasInDirectory:5"); + "testWaitForCachedReplicasInDirectory:2:directive-2"); + waitForCachePoolStats(dfs, + (4+4) * numBlocksPerFile * BLOCK_SIZE, + (4+3) * numBlocksPerFile * BLOCK_SIZE, + 3, 2, + poolInfo, "testWaitForCachedReplicasInDirectory:2:pool"); + // remove and watch numCached go to 0 dfs.removeCacheDirective(id); dfs.removeCacheDirective(id2); waitForCachedBlocks(namenode, 0, 0, - "testWaitForCachedReplicasInDirectory:6"); + "testWaitForCachedReplicasInDirectory:3:blocks"); + waitForCachePoolStats(dfs, + 0, 0, + 0, 0, + poolInfo, "testWaitForCachedReplicasInDirectory:3:pool"); } finally { cluster.shutdown(); } Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCacheAdminConf.xml URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCacheAdminConf.xml?rev=1548309&r1=1548308&r2=1548309&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCacheAdminConf.xml (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCacheAdminConf.xml Thu Dec 5 21:09:30 2013 @@ -399,5 +399,63 @@ </comparators> </test> + <test> <!--Tested --> + <description>Testing listing cache pool statistics</description> + <test-commands> + <cache-admin-command>-addPool foo -owner bob -group bob -mode 0664</cache-admin-command> + <cache-admin-command>-addPool bar -owner alice -group alicegroup -mode 0755</cache-admin-command> + <cache-admin-command>-listPools -stats</cache-admin-command> + </test-commands> + <cleanup-commands> + <cache-admin-command>-removePool foo</cache-admin-command> + <cache-admin-command>-removePool bar</cache-admin-command> + </cleanup-commands> + <comparators> + <comparator> + <type>SubstringComparator</type> + <expected-output>Found 2 results.</expected-output> + </comparator> + <comparator> + <type>SubstringComparator</type> + <expected-output>bar alice alicegroup rwxr-xr-x 100 0 0 0 0</expected-output> + </comparator> + <comparator> + <type>SubstringComparator</type> + <expected-output>foo bob bob rw-rw-r-- 100 0 0 0 0</expected-output> + </comparator> + </comparators> + </test> + + <test> <!--Tested --> + <description>Testing listing cache directive statistics</description> + <test-commands> + <cache-admin-command>-addPool pool1</cache-admin-command> + <cache-admin-command>-addDirective -path /foo -pool pool1 -ttl 2d</cache-admin-command> + <cache-admin-command>-addDirective -path /bar -pool pool1 -ttl 24h</cache-admin-command> + <cache-admin-command>-addDirective -path /baz -replication 2 -pool pool1 -ttl 60m</cache-admin-command> + <cache-admin-command>-listDirectives -pool pool1 -stats</cache-admin-command> + </test-commands> + <cleanup-commands> + <cache-admin-command>-removePool pool1</cache-admin-command> + </cleanup-commands> + <comparators> + <comparator> + <type>SubstringComparator</type> + <expected-output>Found 3 entries</expected-output> + </comparator> + <comparator> + <type>SubstringComparator</type> + <expected-output>/foo 0 0 0 0</expected-output> + </comparator> + <comparator> + <type>SubstringComparator</type> + <expected-output>/bar 0 0 0 0</expected-output> + </comparator> + <comparator> + <type>SubstringComparator</type> + <expected-output>/baz 0 0 0 0</expected-output> + </comparator> + </comparators> + </test> </tests> </configuration>