Author: eli Date: Fri Aug 17 23:24:36 2012 New Revision: 1374474 URL: http://svn.apache.org/viewvc?rev=1374474&view=rev Log: HADOOP-8689. Make trash a server side configuration option. Contributed by Eli Collins
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsServerDefaults.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpConfigKeys.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/LocalConfigKeys.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1374474&r1=1374473&r2=1374474&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Fri Aug 17 23:24:36 2012 @@ -7,6 +7,8 @@ Release 2.0.1-alpha - UNRELEASED HADOOP-8388. Remove unused BlockLocation serialization. (Colin Patrick McCabe via eli) + HADOOP-8689. Make trash a server side configuration option. (eli) + NEW FEATURES HDFS-3042. Automatic failover support for NameNode HA (todd) Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java?rev=1374474&r1=1374473&r2=1374474&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java Fri Aug 17 23:24:36 2012 @@ -661,7 +661,9 @@ public abstract class FileSystem extends 64 * 1024, getDefaultReplication(), conf.getInt("io.file.buffer.size", 4096), - false); + false, + // NB: ignoring the client trash configuration + CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_DEFAULT); } /** Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsServerDefaults.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsServerDefaults.java?rev=1374474&r1=1374473&r2=1374474&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsServerDefaults.java (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsServerDefaults.java Fri Aug 17 23:24:36 2012 @@ -49,19 +49,21 @@ public class FsServerDefaults implements private short replication; private int fileBufferSize; private boolean encryptDataTransfer; + private long trashInterval; public FsServerDefaults() { } public FsServerDefaults(long blockSize, int bytesPerChecksum, int writePacketSize, short replication, int fileBufferSize, - boolean encryptDataTransfer) { + boolean encryptDataTransfer, long trashInterval) { this.blockSize = blockSize; this.bytesPerChecksum = bytesPerChecksum; this.writePacketSize = writePacketSize; this.replication = replication; this.fileBufferSize = fileBufferSize; this.encryptDataTransfer = encryptDataTransfer; + this.trashInterval = trashInterval; } public long getBlockSize() { @@ -88,6 +90,10 @@ public class FsServerDefaults implements return encryptDataTransfer; } + public long getTrashInterval() { + return trashInterval; + } + // ///////////////////////////////////////// // Writable // ///////////////////////////////////////// Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java?rev=1374474&r1=1374473&r2=1374474&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java Fri Aug 17 23:24:36 2012 @@ -34,7 +34,6 @@ import org.apache.commons.logging.LogFac import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.Options.Rename; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; @@ -66,6 +65,7 @@ public class TrashPolicyDefault extends private Path current; private Path homesParent; + private long emptierInterval; public TrashPolicyDefault() { } @@ -79,8 +79,27 @@ public class TrashPolicyDefault extends this.trash = new Path(home, TRASH); this.homesParent = home.getParent(); this.current = new Path(trash, CURRENT); - this.deletionInterval = (long) (conf.getFloat(FS_TRASH_INTERVAL_KEY, - FS_TRASH_INTERVAL_DEFAULT) * MSECS_PER_MINUTE); + long trashInterval = 0; + try { + trashInterval = fs.getServerDefaults(home).getTrashInterval(); + } catch (IOException ioe) { + LOG.warn("Unable to get server defaults", ioe); + } + // If the trash interval is not configured or is disabled on the + // server side then check the config which may be client side. + if (0 == trashInterval) { + this.deletionInterval = (long)(conf.getFloat( + FS_TRASH_INTERVAL_KEY, FS_TRASH_INTERVAL_DEFAULT) + * MSECS_PER_MINUTE); + } else { + this.deletionInterval = trashInterval * MSECS_PER_MINUTE; + } + // For the checkpoint interval use the given config instead of + // checking the server as it's OK if a client starts an emptier + // with a different interval than the server. + this.emptierInterval = (long)(conf.getFloat( + FS_TRASH_CHECKPOINT_INTERVAL_KEY, FS_TRASH_CHECKPOINT_INTERVAL_DEFAULT) + * MSECS_PER_MINUTE); } private Path makeTrashRelativePath(Path basePath, Path rmFilePath) { @@ -89,7 +108,7 @@ public class TrashPolicyDefault extends @Override public boolean isEnabled() { - return (deletionInterval != 0); + return deletionInterval != 0; } @Override @@ -223,7 +242,7 @@ public class TrashPolicyDefault extends @Override public Runnable getEmptier() throws IOException { - return new Emptier(getConf()); + return new Emptier(getConf(), emptierInterval); } private class Emptier implements Runnable { @@ -231,16 +250,14 @@ public class TrashPolicyDefault extends private Configuration conf; private long emptierInterval; - Emptier(Configuration conf) throws IOException { + Emptier(Configuration conf, long emptierInterval) throws IOException { this.conf = conf; - this.emptierInterval = (long) (conf.getFloat(FS_TRASH_CHECKPOINT_INTERVAL_KEY, - FS_TRASH_CHECKPOINT_INTERVAL_DEFAULT) * - MSECS_PER_MINUTE); - if (this.emptierInterval > deletionInterval || - this.emptierInterval == 0) { - LOG.warn("The configured interval for checkpoint is " + - this.emptierInterval + " minutes." + - " Using interval of " + deletionInterval + + this.emptierInterval = emptierInterval; + if (emptierInterval > deletionInterval || emptierInterval == 0) { + LOG.info("The configured checkpoint interval is " + + (emptierInterval / MSECS_PER_MINUTE) + " minutes." + + " Using an interval of " + + (deletionInterval / MSECS_PER_MINUTE) + " minutes that is used for deletion instead"); this.emptierInterval = deletionInterval; } Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpConfigKeys.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpConfigKeys.java?rev=1374474&r1=1374473&r2=1374474&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpConfigKeys.java (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpConfigKeys.java Fri Aug 17 23:24:36 2012 @@ -45,6 +45,7 @@ public class FtpConfigKeys extends Commo "ftp.client-write-packet-size"; public static final int CLIENT_WRITE_PACKET_SIZE_DEFAULT = 64*1024; public static final boolean ENCRYPT_DATA_TRANSFER_DEFAULT = false; + public static final long FS_TRASH_INTERVAL_DEFAULT = 0; protected static FsServerDefaults getServerDefaults() throws IOException { return new FsServerDefaults( @@ -53,7 +54,8 @@ public class FtpConfigKeys extends Commo CLIENT_WRITE_PACKET_SIZE_DEFAULT, REPLICATION_DEFAULT, STREAM_BUFFER_SIZE_DEFAULT, - ENCRYPT_DATA_TRANSFER_DEFAULT); + ENCRYPT_DATA_TRANSFER_DEFAULT, + FS_TRASH_INTERVAL_DEFAULT); } } Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/LocalConfigKeys.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/LocalConfigKeys.java?rev=1374474&r1=1374473&r2=1374474&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/LocalConfigKeys.java (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/LocalConfigKeys.java Fri Aug 17 23:24:36 2012 @@ -44,6 +44,7 @@ public class LocalConfigKeys extends Com "file.client-write-packet-size"; public static final int CLIENT_WRITE_PACKET_SIZE_DEFAULT = 64*1024; public static final boolean ENCRYPT_DATA_TRANSFER_DEFAULT = false; + public static final long FS_TRASH_INTERVAL_DEFAULT = 0; public static FsServerDefaults getServerDefaults() throws IOException { return new FsServerDefaults( @@ -52,7 +53,8 @@ public class LocalConfigKeys extends Com CLIENT_WRITE_PACKET_SIZE_DEFAULT, REPLICATION_DEFAULT, STREAM_BUFFER_SIZE_DEFAULT, - ENCRYPT_DATA_TRANSFER_DEFAULT); + ENCRYPT_DATA_TRANSFER_DEFAULT, + FS_TRASH_INTERVAL_DEFAULT); } } Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml?rev=1374474&r1=1374473&r2=1374474&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml Fri Aug 17 23:24:36 2012 @@ -351,8 +351,12 @@ <name>fs.trash.interval</name> <value>0</value> <description>Number of minutes after which the checkpoint - gets deleted. - If zero, the trash feature is disabled. + gets deleted. If zero, the trash feature is disabled. + This option may be configured both on the server and the + client. If trash is disabled server side then the client + side configuration is checked. If trash is enabled on the + server side then the value configured on the server is + used and the client configuration value is ignored. </description> </property> @@ -360,7 +364,8 @@ <name>fs.trash.checkpoint.interval</name> <value>0</value> <description>Number of minutes between trash checkpoints. - Should be smaller or equal to fs.trash.interval. + Should be smaller or equal to fs.trash.interval. If zero, + the value is set to the value of fs.trash.interval. Every time the checkpointer runs it creates a new checkpoint out of current and removes checkpoints created more than fs.trash.interval minutes ago. Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java?rev=1374474&r1=1374473&r2=1374474&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java Fri Aug 17 23:24:36 2012 @@ -111,10 +111,10 @@ public class TestTrash extends TestCase throws IOException { FileSystem fs = FileSystem.get(conf); - conf.set(FS_TRASH_INTERVAL_KEY, "0"); // disabled + conf.setLong(FS_TRASH_INTERVAL_KEY, 0); // disabled assertFalse(new Trash(conf).isEnabled()); - conf.set(FS_TRASH_INTERVAL_KEY, "10"); // 10 minute + conf.setLong(FS_TRASH_INTERVAL_KEY, 10); // 10 minute assertTrue(new Trash(conf).isEnabled()); FsShell shell = new FsShell(); @@ -435,7 +435,7 @@ public class TestTrash extends TestCase } public static void trashNonDefaultFS(Configuration conf) throws IOException { - conf.set(FS_TRASH_INTERVAL_KEY, "10"); // 10 minute + conf.setLong(FS_TRASH_INTERVAL_KEY, 10); // 10 minute // attempt non-default FileSystem trash { final FileSystem lfs = FileSystem.getLocal(conf); @@ -580,7 +580,7 @@ public class TestTrash extends TestCase FileSystem fs = FileSystem.getLocal(conf); conf.set("fs.defaultFS", fs.getUri().toString()); - conf.set(FS_TRASH_INTERVAL_KEY, "10"); //minutes.. + conf.setLong(FS_TRASH_INTERVAL_KEY, 10); //minutes.. FsShell shell = new FsShell(); shell.setConf(conf); //Path trashRoot = null;