Author: jlowe
Date: Wed Sep 26 17:23:34 2012
New Revision: 1390621
URL: http://svn.apache.org/viewvc?rev=1390621&view=rev
Log:
svn merge -c 1390616 to fix HADOOP-8843. Old trash directories are never
deleted on upgrade from 1.x. Contributed by Jason Lowe
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1390621&r1=1390620&r2=1390621&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Wed Sep 26 17:23:34 2012
@@ -747,6 +747,9 @@ Release 0.23.4 - UNRELEASED
BUG FIXES
+ HADOOP-8843. Old trash directories are never deleted on upgrade
+ from 1.x (jlowe)
+
Release 0.23.3 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java?rev=1390621&r1=1390620&r2=1390621&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
Wed Sep 26 17:23:34 2012
@@ -61,6 +61,9 @@ public class TrashPolicyDefault extends
new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE);
private static final DateFormat CHECKPOINT = new
SimpleDateFormat("yyMMddHHmmss");
+ /** Format of checkpoint directories used prior to Hadoop 0.23. */
+ private static final DateFormat OLD_CHECKPOINT =
+ new SimpleDateFormat("yyMMddHHmm");
private static final int MSECS_PER_MINUTE = 60*1000;
private Path current;
@@ -202,9 +205,7 @@ public class TrashPolicyDefault extends
long time;
try {
- synchronized (CHECKPOINT) {
- time = CHECKPOINT.parse(name).getTime();
- }
+ time = getTimeFromCheckpoint(name);
} catch (ParseException e) {
LOG.warn("Unexpected item in trash: "+dir+". Ignoring.");
continue;
@@ -304,4 +305,22 @@ public class TrashPolicyDefault extends
return (time / interval) * interval;
}
}
+
+ private long getTimeFromCheckpoint(String name) throws ParseException {
+ long time;
+
+ try {
+ synchronized (CHECKPOINT) {
+ time = CHECKPOINT.parse(name).getTime();
+ }
+ } catch (ParseException pe) {
+ // Check for old-style checkpoint directories left over
+ // after an upgrade from Hadoop 1.x
+ synchronized (OLD_CHECKPOINT) {
+ time = OLD_CHECKPOINT.parse(name).getTime();
+ }
+ }
+
+ return time;
+ }
}
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java?rev=1390621&r1=1390620&r2=1390621&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
Wed Sep 26 17:23:34 2012
@@ -26,6 +26,8 @@ import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
import java.net.URI;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
import java.util.HashSet;
import java.util.Set;
@@ -434,6 +436,36 @@ public class TestTrash extends TestCase
output.indexOf("Failed to determine server trash configuration") !=
-1);
}
+ // Verify old checkpoint format is recognized
+ {
+ // emulate two old trash checkpoint directories, one that is old enough
+ // to be deleted on the next expunge and one that isn't.
+ long trashInterval = conf.getLong(FS_TRASH_INTERVAL_KEY,
+ FS_TRASH_INTERVAL_DEFAULT);
+ long now = Time.now();
+ DateFormat oldCheckpointFormat = new SimpleDateFormat("yyMMddHHmm");
+ Path dirToDelete = new Path(trashRoot.getParent(),
+ oldCheckpointFormat.format(now - (trashInterval * 60 * 1000) - 1));
+ Path dirToKeep = new Path(trashRoot.getParent(),
+ oldCheckpointFormat.format(now));
+ mkdir(trashRootFs, dirToDelete);
+ mkdir(trashRootFs, dirToKeep);
+
+ // Clear out trash
+ int rc = -1;
+ try {
+ rc = shell.run(new String [] { "-expunge" } );
+ } catch (Exception e) {
+ System.err.println("Exception raised from fs expunge " +
+ e.getLocalizedMessage());
+ }
+ assertEquals(0, rc);
+ assertFalse("old checkpoint format not recognized",
+ trashRootFs.exists(dirToDelete));
+ assertTrue("old checkpoint format directory should not be removed",
+ trashRootFs.exists(dirToKeep));
+ }
+
}
public static void trashNonDefaultFS(Configuration conf) throws IOException {
