IGNITE-1515: Fixed delete.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/54bb7d76 Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/54bb7d76 Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/54bb7d76 Branch: refs/heads/ignite-1537 Commit: 54bb7d760d9c1261a64fcf3d0be858be7b14444e Parents: be906e3 Author: iveselovskiy <iveselovs...@gridgain.com> Authored: Wed Sep 30 10:05:19 2015 +0300 Committer: vozerov-gridgain <voze...@gridgain.com> Committed: Wed Sep 30 10:05:19 2015 +0300 ---------------------------------------------------------------------- .../java/org/apache/ignite/igfs/IgfsPath.java | 2 +- .../internal/processors/igfs/IgfsFileInfo.java | 2 +- .../internal/processors/igfs/IgfsImpl.java | 57 +---- .../processors/igfs/IgfsMetaManager.java | 251 +++++++++++++++---- .../ignite/igfs/IgfsFragmentizerSelfTest.java | 2 +- .../processors/igfs/IgfsAbstractSelfTest.java | 80 ++++-- .../igfs/IgfsMetaManagerSelfTest.java | 6 - .../processors/igfs/IgfsMetricsSelfTest.java | 2 +- .../processors/igfs/IgfsProcessorSelfTest.java | 29 +-- .../igfs/UniversalFileSystemAdapter.java | 1 - .../processors/hadoop/igfs/HadoopIgfsUtils.java | 36 +++ ...oopFileSystemUniversalFileSystemAdapter.java | 4 +- .../HadoopIgfs20FileSystemAbstractSelfTest.java | 7 +- .../IgniteHadoopFileSystemAbstractSelfTest.java | 5 +- 14 files changed, 345 insertions(+), 139 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/54bb7d76/modules/core/src/main/java/org/apache/ignite/igfs/IgfsPath.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/IgfsPath.java b/modules/core/src/main/java/org/apache/ignite/igfs/IgfsPath.java index a99c1ee..fb0621c 100644 --- a/modules/core/src/main/java/org/apache/ignite/igfs/IgfsPath.java +++ b/modules/core/src/main/java/org/apache/ignite/igfs/IgfsPath.java @@ -50,7 +50,7 @@ public final class IgfsPath implements Comparable<IgfsPath>, Externalizable { private static final char SLASH_CHAR = '/'; /** The directory separator. */ - private static final String SLASH = "/"; + public static final String SLASH = "/"; /** URI representing this path. Should never change after object creation or de-serialization. */ private String path; http://git-wip-us.apache.org/repos/asf/ignite/blob/54bb7d76/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileInfo.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileInfo.java index 116d585..8564500 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileInfo.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileInfo.java @@ -216,7 +216,7 @@ public final class IgfsFileInfo implements Externalizable { * @param listing New directory listing. * @param old Old file info. */ - IgfsFileInfo(Map<String, IgfsListingEntry> listing, IgfsFileInfo old) { + IgfsFileInfo(@Nullable Map<String, IgfsListingEntry> listing, IgfsFileInfo old) { this(old.isDirectory(), old.id, old.blockSize, old.len, old.affKey, listing, old.props, old.fileMap(), old.lockId, false, old.accessTime, old.modificationTime, old.evictExclude()); } http://git-wip-us.apache.org/repos/asf/ignite/blob/54bb7d76/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java index 0dd0307..d5ba95f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java @@ -53,7 +53,6 @@ import org.apache.ignite.events.DiscoveryEvent; import org.apache.ignite.events.Event; import org.apache.ignite.events.IgfsEvent; import org.apache.ignite.igfs.IgfsBlockLocation; -import org.apache.ignite.igfs.IgfsDirectoryNotEmptyException; import org.apache.ignite.igfs.IgfsFile; import org.apache.ignite.igfs.IgfsInvalidPathException; import org.apache.ignite.igfs.IgfsMetrics; @@ -712,6 +711,9 @@ public final class IgfsImpl implements IgfsEx { if (log.isDebugEnabled()) log.debug("Deleting file [path=" + path + ", recursive=" + recursive + ']'); + if (IgfsPath.SLASH.equals(path.toString())) + return false; + IgfsMode mode = resolveMode(path); Set<IgfsMode> childrenModes = modeRslvr.resolveChildrenModes(path); @@ -721,8 +723,11 @@ public final class IgfsImpl implements IgfsEx { FileDescriptor desc = getFileDescriptor(path); if (childrenModes.contains(PRIMARY)) { - if (desc != null) - res = delete0(desc, path.parent(), recursive); + if (desc != null) { + IgniteUuid deletedId = meta.softDelete(path, recursive); + + res = deletedId != null; + } else if (mode == PRIMARY) checkConflictWithPrimary(path); } @@ -750,48 +755,6 @@ public final class IgfsImpl implements IgfsEx { }); } - /** - * Internal procedure for (optionally) recursive file and directory deletion. - * - * @param desc File descriptor of file or directory to delete. - * @param parentPath Parent path. If specified, events will be fired for each deleted file - * or directory. If not specified, events will not be fired. - * @param recursive Recursive deletion flag. - * @return {@code True} if file was successfully deleted. If directory is not empty and - * {@code recursive} flag is false, will return {@code false}. - * @throws IgniteCheckedException In case of error. - */ - private boolean delete0(FileDescriptor desc, @Nullable IgfsPath parentPath, boolean recursive) - throws IgniteCheckedException { - IgfsPath curPath = parentPath == null ? new IgfsPath() : new IgfsPath(parentPath, desc.fileName); - - if (desc.isFile) { - deleteFile(curPath, desc, true); - - return true; - } - else { - if (recursive) { - meta.softDelete(desc.parentId, desc.fileName, desc.fileId); - - return true; - } - else { - Map<String, IgfsListingEntry> infoMap = meta.directoryListing(desc.fileId); - - if (F.isEmpty(infoMap)) { - deleteFile(curPath, desc, true); - - return true; - } - else - // Throw exception if not empty and not recursive. - throw new IgfsDirectoryNotEmptyException("Failed to remove directory (directory is not empty " + - "and recursive flag is not set)"); - } - } - } - /** {@inheritDoc} */ @Override public void mkdirs(IgfsPath path) { mkdirs(path, null); @@ -1454,7 +1417,7 @@ public final class IgfsImpl implements IgfsEx { */ IgniteInternalFuture<?> formatAsync() { try { - IgniteUuid id = meta.softDelete(null, null, ROOT_ID); + IgniteUuid id = meta.format(); if (id == null) return new GridFinishedFuture<Object>(); @@ -1526,6 +1489,8 @@ public final class IgfsImpl implements IgfsEx { * @throws IgniteCheckedException If failed. */ @Nullable private FileDescriptor getFileDescriptor(IgfsPath path) throws IgniteCheckedException { + assert path != null; + List<IgniteUuid> ids = meta.fileIds(path); IgfsFileInfo fileInfo = meta.info(ids.get(ids.size() - 1)); http://git-wip-us.apache.org/repos/asf/ignite/blob/54bb7d76/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java index d283b64..bb6404c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java @@ -34,6 +34,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.SortedSet; import java.util.TreeMap; import java.util.TreeSet; import java.util.concurrent.CountDownLatch; @@ -95,6 +96,20 @@ import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_REA */ @SuppressWarnings("all") public class IgfsMetaManager extends IgfsManager { + /** Comparator for Id sorting. */ + private static final Comparator<IgniteUuid> PATH_ID_SORTING_COMPARATOR + = new Comparator<IgniteUuid>() { + @Override public int compare(IgniteUuid u1, IgniteUuid u2) { + if (u1 == u2) + return 0; + + if (u1 == null) + return -1; + + return u1.compareTo(u2); + } + }; + /** IGFS configuration. */ private FileSystemConfiguration cfg; @@ -376,6 +391,7 @@ public class IgfsMetaManager extends IgfsManager { /** * Gets file info by its ID. + * NB: this method is used both in Tx and out of Tx. * * @param fileId File ID to get details for. * @return File info. @@ -593,12 +609,39 @@ public class IgfsMetaManager extends IgfsManager { } /** + * Answers if the collection is sorted. + * + * @param col The collection to check. + * @param <T> The type of the collection elements. + * @return If the collection sorted. + */ + private static <T extends Comparable<T>> boolean isSorted(Collection<T> col) { + T prev = null; + + for (T t: col) { + if (t == null) + throw new NullPointerException("Collections should not contain nulls"); + + if (prev != null && prev.compareTo(t) > 0) + return false; // disordered. + + prev = t; + } + + return true; + } + + /** * Lock file IDs. + * * @param fileIds File IDs (sorted). * @return Map with lock info. * @throws IgniteCheckedException If failed. */ private Map<IgniteUuid, IgfsFileInfo> lockIds(Collection<IgniteUuid> fileIds) throws IgniteCheckedException { + assert isSorted(fileIds); + assert validTxState(true); + if (log.isDebugEnabled()) log.debug("Locking file ids: " + fileIds); @@ -608,19 +651,34 @@ public class IgfsMetaManager extends IgfsManager { if (log.isDebugEnabled()) log.debug("Locked file ids: " + fileIds); - // Force root ID always exist in cache. - if (fileIds.contains(ROOT_ID) && !map.containsKey(ROOT_ID)) { - IgfsFileInfo info = new IgfsFileInfo(); + // Force root & trash IDs always exist in cache. + addInfoIfNeeded(fileIds, map, ROOT_ID); + addInfoIfNeeded(fileIds, map, TRASH_ID); - id2InfoPrj.putIfAbsent(ROOT_ID, info); + // Returns detail's map for locked IDs. + return map; + } - map = new GridLeanMap<>(map); + /** + * Adds FileInfo into the cache if it is requested in fileIds and is not present in the map. + * + * @param fileIds A list that may contain the id. + * @param map The map that may not contain the id. + * @param id The id to check. + * @throws IgniteCheckedException On error. + */ + private void addInfoIfNeeded(Collection<IgniteUuid> fileIds, Map<IgniteUuid, IgfsFileInfo> map, IgniteUuid id) throws IgniteCheckedException { + assert validTxState(true); - map.put(ROOT_ID, info); - } + if (fileIds.contains(id) && !map.containsKey(id)) { + IgfsFileInfo info = new IgfsFileInfo(id); - // Returns detail's map for locked IDs. - return map; + assert info.listing() != null; + + id2InfoPrj.putIfAbsent(id, info); + + map.put(id, info); + } } /** @@ -779,13 +837,11 @@ public class IgfsMetaManager extends IgfsManager { log.debug("Locking parent id [parentId=" + parentId + ", fileName=" + fileName + ", newFileInfo=" + newFileInfo + ']'); - validTxState(true); + assert validTxState(true); // Lock only parent file ID. IgfsFileInfo parentInfo = info(parentId); - assert validTxState(true); - if (parentInfo == null) throw fsException(new IgfsPathNotFoundException("Failed to lock parent directory (not found): " + parentId)); @@ -798,8 +854,6 @@ public class IgfsMetaManager extends IgfsManager { IgfsListingEntry entry = parentListing.get(fileName); - assert validTxState(true); - if (entry != null) return entry.fileId(); @@ -832,18 +886,7 @@ public class IgfsMetaManager extends IgfsManager { List<IgniteUuid> srcPathIds = fileIds(srcPath); List<IgniteUuid> dstPathIds = fileIds(dstPath); - final Set<IgniteUuid> allIds = new TreeSet<>(new Comparator<IgniteUuid>() { - @Override - public int compare(IgniteUuid u1, IgniteUuid u2) { - if (u1 == u2) - return 0; - - if (u1 == null) - return -1; - - return u1.compareTo(u2); - } - }); + final Set<IgniteUuid> allIds = new TreeSet<>(PATH_ID_SORTING_COMPARATOR); allIds.addAll(srcPathIds); @@ -1009,6 +1052,7 @@ public class IgfsMetaManager extends IgfsManager { private void moveNonTx(IgniteUuid fileId, @Nullable String srcFileName, IgniteUuid srcParentId, String destFileName, IgniteUuid destParentId) throws IgniteCheckedException { assert validTxState(true); + assert fileId != null; assert srcFileName != null; assert srcParentId != null; @@ -1026,8 +1070,6 @@ public class IgfsMetaManager extends IgfsManager { // Lock file ID and parent IDs for this transaction. Map<IgniteUuid, IgfsFileInfo> infoMap = lockIds(srcParentId, fileId, destParentId); - validTxState(true); - IgfsFileInfo srcInfo = infoMap.get(srcParentId); if (srcInfo == null) @@ -1196,6 +1238,66 @@ public class IgfsMetaManager extends IgfsManager { } /** + * Deletes (moves to TRASH) all elements under the root folder. + * + * @return The new Id if the artificially created folder containing all former root + * elements moved to TRASH folder. + * @throws IgniteCheckedException On error. + */ + IgniteUuid format() throws IgniteCheckedException { + if (busyLock.enterBusy()) { + try { + assert validTxState(false); + + final IgniteInternalTx tx = metaCache.txStartEx(PESSIMISTIC, REPEATABLE_READ); + + try { + // NB: We may lock root because its id is less than any other id: + final IgfsFileInfo rootInfo = lockIds(ROOT_ID, TRASH_ID).get(ROOT_ID); + + assert rootInfo != null; + + Map<String, IgfsListingEntry> rootListingMap = rootInfo.listing(); + + assert rootListingMap != null; + + if (rootListingMap.isEmpty()) + return null; // Root is empty, nothing to do. + + // Construct new info and move locked entries from root to it. + Map<String, IgfsListingEntry> transferListing = new HashMap<>(rootListingMap); + + IgfsFileInfo newInfo = new IgfsFileInfo(transferListing); + + id2InfoPrj.put(newInfo.id(), newInfo); + + // Add new info to trash listing. + id2InfoPrj.invoke(TRASH_ID, new UpdateListing(newInfo.id().toString(), + new IgfsListingEntry(newInfo), false)); + + // Remove listing entries from root. + // Note that root directory properties and other attributes are preserved: + id2InfoPrj.put(ROOT_ID, new IgfsFileInfo(null/*listing*/, rootInfo)); + + tx.commit(); + + delWorker.signal(); + + return newInfo.id(); + } + finally { + tx.close(); + } + } + finally { + busyLock.leaveBusy(); + } + } + else + throw new IllegalStateException("Failed to perform format because Grid is stopping."); + } + + /** * Move path to the trash directory. * * @param parentId Parent ID. @@ -1204,26 +1306,86 @@ public class IgfsMetaManager extends IgfsManager { * @return ID of an entry located directly under the trash directory. * @throws IgniteCheckedException If failed. */ - IgniteUuid softDelete(@Nullable IgniteUuid parentId, @Nullable String pathName, IgniteUuid pathId) throws IgniteCheckedException { + IgniteUuid softDelete(final IgfsPath path, final boolean recursive) throws IgniteCheckedException { if (busyLock.enterBusy()) { try { assert validTxState(false); - IgniteInternalTx tx = metaCache.txStartEx(PESSIMISTIC, REPEATABLE_READ); + final SortedSet<IgniteUuid> allIds = new TreeSet<>(PATH_ID_SORTING_COMPARATOR); + + List<IgniteUuid> pathIdList = fileIds(path); + + assert pathIdList.size() > 1; + + final IgniteUuid victimId = pathIdList.get(pathIdList.size() - 1); + + assert !TRASH_ID.equals(victimId) : "TRASH does not have path, it cannot ever be deletion victim."; + assert !ROOT_ID.equals(victimId); // root deletion is prevented in earlier stages. + + allIds.addAll(pathIdList); + + if (allIds.remove(null)) + return null; // A fragment of the path no longer exists. + + boolean added = allIds.add(TRASH_ID); + assert added; + + final IgniteInternalTx tx = metaCache.txStartEx(PESSIMISTIC, REPEATABLE_READ); try { - if (parentId == null) - lockIds(pathId, TRASH_ID); - else - lockIds(parentId, pathId, TRASH_ID); + final Map<IgniteUuid, IgfsFileInfo> infoMap = lockIds(allIds); + + // Directory starure was changed concurrently, so the original path no longer exists: + if (!verifyPathIntegrity(path, pathIdList, infoMap)) + return null; + + final IgfsFileInfo victimInfo = infoMap.get(victimId); + + if (!recursive && victimInfo.isDirectory() && !victimInfo.listing().isEmpty()) + // Throw exception if not empty and not recursive. + throw new IgfsDirectoryNotEmptyException("Failed to remove directory (directory is not " + + "empty and recursive flag is not set)."); + + IgfsFileInfo destInfo = infoMap.get(TRASH_ID); - IgniteUuid resId = softDeleteNonTx(parentId, pathName, pathId); + assert destInfo != null; + + final String srcFileName = path.name(); + + final String destFileName = victimId.toString(); + + assert destInfo.listing().get(destFileName) == null : "Failed to add file name into the " + + "destination directory (file already exists) [destName=" + destFileName + ']'; + + IgfsFileInfo srcParentInfo = infoMap.get(pathIdList.get(pathIdList.size() - 2)); + + assert srcParentInfo != null; + + IgniteUuid srcParentId = srcParentInfo.id(); + assert srcParentId.equals(pathIdList.get(pathIdList.size() - 2)); + + IgfsListingEntry srcEntry = srcParentInfo.listing().get(srcFileName); + + assert srcEntry != null : "Deletion victim not found in parent listing [path=" + path + + ", name=" + srcFileName + ", listing=" + srcParentInfo.listing() + ']'; + + assert victimId.equals(srcEntry.fileId()); + + id2InfoPrj.invoke(srcParentId, new UpdateListing(srcFileName, srcEntry, true)); + + // Add listing entry into the destination parent listing. + id2InfoPrj.invoke(TRASH_ID, new UpdateListing(destFileName, srcEntry, false)); + + if (victimInfo.isFile()) + // Update a file info of the removed file with a file path, + // which will be used by delete worker for event notifications. + id2InfoPrj.invoke(victimId, new UpdatePath(path)); tx.commit(); delWorker.signal(); - return resId; + return victimId; } finally { tx.close(); @@ -1234,8 +1396,8 @@ public class IgfsMetaManager extends IgfsManager { } } else - throw new IllegalStateException("Failed to perform soft delete because Grid is stopping [parentId=" + - parentId + ", pathName=" + pathName + ", pathId=" + pathId + ']'); + throw new IllegalStateException("Failed to perform soft delete because Grid is " + + "stopping [path=" + path + ']'); } /** @@ -1316,6 +1478,7 @@ public class IgfsMetaManager extends IgfsManager { /** * Remove listing entries of the given parent. + * This operation actually deletes directories from TRASH, is used solely by IgfsDeleteWorker. * * @param parentId Parent ID. * @param listing Listing entries. @@ -1403,6 +1566,7 @@ public class IgfsMetaManager extends IgfsManager { /** * Remove entry from the metadata listing. + * Used solely by IgfsDeleteWorker. * * @param parentId Parent ID. * @param name Name. @@ -2394,17 +2558,18 @@ public class IgfsMetaManager extends IgfsManager { if (busyLock.enterBusy()) { try { SynchronizationTask<IgfsFileInfo> task = new SynchronizationTask<IgfsFileInfo>() { - @Override public IgfsFileInfo onSuccess(Map<IgfsPath, IgfsFileInfo> infos) - throws Exception { + @Override public IgfsFileInfo onSuccess(Map<IgfsPath, IgfsFileInfo> infos) throws Exception { if (infos.get(path) == null) return null; fs.update(path, props); - assert path.parent() == null || infos.get(path.parent()) != null; + IgfsFileInfo parentInfo = infos.get(path.parent()); + + assert path.parent() == null || parentInfo != null; - return updatePropertiesNonTx(infos.get(path.parent()).id(), infos.get(path).id(), path.name(), - props); + return updatePropertiesNonTx(parentInfo == null ? null : parentInfo.id(), + infos.get(path).id(), path.name(), props); } @Override public IgfsFileInfo onFailure(@Nullable Exception err) throws IgniteCheckedException { http://git-wip-us.apache.org/repos/asf/ignite/blob/54bb7d76/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerSelfTest.java index ebf91e0..fd4ec17 100644 --- a/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerSelfTest.java @@ -237,7 +237,7 @@ public class IgfsFragmentizerSelfTest extends IgfsFragmentizerAbstractSelfTest { U.sleep(200); } - igfs.delete(new IgfsPath("/"), true); + igfs.format(); igfs.awaitDeletesAsync().get(); http://git-wip-us.apache.org/repos/asf/ignite/blob/54bb7d76/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java index cfa99ff..7e73859 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java @@ -101,10 +101,10 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest { protected static final long BLOCK_SIZE = 32 * 1024 * 1024; /** Default repeat count. */ - protected static final int REPEAT_CNT = 5; + protected static final int REPEAT_CNT = 5; // Diagnostic: ~100; Regression: 5 /** Concurrent operations count. */ - protected static final int OPS_CNT = 16; + protected static final int OPS_CNT = 16; // Diagnostic: ~160; Regression: 16 /** Seed. */ protected static final long SEED = System.currentTimeMillis(); @@ -252,6 +252,8 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest { /** {@inheritDoc} */ @Override protected void afterTest() throws Exception { clear(igfs, igfsSecondary); + + assert igfs.listFiles(new IgfsPath("/")).isEmpty(); } /** {@inheritDoc} */ @@ -923,6 +925,23 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest { } /** + * Tests that root directory properties persist afetr the #format() operation. + * + * @throws Exception If failed. + */ + public void testRootPropertiesPersistAfterFormat() throws Exception { + igfs.update(new IgfsPath("/"), Collections.singletonMap("foo", "moo")); + + igfs.format(); + + IgfsFile file = igfs.info(new IgfsPath("/")); + + Map<String,String> props = file.properties(); + + assertEquals("moo", props.get("foo")); + } + + /** * Test regular file open. * * @throws Exception If failed. @@ -1600,6 +1619,8 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest { * @throws Exception If failed. */ public void testConcurrentMkdirsDelete() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-1541"); + for (int i = 0; i < REPEAT_CNT; i++) { final CyclicBarrier barrier = new CyclicBarrier(2); @@ -1881,18 +1902,10 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest { public void testDeadlocksRename() throws Exception { for (int i = 0; i < REPEAT_CNT; i++) { try { - info(">>>>>> Start deadlock test."); - checkDeadlocks(5, 2, 2, 2, OPS_CNT, 0, 0, 0, 0); - - info(">>>>>> End deadlock test."); } finally { - info(">>>>>> Start cleanup."); - clear(igfs, igfsSecondary); - - info(">>>>>> End cleanup."); } } } @@ -1903,8 +1916,6 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest { * @throws Exception If failed. */ public void testDeadlocksDelete() throws Exception { - fail("https://issues.apache.org/jira/browse/IGNITE-1515"); - for (int i = 0; i < REPEAT_CNT; i++) { try { checkDeadlocks(5, 2, 2, 2, 0, OPS_CNT, 0, 0, 0); @@ -1948,6 +1959,42 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest { } /** + * Ensure that deadlocks do not occur during concurrent delete & rename operations. + * + * @throws Exception If failed. + */ + public void testDeadlocksDeleteRename() throws Exception { + for (int i = 0; i < REPEAT_CNT; i++) { + try { + checkDeadlocks(5, 2, 2, 2, + OPS_CNT, OPS_CNT, 0, 0, 0); + } + finally { + clear(igfs, igfsSecondary); + } + } + } + + /** + * Ensure that deadlocks do not occur during concurrent delete & rename operations. + * + * @throws Exception If failed. + */ + public void testDeadlocksDeleteMkdirs() throws Exception { + fail("https://issues.apache.org/jira/browse/IGNITE-1541"); + + for (int i = 0; i < REPEAT_CNT; i++) { + try { + checkDeadlocks(5, 2, 2, 2, + 0, OPS_CNT, 0, OPS_CNT, 0); + } + finally { + clear(igfs, igfsSecondary); + } + } + } + + /** * Ensure that deadlocks do not occur during concurrent file creation operations. * * @throws Exception If failed. @@ -1969,11 +2016,16 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest { * @throws Exception If failed. */ public void testDeadlocks() throws Exception { - fail("https://issues.apache.org/jira/browse/IGNITE-1515"); + fail("https://issues.apache.org/jira/browse/IGNITE-1541"); for (int i = 0; i < REPEAT_CNT; i++) { try { - checkDeadlocks(5, 2, 2, 2, OPS_CNT, OPS_CNT, OPS_CNT, OPS_CNT, OPS_CNT); + checkDeadlocks(5, 2, 2, 2, + OPS_CNT, // rename + OPS_CNT, // delete + OPS_CNT, // update + OPS_CNT, // mkdirs + OPS_CNT); // create } finally { clear(igfs, igfsSecondary); http://git-wip-us.apache.org/repos/asf/ignite/blob/54bb7d76/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManagerSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManagerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManagerSelfTest.java index 206c9fe..4072636 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManagerSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManagerSelfTest.java @@ -300,25 +300,19 @@ public class IgfsMetaManagerSelfTest extends IgfsCommonAbstractTest { System.out.println("b: " + mgr.directoryListing(b.id())); System.out.println("f3: " + mgr.directoryListing(f3.id())); - //mgr.move(a.id(), "a", ROOT_ID, "a2", ROOT_ID); mgr.move(path("/a"), path("/a2")); - //mgr.move(b.id(), "b", a.id(), "b2", a.id()); mgr.move(path("/a2/b"), path("/a2/b2")); assertNotNull(mgr.info(b.id())); - //mgr.move(f3.id(), "f3", b.id(), "f3-2", a.id()); mgr.move(path("/a2/b2/f3"), path("/a2/b2/f3-2")); assertNotNull(mgr.info(b.id())); - //mgr.move(f3.id(), "f3-2", a.id(), "f3", b.id()); mgr.move(path("/a2/b2/f3-2"), path("/a2/b2/f3")); - //mgr.move(b.id(), "b2", a.id(), "b", a.id()); mgr.move(path("/a2/b2"), path("/a2/b")); - //mgr.move(a.id(), "a2", ROOT_ID, "a", ROOT_ID); mgr.move(path("/a2"), path("/a")); // Validate 'remove' operation. http://git-wip-us.apache.org/repos/asf/ignite/blob/54bb7d76/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetricsSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetricsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetricsSelfTest.java index 8a2e5bf..fb1d6f7 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetricsSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetricsSelfTest.java @@ -348,7 +348,7 @@ public class IgfsMetricsSelfTest extends IgfsCommonAbstractTest { assertEquals(0, m.filesOpenedForRead()); assertEquals(0, m.filesOpenedForWrite()); - fs.delete(new IgfsPath("/"), true); + fs.format(); m = fs.metrics(); http://git-wip-us.apache.org/repos/asf/ignite/blob/54bb7d76/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorSelfTest.java index cb134f8..9c4d832 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorSelfTest.java @@ -389,15 +389,6 @@ public class IgfsProcessorSelfTest extends IgfsCommonAbstractTest { assert paths.size() == 3 : "Unexpected paths: " + paths; - // Delete. - GridTestUtils.assertThrowsInherited(log, new Callable<Object>() { - @Override public Object call() throws Exception { - igfs.delete(path("/"), false); - - return null; - } - }, IgfsException.class, null); - igfs.delete(path("/A1/B1/C1"), false); assertNull(igfs.info(path("/A1/B1/C1"))); @@ -416,19 +407,19 @@ public class IgfsProcessorSelfTest extends IgfsCommonAbstractTest { assertEquals(Arrays.asList(path("/A"), path("/A1"), path("/A2")), sorted(igfs.listPaths(path("/")))); - GridTestUtils.assertThrowsInherited(log, new Callable<Object>() { - @Override public Object call() throws Exception { - igfs.delete(path("/"), false); - - return null; - } - }, IgfsException.class, null); - assertEquals(Arrays.asList(path("/A"), path("/A1"), path("/A2")), sorted(igfs.listPaths(path("/")))); - + // Delete root when it is not empty: igfs.delete(path("/"), true); + igfs.delete(path("/"), false); + + igfs.delete(path("/A"), true); + igfs.delete(path("/A1"), true); + igfs.delete(path("/A2"), true); assertEquals(Collections.<IgfsPath>emptyList(), igfs.listPaths(path("/"))); + // Delete root when it is empty: igfs.delete(path("/"), false); + igfs.delete(path("/"), true); + assertEquals(Collections.<IgfsPath>emptyList(), igfs.listPaths(path("/"))); for (Cache.Entry<Object, Object> e : metaCache) @@ -608,7 +599,7 @@ public class IgfsProcessorSelfTest extends IgfsCommonAbstractTest { assertEquals(text, read("/b")); // Cleanup. - igfs.delete(root, true); + igfs.format(); assertEquals(Collections.<IgfsPath>emptyList(), igfs.listPaths(root)); } http://git-wip-us.apache.org/repos/asf/ignite/blob/54bb7d76/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/UniversalFileSystemAdapter.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/UniversalFileSystemAdapter.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/UniversalFileSystemAdapter.java index 3bf70e2..ba8c164 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/UniversalFileSystemAdapter.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/UniversalFileSystemAdapter.java @@ -28,7 +28,6 @@ import java.util.Map; * To be used solely in tests. */ public interface UniversalFileSystemAdapter { - /** * Gets name of the FS. * @return name of this file system. http://git-wip-us.apache.org/repos/asf/ignite/blob/54bb7d76/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsUtils.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsUtils.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsUtils.java index 3913cbd..fa5cbc5 100644 --- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsUtils.java +++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsUtils.java @@ -20,7 +20,11 @@ package org.apache.ignite.internal.processors.hadoop.igfs; import java.io.FileNotFoundException; import java.io.IOException; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.AbstractFileSystem; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.ParentNotDirectoryException; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathExistsException; import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException; import org.apache.ignite.IgniteCheckedException; @@ -130,6 +134,38 @@ public class HadoopIgfsUtils { } /** + * Deletes all files from the given file system. + * + * @param fs The file system to clean up. + * @throws IOException On error. + */ + public static void clear(FileSystem fs) throws IOException { + // Delete root contents: + FileStatus[] statuses = fs.listStatus(new Path("/")); + + if (statuses != null) { + for (FileStatus stat: statuses) + fs.delete(stat.getPath(), true); + } + } + + /** + * Deletes all files from the given file system. + * + * @param fs The file system to clean up. + * @throws IOException On error. + */ + public static void clear(AbstractFileSystem fs) throws IOException { + // Delete root contents: + FileStatus[] statuses = fs.listStatus(new Path("/")); + + if (statuses != null) { + for (FileStatus stat: statuses) + fs.delete(stat.getPath(), true); + } + } + + /** * Constructor. */ private HadoopIgfsUtils() { http://git-wip-us.apache.org/repos/asf/ignite/blob/54bb7d76/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopFileSystemUniversalFileSystemAdapter.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopFileSystemUniversalFileSystemAdapter.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopFileSystemUniversalFileSystemAdapter.java index 03f0066..608bd25 100644 --- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopFileSystemUniversalFileSystemAdapter.java +++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopFileSystemUniversalFileSystemAdapter.java @@ -26,6 +26,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsUtils; import org.apache.ignite.internal.processors.igfs.IgfsEx; import org.apache.ignite.internal.processors.igfs.UniversalFileSystemAdapter; @@ -33,7 +34,6 @@ import org.apache.ignite.internal.processors.igfs.UniversalFileSystemAdapter; * Universal adapter wrapping {@link org.apache.hadoop.fs.FileSystem} instance. */ public class HadoopFileSystemUniversalFileSystemAdapter implements UniversalFileSystemAdapter { - /** The wrapped filesystem. */ private final FileSystem fileSys; @@ -70,7 +70,7 @@ public class HadoopFileSystemUniversalFileSystemAdapter implements UniversalFile /** {@inheritDoc} */ @Override public void format() throws IOException { - fileSys.delete(new Path("/"), true); + HadoopIgfsUtils.clear(fileSys); } /** {@inheritDoc} */ http://git-wip-us.apache.org/repos/asf/ignite/blob/54bb7d76/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfs20FileSystemAbstractSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfs20FileSystemAbstractSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfs20FileSystemAbstractSelfTest.java index 1235786..c938571 100644 --- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfs20FileSystemAbstractSelfTest.java +++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfs20FileSystemAbstractSelfTest.java @@ -59,6 +59,7 @@ import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.FileSystemConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.hadoop.fs.IgniteHadoopIgfsSecondaryFileSystem; +import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsUtils; import org.apache.ignite.internal.processors.igfs.IgfsCommonAbstractTest; import org.apache.ignite.internal.util.GridConcurrentHashSet; import org.apache.ignite.internal.util.typedef.F; @@ -352,7 +353,7 @@ public abstract class HadoopIgfs20FileSystemAbstractSelfTest extends IgfsCommonA /** {@inheritDoc} */ @Override protected void afterTest() throws Exception { try { - fs.delete(new Path("/"), true); + HadoopIgfsUtils.clear(fs); } catch (Exception ignore) { // No-op. @@ -627,7 +628,9 @@ public abstract class HadoopIgfs20FileSystemAbstractSelfTest extends IgfsCommonA Path root = new Path(fsHome, "/"); - assertTrue(fs.delete(root, true)); + assertFalse(fs.delete(root, true)); + + assertTrue(fs.delete(new Path(fsHome, "/someDir1"), true)); assertPathDoesNotExist(fs, someDir3); assertPathDoesNotExist(fs, new Path(fsHome, "/someDir1/someDir2")); http://git-wip-us.apache.org/repos/asf/ignite/blob/54bb7d76/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java index 5ea841e..2626ebb 100644 --- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java +++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java @@ -300,7 +300,7 @@ public abstract class IgniteHadoopFileSystemAbstractSelfTest extends IgfsCommonA /** {@inheritDoc} */ @Override protected void afterTest() throws Exception { try { - fs.delete(new Path("/"), true); + HadoopIgfsUtils.clear(fs); } catch (Exception ignore) { // No-op. @@ -783,7 +783,8 @@ public abstract class IgniteHadoopFileSystemAbstractSelfTest extends IgfsCommonA Path root = new Path(fsHome, "/"); - assertTrue(fs.delete(root, true)); + assertFalse(fs.delete(root, true)); + assertTrue(fs.delete(new Path("/someDir1"), true)); assertPathDoesNotExist(fs, someDir3); assertPathDoesNotExist(fs, new Path(fsHome, "/someDir1/someDir2"));