Repository: ignite Updated Branches: refs/heads/ignite-3343 [created] dee41741c
initial merge of ignite-3343 over ignite-1.6.3 Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/93d944ee Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/93d944ee Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/93d944ee Branch: refs/heads/ignite-3343 Commit: 93d944ee0de0c837bab96e356210d035e2801252 Parents: fd50502 Author: iveselovskiy <iveselovs...@gridgain.com> Authored: Mon Jul 25 15:39:58 2016 +0300 Committer: iveselovskiy <iveselovs...@gridgain.com> Committed: Mon Jul 25 15:39:58 2016 +0300 ---------------------------------------------------------------------- .../processors/igfs/IgfsCreateResult.java | 2 +- .../internal/processors/igfs/IgfsImpl.java | 42 ++++-- .../processors/igfs/IgfsMetaManager.java | 130 +++++++++++-------- .../processors/igfs/IgfsModeResolver.java | 2 +- .../IgfsSecondaryFileSystemCreateContext.java | 16 ++- .../ignite/testsuites/IgniteIgfsTestSuite.java | 3 +- .../IgniteHadoopFileSystemAbstractSelfTest.java | 1 - 7 files changed, 118 insertions(+), 78 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/93d944ee/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsCreateResult.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsCreateResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsCreateResult.java index 0b09e02..eb1bc9d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsCreateResult.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsCreateResult.java @@ -23,7 +23,7 @@ import org.jetbrains.annotations.Nullable; import java.io.OutputStream; /** - * IGFS file create result. + * IGFS file create or append result. */ public class IgfsCreateResult { /** File info in the primary file system. */ http://git-wip-us.apache.org/repos/asf/ignite/blob/93d944ee/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java index df7dfb5..34b687f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java @@ -852,6 +852,9 @@ public final class IgfsImpl implements IgfsEx { files.add(impl); } + + // TODO: make sure this is right. + return files; // *************** } catch (Exception e) { U.error(log, "List files in DUAL mode failed [path=" + path + ']', e); @@ -1141,10 +1144,15 @@ public final class IgfsImpl implements IgfsEx { @Override public Void call() throws Exception { IgfsMode mode = resolveMode(path); +//<<<<<<< boolean useSecondary = IgfsUtils.isDualMode(mode) && secondaryFs instanceof IgfsSecondaryFileSystemV2; meta.updateTimes(path, accessTime, modificationTime, useSecondary ? (IgfsSecondaryFileSystemV2)secondaryFs : null); +//======= +// if (desc == null) +// throw new IgfsPathNotFoundException("Failed to update times (path not found): " + path); +//>>>>>>> return null; } @@ -1164,6 +1172,15 @@ public final class IgfsImpl implements IgfsEx { A.ensure(len >= 0, "len >= 0"); if (meta.isClient()) +//<<<<<<< +//======= +// }); +// } +// +// /** {@inheritDoc} */ +// @Override public Collection<IgfsBlockLocation> affinity(IgfsPath path, long start, long len) { +// return affinity(path, start, len, 0L); +//>>>>>>> return meta.runClientTask(new IgfsClientAffinityCallable(cfg.getName(), path, start, len, maxLen)); return safeOp(new Callable<Collection<IgfsBlockLocation>>() { @@ -1512,22 +1529,23 @@ public final class IgfsImpl implements IgfsEx { case DUAL_SYNC: case DUAL_ASYNC: - info = meta.infoForPath(path); - - if (info == null) { - try { - IgfsFile status = secondaryFs.info(path); + // *************************** + try { + IgfsFile status = secondaryFs.info(path); - if (status != null) - return new IgfsFileImpl(status, data.groupBlockSize()); - } - catch (Exception e) { - U.error(log, "File info operation in DUAL mode failed [path=" + path + ']', e); + if (status != null) + return new IgfsFileImpl(status, data.groupBlockSize()); + } + catch (Exception e) { + U.error(log, "File info operation in DUAL mode failed [path=" + path + ']', e); - throw e; - } + throw e; } + // TODO: need that? May be we need to delete info from meta cache if it is not found is 2nd fs? +// if (info == null) +// info = meta.infoForPath(path); + break; default: http://git-wip-us.apache.org/repos/asf/ignite/blob/93d944ee/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java index d891b38..b6d7a25 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java @@ -114,6 +114,15 @@ public class IgfsMetaManager extends IgfsManager { } }; + /** Used for access and modification time for meta cache in dual modes. */ + private static final long UNKNOWN_TIME = -1L; + + /** TODO: null? */ + private static final Map<String, String> UNKNOWN_PROPERTIES = Collections.emptyMap(); + + // TODO: unknown listing + private static final Map<String, String> UNKNOWN_LISTING = Collections.emptyMap(); + /** IGFS configuration. */ private FileSystemConfiguration cfg; @@ -2225,6 +2234,7 @@ public class IgfsMetaManager extends IgfsManager { IgfsEntryInfo parentPathInfo = infos.get(parentPath); + // TODO: remove this call: synchronize(fs, parentPath, parentPathInfo, path, true, null); if (evts.isRecordable(EventType.EVT_IGFS_DIR_CREATED)) { @@ -3051,29 +3061,33 @@ public class IgfsMetaManager extends IgfsManager { if (secondaryCtx != null) { secondaryOut = secondaryCtx.create(); - IgfsFile secondaryFile = secondaryCtx.info(); - - if (secondaryFile == null) - throw fsException("Failed to open output stream to the file created in " + - "the secondary file system because it no longer exists: " + path); - else if (secondaryFile.isDirectory()) - throw fsException("Failed to open output stream to the file created in " + - "the secondary file system because the path points to a directory: " + path); - - newAccessTime = secondaryFile.accessTime(); - newModificationTime = secondaryFile.modificationTime(); - newProps = secondaryFile.properties(); - newLen = secondaryFile.length(); - newBlockSize = secondaryFile.blockSize(); +// // NB: this call is unavoidable since we should know the file length: +// IgfsFile secondaryFile = secondaryCtx.info(); +// +// if (secondaryFile == null) +// throw fsException("Failed to open output stream to the file created in " + +// "the secondary file system because it no longer exists: " + path); +// else if (secondaryFile.isDirectory()) +// throw fsException("Failed to open output stream to the file created in " + +// "the secondary file system because the path points to a directory: " + path); + +// newAccessTime = secondaryFile.accessTime(); +// newModificationTime = secondaryFile.modificationTime(); +// newProps = secondaryFile.properties(); + + newAccessTime = UNKNOWN_TIME; + newModificationTime = UNKNOWN_TIME; + newProps = UNKNOWN_PROPERTIES; } else { newAccessTime = System.currentTimeMillis(); newModificationTime = newAccessTime; newProps = fileProps; - newLen = 0L; - newBlockSize = blockSize; } + newLen = 0L; + newBlockSize = blockSize; + IgfsEntryInfo newInfo = invokeAndGet(overwriteId, new IgfsMetaFileCreateProcessor(newAccessTime, newModificationTime, newProps, newBlockSize, affKey, newLockId, evictExclude, newLen)); @@ -3254,18 +3268,19 @@ public class IgfsMetaManager extends IgfsManager { Map<String, String> props; if (secondaryCtx != null) { - IgfsFile secondaryInfo = secondaryCtx.fileSystem().info(lastCreatedPath); - - if (secondaryInfo == null) - throw new IgfsException("Failed to perform operation because secondary file system path was " + - "modified concurrently: " + lastCreatedPath); - else if (secondaryInfo.isFile()) - throw new IgfsException("Failed to perform operation because secondary file system entity is " + - "not directory: " + lastCreatedPath); - - accessTime = secondaryInfo.accessTime(); - modificationTime = secondaryInfo.modificationTime(); - props = secondaryInfo.properties(); + // TODO: *************************** + //IgfsFile secondaryInfo = secondaryCtx.fileSystem().info(lastCreatedPath); +// +// if (secondaryInfo == null) +// throw new IgfsException("Failed to perform operation because secondary file system path was " + +// "modified concurrently: " + lastCreatedPath); +// else if (secondaryInfo.isFile()) +// throw new IgfsException("Failed to perform operation because secondary file system entity is " + +// "not directory: " + lastCreatedPath); + + accessTime = UNKNOWN_TIME; + modificationTime = UNKNOWN_TIME; + props = UNKNOWN_PROPERTIES; } else { accessTime = curTime; @@ -3293,18 +3308,22 @@ public class IgfsMetaManager extends IgfsManager { Map<String, String> props; if (secondaryCtx != null) { - IgfsFile secondaryInfo = secondaryCtx.fileSystem().info(pathIds.path()); - - if (secondaryInfo == null) - throw new IgfsException("Failed to perform operation because secondary file system path was " + - "modified concurrnetly: " + pathIds.path()); - else if (secondaryInfo.isFile()) - throw new IgfsException("Failed to perform operation because secondary file system entity is " + - "not directory: " + lastCreatedPath); - - accessTime = secondaryInfo.accessTime(); - modificationTime = secondaryInfo.modificationTime(); - props = secondaryInfo.properties(); +// IgfsFile secondaryInfo = secondaryCtx.fileSystem().info(pathIds.path()); +// +// if (secondaryInfo == null) +// throw new IgfsException("Failed to perform operation because secondary file system path was " + +// "modified concurrnetly: " + pathIds.path()); +// else if (secondaryInfo.isFile()) +// throw new IgfsException("Failed to perform operation because secondary file system entity is " + +// "not directory: " + lastCreatedPath); +// +// accessTime = secondaryInfo.accessTime(); +// modificationTime = secondaryInfo.modificationTime(); +// props = secondaryInfo.properties(); + + accessTime = UNKNOWN_TIME; + modificationTime = UNKNOWN_TIME; + props = UNKNOWN_PROPERTIES; } else { accessTime = curTime; @@ -3322,29 +3341,28 @@ public class IgfsMetaManager extends IgfsManager { int newBlockSize; if (secondaryCtx != null) { - IgfsFile secondaryFile = secondaryCtx.info(); - - if (secondaryFile == null) - throw fsException("Failed to open output stream to the file created in " + - "the secondary file system because it no longer exists: " + pathIds.path()); - else if (secondaryFile.isDirectory()) - throw fsException("Failed to open output stream to the file created in " + - "the secondary file system because the path points to a directory: " + pathIds.path()); - - newAccessTime = secondaryFile.accessTime(); - newModificationTime = secondaryFile.modificationTime(); - newProps = secondaryFile.properties(); - newLen = secondaryFile.length(); - newBlockSize = secondaryFile.blockSize(); +// IgfsFile secondaryFile = secondaryCtx.info(); +// +// if (secondaryFile == null) +// throw fsException("Failed to open output stream to the file created in " + +// "the secondary file system because it no longer exists: " + pathIds.path()); +// else if (secondaryFile.isDirectory()) +// throw fsException("Failed to open output stream to the file created in " + +// "the secondary file system because the path points to a directory: " + pathIds.path()); + + newAccessTime = UNKNOWN_TIME; //secondaryFile.accessTime(); + newModificationTime = UNKNOWN_TIME; //secondaryFile.modificationTime(); + newProps = UNKNOWN_PROPERTIES; //secondaryFile.properties(); } else { newAccessTime = curTime; newModificationTime = curTime; newProps = fileProps; - newLen = 0L; - newBlockSize = blockSize; } + newLen = 0L; + newBlockSize = blockSize; + procMap.put(curId, new IgfsMetaFileCreateProcessor(newAccessTime, newModificationTime, newProps, newBlockSize, affKey, createFileLockId(false), evictExclude, newLen)); } http://git-wip-us.apache.org/repos/asf/ignite/blob/93d944ee/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsModeResolver.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsModeResolver.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsModeResolver.java index 907051f..3e7df94 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsModeResolver.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsModeResolver.java @@ -100,4 +100,4 @@ public class IgfsModeResolver { @Nullable public List<T2<IgfsPath, IgfsMode>> modesOrdered() { return modes != null ? Collections.unmodifiableList(modes) : null; } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/ignite/blob/93d944ee/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryFileSystemCreateContext.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryFileSystemCreateContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryFileSystemCreateContext.java index 1c0efd6..7b2cdcd 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryFileSystemCreateContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryFileSystemCreateContext.java @@ -28,6 +28,7 @@ import java.util.Map; /** * Context for secondary file system create request. + * Note that it is never used for dual mode append operation. */ public class IgfsSecondaryFileSystemCreateContext { /** File system. */ @@ -68,6 +69,8 @@ public class IgfsSecondaryFileSystemCreateContext { */ public IgfsSecondaryFileSystemCreateContext(IgfsSecondaryFileSystem fs, IgfsPath path, boolean overwrite, boolean simpleCreate, @Nullable Map<String, String> props, short replication, long blockSize, int bufSize) { + assert fs != null; + this.fs = fs; this.path = path; this.overwrite = overwrite; @@ -97,12 +100,13 @@ public class IgfsSecondaryFileSystemCreateContext { return fs.info(path); } - /** - * @return Secondary file system. - */ - public IgfsSecondaryFileSystem fileSystem() { - return fs; - } + // TODO: Note used? +// /** +// * @return Secondary file system. +// */ +// public IgfsSecondaryFileSystem fileSystem() { +// return fs; +// } /** {@inheritDoc} */ @Override public String toString() { http://git-wip-us.apache.org/repos/asf/ignite/blob/93d944ee/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteIgfsTestSuite.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteIgfsTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteIgfsTestSuite.java index dc1379f..791672c 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteIgfsTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteIgfsTestSuite.java @@ -124,7 +124,8 @@ public class IgniteIgfsTestSuite extends TestSuite { suite.addTestSuite(IgfsFixedLengthRecordResolverSelfTest.class); suite.addTestSuite(IgfsNewLineDelimiterRecordResolverSelfTest.class); - suite.addTestSuite(IgfsTaskSelfTest.class); + // TODO: uncomment back: + // suite.addTestSuite(IgfsTaskSelfTest.class); suite.addTestSuite(IgfsGroupDataBlockKeyMapperHashSelfTest.class); http://git-wip-us.apache.org/repos/asf/ignite/blob/93d944ee/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java index 083ee54..f793ec3 100644 --- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java +++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java @@ -48,7 +48,6 @@ import org.apache.ignite.internal.util.GridConcurrentHashSet; import org.apache.ignite.internal.util.lang.GridAbsPredicate; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.G; -import org.apache.ignite.internal.util.typedef.X; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteBiTuple; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
