Return-Path: X-Original-To: apmail-ignite-commits-archive@minotaur.apache.org Delivered-To: apmail-ignite-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 8C1EF19066 for ; Tue, 22 Mar 2016 16:49:38 +0000 (UTC) Received: (qmail 60223 invoked by uid 500); 22 Mar 2016 16:49:38 -0000 Delivered-To: apmail-ignite-commits-archive@ignite.apache.org Received: (qmail 60008 invoked by uid 500); 22 Mar 2016 16:49:38 -0000 Mailing-List: contact commits-help@ignite.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@ignite.apache.org Delivered-To: mailing list commits@ignite.apache.org Received: (qmail 58925 invoked by uid 99); 22 Mar 2016 16:49:37 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 22 Mar 2016 16:49:37 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 8D047E03C7; Tue, 22 Mar 2016 16:49:37 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: ntikhonov@apache.org To: commits@ignite.apache.org Date: Tue, 22 Mar 2016 16:49:58 -0000 Message-Id: <659190d0df954ff9b635f3be6c06bd80@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [22/50] [abbrv] ignite git commit: IGNITE-2842: IGFS: Optimized create/mkdirs operations with help of entry processors. IGNITE-2842: IGFS: Optimized create/mkdirs operations with help of entry processors. Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/ad6bbd9a Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/ad6bbd9a Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/ad6bbd9a Branch: refs/heads/ignite-2004 Commit: ad6bbd9ad55a5cc1fdf526b5d511465628a8dca5 Parents: 297cbcb Author: vozerov-gridgain Authored: Wed Mar 16 12:31:37 2016 +0300 Committer: vozerov-gridgain Committed: Wed Mar 16 12:31:37 2016 +0300 ---------------------------------------------------------------------- .../internal/processors/igfs/IgfsFileInfo.java | 13 + .../processors/igfs/IgfsListingEntry.java | 15 +- .../processors/igfs/IgfsMetaManager.java | 262 ++++++++++++++++--- 3 files changed, 251 insertions(+), 39 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/ad6bbd9a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileInfo.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileInfo.java index ba484bb..13c54ff 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileInfo.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileInfo.java @@ -345,6 +345,19 @@ public final class IgfsFileInfo implements Externalizable { } /** + * Temporal hack to change ID before saving entry to cache. Currently we have too much constructors and adding + * more will make things even worse. Instead, we use this method until directories and files are split into + * separate entities. + * + * @param id ID. + * @deprecated Use only on not-yet-saved entries. + */ + @Deprecated + public void id(IgniteUuid id) { + this.id = id; + } + + /** * @return {@code True} if this is a file. */ public boolean isFile() { http://git-wip-us.apache.org/repos/asf/ignite/blob/ad6bbd9a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsListingEntry.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsListingEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsListingEntry.java index 61d9265..4fe0dca 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsListingEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsListingEntry.java @@ -34,7 +34,7 @@ public class IgfsListingEntry implements Externalizable { /** */ private static final long serialVersionUID = 0L; - /** File id. */ + /** ID. */ private IgniteUuid id; /** Directory marker. */ @@ -48,6 +48,8 @@ public class IgfsListingEntry implements Externalizable { } /** + * Constructor. + * * @param fileInfo File info to construct listing entry from. */ public IgfsListingEntry(IgfsFileInfo fileInfo) { @@ -56,6 +58,17 @@ public class IgfsListingEntry implements Externalizable { } /** + * Constructor. + * + * @param id File ID. + * @param dir Directory marker. + */ + public IgfsListingEntry(IgniteUuid id, boolean dir) { + this.id = id; + this.dir = dir; + } + + /** * @return Entry file ID. */ public IgniteUuid fileId() { http://git-wip-us.apache.org/repos/asf/ignite/blob/ad6bbd9a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java index df69d49..2a85cf8 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java @@ -1868,13 +1868,11 @@ public class IgfsMetaManager extends IgfsManager { * @return New file info. * @throws IgniteCheckedException If failed. */ - private IgfsFileInfo invokeAndGet(IgniteUuid id, EntryProcessor proc) + private IgfsFileInfo invokeAndGet(IgniteUuid id, EntryProcessor proc) throws IgniteCheckedException { validTxState(true); - id2InfoPrj.invoke(id, proc); - - return getInfo(id); + return id2InfoPrj.invoke(id, proc).get(); } /** @@ -3493,7 +3491,7 @@ public class IgfsMetaManager extends IgfsManager { private final SortedSet idSet = new TreeSet(PATH_ID_SORTING_COMPARATOR); /** The middle node properties. */ - private final Map middleProps; + private final Map props; /** The leaf node properties. */ private final Map leafProps; @@ -3557,7 +3555,7 @@ public class IgfsMetaManager extends IgfsManager { * Constructor. * * @param path Path. - * @param middleProps Middle properties. + * @param props Middle properties. * @param leafProps Leaf properties. * @param leafDir Whether leaf is directory or file. * @param blockSize Block size. @@ -3565,13 +3563,13 @@ public class IgfsMetaManager extends IgfsManager { * @param evictExclude Evict exclude flag. * @throws IgniteCheckedException If failed. */ - private DirectoryChainBuilder(IgfsPath path, Map middleProps, Map leafProps, + private DirectoryChainBuilder(IgfsPath path, Map props, Map leafProps, boolean leafDir, int blockSize, @Nullable IgniteUuid affKey, boolean evictExclude) throws IgniteCheckedException { this.path = path; this.components = path.components(); this.idList = fileIds(path); - this.middleProps = middleProps; + this.props = props; this.leafProps = leafProps; this.leafDir = leafDir; this.blockSize = blockSize; @@ -3617,44 +3615,46 @@ public class IgfsMetaManager extends IgfsManager { * Does the main portion of job building the renmaining path. */ public final void doBuild() throws IgniteCheckedException { - IgfsFileInfo childInfo = null; + // Fix current time. It will be used in all created entities. + long createTime = System.currentTimeMillis(); + IgfsListingEntry childInfo = null; String childName = null; - IgfsFileInfo newInfo; - IgniteUuid parentId = null; // This loop creates the missing directory chain from the bottom to the top: for (int i = components.size() - 1; i >= existingIdCnt - 1; i--) { - // Required entry does not exist. - // Create new directory info: + IgniteUuid childId = IgniteUuid.randomUuid(); + boolean childDir; + if (childName == null) { assert childInfo == null; - long t = System.currentTimeMillis(); + if (leafDir) { + childDir = true; - if (leafDir) - newInfo = new IgfsFileInfo(true, leafProps, t, t); - else - newInfo = new IgfsFileInfo(blockSize, 0L, affKey, createFileLockId(false), evictExclude, - leafProps, t, t); + leafInfo = invokeAndGet(childId, new DirectoryCreateProcessor(createTime, leafProps)); + } + else { + childDir = false; - leafInfo = newInfo; + leafInfo = invokeAndGet(childId, new FileCreateProcessor(createTime, leafProps, blockSize, + affKey, createFileLockId(false), evictExclude)); + } } else { assert childInfo != null; - newInfo = new IgfsFileInfo(Collections.singletonMap(childName, - new IgfsListingEntry(childInfo)), middleProps); + childDir = true; + + id2InfoPrj.invoke(childId, new DirectoryCreateProcessor(createTime, props, childName, childInfo)); if (parentId == null) - parentId = newInfo.id(); + parentId = childId; } - id2InfoPrj.put(newInfo.id(), newInfo); - - childInfo = newInfo; + childInfo = new IgfsListingEntry(childId, childDir); childName = components.get(i); } @@ -3665,7 +3665,7 @@ public class IgfsMetaManager extends IgfsManager { leafParentId = parentId; // Now link the newly created directory chain to the lowermost existing parent: - id2InfoPrj.invoke(lowermostExistingId, new ListingAdd(childName, new IgfsListingEntry(childInfo))); + id2InfoPrj.invoke(lowermostExistingId, new ListingAdd(childName, childInfo)); } /** @@ -3692,9 +3692,193 @@ public class IgfsMetaManager extends IgfsManager { } /** + * File create processor. + */ + private static class FileCreateProcessor implements EntryProcessor, + Externalizable { + /** */ + private static final long serialVersionUID = 0L; + + /** Create time. */ + private long createTime; + + /** Properties. */ + private Map props; + + /** Block size. */ + private int blockSize; + + /** Affintiy key. */ + private IgniteUuid affKey; + + /** Lcok ID. */ + private IgniteUuid lockId; + + /** Evict exclude flag. */ + private boolean evictExclude; + + /** + * Constructor. + */ + public FileCreateProcessor() { + // No-op. + } + + /** + * Constructor. + * + * @param createTime Create time. + * @param props Properties. + * @param blockSize Block size. + * @param affKey Affinity key. + * @param lockId Lock ID. + * @param evictExclude Evict exclude flag. + */ + public FileCreateProcessor(long createTime, Map props, int blockSize, + @Nullable IgniteUuid affKey, IgniteUuid lockId, boolean evictExclude) { + this.createTime = createTime; + this.props = props; + this.blockSize = blockSize; + this.affKey = affKey; + this.lockId = lockId; + this.evictExclude = evictExclude; + } + + /** {@inheritDoc} */ + @Override public IgfsFileInfo process(MutableEntry entry, Object... args) + throws EntryProcessorException { + IgfsFileInfo info = new IgfsFileInfo(blockSize, 0L, affKey, lockId, evictExclude, props, + createTime, createTime); + + info.id(entry.getKey()); + + entry.setValue(info); + + return info; + } + + /** {@inheritDoc} */ + @Override public void writeExternal(ObjectOutput out) throws IOException { + out.writeLong(createTime); + U.writeStringMap(out, props); + out.writeInt(blockSize); + out.writeObject(affKey); + out.writeObject(lockId); + out.writeBoolean(evictExclude); + } + + /** {@inheritDoc} */ + @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { + createTime = in.readLong(); + props = U.readStringMap(in); + blockSize = in.readInt(); + affKey = (IgniteUuid)in.readObject(); + lockId = (IgniteUuid)in.readObject(); + evictExclude = in.readBoolean(); + } + } + + /** + * Directory create processor. + */ + private static class DirectoryCreateProcessor implements EntryProcessor, + Externalizable { + /** */ + private static final long serialVersionUID = 0L; + + /** Create time. */ + private long createTime; + + /** Properties. */ + private Map props; + + /** Child name (optional). */ + private String childName; + + /** Child entry (optional. */ + private IgfsListingEntry childEntry; + + /** + * Constructor. + */ + public DirectoryCreateProcessor() { + // No-op. + } + + /** + * Constructor. + * + * @param createTime Create time. + * @param props Properties. + */ + public DirectoryCreateProcessor(long createTime, Map props) { + this(createTime, props, null, null); + } + + /** + * Constructor. + * + * @param createTime Create time. + * @param props Properties. + * @param childName Child name. + * @param childEntry Child entry. + */ + public DirectoryCreateProcessor(long createTime, Map props, String childName, + IgfsListingEntry childEntry) { + this.createTime = createTime; + this.props = props; + this.childName = childName; + this.childEntry = childEntry; + } + + /** {@inheritDoc} */ + @Override public IgfsFileInfo process(MutableEntry entry, Object... args) + throws EntryProcessorException { + + IgfsFileInfo info = new IgfsFileInfo(true, props, createTime, createTime); + + if (childName != null) + info = new IgfsFileInfo(Collections.singletonMap(childName, childEntry), info); + + info.id(entry.getKey()); + + entry.setValue(info); + + return info; + } + + /** {@inheritDoc} */ + @Override public void writeExternal(ObjectOutput out) throws IOException { + out.writeLong(createTime); + U.writeStringMap(out, props); + + if (childName != null) { + out.writeBoolean(true); + + U.writeString(out, childName); + out.writeObject(childEntry); + } + else + out.writeBoolean(false); + } + + /** {@inheritDoc} */ + @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { + createTime = in.readLong(); + props = U.readStringMap(in); + + if (in.readBoolean()) { + childName = U.readString(in); + childEntry = (IgfsListingEntry)in.readObject(); + } + } + } + + /** * File lock entry processor. */ - private static class FileLockProcessor implements EntryProcessor, Externalizable { + private static class FileLockProcessor implements EntryProcessor, + Externalizable { /** */ private static final long serialVersionUID = 0L; @@ -3718,13 +3902,15 @@ public class IgfsMetaManager extends IgfsManager { } /** {@inheritDoc} */ - @Override public Void process(MutableEntry entry, Object... args) + @Override public IgfsFileInfo process(MutableEntry entry, Object... args) throws EntryProcessorException { - IgfsFileInfo old = entry.getValue(); + IgfsFileInfo oldInfo = entry.getValue(); - entry.setValue(new IgfsFileInfo(old, lockId, old.modificationTime())); + IgfsFileInfo newInfo = new IgfsFileInfo(oldInfo, lockId, oldInfo.modificationTime()); - return null; + entry.setValue(newInfo); + + return newInfo; } /** {@inheritDoc} */ @@ -3789,7 +3975,7 @@ public class IgfsMetaManager extends IgfsManager { /** * File reserve space entry processor. */ - private static class FileReserveSpaceProcessor implements EntryProcessor, + private static class FileReserveSpaceProcessor implements EntryProcessor, Externalizable { /** */ private static final long serialVersionUID = 0L; @@ -3819,7 +4005,7 @@ public class IgfsMetaManager extends IgfsManager { } /** {@inheritDoc} */ - @Override public Void process(MutableEntry entry, Object... args) + @Override public IgfsFileInfo process(MutableEntry entry, Object... args) throws EntryProcessorException { IgfsFileInfo oldInfo = entry.getValue(); @@ -3833,7 +4019,7 @@ public class IgfsMetaManager extends IgfsManager { entry.setValue(newInfo); - return null; + return newInfo; } /** {@inheritDoc} */ @@ -3852,7 +4038,7 @@ public class IgfsMetaManager extends IgfsManager { /** * Update properties processor. */ - private static class UpdatePropertiesProcessor implements EntryProcessor, + private static class UpdatePropertiesProcessor implements EntryProcessor, Externalizable { /** */ private static final long serialVersionUID = 0L; @@ -3877,7 +4063,7 @@ public class IgfsMetaManager extends IgfsManager { } /** {@inheritDoc} */ - @Override public Void process(MutableEntry entry, Object... args) + @Override public IgfsFileInfo process(MutableEntry entry, Object... args) throws EntryProcessorException { IgfsFileInfo oldInfo = entry.getValue(); @@ -3898,7 +4084,7 @@ public class IgfsMetaManager extends IgfsManager { entry.setValue(newInfo); - return null; + return newInfo; } /** {@inheritDoc} */