Return-Path: X-Original-To: apmail-ignite-commits-archive@minotaur.apache.org Delivered-To: apmail-ignite-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id B16A61006B for ; Thu, 5 Mar 2015 07:31:11 +0000 (UTC) Received: (qmail 48969 invoked by uid 500); 5 Mar 2015 07:31:11 -0000 Delivered-To: apmail-ignite-commits-archive@ignite.apache.org Received: (qmail 48929 invoked by uid 500); 5 Mar 2015 07:31:11 -0000 Mailing-List: contact commits-help@ignite.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@ignite.incubator.apache.org Delivered-To: mailing list commits@ignite.incubator.apache.org Received: (qmail 48872 invoked by uid 99); 5 Mar 2015 07:31:11 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 05 Mar 2015 07:31:11 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED,T_RP_MATCHES_RCVD X-Spam-Check-By: apache.org Received: from [140.211.11.3] (HELO mail.apache.org) (140.211.11.3) by apache.org (qpsmtpd/0.29) with SMTP; Thu, 05 Mar 2015 07:30:37 +0000 Received: (qmail 45100 invoked by uid 99); 5 Mar 2015 07:30:11 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 05 Mar 2015 07:30:11 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 315A0E107D; Thu, 5 Mar 2015 07:30:11 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: vozerov@apache.org To: commits@ignite.incubator.apache.org Date: Thu, 05 Mar 2015 07:30:52 -0000 Message-Id: In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [43/45] incubator-ignite git commit: IGNITE-386: Squashed changes. X-Virus-Checked: Checked by ClamAV on apache.org http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/6423cf02/modules/core/src/main/java/org/apache/ignite/igfs/IgfsReader.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/IgfsReader.java b/modules/core/src/main/java/org/apache/ignite/igfs/IgfsReader.java deleted file mode 100644 index fff8ca3..0000000 --- a/modules/core/src/main/java/org/apache/ignite/igfs/IgfsReader.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.igfs; - -import java.io.*; - -/** - * The simplest data input interface to read from secondary file system in dual modes. - */ -public interface IgfsReader extends Closeable { - /** - * Read up to the specified number of bytes, from a given position within a file, and return the number of bytes - * read. - * - * @param pos Position in the input stream to seek. - * @param buf Buffer into which data is read. - * @param off Offset in the buffer from which stream data should be written. - * @param len The number of bytes to read. - * @return Total number of bytes read into the buffer, or -1 if there is no more data (EOF). - * @throws IOException In case of any exception. - */ - public int read(long pos, byte[] buf, int off, int len) throws IOException; -} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/6423cf02/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsInputStreamJobAdapter.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsInputStreamJobAdapter.java b/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsInputStreamJobAdapter.java index bf2636f..d7fd866 100644 --- a/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsInputStreamJobAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsInputStreamJobAdapter.java @@ -32,7 +32,7 @@ import java.io.*; */ public abstract class IgfsInputStreamJobAdapter extends IgfsJobAdapter { /** {@inheritDoc} */ - @Override public final Object execute(IgniteFs igfs, IgfsFileRange range, IgfsInputStream in) + @Override public final Object execute(IgniteFileSystem igfs, IgfsFileRange range, IgfsInputStream in) throws IgniteException, IOException { in.seek(range.start()); @@ -48,5 +48,5 @@ public abstract class IgfsInputStreamJobAdapter extends IgfsJobAdapter { * @throws IgniteException If execution failed. * @throws IOException If IO exception encountered while working with stream. */ - public abstract Object execute(IgniteFs igfs, IgfsRangeInputStream in) throws IgniteException, IOException; + public abstract Object execute(IgniteFileSystem igfs, IgfsRangeInputStream in) throws IgniteException, IOException; } http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/6423cf02/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsJob.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsJob.java b/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsJob.java index d12fc79..0fdc0f2 100644 --- a/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsJob.java +++ b/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsJob.java @@ -26,7 +26,7 @@ import java.io.*; * Defines executable unit for {@link IgfsTask}. Before this job is executed, it is assigned one of the * ranges provided by the {@link IgfsRecordResolver} passed to one of the {@code IgniteFs.execute(...)} methods. *

- * {@link #execute(org.apache.ignite.IgniteFs, IgfsFileRange, org.apache.ignite.igfs.IgfsInputStream)} method is given {@link IgfsFileRange} this + * {@link #execute(org.apache.ignite.IgniteFileSystem, IgfsFileRange, org.apache.ignite.igfs.IgfsInputStream)} method is given {@link IgfsFileRange} this * job is expected to operate on, and already opened {@link org.apache.ignite.igfs.IgfsInputStream} for the file this range belongs to. *

* Note that provided input stream has position already adjusted to range start. However, it will not @@ -52,7 +52,7 @@ public interface IgfsJob { * @throws IgniteException If execution failed. * @throws IOException If file system operation resulted in IO exception. */ - public Object execute(IgniteFs igfs, IgfsFileRange range, IgfsInputStream in) throws IgniteException, + public Object execute(IgniteFileSystem igfs, IgfsFileRange range, IgfsInputStream in) throws IgniteException, IOException; /** http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/6423cf02/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsRecordResolver.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsRecordResolver.java b/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsRecordResolver.java index 52afeda..cc1d73f 100644 --- a/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsRecordResolver.java +++ b/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsRecordResolver.java @@ -51,6 +51,6 @@ public interface IgfsRecordResolver extends Serializable { * @throws IgniteException If resolve failed. * @throws IOException If resolve failed. */ - @Nullable public IgfsFileRange resolveRecords(IgniteFs fs, IgfsInputStream stream, + @Nullable public IgfsFileRange resolveRecords(IgniteFileSystem fs, IgfsInputStream stream, IgfsFileRange suggestedRecord) throws IgniteException, IOException; } http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/6423cf02/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsTask.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsTask.java b/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsTask.java index e42b012..9936140 100644 --- a/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsTask.java +++ b/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsTask.java @@ -37,7 +37,7 @@ import java.util.*; *

* Each file participating in IGFS task is split into {@link IgfsFileRange}s first. Normally range is a number of * consequent bytes located on a single node (see {@code IgfssGroupDataBlocksKeyMapper}). In case maximum range size - * is provided (either through {@link org.apache.ignite.configuration.IgfsConfiguration#getMaximumTaskRangeLength()} or {@code IgniteFs.execute()} + * is provided (either through {@link org.apache.ignite.configuration.FileSystemConfiguration#getMaximumTaskRangeLength()} or {@code IgniteFs.execute()} * argument), then ranges could be further divided into smaller chunks. *

* Once file is split into ranges, each range is passed to {@code IgfsTask.createJob()} method in order to create a @@ -88,7 +88,7 @@ public abstract class IgfsTask extends ComputeTaskAdapter, assert ignite != null; assert args != null; - IgniteFs fs = ignite.fileSystem(args.igfsName()); + IgniteFileSystem fs = ignite.fileSystem(args.igfsName()); IgfsProcessorAdapter igfsProc = ((IgniteKernal) ignite).context().igfs(); Map splitMap = new HashMap<>(); http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/6423cf02/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsTaskArgs.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsTaskArgs.java b/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsTaskArgs.java index 7db26ec..5ef5352 100644 --- a/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsTaskArgs.java +++ b/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsTaskArgs.java @@ -33,7 +33,7 @@ import java.util.*; *

  • {@link IgfsRecordResolver} for that task
    • *
  • Flag indicating whether to skip non-existent file paths or throw an exception
    • *
  • User-defined task argument
    • - *
  • Maximum file range length for that task (see {@link org.apache.ignite.configuration.IgfsConfiguration#getMaximumTaskRangeLength()})
    • + *
  • Maximum file range length for that task (see {@link org.apache.ignite.configuration.FileSystemConfiguration#getMaximumTaskRangeLength()})
    • * */ public interface IgfsTaskArgs { http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/6423cf02/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/records/IgfsByteDelimiterRecordResolver.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/records/IgfsByteDelimiterRecordResolver.java b/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/records/IgfsByteDelimiterRecordResolver.java index c1c15a5..c9ed821 100644 --- a/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/records/IgfsByteDelimiterRecordResolver.java +++ b/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/records/IgfsByteDelimiterRecordResolver.java @@ -20,7 +20,6 @@ package org.apache.ignite.igfs.mapreduce.records; import org.apache.ignite.*; import org.apache.ignite.igfs.*; import org.apache.ignite.igfs.mapreduce.*; -import org.apache.ignite.internal.util.*; import org.apache.ignite.internal.util.tostring.*; import org.apache.ignite.internal.util.typedef.*; import org.apache.ignite.internal.util.typedef.internal.*; @@ -78,7 +77,7 @@ public class IgfsByteDelimiterRecordResolver implements IgfsRecordResolver, Exte } /** {@inheritDoc} */ - @Override public IgfsFileRange resolveRecords(IgniteFs fs, IgfsInputStream stream, + @Override public IgfsFileRange resolveRecords(IgniteFileSystem fs, IgfsInputStream stream, IgfsFileRange suggestedRecord) throws IgniteException, IOException { long suggestedStart = suggestedRecord.start(); long suggestedEnd = suggestedStart + suggestedRecord.length(); http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/6423cf02/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/records/IgfsFixedLengthRecordResolver.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/records/IgfsFixedLengthRecordResolver.java b/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/records/IgfsFixedLengthRecordResolver.java index fe22627..bcc8f69 100644 --- a/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/records/IgfsFixedLengthRecordResolver.java +++ b/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/records/IgfsFixedLengthRecordResolver.java @@ -52,7 +52,7 @@ public class IgfsFixedLengthRecordResolver implements IgfsRecordResolver, Extern } /** {@inheritDoc} */ - @Override public IgfsFileRange resolveRecords(IgniteFs fs, IgfsInputStream stream, + @Override public IgfsFileRange resolveRecords(IgniteFileSystem fs, IgfsInputStream stream, IgfsFileRange suggestedRecord) throws IgniteException, IOException { long suggestedEnd = suggestedRecord.start() + suggestedRecord.length(); http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/6423cf02/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystem.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystem.java b/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystem.java new file mode 100644 index 0000000..089a8e3 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystem.java @@ -0,0 +1,201 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.igfs.secondary; + +import org.apache.ignite.*; +import org.apache.ignite.igfs.*; +import org.jetbrains.annotations.*; + +import java.io.*; +import java.util.*; + +/** + * Secondary file system interface. + */ +public interface IgfsSecondaryFileSystem { + /** + * Checks if the specified path exists. + * + * @param path Path to check for existence. + * @return {@code True} if such file exists, otherwise - {@code false}. + * @throws IgniteException In case of error. + */ + public boolean exists(IgfsPath path); + + /** + * Updates file information for the specified path. Existent properties, not listed in the passed collection, + * will not be affected. Other properties will be added or overwritten. Passed properties with {@code null} values + * will be removed from the stored properties or ignored if they don't exist in the file info. + *

    + * When working in {@code DUAL_SYNC} or {@code DUAL_ASYNC} modes only the following properties will be propagated + * to the secondary file system: + *

      + *
    • {@code usrName} - file owner name;
      • + *
    • {@code grpName} - file owner group;
      • + *
    • {@code permission} - Unix-style string representing file permissions.
      • + *
    + * + * @param path File path to set properties for. + * @param props Properties to update. + * @return File information for specified path or {@code null} if such path does not exist. + * @throws IgniteException In case of error. + */ + public IgfsFile update(IgfsPath path, Map props) throws IgniteException; + + /** + * Renames/moves a file. + *

    + * You are free to rename/move data files as you wish, but directories can be only renamed. + * You cannot move the directory between different parent directories. + *

    + * Examples: + *

      + *
    • "/work/file.txt" => "/home/project/Presentation Scenario.txt"
      • + *
    • "/work" => "/work-2012.bkp"
      • + *
    • "/work" => "/backups/work" - such operation is restricted for directories.
      • + *
    + * + * @param src Source file path to rename. + * @param dest Destination file path. If destination path is a directory, then source file will be placed + * into destination directory with original name. + * @throws IgniteException In case of error. + * @throws IgfsFileNotFoundException If source file doesn't exist. + */ + public void rename(IgfsPath src, IgfsPath dest) throws IgniteException; + + /** + * Deletes file. + * + * @param path File path to delete. + * @param recursive Delete non-empty directories recursively. + * @return {@code True} in case of success, {@code false} otherwise. + * @throws IgniteException In case of error. + */ + public boolean delete(IgfsPath path, boolean recursive) throws IgniteException; + + /** + * Creates directories under specified path. + * + * @param path Path of directories chain to create. + * @throws IgniteException In case of error. + */ + public void mkdirs(IgfsPath path) throws IgniteException; + + /** + * Creates directories under specified path with the specified properties. + * + * @param path Path of directories chain to create. + * @param props Metadata properties to set on created directories. + * @throws IgniteException In case of error. + */ + public void mkdirs(IgfsPath path, @Nullable Map props) throws IgniteException; + + /** + * Lists file paths under the specified path. + * + * @param path Path to list files under. + * @return List of files under the specified path. + * @throws IgniteException In case of error. + * @throws IgfsFileNotFoundException If path doesn't exist. + */ + public Collection listPaths(IgfsPath path) throws IgniteException; + + /** + * Lists files under the specified path. + * + * @param path Path to list files under. + * @return List of files under the specified path. + * @throws IgniteException In case of error. + * @throws IgfsFileNotFoundException If path doesn't exist. + */ + public Collection listFiles(IgfsPath path) throws IgniteException; + + /** + * Opens a file for reading. + * + * @param path File path to read. + * @param bufSize Read buffer size (bytes) or {@code zero} to use default value. + * @return File input stream to read data from. + * @throws IgniteException In case of error. + * @throws IgfsFileNotFoundException If path doesn't exist. + */ + public IgfsSecondaryFileSystemPositionedReadable open(IgfsPath path, int bufSize) throws IgniteException; + + /** + * Creates a file and opens it for writing. + * + * @param path File path to create. + * @param overwrite Overwrite file if it already exists. Note: you cannot overwrite an existent directory. + * @return File output stream to write data to. + * @throws IgniteException In case of error. + */ + public OutputStream create(IgfsPath path, boolean overwrite) throws IgniteException; + + /** + * Creates a file and opens it for writing. + * + * @param path File path to create. + * @param bufSize Write buffer size (bytes) or {@code zero} to use default value. + * @param overwrite Overwrite file if it already exists. Note: you cannot overwrite an existent directory. + * @param replication Replication factor. + * @param blockSize Block size. + * @param props File properties to set. + * @return File output stream to write data to. + * @throws IgniteException In case of error. + */ + public OutputStream create(IgfsPath path, int bufSize, boolean overwrite, int replication, long blockSize, + @Nullable Map props) throws IgniteException; + + /** + * Opens an output stream to an existing file for appending data. + * + * @param path File path to append. + * @param bufSize Write buffer size (bytes) or {@code zero} to use default value. + * @param create Create file if it doesn't exist yet. + * @param props File properties to set only in case it file was just created. + * @return File output stream to append data to. + * @throws IgniteException In case of error. + * @throws IgfsFileNotFoundException If path doesn't exist and create flag is {@code false}. + */ + public OutputStream append(IgfsPath path, int bufSize, boolean create, @Nullable Map props) + throws IgniteException; + + /** + * Gets file information for the specified path. + * + * @param path Path to get information for. + * @return File information for specified path or {@code null} if such path does not exist. + * @throws IgniteException In case of error. + */ + public IgfsFile info(IgfsPath path) throws IgniteException; + + /** + * Gets used space in bytes. + * + * @return Used space in bytes. + * @throws IgniteException In case of error. + */ + public long usedSpaceSize() throws IgniteException; + + /** + * Gets the implementation specific properties of file system. + * + * @return Map of properties. + */ + public Map properties(); +} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/6423cf02/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystemPositionedReadable.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystemPositionedReadable.java b/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystemPositionedReadable.java new file mode 100644 index 0000000..3d36236 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystemPositionedReadable.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.igfs.secondary; + +import java.io.*; + +/** + * The simplest data input interface to read from secondary file system. + */ +public interface IgfsSecondaryFileSystemPositionedReadable extends Closeable { + /** + * Read up to the specified number of bytes, from a given position within a file, and return the number of bytes + * read. + * + * @param pos Position in the input stream to seek. + * @param buf Buffer into which data is read. + * @param off Offset in the buffer from which stream data should be written. + * @param len The number of bytes to read. + * @return Total number of bytes read into the buffer, or -1 if there is no more data (EOF). + * @throws IOException In case of any exception. + */ + public int read(long pos, byte[] buf, int off, int len) throws IOException; +} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/6423cf02/modules/core/src/main/java/org/apache/ignite/igfs/secondary/package.html ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/secondary/package.html b/modules/core/src/main/java/org/apache/ignite/igfs/secondary/package.html new file mode 100644 index 0000000..8bd668d --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/igfs/secondary/package.html @@ -0,0 +1,24 @@ + + + + + + + Contains APIs for IGFS secondary file system. + + http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/6423cf02/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java index cb9ffa1..30ba883 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java @@ -282,7 +282,7 @@ public interface GridKernalContext extends Iterable { * * @return Hadoop processor. */ - public IgniteHadoopProcessorAdapter hadoop(); + public HadoopProcessorAdapter hadoop(); /** * Gets utility cache pool. http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/6423cf02/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java index 756c16a..e80df0b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java @@ -230,7 +230,7 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable /** */ @GridToStringExclude - private IgniteHadoopProcessorAdapter hadoopProc; + private HadoopProcessorAdapter hadoopProc; /** */ @GridToStringExclude @@ -456,8 +456,8 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable streamProc = (GridStreamProcessor)comp; else if (comp instanceof GridContinuousProcessor) contProc = (GridContinuousProcessor)comp; - else if (comp instanceof IgniteHadoopProcessorAdapter) - hadoopProc = (IgniteHadoopProcessorAdapter)comp; + else if (comp instanceof HadoopProcessorAdapter) + hadoopProc = (HadoopProcessorAdapter)comp; else if (comp instanceof GridPortableProcessor) portableProc = (GridPortableProcessor)comp; else if (comp instanceof IgnitePluginProcessor) @@ -680,7 +680,7 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable } /** {@inheritDoc} */ - @Override public IgniteHadoopProcessorAdapter hadoop() { + @Override public HadoopProcessorAdapter hadoop() { return hadoopProc; } http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/6423cf02/modules/core/src/main/java/org/apache/ignite/internal/IgniteComponentType.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteComponentType.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteComponentType.java index d0e487a..0e5c1cf 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteComponentType.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteComponentType.java @@ -35,8 +35,8 @@ public enum IgniteComponentType { /** Hadoop. */ HADOOP( - "org.apache.ignite.internal.processors.hadoop.IgniteHadoopNoopProcessor", - "org.apache.ignite.internal.processors.hadoop.IgniteHadoopProcessor", + "org.apache.ignite.internal.processors.hadoop.HadoopNoopProcessor", + "org.apache.ignite.internal.processors.hadoop.HadoopProcessor", "ignite-hadoop" ), http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/6423cf02/modules/core/src/main/java/org/apache/ignite/internal/IgniteEx.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteEx.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteEx.java index 3c35a08..3ede8d5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteEx.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteEx.java @@ -117,14 +117,14 @@ public interface IgniteEx extends Ignite { * @param name IGFS name. * @return IGFS. */ - @Nullable public IgniteFs igfsx(@Nullable String name); + @Nullable public IgniteFileSystem igfsx(@Nullable String name); /** * Get Hadoop facade. * * @return Hadoop. */ - public GridHadoop hadoop(); + public Hadoop hadoop(); /** {@inheritDoc} */ @Override IgniteClusterEx cluster(); http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/6423cf02/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java index b9abdb5..76c070d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java @@ -704,7 +704,7 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable { // Starts lifecycle aware components. U.startLifecycleAware(lifecycleAwares(cfg)); - addHelper(ctx, IGFS_HELPER.create(F.isEmpty(cfg.getIgfsConfiguration()))); + addHelper(ctx, IGFS_HELPER.create(F.isEmpty(cfg.getFileSystemConfiguration()))); startProcessor(ctx, new IgnitePluginProcessor(ctx, cfg), attrs); @@ -756,7 +756,7 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable { startProcessor(ctx, new GridRestProcessor(ctx), attrs); startProcessor(ctx, new GridDataLoaderProcessor(ctx), attrs); startProcessor(ctx, new GridStreamProcessor(ctx), attrs); - startProcessor(ctx, (GridProcessor) IGFS.create(ctx, F.isEmpty(cfg.getIgfsConfiguration())), attrs); + startProcessor(ctx, (GridProcessor) IGFS.create(ctx, F.isEmpty(cfg.getFileSystemConfiguration())), attrs); startProcessor(ctx, new GridContinuousProcessor(ctx), attrs); startProcessor(ctx, (GridProcessor)(cfg.isPeerClassLoadingEnabled() ? IgniteComponentType.HADOOP.create(ctx, true): // No-op when peer class loading is enabled. @@ -2349,11 +2349,11 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable { } /** {@inheritDoc} */ - @Override public IgniteFs fileSystem(String name) { + @Override public IgniteFileSystem fileSystem(String name) { guard(); try{ - IgniteFs fs = ctx.igfs().igfs(name); + IgniteFileSystem fs = ctx.igfs().igfs(name); if (fs == null) throw new IllegalArgumentException("IGFS is not configured: " + name); @@ -2366,7 +2366,7 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable { } /** {@inheritDoc} */ - @Nullable @Override public IgniteFs igfsx(@Nullable String name) { + @Nullable @Override public IgniteFileSystem igfsx(@Nullable String name) { guard(); try { @@ -2378,7 +2378,7 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable { } /** {@inheritDoc} */ - @Override public Collection fileSystems() { + @Override public Collection fileSystems() { guard(); try { @@ -2390,7 +2390,7 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable { } /** {@inheritDoc} */ - @Override public GridHadoop hadoop() { + @Override public Hadoop hadoop() { guard(); try { http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/6423cf02/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java index cb2efbf..2e8cfc1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java @@ -1606,15 +1606,15 @@ public class IgnitionEx { if (myCfg.getPeerClassLoadingLocalClassPathExclude() == null) myCfg.setPeerClassLoadingLocalClassPathExclude(EMPTY_STR_ARR); - IgfsConfiguration[] igfsCfgs = myCfg.getIgfsConfiguration(); + FileSystemConfiguration[] igfsCfgs = myCfg.getFileSystemConfiguration(); if (igfsCfgs != null) { - IgfsConfiguration[] clone = igfsCfgs.clone(); + FileSystemConfiguration[] clone = igfsCfgs.clone(); for (int i = 0; i < igfsCfgs.length; i++) - clone[i] = new IgfsConfiguration(igfsCfgs[i]); + clone[i] = new FileSystemConfiguration(igfsCfgs[i]); - myCfg.setIgfsConfiguration(clone); + myCfg.setFileSystemConfiguration(clone); } StreamerConfiguration[] streamerCfgs = myCfg.getStreamerConfiguration(); http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/6423cf02/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java index 12ea535..fe88012 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java @@ -236,10 +236,10 @@ public abstract class GridCacheAdapter implements GridCache, mxBean = new CacheMetricsMXBeanImpl(this); - IgfsConfiguration[] igfsCfgs = gridCfg.getIgfsConfiguration(); + FileSystemConfiguration[] igfsCfgs = gridCfg.getFileSystemConfiguration(); if (igfsCfgs != null) { - for (IgfsConfiguration igfsCfg : igfsCfgs) { + for (FileSystemConfiguration igfsCfg : igfsCfgs) { if (F.eq(ctx.name(), igfsCfg.getDataCacheName())) { if (!ctx.isNear()) { igfsDataCache = true; http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/6423cf02/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java index 3b2ca47..72c2b49 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java @@ -550,10 +550,10 @@ public class GridCacheProcessor extends GridProcessorAdapter { maxPreloadOrder = validatePreloadOrder(ctx.config().getCacheConfiguration()); // Internal caches which should not be returned to user. - IgfsConfiguration[] igfsCfgs = ctx.grid().configuration().getIgfsConfiguration(); + FileSystemConfiguration[] igfsCfgs = ctx.grid().configuration().getFileSystemConfiguration(); if (igfsCfgs != null) { - for (IgfsConfiguration igfsCfg : igfsCfgs) { + for (FileSystemConfiguration igfsCfg : igfsCfgs) { sysCaches.add(igfsCfg.getMetaCacheName()); sysCaches.add(igfsCfg.getDataCacheName()); } http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/6423cf02/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java index 1e67907..b07c14e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java @@ -1594,10 +1594,10 @@ public class GridCacheUtils { * @return {@code True} in this is IGFS data or meta cache. */ public static boolean isIgfsCache(IgniteConfiguration cfg, @Nullable String cacheName) { - IgfsConfiguration[] igfsCfgs = cfg.getIgfsConfiguration(); + FileSystemConfiguration[] igfsCfgs = cfg.getFileSystemConfiguration(); if (igfsCfgs != null) { - for (IgfsConfiguration igfsCfg : igfsCfgs) { + for (FileSystemConfiguration igfsCfg : igfsCfgs) { // IGFS config probably has not been validated yet => possible NPE, so we check for null. if (igfsCfg != null && (F.eq(cacheName, igfsCfg.getDataCacheName()) || F.eq(cacheName, igfsCfg.getMetaCacheName()))) http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/6423cf02/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoop.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoop.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoop.java deleted file mode 100644 index c262d48..0000000 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoop.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.hadoop; - -import org.apache.ignite.*; -import org.apache.ignite.internal.*; -import org.jetbrains.annotations.*; - -/** - * Hadoop facade providing access to Ignite Hadoop features. - */ -public interface GridHadoop { - /** - * Gets Hadoop module configuration. - * - * @return Hadoop module configuration. - */ - public GridHadoopConfiguration configuration(); - - /** - * Generate next job ID. - * - * @return Next job ID. - */ - public GridHadoopJobId nextJobId(); - - /** - * Submits job to job tracker. - * - * @param jobId Job ID to submit. - * @param jobInfo Job info to submit. - * @return Execution future. - */ - public IgniteInternalFuture submit(GridHadoopJobId jobId, GridHadoopJobInfo jobInfo); - - /** - * Gets Hadoop job execution status. - * - * @param jobId Job ID to get status for. - * @return Job execution status or {@code null} in case job with the given ID is not found. - * @throws IgniteCheckedException If failed. - */ - @Nullable public GridHadoopJobStatus status(GridHadoopJobId jobId) throws IgniteCheckedException; - - /** - * Returns job counters. - * - * @param jobId Job ID to get counters for. - * @return Job counters object. - * @throws IgniteCheckedException If failed. - */ - public GridHadoopCounters counters(GridHadoopJobId jobId) throws IgniteCheckedException; - - /** - * Gets Hadoop finish future for particular job. - * - * @param jobId Job ID. - * @return Job finish future or {@code null} in case job with the given ID is not found. - * @throws IgniteCheckedException If failed. - */ - @Nullable public IgniteInternalFuture finishFuture(GridHadoopJobId jobId) throws IgniteCheckedException; - - /** - * Kills job. - * - * @param jobId Job ID. - * @return {@code True} if job was killed. - * @throws IgniteCheckedException If failed. - */ - public boolean kill(GridHadoopJobId jobId) throws IgniteCheckedException; -} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/6423cf02/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopConfiguration.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopConfiguration.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopConfiguration.java deleted file mode 100644 index f66b95a..0000000 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopConfiguration.java +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.hadoop; - -import org.apache.ignite.internal.util.typedef.internal.*; - -/** - * Hadoop configuration. - */ -public class GridHadoopConfiguration { - /** Default finished job info time-to-live. */ - public static final long DFLT_FINISHED_JOB_INFO_TTL = 10_000; - - /** Default value for external execution flag. */ - public static final boolean DFLT_EXTERNAL_EXECUTION = false; - - /** Default value for the max parallel tasks. */ - public static final int DFLT_MAX_PARALLEL_TASKS = Runtime.getRuntime().availableProcessors(); - - /** Default value for the max task queue size. */ - public static final int DFLT_MAX_TASK_QUEUE_SIZE = 1000; - - /** Map reduce planner. */ - private GridHadoopMapReducePlanner planner; - - /** */ - private boolean extExecution = DFLT_EXTERNAL_EXECUTION; - - /** Finished job info TTL. */ - private long finishedJobInfoTtl = DFLT_FINISHED_JOB_INFO_TTL; - - /** */ - private int maxParallelTasks = DFLT_MAX_PARALLEL_TASKS; - - /** */ - private int maxTaskQueueSize = DFLT_MAX_TASK_QUEUE_SIZE; - - /** - * Default constructor. - */ - public GridHadoopConfiguration() { - // No-op. - } - - /** - * Copy constructor. - * - * @param cfg Configuration to copy. - */ - public GridHadoopConfiguration(GridHadoopConfiguration cfg) { - // Preserve alphabetic order. - extExecution = cfg.isExternalExecution(); - finishedJobInfoTtl = cfg.getFinishedJobInfoTtl(); - planner = cfg.getMapReducePlanner(); - maxParallelTasks = cfg.getMaxParallelTasks(); - maxTaskQueueSize = cfg.getMaxTaskQueueSize(); - } - - /** - * Gets max number of local tasks that may be executed in parallel. - * - * @return Max number of local tasks that may be executed in parallel. - */ - public int getMaxParallelTasks() { - return maxParallelTasks; - } - - /** - * Sets max number of local tasks that may be executed in parallel. - * - * @param maxParallelTasks Max number of local tasks that may be executed in parallel. - */ - public void setMaxParallelTasks(int maxParallelTasks) { - this.maxParallelTasks = maxParallelTasks; - } - - /** - * Gets max task queue size. - * - * @return Max task queue size. - */ - public int getMaxTaskQueueSize() { - return maxTaskQueueSize; - } - - /** - * Sets max task queue size. - * - * @param maxTaskQueueSize Max task queue size. - */ - public void setMaxTaskQueueSize(int maxTaskQueueSize) { - this.maxTaskQueueSize = maxTaskQueueSize; - } - - /** - * Gets finished job info time-to-live in milliseconds. - * - * @return Finished job info time-to-live. - */ - public long getFinishedJobInfoTtl() { - return finishedJobInfoTtl; - } - - /** - * Sets finished job info time-to-live. - * - * @param finishedJobInfoTtl Finished job info time-to-live. - */ - public void setFinishedJobInfoTtl(long finishedJobInfoTtl) { - this.finishedJobInfoTtl = finishedJobInfoTtl; - } - - /** - * Gets external task execution flag. If {@code true}, hadoop job tasks will be executed in an external - * (relative to node) process. - * - * @return {@code True} if external execution. - */ - public boolean isExternalExecution() { - return extExecution; - } - - /** - * Sets external task execution flag. - * - * @param extExecution {@code True} if tasks should be executed in an external process. - * @see #isExternalExecution() - */ - public void setExternalExecution(boolean extExecution) { - this.extExecution = extExecution; - } - - /** - * Gets Hadoop map-reduce planner, a component which defines job execution plan based on job - * configuration and current grid topology. - * - * @return Map-reduce planner. - */ - public GridHadoopMapReducePlanner getMapReducePlanner() { - return planner; - } - - /** - * Sets Hadoop map-reduce planner, a component which defines job execution plan based on job - * configuration and current grid topology. - * - * @param planner Map-reduce planner. - */ - public void setMapReducePlanner(GridHadoopMapReducePlanner planner) { - this.planner = planner; - } - - /** {@inheritDoc} */ - @Override public String toString() { - return S.toString(GridHadoopConfiguration.class, this, super.toString()); - } -} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/6423cf02/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopCounter.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopCounter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopCounter.java deleted file mode 100644 index 83902dd..0000000 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopCounter.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.hadoop; - -/** - * Hadoop counter. - */ -public interface GridHadoopCounter { - /** - * Gets name. - * - * @return Name of the counter. - */ - public String name(); - - /** - * Gets counter group. - * - * @return Counter group's name. - */ - public String group(); - - /** - * Merge the given counter to this counter. - * - * @param cntr Counter to merge into this counter. - */ - public void merge(GridHadoopCounter cntr); -} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/6423cf02/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopCounterWriter.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopCounterWriter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopCounterWriter.java deleted file mode 100644 index af72e69..0000000 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopCounterWriter.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.hadoop; - -import org.apache.ignite.*; - -/** - * The object that writes some system counters to some storage for each running job. This operation is a part of - * whole statistics collection process. - */ -public interface GridHadoopCounterWriter { - /** - * Writes counters of given job to some statistics storage. - * - * @param jobInfo Job info. - * @param jobId Job id. - * @param cntrs Counters. - * @throws IgniteCheckedException If failed. - */ - public void write(GridHadoopJobInfo jobInfo, GridHadoopJobId jobId, GridHadoopCounters cntrs) throws IgniteCheckedException; -} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/6423cf02/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopCounters.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopCounters.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopCounters.java deleted file mode 100644 index 91eb8a1..0000000 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopCounters.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.hadoop; - -import java.util.*; - -/** - * Counters store. - */ -public interface GridHadoopCounters { - /** - * Returns counter for the specified group and counter name. Creates new if it does not exist. - * - * @param grp Counter group name. - * @param name Counter name. - * @param cls Class for new instance creation if it's needed. - * @return The counter that was found or added or {@code null} if create is false. - */ - T counter(String grp, String name, Class cls); - - /** - * Returns all existing counters. - * - * @return Collection of counters. - */ - Collection all(); - - /** - * Merges all counters from another store with existing counters. - * - * @param other Counters to merge with. - */ - void merge(GridHadoopCounters other); -} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/6423cf02/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopFileBlock.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopFileBlock.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopFileBlock.java deleted file mode 100644 index fae111a..0000000 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopFileBlock.java +++ /dev/null @@ -1,162 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.hadoop; - -import org.apache.ignite.internal.util.tostring.*; -import org.apache.ignite.internal.util.typedef.internal.*; - -import java.io.*; -import java.net.*; -import java.util.*; - -/** - * Hadoop file block. - */ -public class GridHadoopFileBlock extends GridHadoopInputSplit { - /** */ - private static final long serialVersionUID = 0L; - - /** */ - @GridToStringInclude - protected URI file; - - /** */ - @GridToStringInclude - protected long start; - - /** */ - @GridToStringInclude - protected long len; - - /** - * Creates new file block. - */ - public GridHadoopFileBlock() { - // No-op. - } - - /** - * Creates new file block. - * - * @param hosts List of hosts where the block resides. - * @param file File URI. - * @param start Start position of the block in the file. - * @param len Length of the block. - */ - public GridHadoopFileBlock(String[] hosts, URI file, long start, long len) { - A.notNull(hosts, "hosts", file, "file"); - - this.hosts = hosts; - this.file = file; - this.start = start; - this.len = len; - } - - /** {@inheritDoc} */ - @Override public void writeExternal(ObjectOutput out) throws IOException { - out.writeObject(file()); - out.writeLong(start()); - out.writeLong(length()); - } - - /** {@inheritDoc} */ - @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { - file = (URI)in.readObject(); - start = in.readLong(); - len = in.readLong(); - } - - /** - * @return Length. - */ - public long length() { - return len; - } - - /** - * @param len New length. - */ - public void length(long len) { - this.len = len; - } - - /** - * @return Start. - */ - public long start() { - return start; - } - - /** - * @param start New start. - */ - public void start(long start) { - this.start = start; - } - - /** - * @return File. - */ - public URI file() { - return file; - } - - /** - * @param file New file. - */ - public void file(URI file) { - this.file = file; - } - - /** - * @param hosts New hosts. - */ - public void hosts(String[] hosts) { - A.notNull(hosts, "hosts"); - - this.hosts = hosts; - } - - /** {@inheritDoc} */ - @Override public boolean equals(Object o) { - if (this == o) - return true; - - if (!(o instanceof GridHadoopFileBlock)) - return false; - - GridHadoopFileBlock that = (GridHadoopFileBlock)o; - - return len == that.len && start == that.start && file.equals(that.file); - } - - /** {@inheritDoc} */ - @Override public int hashCode() { - int res = file.hashCode(); - - res = 31 * res + (int)(start ^ (start >>> 32)); - res = 31 * res + (int)(len ^ (len >>> 32)); - - return res; - } - - /** {@inheritDoc} */ - public String toString() { - return S.toString(GridHadoopFileBlock.class, this, "hosts", Arrays.toString(hosts)); - } -} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/6423cf02/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopInputSplit.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopInputSplit.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopInputSplit.java deleted file mode 100644 index e68a6f5..0000000 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopInputSplit.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.hadoop; - -import java.io.*; - -/** - * Abstract fragment of an input data source. - */ -public abstract class GridHadoopInputSplit implements Externalizable { - /** */ - protected String[] hosts; - - /** - * Array of hosts where this input split resides. - * - * @return Hosts. - */ - public String[] hosts() { - assert hosts != null; - - return hosts; - } - - /** - * This method must be implemented for purpose of internal implementation. - * - * @param obj Another object. - * @return {@code true} If objects are equal. - */ - @Override public abstract boolean equals(Object obj); - - /** - * This method must be implemented for purpose of internal implementation. - * - * @return Hash code of the object. - */ - @Override public abstract int hashCode(); -} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/6423cf02/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJob.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJob.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJob.java deleted file mode 100644 index f7ea105..0000000 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJob.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.hadoop; - -import org.apache.ignite.*; - -import java.util.*; - -/** - * Hadoop job. - */ -public interface GridHadoopJob { - /** - * Gets job ID. - * - * @return Job ID. - */ - public GridHadoopJobId id(); - - /** - * Gets job information. - * - * @return Job information. - */ - public GridHadoopJobInfo info(); - - /** - * Gets collection of input splits for this job. - * - * @return Input splits. - */ - public Collection input() throws IgniteCheckedException; - - /** - * Returns context for task execution. - * - * @param info Task info. - * @return Task Context. - * @throws IgniteCheckedException If failed. - */ - public GridHadoopTaskContext getTaskContext(GridHadoopTaskInfo info) throws IgniteCheckedException; - - /** - * Does all the needed initialization for the job. Will be called on each node where tasks for this job must - * be executed. - *

    - * If job is running in external mode this method will be called on instance in Ignite node with parameter - * {@code false} and on instance in external process with parameter {@code true}. - * - * @param external If {@code true} then this job instance resides in external process. - * @param locNodeId Local node ID. - * @throws IgniteCheckedException If failed. - */ - public void initialize(boolean external, UUID locNodeId) throws IgniteCheckedException; - - /** - * Release all the resources. - *

    - * If job is running in external mode this method will be called on instance in Ignite node with parameter - * {@code false} and on instance in external process with parameter {@code true}. - * - * @param external If {@code true} then this job instance resides in external process. - * @throws IgniteCheckedException If failed. - */ - public void dispose(boolean external) throws IgniteCheckedException; - - /** - * Prepare local environment for the task. - * - * @param info Task info. - * @throws IgniteCheckedException If failed. - */ - public void prepareTaskEnvironment(GridHadoopTaskInfo info) throws IgniteCheckedException; - - /** - * Cleans up local environment of the task. - * - * @param info Task info. - * @throws IgniteCheckedException If failed. - */ - public void cleanupTaskEnvironment(GridHadoopTaskInfo info) throws IgniteCheckedException; - - /** - * Cleans up the job staging directory. - */ - void cleanupStagingDirectory(); -} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/6423cf02/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJobId.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJobId.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJobId.java deleted file mode 100644 index ffc2057..0000000 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJobId.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.hadoop; - -import org.apache.ignite.internal.processors.cache.*; -import org.apache.ignite.internal.util.typedef.internal.*; - -import java.io.*; -import java.util.*; - -/** - * Job ID. - */ -public class GridHadoopJobId implements GridCacheInternal, Externalizable { - /** */ - private static final long serialVersionUID = 0L; - - /** */ - private UUID nodeId; - - /** */ - private int jobId; - - /** - * For {@link Externalizable}. - */ - public GridHadoopJobId() { - // No-op. - } - - /** - * @param nodeId Node ID. - * @param jobId Job ID. - */ - public GridHadoopJobId(UUID nodeId, int jobId) { - this.nodeId = nodeId; - this.jobId = jobId; - } - - public UUID globalId() { - return nodeId; - } - - public int localId() { - return jobId; - } - - /** {@inheritDoc} */ - @Override public void writeExternal(ObjectOutput out) throws IOException { - U.writeUuid(out, nodeId); - out.writeInt(jobId); - } - - /** {@inheritDoc} */ - @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { - nodeId = U.readUuid(in); - jobId = in.readInt(); - } - - /** {@inheritDoc} */ - @Override public boolean equals(Object o) { - if (this == o) - return true; - - if (o == null || getClass() != o.getClass()) - return false; - - GridHadoopJobId that = (GridHadoopJobId) o; - - if (jobId != that.jobId) - return false; - - if (!nodeId.equals(that.nodeId)) - return false; - - return true; - } - - /** {@inheritDoc} */ - @Override public int hashCode() { - return 31 * nodeId.hashCode() + jobId; - } - - /** {@inheritDoc} */ - @Override public String toString() { - return nodeId + "_" + jobId; - } -} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/6423cf02/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJobInfo.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJobInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJobInfo.java deleted file mode 100644 index 9a891f4..0000000 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJobInfo.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.hadoop; - -import org.apache.ignite.*; -import org.jetbrains.annotations.*; - -import java.io.*; - -/** - * Compact job description. - */ -public interface GridHadoopJobInfo extends Serializable { - /** - * Gets optional configuration property for the job. - * - * @param name Property name. - * @return Value or {@code null} if none. - */ - @Nullable public String property(String name); - - /** - * Checks whether job has combiner. - * - * @return {@code true} If job has combiner. - */ - public boolean hasCombiner(); - - /** - * Checks whether job has reducer. - * Actual number of reducers will be in {@link GridHadoopMapReducePlan#reducers()}. - * - * @return Number of reducer. - */ - public boolean hasReducer(); - - /** - * Creates new job instance for the given ID. - * {@link GridHadoopJobInfo} is reusable for multiple jobs while {@link GridHadoopJob} is for one job execution. - * This method will be called once for the same ID on one node, though it can be called on the same host - * multiple times from different processes (in case of multiple nodes on the same host or external execution). - * - * @param jobId Job ID. - * @param log Logger. - * @return Job. - * @throws IgniteCheckedException If failed. - */ - GridHadoopJob createJob(GridHadoopJobId jobId, IgniteLogger log) throws IgniteCheckedException; - - /** - * @return Number of reducers configured for job. - */ - public int reducers(); - - /** - * Gets job name. - * - * @return Job name. - */ - public String jobName(); - - /** - * Gets user name. - * - * @return User name. - */ - public String user(); -} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/6423cf02/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJobPhase.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJobPhase.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJobPhase.java deleted file mode 100644 index cc122bb..0000000 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJobPhase.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.hadoop; - -/** - * Job run phase. - */ -public enum GridHadoopJobPhase { - /** Job is running setup task. */ - PHASE_SETUP, - - /** Job is running map and combine tasks. */ - PHASE_MAP, - - /** Job has finished all map tasks and running reduce tasks. */ - PHASE_REDUCE, - - /** Job is stopping due to exception during any of the phases. */ - PHASE_CANCELLING, - - /** Job has finished execution. */ - PHASE_COMPLETE -} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/6423cf02/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJobProperty.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJobProperty.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJobProperty.java deleted file mode 100644 index 0ece051..0000000 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJobProperty.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.hadoop; - -import org.jetbrains.annotations.*; - -/** - * Enumeration of optional properties supported by Ignite for Apache Hadoop. - */ -public enum GridHadoopJobProperty { - /** - * Initial size for hashmap which stores output of mapper and will be used as input of combiner. - *

    - * Setting it right allows to avoid rehashing. - */ - COMBINER_HASHMAP_SIZE, - - /** - * Initial size for hashmap which stores output of mapper or combiner and will be used as input of reducer. - *

    - * Setting it right allows to avoid rehashing. - */ - PARTITION_HASHMAP_SIZE, - - /** - * Specifies number of concurrently running mappers for external execution mode. - *

    - * If not specified, defaults to {@code Runtime.getRuntime().availableProcessors()}. - */ - EXTERNAL_CONCURRENT_MAPPERS, - - /** - * Specifies number of concurrently running reducers for external execution mode. - *

    - * If not specified, defaults to {@code Runtime.getRuntime().availableProcessors()}. - */ - EXTERNAL_CONCURRENT_REDUCERS, - - /** - * Delay in milliseconds after which Ignite server will reply job status. - */ - JOB_STATUS_POLL_DELAY, - - /** - * Size in bytes of single memory page which will be allocated for data structures in shuffle. - *

    - * By default is {@code 32 * 1024}. - */ - SHUFFLE_OFFHEAP_PAGE_SIZE, - - /** - * If set to {@code true} then input for combiner will not be sorted by key. - * Internally hash-map will be used instead of sorted one, so {@link Object#equals(Object)} - * and {@link Object#hashCode()} methods of key must be implemented consistently with - * comparator for that type. Grouping comparator is not supported if this setting is {@code true}. - *

    - * By default is {@code false}. - */ - SHUFFLE_COMBINER_NO_SORTING, - - /** - * If set to {@code true} then input for reducer will not be sorted by key. - * Internally hash-map will be used instead of sorted one, so {@link Object#equals(Object)} - * and {@link Object#hashCode()} methods of key must be implemented consistently with - * comparator for that type. Grouping comparator is not supported if this setting is {@code true}. - *

    - * By default is {@code false}. - */ - SHUFFLE_REDUCER_NO_SORTING; - - /** */ - private final String ptyName; - - /** - * - */ - GridHadoopJobProperty() { - ptyName = "ignite." + name().toLowerCase().replace('_', '.'); - } - - /** - * @return Property name. - */ - public String propertyName() { - return ptyName; - } - - /** - * @param jobInfo Job info. - * @param pty Property. - * @param dflt Default value. - * @return Property value. - */ - public static String get(GridHadoopJobInfo jobInfo, GridHadoopJobProperty pty, @Nullable String dflt) { - String res = jobInfo.property(pty.propertyName()); - - return res == null ? dflt : res; - } - - /** - * @param jobInfo Job info. - * @param pty Property. - * @param dflt Default value. - * @return Property value. - */ - public static int get(GridHadoopJobInfo jobInfo, GridHadoopJobProperty pty, int dflt) { - String res = jobInfo.property(pty.propertyName()); - - return res == null ? dflt : Integer.parseInt(res); - } - - /** - * @param jobInfo Job info. - * @param pty Property. - * @param dflt Default value. - * @return Property value. - */ - public static boolean get(GridHadoopJobInfo jobInfo, GridHadoopJobProperty pty, boolean dflt) { - String res = jobInfo.property(pty.propertyName()); - - return res == null ? dflt : Boolean.parseBoolean(res); - } -} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/6423cf02/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJobStatus.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJobStatus.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJobStatus.java deleted file mode 100644 index 02ea883..0000000 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopJobStatus.java +++ /dev/null @@ -1,207 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.hadoop; - -import org.apache.ignite.internal.util.typedef.internal.*; - -import java.io.*; - -/** - * Hadoop job status. - */ -public class GridHadoopJobStatus implements Externalizable { - /** */ - private static final long serialVersionUID = 0L; - - /** Job ID. */ - private GridHadoopJobId jobId; - - /** Job name. */ - private String jobName; - - /** User. */ - private String usr; - - /** Pending mappers count. */ - private int pendingMapperCnt; - - /** Pending reducers count. */ - private int pendingReducerCnt; - - /** Total mappers count. */ - private int totalMapperCnt; - - /** Total reducers count. */ - private int totalReducerCnt; - /** Phase. */ - private GridHadoopJobPhase jobPhase; - - /** */ - private boolean failed; - - /** Version. */ - private long ver; - - /** - * {@link Externalizable} support. - */ - public GridHadoopJobStatus() { - // No-op. - } - - /** - * Constructor. - * - * @param jobId Job ID. - * @param jobName Job name. - * @param usr User. - * @param pendingMapperCnt Pending mappers count. - * @param pendingReducerCnt Pending reducers count. - * @param totalMapperCnt Total mappers count. - * @param totalReducerCnt Total reducers count. - * @param jobPhase Job phase. - * @param failed Failed. - * @param ver Version. - */ - public GridHadoopJobStatus( - GridHadoopJobId jobId, - String jobName, - String usr, - int pendingMapperCnt, - int pendingReducerCnt, - int totalMapperCnt, - int totalReducerCnt, - GridHadoopJobPhase jobPhase, - boolean failed, - long ver - ) { - this.jobId = jobId; - this.jobName = jobName; - this.usr = usr; - this.pendingMapperCnt = pendingMapperCnt; - this.pendingReducerCnt = pendingReducerCnt; - this.totalMapperCnt = totalMapperCnt; - this.totalReducerCnt = totalReducerCnt; - this.jobPhase = jobPhase; - this.failed = failed; - this.ver = ver; - } - - /** - * @return Job ID. - */ - public GridHadoopJobId jobId() { - return jobId; - } - - /** - * @return Job name. - */ - public String jobName() { - return jobName; - } - - /** - * @return User. - */ - public String user() { - return usr; - } - - /** - * @return Pending mappers count. - */ - public int pendingMapperCnt() { - return pendingMapperCnt; - } - - /** - * @return Pending reducers count. - */ - public int pendingReducerCnt() { - return pendingReducerCnt; - } - - /** - * @return Total mappers count. - */ - public int totalMapperCnt() { - return totalMapperCnt; - } - - /** - * @return Total reducers count. - */ - public int totalReducerCnt() { - return totalReducerCnt; - } - - /** - * @return Version. - */ - public long version() { - return ver; - } - - /** - * @return Job phase. - */ - public GridHadoopJobPhase jobPhase() { - return jobPhase; - } - - /** - * @return {@code true} If the job failed. - */ - public boolean isFailed() { - return failed; - } - - /** {@inheritDoc} */ - @Override public String toString() { - return S.toString(GridHadoopJobStatus.class, this); - } - - /** {@inheritDoc} */ - @Override public void writeExternal(ObjectOutput out) throws IOException { - out.writeObject(jobId); - U.writeString(out, jobName); - U.writeString(out, usr); - out.writeInt(pendingMapperCnt); - out.writeInt(pendingReducerCnt); - out.writeInt(totalMapperCnt); - out.writeInt(totalReducerCnt); - out.writeObject(jobPhase); - out.writeBoolean(failed); - out.writeLong(ver); - } - - /** {@inheritDoc} */ - @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { - jobId = (GridHadoopJobId)in.readObject(); - jobName = U.readString(in); - usr = U.readString(in); - pendingMapperCnt = in.readInt(); - pendingReducerCnt = in.readInt(); - totalMapperCnt = in.readInt(); - totalReducerCnt = in.readInt(); - jobPhase = (GridHadoopJobPhase)in.readObject(); - failed = in.readBoolean(); - ver = in.readLong(); - } -} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/6423cf02/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopMapReducePlan.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopMapReducePlan.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopMapReducePlan.java deleted file mode 100644 index 2fd5160..0000000 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/GridHadoopMapReducePlan.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.hadoop; - -import org.jetbrains.annotations.*; - -import java.io.*; -import java.util.*; - -/** - * Map-reduce job execution plan. - */ -public interface GridHadoopMapReducePlan extends Serializable { - /** - * Gets collection of file blocks for which mappers should be executed. - * - * @param nodeId Node ID to check. - * @return Collection of file blocks or {@code null} if no mappers should be executed on given node. - */ - @Nullable public Collection mappers(UUID nodeId); - - /** - * Gets reducer IDs that should be started on given node. - * - * @param nodeId Node ID to check. - * @return Array of reducer IDs. - */ - @Nullable public int[] reducers(UUID nodeId); - - /** - * Gets collection of all node IDs involved in map part of job execution. - * - * @return Collection of node IDs. - */ - public Collection mapperNodeIds(); - - /** - * Gets collection of all node IDs involved in reduce part of job execution. - * - * @return Collection of node IDs. - */ - public Collection reducerNodeIds(); - - /** - * Gets overall number of mappers for the job. - * - * @return Number of mappers. - */ - public int mappers(); - - /** - * Gets overall number of reducers for the job. - * - * @return Number of reducers. - */ - public int reducers(); - - /** - * Gets node ID for reducer. - * - * @param reducer Reducer. - * @return Node ID. - */ - public UUID nodeForReducer(int reducer); -}