ignite-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sboi...@apache.org
Subject [46/47] ignite git commit: IGNITE-3912: Hadoop: Implemented new class loading architecture for embedded execution mode.
Date Mon, 26 Sep 2016 11:25:27 GMT
IGNITE-3912: Hadoop: Implemented new class loading architecture for embedded execution mode.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/8032fc2c
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/8032fc2c
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/8032fc2c

Branch: refs/heads/ignite-3967
Commit: 8032fc2c8a7cf9f404eb75c65164bb2900aab79d
Parents: 548fe6a
Author: vozerov-gridgain <vozerov@gridgain.com>
Authored: Mon Sep 26 12:31:30 2016 +0300
Committer: vozerov-gridgain <vozerov@gridgain.com>
Committed: Mon Sep 26 12:31:34 2016 +0300

----------------------------------------------------------------------
 bin/include/setenv.bat                          |     8 -
 bin/include/setenv.sh                           |    31 -
 .../ignite/internal/GridKernalContext.java      |     8 +
 .../ignite/internal/GridKernalContextImpl.java  |    12 +
 .../ignite/internal/IgniteComponentType.java    |    11 +-
 .../apache/ignite/internal/IgniteKernal.java    |     8 +-
 .../processors/hadoop/HadoopClassLoader.java    |   487 +
 .../processors/hadoop/HadoopClasspathUtils.java |   424 +
 .../processors/hadoop/HadoopDefaultJobInfo.java |   156 +
 .../processors/hadoop/HadoopHelper.java         |    55 +
 .../processors/hadoop/HadoopJobInfo.java        |     4 +-
 .../processors/hadoop/HadoopLocations.java      |   123 +
 .../processors/hadoop/HadoopNoopHelper.java     |    66 +
 .../processors/hadoop/HadoopNoopProcessor.java  |     4 +-
 .../internal/processors/igfs/IgfsImpl.java      |     5 +-
 .../processors/igfs/IgfsKernalContextAware.java |    32 +
 .../hadoop/fs/BasicHadoopFileSystemFactory.java |   125 +-
 .../fs/CachingHadoopFileSystemFactory.java      |    54 +-
 .../hadoop/fs/HadoopFileSystemFactory.java      |    11 +-
 .../fs/IgniteHadoopFileSystemCounterWriter.java |    79 +-
 .../fs/IgniteHadoopIgfsSecondaryFileSystem.java |   402 +-
 .../fs/KerberosHadoopFileSystemFactory.java     |    77 +-
 .../hadoop/fs/v1/IgniteHadoopFileSystem.java    |    47 +-
 .../hadoop/fs/v2/IgniteHadoopFileSystem.java    |    47 +-
 .../IgniteHadoopClientProtocolProvider.java     |    11 +-
 .../mapreduce/IgniteHadoopMapReducePlanner.java |    22 +-
 .../IgniteHadoopWeightedMapReducePlanner.java   |     4 +-
 .../ignite/hadoop/util/UserNameMapper.java      |     4 +-
 .../processors/hadoop/HadoopClassLoader.java    |   964 --
 .../processors/hadoop/HadoopClasspathMain.java  |    44 -
 .../processors/hadoop/HadoopClasspathUtils.java |   461 -
 .../processors/hadoop/HadoopCommonUtils.java    |   154 +
 .../processors/hadoop/HadoopContext.java        |     1 -
 .../processors/hadoop/HadoopDefaultJobInfo.java |   156 -
 .../processors/hadoop/HadoopExternalSplit.java  |    88 +
 .../processors/hadoop/HadoopHelperImpl.java     |   120 +
 .../processors/hadoop/HadoopLocations.java      |   123 -
 .../hadoop/HadoopMapReduceCounterGroup.java     |   123 -
 .../hadoop/HadoopMapReduceCounters.java         |   228 -
 .../processors/hadoop/HadoopProcessor.java      |    32 +-
 .../processors/hadoop/HadoopSplitWrapper.java   |   119 +
 .../internal/processors/hadoop/HadoopUtils.java |   443 -
 .../hadoop/counter/HadoopCounterAdapter.java    |     1 +
 .../counter/HadoopPerformanceCounter.java       |    12 +-
 .../hadoop/delegate/HadoopDelegateUtils.java    |   138 +
 .../HadoopFileSystemCounterWriterDelegate.java  |    36 +
 .../HadoopFileSystemFactoryDelegate.java        |    36 +
 .../HadoopIgfsSecondaryFileSystemDelegate.java  |    28 +
 .../hadoop/fs/HadoopFileSystemCacheUtils.java   |   242 -
 .../hadoop/fs/HadoopFileSystemsUtils.java       |    51 -
 .../hadoop/fs/HadoopLazyConcurrentMap.java      |   212 -
 .../hadoop/fs/HadoopLocalFileSystemV1.java      |    39 -
 .../hadoop/fs/HadoopLocalFileSystemV2.java      |    88 -
 .../processors/hadoop/fs/HadoopParameters.java  |    94 -
 .../hadoop/fs/HadoopRawLocalFileSystem.java     |   314 -
 .../processors/hadoop/igfs/HadoopIgfs.java      |   202 -
 .../igfs/HadoopIgfsCommunicationException.java  |    57 -
 .../processors/hadoop/igfs/HadoopIgfsEx.java    |    93 -
 .../hadoop/igfs/HadoopIgfsFuture.java           |    97 -
 .../hadoop/igfs/HadoopIgfsInProc.java           |   510 -
 .../hadoop/igfs/HadoopIgfsInputStream.java      |   629 -
 .../processors/hadoop/igfs/HadoopIgfsIo.java    |    76 -
 .../processors/hadoop/igfs/HadoopIgfsIpcIo.java |   624 -
 .../hadoop/igfs/HadoopIgfsIpcIoListener.java    |    36 -
 .../hadoop/igfs/HadoopIgfsJclLogger.java        |   116 -
 .../hadoop/igfs/HadoopIgfsOutProc.java          |   524 -
 .../hadoop/igfs/HadoopIgfsOutputStream.java     |   201 -
 .../hadoop/igfs/HadoopIgfsProperties.java       |    86 -
 .../hadoop/igfs/HadoopIgfsProxyInputStream.java |   337 -
 .../igfs/HadoopIgfsProxyOutputStream.java       |   165 -
 ...fsSecondaryFileSystemPositionedReadable.java |   105 -
 .../hadoop/igfs/HadoopIgfsStreamDelegate.java   |    96 -
 .../igfs/HadoopIgfsStreamEventListener.java     |    39 -
 .../processors/hadoop/igfs/HadoopIgfsUtils.java |   174 -
 .../hadoop/igfs/HadoopIgfsWrapper.java          |   552 -
 .../impl/HadoopMapReduceCounterGroup.java       |   124 +
 .../hadoop/impl/HadoopMapReduceCounters.java    |   229 +
 .../processors/hadoop/impl/HadoopUtils.java     |   328 +
 .../HadoopBasicFileSystemFactoryDelegate.java   |   164 +
 .../HadoopCachingFileSystemFactoryDelegate.java |    75 +
 .../HadoopDefaultFileSystemFactoryDelegate.java |    62 +
 ...doopFileSystemCounterWriterDelegateImpl.java |   108 +
 ...doopIgfsSecondaryFileSystemDelegateImpl.java |   472 +
 ...HadoopKerberosFileSystemFactoryDelegate.java |   112 +
 .../impl/fs/HadoopFileSystemCacheUtils.java     |   243 +
 .../hadoop/impl/fs/HadoopFileSystemsUtils.java  |    51 +
 .../hadoop/impl/fs/HadoopLazyConcurrentMap.java |   210 +
 .../hadoop/impl/fs/HadoopLocalFileSystemV1.java |    40 +
 .../hadoop/impl/fs/HadoopLocalFileSystemV2.java |    89 +
 .../hadoop/impl/fs/HadoopParameters.java        |    94 +
 .../impl/fs/HadoopRawLocalFileSystem.java       |   315 +
 .../processors/hadoop/impl/igfs/HadoopIgfs.java |   203 +
 .../igfs/HadoopIgfsCommunicationException.java  |    57 +
 .../hadoop/impl/igfs/HadoopIgfsEx.java          |    94 +
 .../hadoop/impl/igfs/HadoopIgfsFuture.java      |    97 +
 .../hadoop/impl/igfs/HadoopIgfsInProc.java      |   511 +
 .../hadoop/impl/igfs/HadoopIgfsInputStream.java |   630 +
 .../hadoop/impl/igfs/HadoopIgfsIo.java          |    76 +
 .../hadoop/impl/igfs/HadoopIgfsIpcIo.java       |   625 +
 .../impl/igfs/HadoopIgfsIpcIoListener.java      |    36 +
 .../hadoop/impl/igfs/HadoopIgfsJclLogger.java   |   116 +
 .../hadoop/impl/igfs/HadoopIgfsOutProc.java     |   525 +
 .../impl/igfs/HadoopIgfsOutputStream.java       |   202 +
 .../hadoop/impl/igfs/HadoopIgfsProperties.java  |    87 +
 .../impl/igfs/HadoopIgfsProxyInputStream.java   |   338 +
 .../impl/igfs/HadoopIgfsProxyOutputStream.java  |   166 +
 ...fsSecondaryFileSystemPositionedReadable.java |   106 +
 .../impl/igfs/HadoopIgfsStreamDelegate.java     |    96 +
 .../igfs/HadoopIgfsStreamEventListener.java     |    39 +
 .../hadoop/impl/igfs/HadoopIgfsUtils.java       |   175 +
 .../hadoop/impl/igfs/HadoopIgfsWrapper.java     |   554 +
 .../hadoop/impl/proto/HadoopClientProtocol.java |   354 +
 .../hadoop/impl/v1/HadoopV1CleanupTask.java     |    65 +
 .../hadoop/impl/v1/HadoopV1Counter.java         |   107 +
 .../hadoop/impl/v1/HadoopV1MapTask.java         |   122 +
 .../hadoop/impl/v1/HadoopV1OutputCollector.java |   138 +
 .../hadoop/impl/v1/HadoopV1Partitioner.java     |    44 +
 .../hadoop/impl/v1/HadoopV1ReduceTask.java      |   101 +
 .../hadoop/impl/v1/HadoopV1Reporter.java        |    81 +
 .../hadoop/impl/v1/HadoopV1SetupTask.java       |    57 +
 .../hadoop/impl/v1/HadoopV1Splitter.java        |   103 +
 .../processors/hadoop/impl/v1/HadoopV1Task.java |    98 +
 .../processors/hadoop/impl/v2/HadoopDaemon.java |   126 +
 .../impl/v2/HadoopSerializationWrapper.java     |   139 +
 .../impl/v2/HadoopShutdownHookManager.java      |    98 +
 .../hadoop/impl/v2/HadoopV2CleanupTask.java     |    73 +
 .../hadoop/impl/v2/HadoopV2Context.java         |   244 +
 .../hadoop/impl/v2/HadoopV2Counter.java         |    89 +
 .../processors/hadoop/impl/v2/HadoopV2Job.java  |   452 +
 .../impl/v2/HadoopV2JobResourceManager.java     |   324 +
 .../hadoop/impl/v2/HadoopV2MapTask.java         |    99 +
 .../hadoop/impl/v2/HadoopV2Partitioner.java     |    44 +
 .../hadoop/impl/v2/HadoopV2ReduceTask.java      |    91 +
 .../hadoop/impl/v2/HadoopV2SetupTask.java       |    66 +
 .../hadoop/impl/v2/HadoopV2Splitter.java        |   112 +
 .../processors/hadoop/impl/v2/HadoopV2Task.java |   186 +
 .../hadoop/impl/v2/HadoopV2TaskContext.java     |   563 +
 .../impl/v2/HadoopWritableSerialization.java    |    76 +
 .../hadoop/jobtracker/HadoopJobTracker.java     |    23 +-
 .../planner/HadoopDefaultMapReducePlan.java     |     7 +-
 .../hadoop/proto/HadoopClientProtocol.java      |   349 -
 .../child/HadoopChildProcessRunner.java         |    16 +-
 .../hadoop/v1/HadoopV1CleanupTask.java          |    64 -
 .../processors/hadoop/v1/HadoopV1Counter.java   |   106 -
 .../processors/hadoop/v1/HadoopV1MapTask.java   |   122 -
 .../hadoop/v1/HadoopV1OutputCollector.java      |   137 -
 .../hadoop/v1/HadoopV1Partitioner.java          |    44 -
 .../hadoop/v1/HadoopV1ReduceTask.java           |   101 -
 .../processors/hadoop/v1/HadoopV1Reporter.java  |    81 -
 .../processors/hadoop/v1/HadoopV1SetupTask.java |    56 -
 .../processors/hadoop/v1/HadoopV1Splitter.java  |   102 -
 .../processors/hadoop/v1/HadoopV1Task.java      |    97 -
 .../processors/hadoop/v2/HadoopDaemon.java      |   126 -
 .../hadoop/v2/HadoopExternalSplit.java          |    89 -
 .../hadoop/v2/HadoopSerializationWrapper.java   |   138 -
 .../hadoop/v2/HadoopShutdownHookManager.java    |    98 -
 .../hadoop/v2/HadoopSplitWrapper.java           |   119 -
 .../hadoop/v2/HadoopV2CleanupTask.java          |    72 -
 .../processors/hadoop/v2/HadoopV2Context.java   |   243 -
 .../processors/hadoop/v2/HadoopV2Counter.java   |    88 -
 .../processors/hadoop/v2/HadoopV2Job.java       |   445 -
 .../hadoop/v2/HadoopV2JobResourceManager.java   |   323 -
 .../processors/hadoop/v2/HadoopV2MapTask.java   |    99 -
 .../hadoop/v2/HadoopV2Partitioner.java          |    44 -
 .../hadoop/v2/HadoopV2ReduceTask.java           |    91 -
 .../processors/hadoop/v2/HadoopV2SetupTask.java |    65 -
 .../processors/hadoop/v2/HadoopV2Splitter.java  |   111 -
 .../processors/hadoop/v2/HadoopV2Task.java      |   185 -
 .../hadoop/v2/HadoopV2TaskContext.java          |   560 -
 .../hadoop/v2/HadoopWritableSerialization.java  |    75 -
 .../HadoopClientProtocolEmbeddedSelfTest.java   |    35 -
 .../hadoop/HadoopClientProtocolSelfTest.java    |   654 -
 .../hadoop/cache/HadoopTxConfigCacheTest.java   |    42 -
 ...KerberosHadoopFileSystemFactorySelfTest.java |   121 -
 .../util/BasicUserNameMapperSelfTest.java       |   133 -
 .../util/ChainedUserNameMapperSelfTest.java     |   107 -
 .../util/KerberosUserNameMapperSelfTest.java    |    99 -
 .../ignite/igfs/Hadoop1DualAbstractTest.java    |   158 -
 .../igfs/Hadoop1OverIgfsDualAsyncTest.java      |    30 -
 .../igfs/Hadoop1OverIgfsDualSyncTest.java       |    30 -
 .../igfs/HadoopFIleSystemFactorySelfTest.java   |   317 -
 .../HadoopIgfs20FileSystemAbstractSelfTest.java |  2040 ---
 ...Igfs20FileSystemLoopbackPrimarySelfTest.java |    74 -
 ...oopIgfs20FileSystemShmemPrimarySelfTest.java |    74 -
 .../igfs/HadoopIgfsDualAbstractSelfTest.java    |   321 -
 .../igfs/HadoopIgfsDualAsyncSelfTest.java       |    32 -
 .../ignite/igfs/HadoopIgfsDualSyncSelfTest.java |    32 -
 ...adoopIgfsSecondaryFileSystemTestAdapter.java |   149 -
 ...oopSecondaryFileSystemConfigurationTest.java |   575 -
 .../apache/ignite/igfs/IgfsEventsTestSuite.java |   285 -
 .../igfs/IgfsNearOnlyMultiNodeSelfTest.java     |   223 -
 .../IgniteHadoopFileSystemAbstractSelfTest.java |  2432 ---
 .../IgniteHadoopFileSystemClientSelfTest.java   |   212 -
 ...IgniteHadoopFileSystemHandshakeSelfTest.java |   389 -
 .../IgniteHadoopFileSystemIpcCacheSelfTest.java |   214 -
 .../IgniteHadoopFileSystemLoggerSelfTest.java   |   298 -
 ...niteHadoopFileSystemLoggerStateSelfTest.java |   329 -
 ...adoopFileSystemLoopbackAbstractSelfTest.java |    46 -
 ...SystemLoopbackEmbeddedDualAsyncSelfTest.java |    33 -
 ...eSystemLoopbackEmbeddedDualSyncSelfTest.java |    33 -
 ...leSystemLoopbackEmbeddedPrimarySelfTest.java |    33 -
 ...SystemLoopbackEmbeddedSecondarySelfTest.java |    34 -
 ...SystemLoopbackExternalDualAsyncSelfTest.java |    33 -
 ...eSystemLoopbackExternalDualSyncSelfTest.java |    33 -
 ...leSystemLoopbackExternalPrimarySelfTest.java |    33 -
 ...SystemLoopbackExternalSecondarySelfTest.java |    34 -
 ...condaryFileSystemInitializationSelfTest.java |   214 -
 ...teHadoopFileSystemShmemAbstractSelfTest.java |    91 -
 ...ileSystemShmemEmbeddedDualAsyncSelfTest.java |    33 -
 ...FileSystemShmemEmbeddedDualSyncSelfTest.java |    33 -
 ...pFileSystemShmemEmbeddedPrimarySelfTest.java |    33 -
 ...ileSystemShmemEmbeddedSecondarySelfTest.java |    33 -
 ...ileSystemShmemExternalDualAsyncSelfTest.java |    33 -
 ...FileSystemShmemExternalDualSyncSelfTest.java |    33 -
 ...pFileSystemShmemExternalPrimarySelfTest.java |    33 -
 ...ileSystemShmemExternalSecondarySelfTest.java |    33 -
 .../hadoop/HadoopAbstractMapReduceTest.java     |   429 -
 .../hadoop/HadoopAbstractSelfTest.java          |   239 -
 .../hadoop/HadoopAbstractWordCountTest.java     |   175 -
 .../hadoop/HadoopClassLoaderTest.java           |   110 -
 .../hadoop/HadoopCommandLineTest.java           |   474 -
 .../HadoopDefaultMapReducePlannerSelfTest.java  |   615 -
 .../processors/hadoop/HadoopErrorSimulator.java |   326 -
 .../hadoop/HadoopFileSystemsTest.java           |   155 -
 .../processors/hadoop/HadoopGroupingTest.java   |   307 -
 .../hadoop/HadoopJobTrackerSelfTest.java        |   345 -
 .../hadoop/HadoopMapReduceEmbeddedSelfTest.java |   253 -
 .../HadoopMapReduceErrorResilienceTest.java     |   154 -
 .../processors/hadoop/HadoopMapReduceTest.java  |    66 -
 .../hadoop/HadoopNoHadoopMapReduceTest.java     |    47 -
 .../processors/hadoop/HadoopPlannerMockJob.java |   168 -
 .../hadoop/HadoopPopularWordsTest.java          |   298 -
 .../HadoopSerializationWrapperSelfTest.java     |    79 -
 .../processors/hadoop/HadoopSharedMap.java      |     1 +
 .../hadoop/HadoopSnappyFullMapReduceTest.java   |    36 -
 .../processors/hadoop/HadoopSnappyTest.java     |   102 -
 .../hadoop/HadoopSortingExternalTest.java       |    46 -
 .../processors/hadoop/HadoopSortingTest.java    |   303 -
 .../hadoop/HadoopSplitWrapperSelfTest.java      |    72 -
 .../processors/hadoop/HadoopStartup.java        |    54 -
 .../hadoop/HadoopTaskExecutionSelfTest.java     |   567 -
 .../hadoop/HadoopTasksAllVersionsTest.java      |   260 -
 .../processors/hadoop/HadoopTasksV1Test.java    |    58 -
 .../processors/hadoop/HadoopTasksV2Test.java    |    77 -
 .../hadoop/HadoopTestClassLoader.java           |   106 +
 .../hadoop/HadoopTestRoundRobinMrPlanner.java   |    71 -
 .../hadoop/HadoopTestTaskContext.java           |   228 -
 .../processors/hadoop/HadoopTestUtils.java      |   178 -
 .../hadoop/HadoopUserLibsSelfTest.java          |   260 -
 .../processors/hadoop/HadoopV2JobSelfTest.java  |   100 -
 .../hadoop/HadoopValidationSelfTest.java        |    53 -
 .../HadoopWeightedMapReducePlannerTest.java     |   599 -
 .../HadoopWeightedPlannerMapReduceTest.java     |    38 -
 .../hadoop/books/alice-in-wonderland.txt        |  3735 -----
 .../processors/hadoop/books/art-of-war.txt      |  6982 ---------
 .../hadoop/books/huckleberry-finn.txt           | 11733 ---------------
 .../processors/hadoop/books/sherlock-holmes.txt | 13052 -----------------
 .../processors/hadoop/books/tom-sawyer.txt      |  8858 -----------
 .../hadoop/deps/CircularWIthHadoop.java         |    32 -
 .../hadoop/deps/CircularWithoutHadoop.java      |    27 -
 .../processors/hadoop/deps/WithCast.java        |    41 -
 .../hadoop/deps/WithClassAnnotation.java        |    28 -
 .../hadoop/deps/WithConstructorInvocation.java  |    31 -
 .../processors/hadoop/deps/WithExtends.java     |    27 -
 .../processors/hadoop/deps/WithField.java       |    29 -
 .../processors/hadoop/deps/WithImplements.java  |    36 -
 .../hadoop/deps/WithIndirectField.java          |    27 -
 .../processors/hadoop/deps/WithInitializer.java |    33 -
 .../processors/hadoop/deps/WithInnerClass.java  |    31 -
 .../hadoop/deps/WithLocalVariable.java          |    38 -
 .../hadoop/deps/WithMethodAnnotation.java       |    32 -
 .../hadoop/deps/WithMethodArgument.java         |    31 -
 .../hadoop/deps/WithMethodCheckedException.java |    31 -
 .../hadoop/deps/WithMethodInvocation.java       |    31 -
 .../hadoop/deps/WithMethodReturnType.java       |    31 -
 .../hadoop/deps/WithMethodRuntimeException.java |    31 -
 .../processors/hadoop/deps/WithOuterClass.java  |    38 -
 .../hadoop/deps/WithParameterAnnotation.java    |    31 -
 .../processors/hadoop/deps/WithStaticField.java |    29 -
 .../hadoop/deps/WithStaticInitializer.java      |    34 -
 .../processors/hadoop/deps/Without.java         |    25 -
 .../hadoop/examples/HadoopWordCount1.java       |    94 -
 .../hadoop/examples/HadoopWordCount1Map.java    |    79 -
 .../hadoop/examples/HadoopWordCount1Reduce.java |    61 -
 .../hadoop/examples/HadoopWordCount2.java       |   111 -
 .../examples/HadoopWordCount2Combiner.java      |    45 -
 .../hadoop/examples/HadoopWordCount2Mapper.java |    88 -
 .../examples/HadoopWordCount2Reducer.java       |   113 -
 .../impl/HadoopAbstractMapReduceTest.java       |   430 +
 .../hadoop/impl/HadoopAbstractSelfTest.java     |   239 +
 .../impl/HadoopAbstractWordCountTest.java       |   175 +
 .../hadoop/impl/HadoopCommandLineTest.java      |   476 +
 .../HadoopDefaultMapReducePlannerSelfTest.java  |   619 +
 .../hadoop/impl/HadoopErrorSimulator.java       |   326 +
 .../hadoop/impl/HadoopFileSystemsTest.java      |   155 +
 .../hadoop/impl/HadoopGroupingTest.java         |   302 +
 .../hadoop/impl/HadoopJobTrackerSelfTest.java   |   334 +
 .../impl/HadoopMapReduceEmbeddedSelfTest.java   |   249 +
 .../HadoopMapReduceErrorResilienceTest.java     |   154 +
 .../hadoop/impl/HadoopMapReduceTest.java        |    66 +
 .../impl/HadoopNoHadoopMapReduceTest.java       |    47 +
 .../hadoop/impl/HadoopPlannerMockJob.java       |   175 +
 .../hadoop/impl/HadoopPopularWordsTest.java     |   298 +
 .../HadoopSerializationWrapperSelfTest.java     |    80 +
 .../impl/HadoopSnappyFullMapReduceTest.java     |    36 +
 .../hadoop/impl/HadoopSnappyTest.java           |   104 +
 .../hadoop/impl/HadoopSortingExternalTest.java  |    46 +
 .../hadoop/impl/HadoopSortingTest.java          |   304 +
 .../hadoop/impl/HadoopSplitWrapperSelfTest.java |    72 +
 .../processors/hadoop/impl/HadoopStartup.java   |    54 +
 .../impl/HadoopTaskExecutionSelfTest.java       |   550 +
 .../hadoop/impl/HadoopTasksAllVersionsTest.java |   264 +
 .../hadoop/impl/HadoopTasksV1Test.java          |    62 +
 .../hadoop/impl/HadoopTasksV2Test.java          |    81 +
 .../impl/HadoopTestRoundRobinMrPlanner.java     |    75 +
 .../hadoop/impl/HadoopTestTaskContext.java      |   233 +
 .../processors/hadoop/impl/HadoopTestUtils.java |   178 +
 .../hadoop/impl/HadoopTxConfigCacheTest.java    |    42 +
 .../hadoop/impl/HadoopUserLibsSelfTest.java     |   261 +
 .../hadoop/impl/HadoopV2JobSelfTest.java        |   108 +
 .../hadoop/impl/HadoopValidationSelfTest.java   |    53 +
 .../HadoopWeightedMapReducePlannerTest.java     |   602 +
 .../HadoopWeightedPlannerMapReduceTest.java     |    38 +
 .../hadoop/impl/books/alice-in-wonderland.txt   |  3735 +++++
 .../processors/hadoop/impl/books/art-of-war.txt |  6982 +++++++++
 .../hadoop/impl/books/huckleberry-finn.txt      | 11733 +++++++++++++++
 .../hadoop/impl/books/sherlock-holmes.txt       | 13052 +++++++++++++++++
 .../processors/hadoop/impl/books/tom-sawyer.txt |  8858 +++++++++++
 .../HadoopClientProtocolEmbeddedSelfTest.java   |    35 +
 .../client/HadoopClientProtocolSelfTest.java    |   654 +
 .../hadoop/impl/examples/HadoopWordCount1.java  |    94 +
 .../impl/examples/HadoopWordCount1Map.java      |    79 +
 .../impl/examples/HadoopWordCount1Reduce.java   |    61 +
 .../hadoop/impl/examples/HadoopWordCount2.java  |   111 +
 .../impl/examples/HadoopWordCount2Combiner.java |    45 +
 .../impl/examples/HadoopWordCount2Mapper.java   |    88 +
 .../impl/examples/HadoopWordCount2Reducer.java  |   113 +
 ...KerberosHadoopFileSystemFactorySelfTest.java |   126 +
 .../impl/igfs/Hadoop1DualAbstractTest.java      |   157 +
 .../impl/igfs/Hadoop1OverIgfsDualAsyncTest.java |    32 +
 .../impl/igfs/Hadoop1OverIgfsDualSyncTest.java  |    32 +
 .../igfs/HadoopFIleSystemFactorySelfTest.java   |   345 +
 .../HadoopIgfs20FileSystemAbstractSelfTest.java |  2047 +++
 ...Igfs20FileSystemLoopbackPrimarySelfTest.java |    77 +
 ...oopIgfs20FileSystemShmemPrimarySelfTest.java |    77 +
 .../igfs/HadoopIgfsDualAbstractSelfTest.java    |   328 +
 .../impl/igfs/HadoopIgfsDualAsyncSelfTest.java  |    32 +
 .../impl/igfs/HadoopIgfsDualSyncSelfTest.java   |    32 +
 ...adoopIgfsSecondaryFileSystemTestAdapter.java |   153 +
 ...oopSecondaryFileSystemConfigurationTest.java |   583 +
 .../hadoop/impl/igfs/IgfsEventsTestSuite.java   |   289 +
 .../igfs/IgfsNearOnlyMultiNodeSelfTest.java     |   226 +
 .../IgniteHadoopFileSystemAbstractSelfTest.java |  2435 +++
 .../IgniteHadoopFileSystemClientSelfTest.java   |   216 +
 ...IgniteHadoopFileSystemHandshakeSelfTest.java |   393 +
 .../IgniteHadoopFileSystemIpcCacheSelfTest.java |   215 +
 .../IgniteHadoopFileSystemLoggerSelfTest.java   |   299 +
 ...niteHadoopFileSystemLoggerStateSelfTest.java |   332 +
 ...adoopFileSystemLoopbackAbstractSelfTest.java |    50 +
 ...SystemLoopbackEmbeddedDualAsyncSelfTest.java |    33 +
 ...eSystemLoopbackEmbeddedDualSyncSelfTest.java |    33 +
 ...leSystemLoopbackEmbeddedPrimarySelfTest.java |    33 +
 ...SystemLoopbackEmbeddedSecondarySelfTest.java |    34 +
 ...SystemLoopbackExternalDualAsyncSelfTest.java |    33 +
 ...eSystemLoopbackExternalDualSyncSelfTest.java |    33 +
 ...leSystemLoopbackExternalPrimarySelfTest.java |    33 +
 ...SystemLoopbackExternalSecondarySelfTest.java |    34 +
 ...condaryFileSystemInitializationSelfTest.java |   217 +
 ...teHadoopFileSystemShmemAbstractSelfTest.java |    94 +
 ...ileSystemShmemEmbeddedDualAsyncSelfTest.java |    33 +
 ...FileSystemShmemEmbeddedDualSyncSelfTest.java |    33 +
 ...pFileSystemShmemEmbeddedPrimarySelfTest.java |    33 +
 ...ileSystemShmemEmbeddedSecondarySelfTest.java |    33 +
 ...ileSystemShmemExternalDualAsyncSelfTest.java |    33 +
 ...FileSystemShmemExternalDualSyncSelfTest.java |    33 +
 ...pFileSystemShmemExternalPrimarySelfTest.java |    33 +
 ...ileSystemShmemExternalSecondarySelfTest.java |    33 +
 .../collections/HadoopAbstractMapTest.java      |   175 +
 .../HadoopConcurrentHashMultimapSelftest.java   |   280 +
 .../collections/HadoopHashMapSelfTest.java      |   133 +
 .../collections/HadoopSkipListSelfTest.java     |   320 +
 .../streams/HadoopDataStreamSelfTest.java       |   153 +
 .../taskexecutor/HadoopExecutorServiceTest.java |   119 +
 .../HadoopExternalTaskExecutionSelfTest.java    |   232 +
 .../HadoopExternalCommunicationSelfTest.java    |   222 +
 .../impl/util/BasicUserNameMapperSelfTest.java  |   134 +
 .../util/ChainedUserNameMapperSelfTest.java     |   111 +
 .../util/KerberosUserNameMapperSelfTest.java    |   100 +
 .../collections/HadoopAbstractMapTest.java      |   174 -
 .../HadoopConcurrentHashMultimapSelftest.java   |   278 -
 .../collections/HadoopHashMapSelfTest.java      |   131 -
 .../collections/HadoopSkipListSelfTest.java     |   318 -
 .../streams/HadoopDataStreamSelfTest.java       |   150 -
 .../hadoop/state/HadoopGroupingTestState.java   |    40 +
 .../state/HadoopJobTrackerSelfTestState.java    |    45 +
 .../HadoopMapReduceEmbeddedSelfTestState.java   |    32 +
 .../HadoopTaskExecutionSelfTestValues.java      |    51 +
 .../taskexecutor/HadoopExecutorServiceTest.java |   118 -
 .../HadoopExternalTaskExecutionSelfTest.java    |   232 -
 .../HadoopExternalCommunicationSelfTest.java    |   220 -
 .../testsuites/IgniteHadoopTestSuite.java       |   114 +-
 .../IgniteIgfsLinuxAndMacOSTestSuite.java       |    25 +-
 402 files changed, 78397 insertions(+), 78325 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/8032fc2c/bin/include/setenv.bat
----------------------------------------------------------------------
diff --git a/bin/include/setenv.bat b/bin/include/setenv.bat
index 9d55521..82b8fce 100644
--- a/bin/include/setenv.bat
+++ b/bin/include/setenv.bat
@@ -43,16 +43,8 @@ set IGNITE_LIBS=%IGNITE_HOME%\libs\*
 
 for /D %%F in (%IGNITE_HOME%\libs\*) do if not "%%F" == "%IGNITE_HOME%\libs\optional" call :concat %%F\*
 
-if exist %IGNITE_HOME%\libs\ignite-hadoop set HADOOP_EDITION=1
-
 if defined USER_LIBS set IGNITE_LIBS=%USER_LIBS%;%IGNITE_LIBS%
 
-if "%HADOOP_EDITION%" == "1" FOR /F "delims=" %%i IN ('%JAVA_HOME%\bin\java.exe -cp %IGNITE_HOME%\libs\ignite-hadoop\* org.apache.ignite.internal.processors.hadoop.HadoopClasspathMain ";"' ) DO set IGNITE_HADOOP_CLASSPATH=%%i
-
-if "%IGNITE_HADOOP_CLASSPATH%" == "" goto :eof
-
-set IGNITE_LIBS=%IGNITE_LIBS%;%IGNITE_HADOOP_CLASSPATH%
-
 goto :eof
 
 :concat

http://git-wip-us.apache.org/repos/asf/ignite/blob/8032fc2c/bin/include/setenv.sh
----------------------------------------------------------------------
diff --git a/bin/include/setenv.sh b/bin/include/setenv.sh
index a85cba3..e088c08 100755
--- a/bin/include/setenv.sh
+++ b/bin/include/setenv.sh
@@ -61,39 +61,8 @@ do
     if [ -d ${file} ] && [ "${file}" != "${IGNITE_HOME}"/libs/optional ]; then
         IGNITE_LIBS=${IGNITE_LIBS}${SEP}${file}/*
     fi
-
-    if [ -d ${file} ] && [ "${file}" == "${IGNITE_HOME}"/libs/ignite-hadoop ]; then
-        HADOOP_EDITION=1
-    fi
 done
 
 if [ "${USER_LIBS}" != "" ]; then
     IGNITE_LIBS=${USER_LIBS}${SEP}${IGNITE_LIBS}
 fi
-
-if [ "${HADOOP_EDITION}" == "1" ]; then
-    # Resolve constants.
-    HADOOP_DEFAULTS="/etc/default/hadoop"
-
-    #
-    # Resolve the rest of Hadoop environment variables.
-    #
-    if [[ -z "${HADOOP_COMMON_HOME}" || -z "${HADOOP_HDFS_HOME}" || -z "${HADOOP_MAPRED_HOME}" ]]; then
-        if [ -f "$HADOOP_DEFAULTS" ]; then
-            source "$HADOOP_DEFAULTS"
-        fi
-    fi
-
-    IGNITE_HADOOP_CLASSPATH=$( "$JAVA" -cp "${IGNITE_HOME}"/libs/ignite-hadoop/'*' \
-        org.apache.ignite.internal.processors.hadoop.HadoopClasspathMain ":" )
-
-    statusCode=${?}
-
-    if [ "${statusCode}" -ne 0 ]; then
-       exit ${statusCode}
-    fi
-
-    unset statusCode
-
-    IGNITE_LIBS=${IGNITE_LIBS}${SEP}${IGNITE_HADOOP_CLASSPATH}
-fi

http://git-wip-us.apache.org/repos/asf/ignite/blob/8032fc2c/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
index 3eaef1e..b123a4a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
@@ -44,6 +44,7 @@ import org.apache.ignite.internal.processors.continuous.GridContinuousProcessor;
 import org.apache.ignite.internal.processors.datastreamer.DataStreamProcessor;
 import org.apache.ignite.internal.processors.datastructures.DataStructuresProcessor;
 import org.apache.ignite.internal.processors.hadoop.HadoopProcessorAdapter;
+import org.apache.ignite.internal.processors.hadoop.HadoopHelper;
 import org.apache.ignite.internal.processors.igfs.IgfsHelper;
 import org.apache.ignite.internal.processors.igfs.IgfsProcessorAdapter;
 import org.apache.ignite.internal.processors.job.GridJobProcessor;
@@ -285,6 +286,13 @@ public interface GridKernalContext extends Iterable<GridComponent> {
     public HadoopProcessorAdapter hadoop();
 
     /**
+     * Gets Hadoop helper.
+     *
+     * @return Hadoop helper.
+     */
+    public HadoopHelper hadoopHelper();
+
+    /**
      * Gets utility cache pool.
      *
      * @return Utility cache pool.

http://git-wip-us.apache.org/repos/asf/ignite/blob/8032fc2c/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
index 1ff4543..eb214e8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
@@ -60,6 +60,7 @@ import org.apache.ignite.internal.processors.continuous.GridContinuousProcessor;
 import org.apache.ignite.internal.processors.datastreamer.DataStreamProcessor;
 import org.apache.ignite.internal.processors.datastructures.DataStructuresProcessor;
 import org.apache.ignite.internal.processors.hadoop.HadoopProcessorAdapter;
+import org.apache.ignite.internal.processors.hadoop.HadoopHelper;
 import org.apache.ignite.internal.processors.igfs.IgfsHelper;
 import org.apache.ignite.internal.processors.igfs.IgfsProcessorAdapter;
 import org.apache.ignite.internal.processors.job.GridJobProcessor;
@@ -238,6 +239,10 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable
 
     /** */
     @GridToStringInclude
+    private HadoopHelper hadoopHelper;
+
+    /** */
+    @GridToStringInclude
     private GridSegmentationProcessor segProc;
 
     /** */
@@ -541,6 +546,8 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable
 
         if (helper instanceof IgfsHelper)
             igfsHelper = (IgfsHelper)helper;
+        else if (helper instanceof HadoopHelper)
+            hadoopHelper = (HadoopHelper)helper;
         else
             assert false : "Unknown helper class: " + helper.getClass();
     }
@@ -733,6 +740,11 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable
     }
 
     /** {@inheritDoc} */
+    @Override public HadoopHelper hadoopHelper() {
+        return hadoopHelper;
+    }
+
+    /** {@inheritDoc} */
     @Override public GridContinuousProcessor continuous() {
         return contProc;
     }

http://git-wip-us.apache.org/repos/asf/ignite/blob/8032fc2c/modules/core/src/main/java/org/apache/ignite/internal/IgniteComponentType.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteComponentType.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteComponentType.java
index 76e495f..0cd2fc1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteComponentType.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteComponentType.java
@@ -41,6 +41,13 @@ public enum IgniteComponentType {
         "ignite-hadoop"
     ),
 
+    /** Hadoop Helper component. */
+    HADOOP_HELPER(
+        "org.apache.ignite.internal.processors.hadoop.HadoopNoopHelper",
+        "org.apache.ignite.internal.processors.hadoop.HadoopHelperImpl",
+        "ignite-hadoop"
+    ),
+
     /** IGFS helper component. */
     IGFS_HELPER(
         "org.apache.ignite.internal.processors.igfs.IgfsNoopHelper",
@@ -160,7 +167,7 @@ public enum IgniteComponentType {
      * @return Created component.
      * @throws IgniteCheckedException If failed.
      */
-    public <T extends GridComponent> T create(GridKernalContext ctx, boolean noOp) throws IgniteCheckedException {
+    public <T> T create(GridKernalContext ctx, boolean noOp) throws IgniteCheckedException {
         return create0(ctx, noOp ? noOpClsName : clsName);
     }
 
@@ -172,7 +179,7 @@ public enum IgniteComponentType {
      * @return Created component.
      * @throws IgniteCheckedException If failed.
      */
-    public <T extends GridComponent> T createIfInClassPath(GridKernalContext ctx, boolean mandatory)
+    public <T> T createIfInClassPath(GridKernalContext ctx, boolean mandatory)
         throws IgniteCheckedException {
         String cls = clsName;
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/8032fc2c/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index c5d2748..6c5a628 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -115,6 +115,7 @@ import org.apache.ignite.internal.processors.continuous.GridContinuousProcessor;
 import org.apache.ignite.internal.processors.datastreamer.DataStreamProcessor;
 import org.apache.ignite.internal.processors.datastructures.DataStructuresProcessor;
 import org.apache.ignite.internal.processors.hadoop.Hadoop;
+import org.apache.ignite.internal.processors.hadoop.HadoopClassLoader;
 import org.apache.ignite.internal.processors.hadoop.HadoopProcessorAdapter;
 import org.apache.ignite.internal.processors.job.GridJobProcessor;
 import org.apache.ignite.internal.processors.jobmetrics.GridJobMetricsProcessor;
@@ -188,6 +189,7 @@ import static org.apache.ignite.internal.GridKernalState.STARTED;
 import static org.apache.ignite.internal.GridKernalState.STARTING;
 import static org.apache.ignite.internal.GridKernalState.STOPPED;
 import static org.apache.ignite.internal.GridKernalState.STOPPING;
+import static org.apache.ignite.internal.IgniteComponentType.HADOOP_HELPER;
 import static org.apache.ignite.internal.IgniteComponentType.IGFS;
 import static org.apache.ignite.internal.IgniteComponentType.IGFS_HELPER;
 import static org.apache.ignite.internal.IgniteComponentType.SCHEDULE;
@@ -821,6 +823,8 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
 
             addHelper(IGFS_HELPER.create(F.isEmpty(cfg.getFileSystemConfiguration())));
 
+            addHelper(HADOOP_HELPER.createIfInClassPath(ctx, false));
+
             startProcessor(new IgnitePluginProcessor(ctx, cfg, plugins));
 
             // Off-heap processor has no dependencies.
@@ -881,7 +885,7 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
             startProcessor(new DataStreamProcessor(ctx));
             startProcessor((GridProcessor)IGFS.create(ctx, F.isEmpty(cfg.getFileSystemConfiguration())));
             startProcessor(new GridContinuousProcessor(ctx));
-            startProcessor((GridProcessor)createHadoopComponent());
+            startProcessor(createHadoopComponent());
             startProcessor(new DataStructuresProcessor(ctx));
             startProcessor(createComponent(PlatformProcessor.class, ctx));
 
@@ -1148,7 +1152,7 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
         else {
             HadoopProcessorAdapter cmp = null;
 
-            if (IgniteComponentType.HADOOP.inClassPath() && cfg.isPeerClassLoadingEnabled()) {
+            if (!ctx.hadoopHelper().isNoOp() && cfg.isPeerClassLoadingEnabled()) {
                 U.warn(log, "Hadoop module is found in classpath, but will not be started because peer class " +
                     "loading is enabled (set IgniteConfiguration.peerClassLoadingEnabled to \"false\" if you want " +
                     "to use Hadoop module).");

http://git-wip-us.apache.org/repos/asf/ignite/blob/8032fc2c/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
new file mode 100644
index 0000000..cd94c89
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClassLoader.java
@@ -0,0 +1,487 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.internal.util.ClassCache;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.jetbrains.annotations.Nullable;
+import org.jsr166.ConcurrentHashMap8;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.Map;
+import java.util.Vector;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+/**
+ * Class loader allowing explicitly load classes without delegation to parent class loader.
+ * Also supports class parsing for finding dependencies which contain transitive dependencies
+ * unavailable for parent.
+ */
+public class HadoopClassLoader extends URLClassLoader implements ClassCache {
+    /** Hadoop class name: Daemon. */
+    public static final String CLS_DAEMON = "org.apache.hadoop.util.Daemon";
+
+    /** Hadoop class name: ShutdownHookManager. */
+    public static final String CLS_SHUTDOWN_HOOK_MANAGER = "org.apache.hadoop.util.ShutdownHookManager";
+
+    /** Hadoop class name: Daemon replacement. */
+    public static final String CLS_DAEMON_REPLACE = "org.apache.ignite.internal.processors.hadoop.impl.v2.HadoopDaemon";
+
+    /** Hadoop class name: ShutdownHookManager replacement. */
+    public static final String CLS_SHUTDOWN_HOOK_MANAGER_REPLACE =
+        "org.apache.ignite.internal.processors.hadoop.impl.v2.HadoopShutdownHookManager";
+
+    /** */
+    private static final URLClassLoader APP_CLS_LDR = (URLClassLoader)HadoopClassLoader.class.getClassLoader();
+
+    /** */
+    private static final Collection<URL> appJars = F.asList(APP_CLS_LDR.getURLs());
+
+    /** Mutex for native libraries initialization. */
+    private static final Object LIBS_MUX = new Object();
+
+    /** Predefined native libraries to load. */
+    private static final Collection<String> PREDEFINED_NATIVE_LIBS;
+
+    /** Native libraries. */
+    private static Collection<Object> NATIVE_LIBS;
+
+    /** */
+    private static volatile Collection<URL> hadoopJars;
+
+    /** */
+    private static final Map<String, byte[]> bytesCache = new ConcurrentHashMap8<>();
+
+    /** Class cache. */
+    private final ConcurrentMap<String, Class> cacheMap = new ConcurrentHashMap<>();
+
+    /** Diagnostic name of this class loader. */
+    @SuppressWarnings({"FieldCanBeLocal", "UnusedDeclaration"})
+    private final String name;
+
+    /** Igfs Helper. */
+    private final HadoopHelper helper;
+
+    static {
+        // We are very parallel capable.
+        registerAsParallelCapable();
+
+        PREDEFINED_NATIVE_LIBS = new HashSet<>();
+
+        PREDEFINED_NATIVE_LIBS.add("hadoop");
+        PREDEFINED_NATIVE_LIBS.add("MapRClient");
+    }
+
+    /**
+     * Gets name for the task class loader. Task class loader
+     * @param info The task info.
+     * @param prefix Get only prefix (without task type and number)
+     * @return The class loader name.
+     */
+    public static String nameForTask(HadoopTaskInfo info, boolean prefix) {
+        if (prefix)
+            return "hadoop-task-" + info.jobId() + "-";
+        else
+            return "hadoop-task-" + info.jobId() + "-" + info.type() + "-" + info.taskNumber();
+    }
+
+    /**
+     * Constructor.
+     *
+     * @param urls Urls.
+     * @param name Classloader name.
+     * @param libNames Optional additional native library names to be linked from parent classloader.
+     */
+    public HadoopClassLoader(URL[] urls, String name, @Nullable String[] libNames, HadoopHelper helper) {
+        super(addHadoopUrls(urls), APP_CLS_LDR);
+
+        assert !(getParent() instanceof HadoopClassLoader);
+
+        this.name = name;
+        this.helper = helper;
+
+        initializeNativeLibraries(libNames);
+    }
+
+    /**
+     * Workaround to load native Hadoop libraries. Java doesn't allow native libraries to be loaded from different
+     * classloaders. But we load Hadoop classes many times and one of these classes - {@code NativeCodeLoader} - tries
+     * to load the same native library over and over again.
+     * <p>
+     * To fix the problem, we force native library load in parent class loader and then "link" handle to this native
+     * library to our class loader. As a result, our class loader will think that the library is already loaded and will
+     * be able to link native methods.
+     *
+     * @see <a href="http://docs.oracle.com/javase/1.5.0/docs/guide/jni/spec/invocation.html#library_version">
+     *     JNI specification</a>
+     */
+    @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
+    private void initializeNativeLibraries(@Nullable String[] usrLibs) {
+        Collection<Object> res;
+
+        synchronized (LIBS_MUX) {
+            if (NATIVE_LIBS == null) {
+                LinkedList<NativeLibrary> libs = new LinkedList<>();
+
+                for (String lib : PREDEFINED_NATIVE_LIBS)
+                    libs.add(new NativeLibrary(lib, true));
+
+                if (!F.isEmpty(usrLibs)) {
+                    for (String usrLib : usrLibs)
+                        libs.add(new NativeLibrary(usrLib, false));
+                }
+
+                NATIVE_LIBS = initializeNativeLibraries0(libs);
+            }
+
+            res = NATIVE_LIBS;
+        }
+
+        // Link libraries to class loader.
+        Vector<Object> ldrLibs = nativeLibraries(this);
+
+        synchronized (ldrLibs) {
+            ldrLibs.addAll(res);
+        }
+    }
+
+    /**
+     * Initialize native libraries.
+     *
+     * @param libs Libraries to initialize.
+     * @return Initialized libraries.
+     */
+    @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
+    private static Collection<Object> initializeNativeLibraries0(Collection<NativeLibrary> libs) {
+        assert Thread.holdsLock(LIBS_MUX);
+
+        Collection<Object> res = new HashSet<>();
+
+        for (NativeLibrary lib : libs) {
+            String libName = lib.name;
+
+            File libFile = new File(libName);
+
+            try {
+                // Load library.
+                if (libFile.isAbsolute())
+                    System.load(libName);
+                else
+                    System.loadLibrary(libName);
+
+                // Find library in class loader internals.
+                Object libObj = null;
+
+                ClassLoader ldr = APP_CLS_LDR;
+
+                while (ldr != null) {
+                    Vector<Object> ldrLibObjs = nativeLibraries(ldr);
+
+                    synchronized (ldrLibObjs) {
+                        for (Object ldrLibObj : ldrLibObjs) {
+                            String name = nativeLibraryName(ldrLibObj);
+
+                            if (libFile.isAbsolute()) {
+                                if (F.eq(name, libFile.getCanonicalPath())) {
+                                    libObj = ldrLibObj;
+
+                                    break;
+                                }
+                            } else {
+                                if (name.contains(libName)) {
+                                    libObj = ldrLibObj;
+
+                                    break;
+                                }
+                            }
+                        }
+                    }
+
+                    if (libObj != null)
+                        break;
+
+                    ldr = ldr.getParent();
+                }
+
+                if (libObj == null)
+                    throw new IgniteException("Failed to find loaded library: " + libName);
+
+                res.add(libObj);
+            }
+            catch (UnsatisfiedLinkError e) {
+                if (!lib.optional)
+                    throw e;
+            }
+            catch (IOException e) {
+                throw new IgniteException("Failed to initialize native libraries due to unexpected exception.", e);
+            }
+        }
+
+        return res;
+    }
+
+    /**
+     * Get native libraries collection for the given class loader.
+     *
+     * @param ldr Class loaded.
+     * @return Native libraries.
+     */
+    private static Vector<Object> nativeLibraries(ClassLoader ldr) {
+        assert ldr != null;
+
+        return U.field(ldr, "nativeLibraries");
+    }
+
+    /**
+     * Get native library name.
+     *
+     * @param lib Library.
+     * @return Name.
+     */
+    private static String nativeLibraryName(Object lib) {
+        assert lib != null;
+
+        return U.field(lib, "name");
+    }
+
+    /** {@inheritDoc} */
+    @Override protected Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundException {
+        try {
+            // Always load Hadoop classes explicitly, since Hadoop can be available in App classpath.
+            if (name.equals(CLS_SHUTDOWN_HOOK_MANAGER))  // Dirty hack to get rid of Hadoop shutdown hooks.
+                return loadReplace(name, CLS_SHUTDOWN_HOOK_MANAGER_REPLACE);
+            else if (name.equals(CLS_DAEMON))
+                // We replace this in order to be able to forcibly stop some daemon threads
+                // that otherwise never stop (e.g. PeerCache runnables):
+                return loadReplace(name, CLS_DAEMON_REPLACE);
+
+            // For Ignite Hadoop and IGFS classes we have to check if they depend on Hadoop.
+            if (loadByCurrentClassloader(name))
+                return loadClassExplicitly(name, resolve);
+
+            return super.loadClass(name, resolve);
+        }
+        catch (NoClassDefFoundError | ClassNotFoundException e) {
+            throw new ClassNotFoundException("Failed to load class: " + name, e);
+        }
+    }
+
+    /**
+     * Load a class replacing it with our own implementation.
+     *
+     * @param originalName Name.
+     * @param replaceName Replacement.
+     * @return Class.
+     */
+    private Class<?> loadReplace(final String originalName, final String replaceName) {
+        synchronized (getClassLoadingLock(originalName)) {
+            // First, check if the class has already been loaded
+            Class c = findLoadedClass(originalName);
+
+            if (c != null)
+                return c;
+
+            byte[] bytes = bytesCache.get(originalName);
+
+            if (bytes == null) {
+                InputStream in = helper.loadClassBytes(this, replaceName);
+
+                if (in == null)
+                    throw new IgniteException("Failed to replace class [originalName=" + originalName +
+                        ", replaceName=" +  replaceName + ']');
+
+                bytes = helper.loadReplace(in, originalName, replaceName);
+
+                bytesCache.put(originalName, bytes);
+            }
+
+            return defineClass(originalName, bytes, 0, bytes.length);
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public Class<?> getFromCache(String clsName) throws ClassNotFoundException {
+        Class<?> cls = cacheMap.get(clsName);
+
+        if (cls == null) {
+            Class old = cacheMap.putIfAbsent(clsName, cls = Class.forName(clsName, true, this));
+
+            if (old != null)
+                cls = old;
+        }
+
+        return cls;
+    }
+
+    /**
+     * Check whether file must be loaded with current class loader, or normal delegation model should be used.
+     * <p>
+     * Override is only necessary for Ignite classes which have direct or transitive dependencies on Hadoop classes.
+     * These are all classes from "org.apache.ignite.internal.processors.hadoop.impl" package,
+     * and these are several well-know classes from "org.apache.ignite.hadoop" package.
+     *
+     * @param clsName Class name.
+     * @return Whether class must be loaded by current classloader without delegation.
+     */
+    @SuppressWarnings("RedundantIfStatement")
+    public static boolean loadByCurrentClassloader(String clsName) {
+        // All impl classes.
+        if (clsName.startsWith("org.apache.ignite.internal.processors.hadoop.impl"))
+            return true;
+
+        // Several classes from public API.
+        if (clsName.startsWith("org.apache.ignite.hadoop")) {
+            // We use "contains" instead of "equals" to handle subclasses properly.
+            if (clsName.contains("org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem") ||
+                clsName.contains("org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem") ||
+                clsName.contains("org.apache.ignite.hadoop.mapreduce.IgniteHadoopClientProtocolProvider"))
+                return true;
+        }
+
+        return false;
+    }
+
+    /**
+     * @param name Class name.
+     * @param resolve Resolve class.
+     * @return Class.
+     * @throws ClassNotFoundException If failed.
+     */
+    private Class<?> loadClassExplicitly(String name, boolean resolve) throws ClassNotFoundException {
+        synchronized (getClassLoadingLock(name)) {
+            // First, check if the class has already been loaded
+            Class c = findLoadedClass(name);
+
+            if (c == null) {
+                long t1 = System.nanoTime();
+
+                c = findClass(name);
+
+                // this is the defining class loader; record the stats
+                sun.misc.PerfCounter.getFindClassTime().addElapsedTimeFrom(t1);
+                sun.misc.PerfCounter.getFindClasses().increment();
+            }
+
+            if (resolve)
+                resolveClass(c);
+
+            return c;
+        }
+    }
+
+    /**
+     * @param urls URLs.
+     * @return URLs.
+     */
+    private static URL[] addHadoopUrls(URL[] urls) {
+        Collection<URL> hadoopJars;
+
+        try {
+            hadoopJars = hadoopUrls();
+        }
+        catch (IgniteCheckedException e) {
+            throw new RuntimeException(e);
+        }
+
+        ArrayList<URL> list = new ArrayList<>(hadoopJars.size() + appJars.size() + (urls == null ? 0 : urls.length));
+
+        list.addAll(appJars);
+        list.addAll(hadoopJars);
+
+        if (!F.isEmpty(urls))
+            list.addAll(F.asList(urls));
+
+        return list.toArray(new URL[list.size()]);
+    }
+
+    /**
+     * @return Collection of jar URLs.
+     * @throws IgniteCheckedException If failed.
+     */
+    public static Collection<URL> hadoopUrls() throws IgniteCheckedException {
+        Collection<URL> hadoopUrls = hadoopJars;
+
+        if (hadoopUrls != null)
+            return hadoopUrls;
+
+        synchronized (HadoopClassLoader.class) {
+            hadoopUrls = hadoopJars;
+
+            if (hadoopUrls != null)
+                return hadoopUrls;
+
+            try {
+                hadoopUrls = HadoopClasspathUtils.classpathForClassLoader();
+            }
+            catch (IOException e) {
+                throw new IgniteCheckedException("Failed to resolve Hadoop JAR locations: " + e.getMessage(), e);
+            }
+
+            hadoopJars = hadoopUrls;
+
+            return hadoopUrls;
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(HadoopClassLoader.class, this);
+    }
+
+    /**
+     * Getter for name field.
+     */
+    public String name() {
+        return name;
+    }
+
+    /**
+     * Native library abstraction.
+     */
+    private static class NativeLibrary {
+        /** Library name. */
+        private final String name;
+
+        /** Whether library is optional. */
+        private final boolean optional;
+
+        /**
+         * Constructor.
+         *
+         * @param name Library name.
+         * @param optional Optional flag.
+         */
+        public NativeLibrary(String name, boolean optional) {
+            this.name = name;
+            this.optional = optional;
+        }
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ignite/blob/8032fc2c/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
new file mode 100644
index 0000000..7579ddb
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopClasspathUtils.java
@@ -0,0 +1,424 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * Hadoop classpath utilities.
+ */
+public class HadoopClasspathUtils {
+    /** Prefix directory. */
+    public static final String PREFIX = "HADOOP_PREFIX";
+
+    /** Hadoop home directory. */
+    public static final String HOME = "HADOOP_HOME";
+
+    /** Hadoop common directory. */
+    public static final String COMMON_HOME = "HADOOP_COMMON_HOME";
+
+    /** Hadoop HDFS directory. */
+    public static final String HDFS_HOME = "HADOOP_HDFS_HOME";
+
+    /** Hadoop mapred directory. */
+    public static final String MAPRED_HOME = "HADOOP_MAPRED_HOME";
+
+    /** Arbitrary additional dependencies. Compliant with standard Java classpath resolution. */
+    public static final String HADOOP_USER_LIBS = "HADOOP_USER_LIBS";
+
+    /** Empty string. */
+    private static final String EMPTY_STR = "";
+
+    /**
+     * Gets Hadoop class path as a list of URLs (for in-process class loader usage).
+     *
+     * @return List of class path URLs.
+     * @throws IOException If failed.
+     */
+    public static List<URL> classpathForClassLoader() throws IOException {
+        List<URL> res = new ArrayList<>();
+
+        for (SearchDirectory dir : classpathDirectories()) {
+            for (File file : dir.files()) {
+                try {
+                    res.add(file.toURI().toURL());
+                }
+                catch (MalformedURLException e) {
+                    throw new IOException("Failed to convert file path to URL: " + file.getPath());
+                }
+            }
+        }
+
+        return res;
+    }
+
+    /**
+     * Gets Hadoop locations.
+     *
+     * @return The locations as determined from the environment.
+     */
+    public static HadoopLocations locations() throws IOException {
+        // Query environment.
+        String hadoopHome = systemOrEnv(PREFIX, systemOrEnv(HOME, EMPTY_STR));
+
+        String commonHome = systemOrEnv(COMMON_HOME, EMPTY_STR);
+        String hdfsHome = systemOrEnv(HDFS_HOME, EMPTY_STR);
+        String mapredHome = systemOrEnv(MAPRED_HOME, EMPTY_STR);
+
+        // If any composite location is defined, use only them.
+        if (!isEmpty(commonHome) || !isEmpty(hdfsHome) || !isEmpty(mapredHome)) {
+            HadoopLocations res = new HadoopLocations(hadoopHome, commonHome, hdfsHome, mapredHome);
+
+            if (res.valid())
+                return res;
+            else
+                throw new IOException("Failed to resolve Hadoop classpath because some environment variables are " +
+                    "either undefined or point to nonexistent directories [" +
+                    "[env=" + COMMON_HOME + ", value=" + commonHome + ", exists=" + res.commonExists() + "], " +
+                    "[env=" + HDFS_HOME + ", value=" + hdfsHome + ", exists=" + res.hdfsExists() + "], " +
+                    "[env=" + MAPRED_HOME + ", value=" + mapredHome + ", exists=" + res.mapredExists() + "]]");
+        }
+        else if (!isEmpty(hadoopHome)) {
+            // All further checks will be based on HADOOP_HOME, so check for it's existence.
+            if (!exists(hadoopHome))
+                throw new IOException("Failed to resolve Hadoop classpath because " + HOME + " environment " +
+                    "variable points to nonexistent directory: " + hadoopHome);
+
+            // Probe Apache Hadoop.
+            HadoopLocations res = new HadoopLocations(
+                hadoopHome,
+                hadoopHome + "/share/hadoop/common",
+                hadoopHome + "/share/hadoop/hdfs",
+                hadoopHome + "/share/hadoop/mapreduce"
+            );
+
+            if (res.valid())
+                return res;
+
+            // Probe CDH.
+            res = new HadoopLocations(
+                hadoopHome,
+                hadoopHome,
+                hadoopHome + "/../hadoop-hdfs",
+                hadoopHome + "/../hadoop-mapreduce"
+            );
+
+            if (res.valid())
+                return res;
+
+            // Probe HDP.
+            res = new HadoopLocations(
+                hadoopHome,
+                hadoopHome,
+                hadoopHome + "/../hadoop-hdfs-client",
+                hadoopHome + "/../hadoop-mapreduce-client"
+            );
+
+            if (res.valid())
+                return res;
+
+            // Failed.
+            throw new IOException("Failed to resolve Hadoop classpath because " + HOME + " environment variable " +
+                "is either invalid or points to non-standard Hadoop distribution: " + hadoopHome);
+        }
+        else {
+            // Advise to set HADOOP_HOME only as this is preferred way to configure classpath.
+            throw new IOException("Failed to resolve Hadoop classpath (please define " + HOME + " environment " +
+                "variable and point it to your Hadoop distribution).");
+        }
+    }
+
+    /**
+     * Gets base directories to discover classpath elements in.
+     *
+     * @return Collection of directory and mask pairs.
+     * @throws IOException if a mandatory classpath location is not found.
+     */
+    private static Collection<SearchDirectory> classpathDirectories() throws IOException {
+        HadoopLocations loc = locations();
+
+        Collection<SearchDirectory> res = new ArrayList<>();
+
+        // Add libraries from Hadoop distribution:
+        res.add(new SearchDirectory(new File(loc.common(), "lib"), AcceptAllDirectoryFilter.INSTANCE));
+        res.add(new SearchDirectory(new File(loc.hdfs(), "lib"), AcceptAllDirectoryFilter.INSTANCE));
+        res.add(new SearchDirectory(new File(loc.mapred(), "lib"), AcceptAllDirectoryFilter.INSTANCE));
+
+        res.add(new SearchDirectory(new File(loc.common()), new PrefixDirectoryFilter("hadoop-common-")));
+        res.add(new SearchDirectory(new File(loc.common()), new PrefixDirectoryFilter("hadoop-auth-")));
+
+        res.add(new SearchDirectory(new File(loc.hdfs()), new PrefixDirectoryFilter("hadoop-hdfs-")));
+
+        res.add(new SearchDirectory(new File(loc.mapred()),
+            new PrefixDirectoryFilter("hadoop-mapreduce-client-common")));
+        res.add(new SearchDirectory(new File(loc.mapred()),
+            new PrefixDirectoryFilter("hadoop-mapreduce-client-core")));
+
+        // Add user provided libs:
+        res.addAll(parseUserLibs());
+
+        return res;
+    }
+
+    /**
+     * Parse user libs.
+     *
+     * @return Parsed libs search patterns.
+     * @throws IOException If failed.
+     */
+    public static Collection<SearchDirectory> parseUserLibs() throws IOException {
+        return parseUserLibs(systemOrEnv(HADOOP_USER_LIBS, null));
+    }
+
+    /**
+     * Parse user libs.
+     *
+     * @param str String.
+     * @return Result.
+     * @throws IOException If failed.
+     */
+    public static Collection<SearchDirectory> parseUserLibs(String str) throws IOException {
+        Collection<SearchDirectory> res = new LinkedList<>();
+
+        if (!isEmpty(str)) {
+            String[] tokens = normalize(str).split(File.pathSeparator);
+
+            for (String token : tokens) {
+                // Skip empty tokens.
+                if (isEmpty(token))
+                    continue;
+
+                File file = new File(token);
+                File dir = file.getParentFile();
+
+                if (token.endsWith("*")) {
+                    assert dir != null;
+
+                    res.add(new SearchDirectory(dir, AcceptAllDirectoryFilter.INSTANCE, false));
+                }
+                else {
+                    // Met "/" or "C:\" pattern, nothing to do with it.
+                    if (dir == null)
+                        continue;
+
+                    res.add(new SearchDirectory(dir, new ExactDirectoryFilter(file.getName()), false));
+                }
+            }
+        }
+
+        return res;
+    }
+
+    /**
+     * Get system property or environment variable with the given name.
+     *
+     * @param name Variable name.
+     * @param dflt Default value.
+     * @return Value.
+     */
+    private static String systemOrEnv(String name, String dflt) {
+        String res = System.getProperty(name);
+
+        if (res == null)
+            res = System.getenv(name);
+
+        return res != null ? res : dflt;
+    }
+
+    /**
+     * Answers if the given path denotes existing directory.
+     *
+     * @param path The directory path.
+     * @return {@code True} if the given path denotes an existing directory.
+     */
+    public static boolean exists(String path) {
+        if (path == null)
+            return false;
+
+        Path p = Paths.get(path);
+
+        return Files.exists(p) && Files.isDirectory(p) && Files.isReadable(p);
+    }
+
+    /**
+     * Check if string is empty.
+     *
+     * @param val Value.
+     * @return {@code True} if empty.
+     */
+    private static boolean isEmpty(String val) {
+        return val == null || val.isEmpty();
+    }
+
+    /**
+     * NOramlize the string.
+     *
+     * @param str String.
+     * @return Normalized string.
+     */
+    private static String normalize(String str) {
+        assert str != null;
+
+        return str.trim().toLowerCase();
+    }
+
+    /**
+     * Simple pair-like structure to hold directory name and a mask assigned to it.
+     */
+    public static class SearchDirectory {
+        /** File. */
+        private final File dir;
+
+        /** Filter. */
+        private final DirectoryFilter filter;
+
+        /** Whether directory must exist. */
+        private final boolean strict;
+
+        /**
+         * Constructor for directory search with strict rule.
+         *
+         * @param dir Directory.
+         * @param filter Filter.
+         * @throws IOException If failed.
+         */
+        private SearchDirectory(File dir, DirectoryFilter filter) throws IOException {
+            this(dir, filter, true);
+        }
+
+        /**
+         * Constructor.
+         *
+         * @param dir Directory.
+         * @param filter Filter.
+         * @param strict Whether directory must exist.
+         * @throws IOException If failed.
+         */
+        private SearchDirectory(File dir, DirectoryFilter filter, boolean strict) throws IOException {
+            this.dir = dir;
+            this.filter = filter;
+            this.strict = strict;
+
+            if (strict && !exists(dir.getAbsolutePath()))
+                throw new IOException("Directory cannot be read: " + dir.getAbsolutePath());
+        }
+
+        /**
+         * @return Child files.
+         */
+        public File[] files() throws IOException {
+            File[] files = dir.listFiles(new FilenameFilter() {
+                @Override public boolean accept(File dir, String name) {
+                    return filter.test(name);
+                }
+            });
+
+            if (files == null) {
+                if (strict)
+                    throw new IOException("Failed to get directory files [dir=" + dir + ']');
+                else
+                    return new File[0];
+            }
+            else
+                return files;
+        }
+    }
+
+    /**
+     * Directory filter interface.
+     */
+    public static interface DirectoryFilter {
+        /**
+         * Test if file with this name should be included.
+         *
+         * @param name File name.
+         * @return {@code True} if passed.
+         */
+        public boolean test(String name);
+    }
+
+    /**
+     * Filter to accept all files.
+     */
+    public static class AcceptAllDirectoryFilter implements DirectoryFilter {
+        /** Singleton instance. */
+        public static final AcceptAllDirectoryFilter INSTANCE = new AcceptAllDirectoryFilter();
+
+        /** {@inheritDoc} */
+        @Override public boolean test(String name) {
+            return true;
+        }
+    }
+
+    /**
+     * Filter which uses prefix to filter files.
+     */
+    public static class PrefixDirectoryFilter implements DirectoryFilter {
+        /** Prefix. */
+        private final String prefix;
+
+        /**
+         * Constructor.
+         *
+         * @param prefix Prefix.
+         */
+        public PrefixDirectoryFilter(String prefix) {
+            assert prefix != null;
+
+            this.prefix = normalize(prefix);
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean test(String name) {
+            return normalize(name).startsWith(prefix);
+        }
+    }
+
+    /**
+     * Filter which uses exact comparison.
+     */
+    public static class ExactDirectoryFilter implements DirectoryFilter {
+        /** Name. */
+        private final String name;
+
+        /**
+         * Constructor.
+         *
+         * @param name Name.
+         */
+        public ExactDirectoryFilter(String name) {
+            this.name = normalize(name);
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean test(String name) {
+            return normalize(name).equals(this.name);
+        }
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ignite/blob/8032fc2c/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopDefaultJobInfo.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopDefaultJobInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopDefaultJobInfo.java
new file mode 100644
index 0000000..ae17ac8
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopDefaultJobInfo.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import java.io.Externalizable;
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.lang.reflect.Constructor;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Hadoop job info based on default Hadoop configuration.
+ */
+public class HadoopDefaultJobInfo implements HadoopJobInfo, Externalizable {
+    /** */
+    private static final long serialVersionUID = 5489900236464999951L;
+
+    /** {@code true} If job has combiner. */
+    private boolean hasCombiner;
+
+    /** Number of reducers configured for job. */
+    private int numReduces;
+
+    /** Configuration. */
+    private Map<String,String> props = new HashMap<>();
+
+    /** Job name. */
+    private String jobName;
+
+    /** User name. */
+    private String user;
+
+    /**
+     * Default constructor required by {@link Externalizable}.
+     */
+    public HadoopDefaultJobInfo() {
+        // No-op.
+    }
+
+    /**
+     * Constructor.
+     *
+     * @param jobName Job name.
+     * @param user User name.
+     * @param hasCombiner {@code true} If job has combiner.
+     * @param numReduces Number of reducers configured for job.
+     * @param props All other properties of the job.
+     */
+    public HadoopDefaultJobInfo(String jobName, String user, boolean hasCombiner, int numReduces,
+        Map<String, String> props) {
+        this.jobName = jobName;
+        this.user = user;
+        this.hasCombiner = hasCombiner;
+        this.numReduces = numReduces;
+        this.props = props;
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public String property(String name) {
+        return props.get(name);
+    }
+
+    /** {@inheritDoc} */
+    @Override public HadoopJob createJob(Class<? extends HadoopJob> jobCls, HadoopJobId jobId, IgniteLogger log,
+        @Nullable String[] libNames, HadoopHelper helper) throws IgniteCheckedException {
+        assert jobCls != null;
+
+        try {
+            Constructor<? extends HadoopJob> constructor = jobCls.getConstructor(HadoopJobId.class,
+                HadoopDefaultJobInfo.class, IgniteLogger.class, String[].class, HadoopHelper.class);
+
+            return constructor.newInstance(jobId, this, log, libNames, helper);
+        }
+        catch (Throwable t) {
+            if (t instanceof Error)
+                throw (Error)t;
+            
+            throw new IgniteCheckedException(t);
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean hasCombiner() {
+        return hasCombiner;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean hasReducer() {
+        return reducers() > 0;
+    }
+
+    /** {@inheritDoc} */
+    @Override public int reducers() {
+        return numReduces;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String jobName() {
+        return jobName;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String user() {
+        return user;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void writeExternal(ObjectOutput out) throws IOException {
+        U.writeString(out, jobName);
+        U.writeString(out, user);
+
+        out.writeBoolean(hasCombiner);
+        out.writeInt(numReduces);
+
+        U.writeStringMap(out, props);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+        jobName = U.readString(in);
+        user = U.readString(in);
+
+        hasCombiner = in.readBoolean();
+        numReduces = in.readInt();
+
+        props = U.readStringMap(in);
+    }
+
+    /**
+     * @return Properties of the job.
+     */
+    public Map<String, String> properties() {
+        return props;
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ignite/blob/8032fc2c/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopHelper.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopHelper.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopHelper.java
new file mode 100644
index 0000000..a8fee79
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopHelper.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import java.io.InputStream;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * IGFS utility processor adapter.
+ */
+public interface HadoopHelper {
+    /**
+     * @return Whether this is no-op implementation.
+     */
+    public boolean isNoOp();
+
+    /**
+     * Get common Hadoop class loader.
+     *
+     * @return Common Hadoop class loader.
+     */
+    public HadoopClassLoader commonClassLoader();
+
+    /**
+     * Load special replacement and impersonate.
+     *
+     * @param in Input stream.
+     * @param originalName Original class name.
+     * @param replaceName Replacer class name.
+     * @return Result.
+     */
+    public byte[] loadReplace(InputStream in, final String originalName, final String replaceName);
+
+    /**
+     * @param ldr Loader.
+     * @param clsName Class.
+     * @return Input stream.
+     */
+    @Nullable public InputStream loadClassBytes(ClassLoader ldr, String clsName);
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ignite/blob/8032fc2c/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobInfo.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobInfo.java
index a3b1bb6..853c63d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobInfo.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopJobInfo.java
@@ -59,11 +59,13 @@ public interface HadoopJobInfo extends Serializable {
      * @param jobId Job ID.
      * @param log Logger.
      * @param libNames Optional additional native library names.
+     * @param helper HadoopHelper.
      * @return Job.
      * @throws IgniteCheckedException If failed.
      */
     public HadoopJob createJob(Class<? extends HadoopJob> jobCls,
-        HadoopJobId jobId, IgniteLogger log, @Nullable String[] libNames) throws IgniteCheckedException;
+        HadoopJobId jobId, IgniteLogger log, @Nullable String[] libNames, HadoopHelper helper)
+            throws IgniteCheckedException;
 
     /**
      * @return Number of reducers configured for job.

http://git-wip-us.apache.org/repos/asf/ignite/blob/8032fc2c/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java
new file mode 100644
index 0000000..a90007f
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopLocations.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+/**
+ * Simple structure to hold Hadoop directory locations.
+ */
+public class HadoopLocations {
+    /** Hadoop home. */
+    private final String home;
+
+    /** Common home. */
+    private final String common;
+
+    /** HDFS home. */
+    private final String hdfs;
+
+    /** Mapred home. */
+    private final String mapred;
+
+    /** Whether common home exists. */
+    private final boolean commonExists;
+
+    /** Whether HDFS home exists. */
+    private final boolean hdfsExists;
+
+    /** Whether mapred home exists. */
+    private final boolean mapredExists;
+
+    /**
+     * Constructor.
+     *
+     * @param home Hadoop home.
+     * @param common Common home.
+     * @param hdfs HDFS home.
+     * @param mapred Mapred home.
+     */
+    public HadoopLocations(String home, String common, String hdfs, String mapred) {
+        assert common != null && hdfs != null && mapred != null;
+
+        this.home = home;
+        this.common = common;
+        this.hdfs = hdfs;
+        this.mapred = mapred;
+
+        commonExists = HadoopClasspathUtils.exists(common);
+        hdfsExists = HadoopClasspathUtils.exists(hdfs);
+        mapredExists = HadoopClasspathUtils.exists(mapred);
+    }
+
+    /**
+     * @return Hadoop home.
+     */
+    public String home() {
+        return home;
+    }
+
+    /**
+     * @return Common home.
+     */
+    public String common() {
+        return common;
+    }
+
+    /**
+     * @return HDFS home.
+     */
+    public String hdfs() {
+        return hdfs;
+    }
+
+    /**
+     * @return Mapred home.
+     */
+    public String mapred() {
+        return mapred;
+    }
+
+    /**
+     * @return Whether common home exists.
+     */
+    public boolean commonExists() {
+        return commonExists;
+    }
+
+    /**
+     * @return Whether HDFS home exists.
+     */
+    public boolean hdfsExists() {
+        return hdfsExists;
+    }
+
+    /**
+     * @return Whether mapred home exists.
+     */
+    public boolean mapredExists() {
+        return mapredExists;
+    }
+
+    /**
+     * Whether all required directories exists.
+     *
+     * @return {@code True} if exists.
+     */
+    public boolean valid() {
+        return commonExists && hdfsExists && mapredExists;
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ignite/blob/8032fc2c/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopNoopHelper.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopNoopHelper.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopNoopHelper.java
new file mode 100644
index 0000000..d3348ca
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopNoopHelper.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.ignite.internal.GridKernalContext;
+import org.jetbrains.annotations.Nullable;
+
+import java.io.InputStream;
+
+/**
+ * Noop Hadoop Helper implementation.
+ */
+@SuppressWarnings("unused")
+public class HadoopNoopHelper implements HadoopHelper {
+    /**
+     * Constructor.
+     *
+     * @param ctx Kernal context.
+     */
+    @SuppressWarnings("UnusedParameters")
+    public HadoopNoopHelper(GridKernalContext ctx) {
+        // No-op.
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isNoOp() {
+        return true;
+    }
+
+    /** {@inheritDoc} */
+    @Override public HadoopClassLoader commonClassLoader() {
+        throw unsupported();
+    }
+
+    /** {@inheritDoc} */
+    @Override public byte[] loadReplace(InputStream in, String originalName, String replaceName) {
+        throw unsupported();
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public InputStream loadClassBytes(ClassLoader ldr, String clsName) {
+        throw unsupported();
+    }
+
+    /**
+     * @return Exception.
+     */
+    private static UnsupportedOperationException unsupported() {
+        throw new UnsupportedOperationException("Operation is unsupported (Hadoop module is not in the classpath).");
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/8032fc2c/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopNoopProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopNoopProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopNoopProcessor.java
index 501870a..fa4ab47 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopNoopProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopNoopProcessor.java
@@ -79,8 +79,8 @@ public class HadoopNoopProcessor extends HadoopProcessorAdapter {
      * Creates an exception to be uniformly thrown from all the methods.
      */
     private IllegalStateException createException() {
-        return new IllegalStateException("Hadoop module is not loaded (please ensure that ignite-hadoop.jar is in " +
-            "classpath and IgniteConfiguration.peerClassLoadingEnabled is set to false).");
+        return new IllegalStateException("Hadoop module is not loaded (please ensure that ignite-hadoop.jar is " +
+            "in libs and IgniteConfiguration.peerClassLoadingEnabled is set to false).");
     }
 
     /** {@inheritDoc} */

http://git-wip-us.apache.org/repos/asf/ignite/blob/8032fc2c/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
index 6ff1f8f..1dd12d9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
@@ -183,8 +183,11 @@ public final class IgfsImpl implements IgfsEx {
         data = igfsCtx.data();
         secondaryFs = cfg.getSecondaryFileSystem();
 
+        if (secondaryFs instanceof IgfsKernalContextAware)
+            ((IgfsKernalContextAware)secondaryFs).setKernalContext(igfsCtx.kernalContext());
+
         if (secondaryFs instanceof LifecycleAware)
-            ((LifecycleAware) secondaryFs).start();
+            ((LifecycleAware)secondaryFs).start();
 
         /* Default IGFS mode. */
         IgfsMode dfltMode;

http://git-wip-us.apache.org/repos/asf/ignite/blob/8032fc2c/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsKernalContextAware.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsKernalContextAware.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsKernalContextAware.java
new file mode 100644
index 0000000..7f59db4
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsKernalContextAware.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.igfs;
+
+import org.apache.ignite.internal.GridKernalContext;
+
+/**
+ * Indicates whether particular file system accepts kernal context.
+ */
+public interface IgfsKernalContextAware {
+    /**
+     * Set kernal context.
+     *
+     * @param ctx Kernal context.
+     */
+    public void setKernalContext(GridKernalContext ctx);
+}


Mime
View raw message