ignite-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From voze...@apache.org
Subject [02/46] ignite git commit: IGNITE-3953: Hadoop: Merged back both modules.
Date Mon, 26 Sep 2016 07:16:58 GMT
http://git-wip-us.apache.org/repos/asf/ignite/blob/4c1739ce/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/util/KerberosUserNameMapperSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/util/KerberosUserNameMapperSelfTest.java
b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/util/KerberosUserNameMapperSelfTest.java
new file mode 100644
index 0000000..bd76b51
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/util/KerberosUserNameMapperSelfTest.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.impl.util;
+
+import org.apache.ignite.hadoop.util.KerberosUserNameMapper;
+import org.apache.ignite.internal.processors.igfs.IgfsUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Tests for Kerberos name mapper.
+ */
+public class KerberosUserNameMapperSelfTest extends GridCommonAbstractTest {
+    /** Test instance. */
+    private static final String INSTANCE = "test_instance";
+
+    /** Test realm. */
+    private static final String REALM = "test_realm";
+
+    /**
+     * Test mapper without instance and realm components.
+     *
+     * @throws Exception If failed.
+     */
+    public void testMapper() throws Exception {
+        KerberosUserNameMapper mapper = create(null, null);
+
+        assertEquals(IgfsUtils.fixUserName(null), mapper.map(null));
+        assertEquals("test", mapper.map("test"));
+    }
+
+    /**
+     * Test mapper with instance component.
+     *
+     * @throws Exception If failed.
+     */
+    public void testMapperInstance() throws Exception {
+        KerberosUserNameMapper mapper = create(INSTANCE, null);
+
+        assertEquals(IgfsUtils.fixUserName(null) + "/" + INSTANCE, mapper.map(null));
+        assertEquals("test" + "/" + INSTANCE, mapper.map("test"));
+    }
+
+    /**
+     * Test mapper with realm.
+     *
+     * @throws Exception If failed.
+     */
+    public void testMapperRealm() throws Exception {
+        KerberosUserNameMapper mapper = create(null, REALM);
+
+        assertEquals(IgfsUtils.fixUserName(null) + "@" + REALM, mapper.map(null));
+        assertEquals("test" + "@" + REALM, mapper.map("test"));
+    }
+
+    /**
+     * Test mapper with instance and realm components.
+     *
+     * @throws Exception If failed.
+     */
+    public void testMapperInstanceAndRealm() throws Exception {
+        KerberosUserNameMapper mapper = create(INSTANCE, REALM);
+
+        assertEquals(IgfsUtils.fixUserName(null) + "/" + INSTANCE + "@" + REALM, mapper.map(null));
+        assertEquals("test" + "/" + INSTANCE + "@" + REALM, mapper.map("test"));
+    }
+
+    /**
+     * Create mapper.
+     *
+     * @param instance Instance.
+     * @param realm Realm.
+     * @return Mapper.
+     */
+    private KerberosUserNameMapper create(@Nullable String instance, @Nullable String realm)
{
+        KerberosUserNameMapper mapper = new KerberosUserNameMapper();
+
+        mapper.setInstance(instance);
+        mapper.setRealm(realm);
+
+        mapper.start();
+
+        return mapper;
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/4c1739ce/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/state/HadoopGroupingTestState.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/state/HadoopGroupingTestState.java
b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/state/HadoopGroupingTestState.java
new file mode 100644
index 0000000..af82d49
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/state/HadoopGroupingTestState.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.state;
+
+import org.apache.ignite.internal.processors.hadoop.HadoopSharedMap;
+import org.apache.ignite.internal.util.GridConcurrentHashSet;
+
+import java.util.Collection;
+import java.util.UUID;
+
+/**
+ * Shared state for HadoopGroupingTest.
+ */
+public class HadoopGroupingTestState {
+    /** Values. */
+    private static final GridConcurrentHashSet<UUID> vals = HadoopSharedMap.map(HadoopGroupingTestState.class)
+        .put("vals", new GridConcurrentHashSet<UUID>());
+
+    /**
+     * @return Values.
+     */
+    public static Collection<UUID> values() {
+        return vals;
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/4c1739ce/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/state/HadoopJobTrackerSelfTestState.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/state/HadoopJobTrackerSelfTestState.java
b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/state/HadoopJobTrackerSelfTestState.java
new file mode 100644
index 0000000..4183f4e
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/state/HadoopJobTrackerSelfTestState.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.state;
+
+import org.apache.ignite.internal.processors.hadoop.HadoopSharedMap;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * Shared state for HadoopJobTrackerSelfTest.
+ */
+public class HadoopJobTrackerSelfTestState {
+    /** */
+    private static HadoopSharedMap m = HadoopSharedMap.map(HadoopJobTrackerSelfTestState.class);
+
+    /** Map task execution count. */
+    public static final AtomicInteger mapExecCnt = m.put("mapExecCnt", new AtomicInteger());
+
+    /** Reduce task execution count. */
+    public static final AtomicInteger reduceExecCnt = m.put("reduceExecCnt", new AtomicInteger());
+
+    /** Reduce task execution count. */
+    public static final AtomicInteger combineExecCnt = m.put("combineExecCnt", new AtomicInteger());
+
+    /** */
+    public static final Map<String, CountDownLatch> latch = m.put("latch", new HashMap<String,
CountDownLatch>());
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/4c1739ce/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/state/HadoopMapReduceEmbeddedSelfTestState.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/state/HadoopMapReduceEmbeddedSelfTestState.java
b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/state/HadoopMapReduceEmbeddedSelfTestState.java
new file mode 100644
index 0000000..ae6ce82
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/state/HadoopMapReduceEmbeddedSelfTestState.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.state;
+
+import org.apache.ignite.internal.processors.hadoop.HadoopSharedMap;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Sharead state for HadoopMapReduceEmbeddedSelfTest.
+ */
+public class HadoopMapReduceEmbeddedSelfTestState {
+    /** */
+    public static Map<String, Boolean> flags = HadoopSharedMap.map(HadoopMapReduceEmbeddedSelfTestState.class)
+        .put("flags", new HashMap<String, Boolean>());
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/4c1739ce/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/state/HadoopTaskExecutionSelfTestValues.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/state/HadoopTaskExecutionSelfTestValues.java
b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/state/HadoopTaskExecutionSelfTestValues.java
new file mode 100644
index 0000000..beefa68
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/state/HadoopTaskExecutionSelfTestValues.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.state;
+
+import org.apache.ignite.internal.processors.hadoop.HadoopSharedMap;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * Shared state for HadoopTaskExecutionSelfTest.
+ */
+public class HadoopTaskExecutionSelfTestValues {
+    /** */
+    private static HadoopSharedMap m = HadoopSharedMap.map(HadoopTaskExecutionSelfTestValues.class);
+
+    /** Line count. */
+    public static final AtomicInteger totalLineCnt = m.put("totalLineCnt", new AtomicInteger());
+
+    /** Executed tasks. */
+    public static final AtomicInteger executedTasks = m.put("executedTasks", new AtomicInteger());
+
+    /** Cancelled tasks. */
+    public static final AtomicInteger cancelledTasks = m.put("cancelledTasks", new AtomicInteger());
+
+    /** Working directory of each task. */
+    public static final Map<String, String> taskWorkDirs = m.put("taskWorkDirs",
+        new ConcurrentHashMap<String, String>());
+
+    /** Mapper id to fail. */
+    public static final AtomicInteger failMapperId = m.put("failMapperId", new AtomicInteger());
+
+    /** Number of splits of the current input. */
+    public static final AtomicInteger splitsCount = m.put("splitsCount", new AtomicInteger());
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/4c1739ce/modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite.java
b/modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite.java
new file mode 100644
index 0000000..bbd92d1
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite.java
@@ -0,0 +1,352 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.testsuites;
+
+import junit.framework.TestSuite;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.internal.processors.hadoop.HadoopTestClassLoader;
+import org.apache.ignite.internal.processors.hadoop.impl.client.HadoopClientProtocolEmbeddedSelfTest;
+import org.apache.ignite.internal.processors.hadoop.impl.client.HadoopClientProtocolSelfTest;
+import org.apache.ignite.internal.processors.hadoop.impl.HadoopTxConfigCacheTest;
+import org.apache.ignite.internal.processors.hadoop.impl.fs.KerberosHadoopFileSystemFactorySelfTest;
+import org.apache.ignite.internal.processors.hadoop.impl.util.BasicUserNameMapperSelfTest;
+import org.apache.ignite.internal.processors.hadoop.impl.util.ChainedUserNameMapperSelfTest;
+import org.apache.ignite.internal.processors.hadoop.impl.util.KerberosUserNameMapperSelfTest;
+import org.apache.ignite.internal.processors.hadoop.impl.igfs.Hadoop1OverIgfsDualAsyncTest;
+import org.apache.ignite.internal.processors.hadoop.impl.igfs.Hadoop1OverIgfsDualSyncTest;
+import org.apache.ignite.internal.processors.hadoop.impl.igfs.HadoopFIleSystemFactorySelfTest;
+import org.apache.ignite.internal.processors.hadoop.impl.igfs.HadoopIgfs20FileSystemLoopbackPrimarySelfTest;
+import org.apache.ignite.internal.processors.hadoop.impl.igfs.HadoopIgfsDualAsyncSelfTest;
+import org.apache.ignite.internal.processors.hadoop.impl.igfs.HadoopIgfsDualSyncSelfTest;
+import org.apache.ignite.internal.processors.hadoop.impl.igfs.HadoopSecondaryFileSystemConfigurationTest;
+import org.apache.ignite.internal.processors.hadoop.impl.igfs.IgfsEventsTestSuite;
+import org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemClientSelfTest;
+import org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemHandshakeSelfTest;
+import org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemLoggerSelfTest;
+import org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemLoggerStateSelfTest;
+import org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemLoopbackEmbeddedDualAsyncSelfTest;
+import org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemLoopbackEmbeddedDualSyncSelfTest;
+import org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemLoopbackEmbeddedPrimarySelfTest;
+import org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemLoopbackEmbeddedSecondarySelfTest;
+import org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemLoopbackExternalDualAsyncSelfTest;
+import org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemLoopbackExternalDualSyncSelfTest;
+import org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemLoopbackExternalPrimarySelfTest;
+import org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemLoopbackExternalSecondarySelfTest;
+import org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemSecondaryFileSystemInitializationSelfTest;
+import org.apache.ignite.internal.processors.hadoop.impl.HadoopCommandLineTest;
+import org.apache.ignite.internal.processors.hadoop.impl.HadoopDefaultMapReducePlannerSelfTest;
+import org.apache.ignite.internal.processors.hadoop.impl.HadoopFileSystemsTest;
+import org.apache.ignite.internal.processors.hadoop.impl.HadoopGroupingTest;
+import org.apache.ignite.internal.processors.hadoop.impl.HadoopJobTrackerSelfTest;
+import org.apache.ignite.internal.processors.hadoop.impl.HadoopMapReduceEmbeddedSelfTest;
+import org.apache.ignite.internal.processors.hadoop.impl.HadoopMapReduceErrorResilienceTest;
+import org.apache.ignite.internal.processors.hadoop.impl.HadoopMapReduceTest;
+import org.apache.ignite.internal.processors.hadoop.impl.HadoopNoHadoopMapReduceTest;
+import org.apache.ignite.internal.processors.hadoop.impl.HadoopSerializationWrapperSelfTest;
+import org.apache.ignite.internal.processors.hadoop.impl.HadoopSnappyFullMapReduceTest;
+import org.apache.ignite.internal.processors.hadoop.impl.HadoopSnappyTest;
+import org.apache.ignite.internal.processors.hadoop.impl.HadoopSortingTest;
+import org.apache.ignite.internal.processors.hadoop.impl.HadoopSplitWrapperSelfTest;
+import org.apache.ignite.internal.processors.hadoop.impl.HadoopTaskExecutionSelfTest;
+import org.apache.ignite.internal.processors.hadoop.impl.HadoopTasksV1Test;
+import org.apache.ignite.internal.processors.hadoop.impl.HadoopTasksV2Test;
+import org.apache.ignite.internal.processors.hadoop.impl.HadoopUserLibsSelfTest;
+import org.apache.ignite.internal.processors.hadoop.impl.HadoopV2JobSelfTest;
+import org.apache.ignite.internal.processors.hadoop.impl.HadoopValidationSelfTest;
+import org.apache.ignite.internal.processors.hadoop.impl.HadoopWeightedMapReducePlannerTest;
+import org.apache.ignite.internal.processors.hadoop.impl.HadoopWeightedPlannerMapReduceTest;
+import org.apache.ignite.internal.processors.hadoop.impl.shuffle.collections.HadoopConcurrentHashMultimapSelftest;
+import org.apache.ignite.internal.processors.hadoop.impl.shuffle.collections.HadoopHashMapSelfTest;
+import org.apache.ignite.internal.processors.hadoop.impl.shuffle.collections.HadoopSkipListSelfTest;
+import org.apache.ignite.internal.processors.hadoop.impl.shuffle.streams.HadoopDataStreamSelfTest;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.net.URL;
+import java.net.URLConnection;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.List;
+
+import static org.apache.ignite.testframework.GridTestUtils.modeToPermissionSet;
+
+/**
+ * Test suite for Hadoop Map Reduce engine.
+ */
+public class IgniteHadoopTestSuite extends TestSuite {
+    /**
+     * @return Test suite.
+     * @throws Exception Thrown in case of the failure.
+     */
+    public static TestSuite suite() throws Exception {
+        downloadHadoop();
+        downloadHive();
+
+        final ClassLoader ldr = new HadoopTestClassLoader();
+
+        TestSuite suite = new TestSuite("Ignite Hadoop MR Test Suite");
+
+        suite.addTest(new TestSuite(ldr.loadClass(HadoopUserLibsSelfTest.class.getName())));
+
+        suite.addTest(new TestSuite(ldr.loadClass(HadoopDefaultMapReducePlannerSelfTest.class.getName())));
+        suite.addTest(new TestSuite(ldr.loadClass(HadoopWeightedMapReducePlannerTest.class.getName())));
+
+        suite.addTest(new TestSuite(ldr.loadClass(BasicUserNameMapperSelfTest.class.getName())));
+        suite.addTest(new TestSuite(ldr.loadClass(KerberosUserNameMapperSelfTest.class.getName())));
+        suite.addTest(new TestSuite(ldr.loadClass(ChainedUserNameMapperSelfTest.class.getName())));
+
+        suite.addTest(new TestSuite(ldr.loadClass(KerberosHadoopFileSystemFactorySelfTest.class.getName())));
+
+        suite.addTest(new TestSuite(ldr.loadClass(HadoopSnappyTest.class.getName())));
+        suite.addTest(new TestSuite(ldr.loadClass(HadoopSnappyFullMapReduceTest.class.getName())));
+
+        suite.addTest(new TestSuite(ldr.loadClass(HadoopIgfs20FileSystemLoopbackPrimarySelfTest.class.getName())));
+
+        suite.addTest(new TestSuite(ldr.loadClass(HadoopIgfsDualSyncSelfTest.class.getName())));
+        suite.addTest(new TestSuite(ldr.loadClass(HadoopIgfsDualAsyncSelfTest.class.getName())));
+
+        suite.addTest(new TestSuite(ldr.loadClass(Hadoop1OverIgfsDualSyncTest.class.getName())));
+        suite.addTest(new TestSuite(ldr.loadClass(Hadoop1OverIgfsDualAsyncTest.class.getName())));
+
+        suite.addTest(new TestSuite(ldr.loadClass(HadoopFIleSystemFactorySelfTest.class.getName())));
+
+        suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemLoopbackExternalPrimarySelfTest.class.getName())));
+        suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemLoopbackExternalSecondarySelfTest.class.getName())));
+        suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemLoopbackExternalDualSyncSelfTest.class.getName())));
+        suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemLoopbackExternalDualAsyncSelfTest.class.getName())));
+        suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemLoopbackEmbeddedPrimarySelfTest.class.getName())));
+        suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemLoopbackEmbeddedSecondarySelfTest.class.getName())));
+        suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemLoopbackEmbeddedDualSyncSelfTest.class.getName())));
+        suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemLoopbackEmbeddedDualAsyncSelfTest.class.getName())));
+
+        suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemSecondaryFileSystemInitializationSelfTest.class.getName())));
+
+        suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemClientSelfTest.class.getName())));
+
+        suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemLoggerStateSelfTest.class.getName())));
+        suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemLoggerSelfTest.class.getName())));
+
+        suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemHandshakeSelfTest.class.getName())));
+
+        suite.addTest(IgfsEventsTestSuite.suiteNoarchOnly());
+
+        suite.addTest(new TestSuite(ldr.loadClass(HadoopFileSystemsTest.class.getName())));
+
+        suite.addTest(new TestSuite(ldr.loadClass(HadoopValidationSelfTest.class.getName())));
+
+        suite.addTest(new TestSuite(ldr.loadClass(HadoopJobTrackerSelfTest.class.getName())));
+
+        suite.addTest(new TestSuite(ldr.loadClass(HadoopHashMapSelfTest.class.getName())));
+        suite.addTest(new TestSuite(ldr.loadClass(HadoopDataStreamSelfTest.class.getName())));
+        suite.addTest(new TestSuite(ldr.loadClass(HadoopConcurrentHashMultimapSelftest.class.getName())));
+
+        suite.addTest(new TestSuite(ldr.loadClass(HadoopSkipListSelfTest.class.getName())));
+
+        suite.addTest(new TestSuite(ldr.loadClass(HadoopTaskExecutionSelfTest.class.getName())));
+
+        suite.addTest(new TestSuite(ldr.loadClass(HadoopV2JobSelfTest.class.getName())));
+
+        suite.addTest(new TestSuite(ldr.loadClass(HadoopSerializationWrapperSelfTest.class.getName())));
+        suite.addTest(new TestSuite(ldr.loadClass(HadoopSplitWrapperSelfTest.class.getName())));
+
+        suite.addTest(new TestSuite(ldr.loadClass(HadoopTasksV1Test.class.getName())));
+        suite.addTest(new TestSuite(ldr.loadClass(HadoopTasksV2Test.class.getName())));
+
+        suite.addTest(new TestSuite(ldr.loadClass(HadoopMapReduceTest.class.getName())));
+        suite.addTest(new TestSuite(ldr.loadClass(HadoopWeightedPlannerMapReduceTest.class.getName())));
+        suite.addTest(new TestSuite(ldr.loadClass(HadoopNoHadoopMapReduceTest.class.getName())));
+        suite.addTest(new TestSuite(ldr.loadClass(HadoopMapReduceErrorResilienceTest.class.getName())));
+
+        suite.addTest(new TestSuite(ldr.loadClass(HadoopMapReduceEmbeddedSelfTest.class.getName())));
+
+        suite.addTest(new TestSuite(ldr.loadClass(HadoopSortingTest.class.getName())));
+
+//        suite.addTest(new TestSuite(ldr.loadClass(HadoopExternalTaskExecutionSelfTest.class.getName())));
+//        suite.addTest(new TestSuite(ldr.loadClass(HadoopExternalCommunicationSelfTest.class.getName())));
+//        suite.addTest(new TestSuite(ldr.loadClass(HadoopSortingExternalTest.class.getName())));
+
+        suite.addTest(new TestSuite(ldr.loadClass(HadoopGroupingTest.class.getName())));
+
+        suite.addTest(new TestSuite(ldr.loadClass(HadoopClientProtocolSelfTest.class.getName())));
+        suite.addTest(new TestSuite(ldr.loadClass(HadoopClientProtocolEmbeddedSelfTest.class.getName())));
+
+        suite.addTest(new TestSuite(ldr.loadClass(HadoopCommandLineTest.class.getName())));
+
+        suite.addTest(new TestSuite(ldr.loadClass(HadoopSecondaryFileSystemConfigurationTest.class.getName())));
+
+        suite.addTest(new TestSuite(ldr.loadClass(HadoopTxConfigCacheTest.class.getName())));
+
+        return suite;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public static void downloadHive() throws Exception {
+        String ver = IgniteSystemProperties.getString("hive.version", "1.2.1");
+
+        X.println("Will use Hive version: " + ver);
+
+        String downloadPath = "hive/hive-" + ver + "/apache-hive-" + ver + "-bin.tar.gz";
+
+        download("Hive", "HIVE_HOME", downloadPath, "apache-hive-" + ver + "-bin");
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public static void downloadHadoop() throws Exception {
+        String ver = IgniteSystemProperties.getString("hadoop.version", "2.4.1");
+
+        X.println("Will use Hadoop version: " + ver);
+
+        String downloadPath = "hadoop/core/hadoop-" + ver + "/hadoop-" + ver + ".tar.gz";
+
+        download("Hadoop", "HADOOP_HOME", downloadPath, "hadoop-" + ver);
+    }
+
+    /**
+     *  Downloads and extracts an Apache product.
+     *
+     * @param appName Name of application for log messages.
+     * @param homeVariable Pointer to home directory of the component.
+     * @param downloadPath Relative download path of tar package.
+     * @param destName Local directory name to install component.
+     * @throws Exception If failed.
+     */
+    private static void download(String appName, String homeVariable, String downloadPath,
String destName)
+        throws Exception {
+        String homeVal = IgniteSystemProperties.getString(homeVariable);
+
+        if (!F.isEmpty(homeVal) && new File(homeVal).isDirectory()) {
+            X.println(homeVariable + " is set to: " + homeVal);
+
+            return;
+        }
+
+        List<String> urls = F.asList(
+            "http://archive.apache.org/dist/",
+            "http://apache-mirror.rbc.ru/pub/apache/",
+            "http://www.eu.apache.org/dist/",
+            "http://www.us.apache.org/dist/");
+
+        String tmpPath = System.getProperty("java.io.tmpdir");
+
+        X.println("tmp: " + tmpPath);
+
+        final File install = new File(tmpPath + File.separatorChar + "__hadoop");
+
+        final File home = new File(install, destName);
+
+        X.println("Setting " + homeVariable + " to " + home.getAbsolutePath());
+
+        System.setProperty(homeVariable, home.getAbsolutePath());
+
+        final File successFile = new File(home, "__success");
+
+        if (home.exists()) {
+            if (successFile.exists()) {
+                X.println(appName + " distribution already exists.");
+
+                return;
+            }
+
+            X.println(appName + " distribution is invalid and it will be deleted.");
+
+            if (!U.delete(home))
+                throw new IOException("Failed to delete directory: " + home.getAbsolutePath());
+        }
+
+        for (String url : urls) {
+            if (!(install.exists() || install.mkdirs()))
+                throw new IOException("Failed to create directory: " + install.getAbsolutePath());
+
+            URL u = new URL(url + downloadPath);
+
+            X.println("Attempting to download from: " + u);
+
+            try {
+                URLConnection c = u.openConnection();
+
+                c.connect();
+
+                try (TarArchiveInputStream in = new TarArchiveInputStream(new GzipCompressorInputStream(
+                    new BufferedInputStream(c.getInputStream(), 32 * 1024)))) {
+
+                    TarArchiveEntry entry;
+
+                    while ((entry = in.getNextTarEntry()) != null) {
+                        File dest = new File(install, entry.getName());
+
+                        if (entry.isDirectory()) {
+                            if (!dest.mkdirs())
+                                throw new IllegalStateException();
+                        }
+                        else if (entry.isSymbolicLink()) {
+                            // Important: in Hadoop installation there are symlinks, we need
to create them:
+                            Path theLinkItself = Paths.get(install.getAbsolutePath(), entry.getName());
+
+                            Path linkTarget = Paths.get(entry.getLinkName());
+
+                            Files.createSymbolicLink(theLinkItself, linkTarget);
+                        }
+                        else {
+                            File parent = dest.getParentFile();
+
+                            if (!(parent.exists() || parent.mkdirs()))
+                                throw new IllegalStateException();
+
+                            X.print(" [" + dest);
+
+                            try (BufferedOutputStream out = new BufferedOutputStream(new
FileOutputStream(dest, false),
+                                    128 * 1024)) {
+                                U.copy(in, out);
+
+                                out.flush();
+                            }
+
+                            Files.setPosixFilePermissions(dest.toPath(), modeToPermissionSet(entry.getMode()));
+
+                            X.println("]");
+                        }
+                    }
+                }
+
+                if (successFile.createNewFile())
+                    return;
+            }
+            catch (Exception e) {
+                e.printStackTrace();
+
+                U.delete(home);
+            }
+        }
+
+        throw new IllegalStateException("Failed to install " + appName + ".");
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ignite/blob/4c1739ce/modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteIgfsLinuxAndMacOSTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteIgfsLinuxAndMacOSTestSuite.java
b/modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteIgfsLinuxAndMacOSTestSuite.java
new file mode 100644
index 0000000..21d78ab
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteIgfsLinuxAndMacOSTestSuite.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.testsuites;
+
+import junit.framework.TestSuite;
+import org.apache.ignite.internal.processors.hadoop.HadoopTestClassLoader;
+import org.apache.ignite.internal.processors.hadoop.impl.igfs.HadoopIgfs20FileSystemShmemPrimarySelfTest;
+import org.apache.ignite.internal.processors.hadoop.impl.igfs.IgfsEventsTestSuite;
+import org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemIpcCacheSelfTest;
+import org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemShmemEmbeddedDualAsyncSelfTest;
+import org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemShmemEmbeddedDualSyncSelfTest;
+import org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemShmemEmbeddedPrimarySelfTest;
+import org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemShmemEmbeddedSecondarySelfTest;
+import org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemShmemExternalDualAsyncSelfTest;
+import org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemShmemExternalDualSyncSelfTest;
+import org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemShmemExternalPrimarySelfTest;
+import org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemShmemExternalSecondarySelfTest;
+import org.apache.ignite.internal.processors.igfs.IgfsServerManagerIpcEndpointRegistrationOnLinuxAndMacSelfTest;
+
+import static org.apache.ignite.testsuites.IgniteHadoopTestSuite.downloadHadoop;
+
+/**
+ * Test suite for Hadoop file system over Ignite cache.
+ * Contains tests which works on Linux and Mac OS platform only.
+ */
+public class IgniteIgfsLinuxAndMacOSTestSuite extends TestSuite {
+    /**
+     * @return Test suite.
+     * @throws Exception Thrown in case of the failure.
+     */
+    public static TestSuite suite() throws Exception {
+        downloadHadoop();
+
+        final ClassLoader ldr = new HadoopTestClassLoader();
+
+        TestSuite suite = new TestSuite("Ignite IGFS Test Suite For Linux And Mac OS");
+
+        suite.addTest(new TestSuite(ldr.loadClass(IgfsServerManagerIpcEndpointRegistrationOnLinuxAndMacSelfTest.class.getName())));
+
+        suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemShmemExternalPrimarySelfTest.class.getName())));
+        suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemShmemExternalSecondarySelfTest.class.getName())));
+        suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemShmemExternalDualSyncSelfTest.class.getName())));
+        suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemShmemExternalDualAsyncSelfTest.class.getName())));
+
+        suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemShmemEmbeddedPrimarySelfTest.class.getName())));
+        suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemShmemEmbeddedSecondarySelfTest.class.getName())));
+        suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemShmemEmbeddedDualSyncSelfTest.class.getName())));
+        suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemShmemEmbeddedDualAsyncSelfTest.class.getName())));
+
+        suite.addTest(new TestSuite(ldr.loadClass(IgniteHadoopFileSystemIpcCacheSelfTest.class.getName())));
+
+        suite.addTest(new TestSuite(ldr.loadClass(HadoopIgfs20FileSystemShmemPrimarySelfTest.class.getName())));
+
+        suite.addTest(IgfsEventsTestSuite.suite());
+
+        return suite;
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ignite/blob/4c1739ce/modules/web-console/src/main/js/app/data/pom-dependencies.json
----------------------------------------------------------------------
diff --git a/modules/web-console/src/main/js/app/data/pom-dependencies.json b/modules/web-console/src/main/js/app/data/pom-dependencies.json
index af30b85..7ab6c1b 100644
--- a/modules/web-console/src/main/js/app/data/pom-dependencies.json
+++ b/modules/web-console/src/main/js/app/data/pom-dependencies.json
@@ -8,7 +8,6 @@
     "Log4j2": {"artifactId": "ignite-log4j2"},
     "JCL": {"artifactId": "ignite-jcl"},
     "HadoopIgfsJcl": {"artifactId": "ignite-hadoop"},
-    "HadoopImplIgfsJcl": {"artifactId": "ignite-hadoop-impl"},
     "SLF4J": {"artifactId": "ignite-slf4j"},
 
     "Generic": {"groupId": "com.mchange", "artifactId": "c3p0", "version": "0.9.5.1"},

http://git-wip-us.apache.org/repos/asf/ignite/blob/4c1739ce/modules/web-console/src/main/js/app/modules/configuration/generator/Pom.service.js
----------------------------------------------------------------------
diff --git a/modules/web-console/src/main/js/app/modules/configuration/generator/Pom.service.js
b/modules/web-console/src/main/js/app/modules/configuration/generator/Pom.service.js
index eee19e3..674c16e 100644
--- a/modules/web-console/src/main/js/app/modules/configuration/generator/Pom.service.js
+++ b/modules/web-console/src/main/js/app/modules/configuration/generator/Pom.service.js
@@ -184,10 +184,8 @@ class GeneratorPom {
         if (dep)
             this.addDependency(deps, 'org.apache.ignite', dep.artifactId, igniteVersion);
 
-        if (_.find(cluster.igfss, (igfs) => igfs.secondaryFileSystemEnabled)) {
+        if (_.find(cluster.igfss, (igfs) => igfs.secondaryFileSystemEnabled))
             this.addDependency(deps, 'org.apache.ignite', 'ignite-hadoop', igniteVersion);
-            this.addDependency(deps, 'org.apache.ignite', 'ignite-hadoop-impl', igniteVersion);
-        }
 
         if (_.find(caches, blobStoreFactory))
             this.addDependency(deps, 'org.apache.ignite', 'ignite-hibernate', igniteVersion);

http://git-wip-us.apache.org/repos/asf/ignite/blob/4c1739ce/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 4f47968..2c7bad1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -52,7 +52,6 @@
         <module>modules/tools</module>
         <module>modules/core</module>
         <module>modules/hadoop</module>
-        <module>modules/hadoop-impl</module>
         <module>modules/extdata/p2p</module>
         <module>modules/extdata/uri</module>
         <module>modules/clients</module>


Mime
View raw message