Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id DDBA8200B89 for ; Wed, 21 Sep 2016 16:52:56 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id DC527160B0C; Wed, 21 Sep 2016 14:52:56 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 49E98160AEA for ; Wed, 21 Sep 2016 16:52:55 +0200 (CEST) Received: (qmail 5580 invoked by uid 500); 21 Sep 2016 14:52:54 -0000 Mailing-List: contact commits-help@ignite.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@ignite.apache.org Delivered-To: mailing list commits@ignite.apache.org Received: (qmail 5125 invoked by uid 99); 21 Sep 2016 14:52:54 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 21 Sep 2016 14:52:54 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 0B3DBEF99F; Wed, 21 Sep 2016 14:52:54 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: vozerov@apache.org To: commits@ignite.apache.org Date: Wed, 21 Sep 2016 14:53:09 -0000 Message-Id: In-Reply-To: <07c11c42090e46f6bdd8ca62f593b415@git.apache.org> References: <07c11c42090e46f6bdd8ca62f593b415@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [17/92] [abbrv] [partial] ignite git commit: Moving classes around. archived-at: Wed, 21 Sep 2016 14:52:57 -0000 http://git-wip-us.apache.org/repos/asf/ignite/blob/67b4da70/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/HadoopWordCount2Reducer.java ---------------------------------------------------------------------- diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/HadoopWordCount2Reducer.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/HadoopWordCount2Reducer.java deleted file mode 100644 index e780170..0000000 --- a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/examples/HadoopWordCount2Reducer.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.hadoop.examples; - -import java.io.IOException; -import org.apache.hadoop.conf.Configurable; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.mapreduce.Reducer; -import org.apache.ignite.internal.processors.hadoop.HadoopErrorSimulator; - -/** - * Combiner and Reducer phase of WordCount job. - */ -public class HadoopWordCount2Reducer extends Reducer implements Configurable { - /** Writable container for writing sum of word counts. */ - private IntWritable totalWordCnt = new IntWritable(); - - /** Flag is to check that mapper was configured before run. */ - private boolean wasConfigured; - - /** Flag is to check that mapper was set up before run. */ - private boolean wasSetUp; - - /** {@inheritDoc} */ - @Override public void reduce(Text key, Iterable values, Context ctx) throws IOException, InterruptedException { - assert wasConfigured : "Reducer should be configured"; - assert wasSetUp : "Reducer should be set up"; - - int wordCnt = 0; - - for (IntWritable value : values) - wordCnt += value.get(); - - totalWordCnt.set(wordCnt); - - ctx.write(key, totalWordCnt); - - reduceError(); - } - - /** - * Simulates reduce error if needed. - */ - protected void reduceError() throws IOException, InterruptedException { - HadoopErrorSimulator.instance().onReduce(); - } - - /** {@inheritDoc} */ - @Override protected void setup(Context context) throws IOException, InterruptedException { - super.setup(context); - - wasSetUp = true; - - setupError(); - } - - /** - * Simulates setup error if needed. - */ - protected void setupError() throws IOException, InterruptedException { - HadoopErrorSimulator.instance().onReduceSetup(); - } - - /** {@inheritDoc} */ - @Override protected void cleanup(Context context) throws IOException, InterruptedException { - super.cleanup(context); - - cleanupError(); - } - - /** - * Simulates cleanup error if needed. - */ - protected void cleanupError() throws IOException, InterruptedException { - HadoopErrorSimulator.instance().onReduceCleanup(); - } - - /** {@inheritDoc} */ - @Override public void setConf(Configuration conf) { - wasConfigured = true; - - configError(); - } - - /** - * Simulates configuration error if needed. - */ - protected void configError() { - HadoopErrorSimulator.instance().onReduceConfigure(); - } - - /** {@inheritDoc} */ - @Override public Configuration getConf() { - return null; - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/67b4da70/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/fs/KerberosHadoopFileSystemFactorySelfTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/fs/KerberosHadoopFileSystemFactorySelfTest.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/fs/KerberosHadoopFileSystemFactorySelfTest.java deleted file mode 100644 index ab6376f..0000000 --- a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/fs/KerberosHadoopFileSystemFactorySelfTest.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.hadoop.fs; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.ObjectInput; -import java.io.ObjectInputStream; -import java.io.ObjectOutput; -import java.io.ObjectOutputStream; -import java.util.concurrent.Callable; - -import org.apache.ignite.hadoop.fs.KerberosHadoopFileSystemFactory; -import org.apache.ignite.internal.processors.hadoop.common.delegate.HadoopDelegateUtils; -import org.apache.ignite.internal.processors.hadoop.common.delegate.HadoopFileSystemFactoryDelegate; -import org.apache.ignite.testframework.GridTestUtils; -import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; -import org.junit.Assert; - -/** - * Tests KerberosHadoopFileSystemFactory. - */ -public class KerberosHadoopFileSystemFactorySelfTest extends GridCommonAbstractTest { - /** - * Test parameters validation. - * - * @throws Exception If failed. - */ - public void testParameters() throws Exception { - checkParameters(null, null, -1); - - checkParameters(null, null, 100); - checkParameters(null, "b", -1); - checkParameters("a", null, -1); - - checkParameters(null, "b", 100); - checkParameters("a", null, 100); - checkParameters("a", "b", -1); - } - - /** - * Check parameters. - * - * @param keyTab Key tab. - * @param keyTabPrincipal Key tab principal. - * @param reloginInterval Re-login interval. - */ - @SuppressWarnings("ThrowableResultOfMethodCallIgnored") - private void checkParameters(String keyTab, String keyTabPrincipal, long reloginInterval) { - final KerberosHadoopFileSystemFactory fac = new KerberosHadoopFileSystemFactory(); - - fac.setKeyTab(keyTab); - fac.setKeyTabPrincipal(keyTabPrincipal); - fac.setReloginInterval(reloginInterval); - - GridTestUtils.assertThrows(null, new Callable() { - @Override public Object call() throws Exception { - HadoopFileSystemFactoryDelegate delegate = HadoopDelegateUtils.fileSystemFactoryDelegate(fac); - - delegate.start(); - - return null; - } - }, IllegalArgumentException.class, null); - } - - /** - * Checks serializatuion and deserialization of the secure factory. - * - * @throws Exception If failed. - */ - public void testSerialization() throws Exception { - KerberosHadoopFileSystemFactory fac = new KerberosHadoopFileSystemFactory(); - - checkSerialization(fac); - - fac = new KerberosHadoopFileSystemFactory(); - - fac.setUri("igfs://igfs@localhost:10500/"); - fac.setConfigPaths("/a/core-sute.xml", "/b/mapred-site.xml"); - fac.setKeyTabPrincipal("foo"); - fac.setKeyTab("/etc/krb5.keytab"); - fac.setReloginInterval(30 * 60 * 1000L); - - checkSerialization(fac); - } - - /** - * Serializes the factory, - * - * @param fac The facory to check. - * @throws Exception If failed. - */ - private void checkSerialization(KerberosHadoopFileSystemFactory fac) throws Exception { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - - ObjectOutput oo = new ObjectOutputStream(baos); - - oo.writeObject(fac); - - ObjectInput in = new ObjectInputStream(new ByteArrayInputStream(baos.toByteArray())); - - KerberosHadoopFileSystemFactory fac2 = (KerberosHadoopFileSystemFactory)in.readObject(); - - assertEquals(fac.getUri(), fac2.getUri()); - Assert.assertArrayEquals(fac.getConfigPaths(), fac2.getConfigPaths()); - assertEquals(fac.getKeyTab(), fac2.getKeyTab()); - assertEquals(fac.getKeyTabPrincipal(), fac2.getKeyTabPrincipal()); - assertEquals(fac.getReloginInterval(), fac2.getReloginInterval()); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/67b4da70/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/igfs/Hadoop1DualAbstractTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/igfs/Hadoop1DualAbstractTest.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/igfs/Hadoop1DualAbstractTest.java deleted file mode 100644 index 0d01687..0000000 --- a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/igfs/Hadoop1DualAbstractTest.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.hadoop.igfs; - -import java.io.IOException; -import org.apache.hadoop.conf.Configuration; -import org.apache.ignite.IgniteException; -import org.apache.ignite.hadoop.fs.CachingHadoopFileSystemFactory; -import org.apache.ignite.hadoop.fs.IgniteHadoopIgfsSecondaryFileSystem; -import org.apache.ignite.hadoop.util.ChainedUserNameMapper; -import org.apache.ignite.hadoop.util.KerberosUserNameMapper; -import org.apache.ignite.hadoop.util.UserNameMapper; -import org.apache.ignite.igfs.IgfsIpcEndpointConfiguration; -import org.apache.ignite.igfs.IgfsIpcEndpointType; -import org.apache.ignite.igfs.IgfsMode; -import org.apache.ignite.igfs.secondary.IgfsSecondaryFileSystem; -import org.apache.ignite.internal.processors.igfs.IgfsDualAbstractSelfTest; -import org.apache.ignite.lifecycle.LifecycleAware; -import org.jetbrains.annotations.Nullable; - -import static org.apache.ignite.IgniteFileSystem.IGFS_SCHEME; -import static org.apache.ignite.igfs.IgfsMode.PRIMARY; - -/** - * Abstract test for Hadoop 1.0 file system stack. - */ -public abstract class Hadoop1DualAbstractTest extends IgfsDualAbstractSelfTest { - /** Secondary grid name */ - private static final String GRID_NAME = "grid_secondary"; - - /** Secondary file system name */ - private static final String IGFS_NAME = "igfs_secondary"; - - /** Secondary file system REST endpoint port */ - private static final int PORT = 11500; - - /** Secondary file system REST endpoint configuration map. */ - private static final IgfsIpcEndpointConfiguration SECONDARY_REST_CFG = new IgfsIpcEndpointConfiguration() {{ - setType(IgfsIpcEndpointType.TCP); - setPort(PORT); - }}; - - /** Secondary file system authority. */ - private static final String SECONDARY_AUTHORITY = IGFS_NAME + ":" + GRID_NAME + "@127.0.0.1:" + PORT; - - /** Secondary Fs configuration full path. */ - protected String secondaryConfFullPath; - - /** Secondary Fs URI. */ - protected String secondaryUri; - - /** Constructor. */ - public Hadoop1DualAbstractTest(IgfsMode mode) { - super(mode); - } - - /** - * Creates secondary filesystems. - * @return IgfsSecondaryFileSystem - * @throws Exception On failure. - */ - @Override protected IgfsSecondaryFileSystem createSecondaryFileSystemStack() throws Exception { - startUnderlying(); - - prepareConfiguration(); - - KerberosUserNameMapper mapper1 = new KerberosUserNameMapper(); - - mapper1.setRealm("TEST.COM"); - - TestUserNameMapper mapper2 = new TestUserNameMapper(); - - ChainedUserNameMapper mapper = new ChainedUserNameMapper(); - - mapper.setMappers(mapper1, mapper2); - - CachingHadoopFileSystemFactory factory = new CachingHadoopFileSystemFactory(); - - factory.setUri(secondaryUri); - factory.setConfigPaths(secondaryConfFullPath); - factory.setUserNameMapper(mapper); - - IgniteHadoopIgfsSecondaryFileSystem second = new IgniteHadoopIgfsSecondaryFileSystem(); - - second.setFileSystemFactory(factory); - - igfsSecondary = new HadoopIgfsSecondaryFileSystemTestAdapter(factory); - - return second; - } - - /** - * Starts underlying Ignite process. - * @throws IOException On failure. - */ - protected void startUnderlying() throws Exception { - startGridWithIgfs(GRID_NAME, IGFS_NAME, PRIMARY, null, SECONDARY_REST_CFG, secondaryIpFinder); - } - - /** - * Prepares Fs configuration. - * @throws IOException On failure. - */ - protected void prepareConfiguration() throws IOException { - Configuration secondaryConf = HadoopSecondaryFileSystemConfigurationTest.configuration(IGFS_SCHEME, SECONDARY_AUTHORITY, true, true); - - secondaryConf.setInt("fs.igfs.block.size", 1024); - - secondaryConfFullPath = HadoopSecondaryFileSystemConfigurationTest.writeConfiguration(secondaryConf, HadoopSecondaryFileSystemConfigurationTest.SECONDARY_CFG_PATH); - - secondaryUri = HadoopSecondaryFileSystemConfigurationTest.mkUri(IGFS_SCHEME, SECONDARY_AUTHORITY); - } - - /** - * Test user name mapper. - */ - private static class TestUserNameMapper implements UserNameMapper, LifecycleAware { - /** */ - private static final long serialVersionUID = 0L; - - /** Started flag. */ - private boolean started; - - /** {@inheritDoc} */ - @Nullable @Override public String map(String name) { - assert started; - assert name != null && name.contains("@"); - - return name.substring(0, name.indexOf("@")); - } - - /** {@inheritDoc} */ - @Override public void start() throws IgniteException { - started = true; - } - - /** {@inheritDoc} */ - @Override public void stop() throws IgniteException { - // No-op. - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/67b4da70/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/igfs/Hadoop1OverIgfsDualAsyncTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/igfs/Hadoop1OverIgfsDualAsyncTest.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/igfs/Hadoop1OverIgfsDualAsyncTest.java deleted file mode 100644 index 89a4148..0000000 --- a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/igfs/Hadoop1OverIgfsDualAsyncTest.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.hadoop.igfs; - -import org.apache.ignite.igfs.IgfsMode; - -/** - * DUAL_ASYNC mode test. - */ -public class Hadoop1OverIgfsDualAsyncTest extends Hadoop1DualAbstractTest { - /** - * Constructor. - */ - public Hadoop1OverIgfsDualAsyncTest() { - super(IgfsMode.DUAL_ASYNC); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/67b4da70/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/igfs/Hadoop1OverIgfsDualSyncTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/igfs/Hadoop1OverIgfsDualSyncTest.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/igfs/Hadoop1OverIgfsDualSyncTest.java deleted file mode 100644 index 2b85783..0000000 --- a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/igfs/Hadoop1OverIgfsDualSyncTest.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.hadoop.igfs; - -import org.apache.ignite.igfs.IgfsMode; - -/** - * DUAL_SYNC mode. - */ -public class Hadoop1OverIgfsDualSyncTest extends Hadoop1DualAbstractTest { - /** - * Constructor. - */ - public Hadoop1OverIgfsDualSyncTest() { - super(IgfsMode.DUAL_SYNC); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/67b4da70/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopFIleSystemFactorySelfTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopFIleSystemFactorySelfTest.java b/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopFIleSystemFactorySelfTest.java deleted file mode 100644 index 85ef32c..0000000 --- a/modules/hadoop-impl/src/test/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopFIleSystemFactorySelfTest.java +++ /dev/null @@ -1,345 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.hadoop.igfs; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.ignite.IgniteException; -import org.apache.ignite.cache.CacheWriteSynchronizationMode; -import org.apache.ignite.configuration.CacheConfiguration; -import org.apache.ignite.configuration.FileSystemConfiguration; -import org.apache.ignite.configuration.IgniteConfiguration; -import org.apache.ignite.hadoop.fs.CachingHadoopFileSystemFactory; -import org.apache.ignite.hadoop.fs.HadoopFileSystemFactory; -import org.apache.ignite.hadoop.fs.IgniteHadoopIgfsSecondaryFileSystem; -import org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem; -import org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper; -import org.apache.ignite.igfs.IgfsIpcEndpointConfiguration; -import org.apache.ignite.igfs.IgfsIpcEndpointType; -import org.apache.ignite.igfs.IgfsMode; -import org.apache.ignite.igfs.IgfsPath; -import org.apache.ignite.igfs.secondary.IgfsSecondaryFileSystem; -import org.apache.ignite.internal.processors.hadoop.common.delegate.HadoopDelegateUtils; -import org.apache.ignite.internal.processors.hadoop.common.delegate.HadoopFileSystemFactoryDelegate; -import org.apache.ignite.internal.processors.igfs.IgfsCommonAbstractTest; -import org.apache.ignite.internal.processors.igfs.IgfsEx; -import org.apache.ignite.internal.util.typedef.G; -import org.apache.ignite.internal.util.typedef.internal.U; -import org.apache.ignite.lifecycle.LifecycleAware; -import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; -import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; -import org.jetbrains.annotations.Nullable; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.net.URI; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; -import static org.apache.ignite.cache.CacheMode.PARTITIONED; -import static org.apache.ignite.cache.CacheMode.REPLICATED; - -/** - * Tests for Hadoop file system factory. - */ -public class HadoopFIleSystemFactorySelfTest extends IgfsCommonAbstractTest { - /** Amount of "start" invocations */ - private static final AtomicInteger START_CNT = new AtomicInteger(); - - /** Amount of "stop" invocations */ - private static final AtomicInteger STOP_CNT = new AtomicInteger(); - - /** Path to secondary file system configuration. */ - private static final String SECONDARY_CFG_PATH = "/work/core-site-HadoopFIleSystemFactorySelfTest.xml"; - - /** IGFS path for DUAL mode. */ - private static final Path PATH_DUAL = new Path("/ignite/sync/test_dir"); - - /** IGFS path for PROXY mode. */ - private static final Path PATH_PROXY = new Path("/ignite/proxy/test_dir"); - - /** IGFS path for DUAL mode. */ - private static final IgfsPath IGFS_PATH_DUAL = new IgfsPath("/ignite/sync/test_dir"); - - /** IGFS path for PROXY mode. */ - private static final IgfsPath IGFS_PATH_PROXY = new IgfsPath("/ignite/proxy/test_dir"); - - /** Secondary IGFS. */ - private IgfsEx secondary; - - /** Primary IGFS. */ - private IgfsEx primary; - - /** {@inheritDoc} */ - @Override protected void beforeTest() throws Exception { - super.beforeTest(); - - START_CNT.set(0); - STOP_CNT.set(0); - - secondary = startSecondary(); - primary = startPrimary(); - } - - /** {@inheritDoc} */ - @Override protected void afterTest() throws Exception { - super.afterTest(); - - secondary = null; - primary = null; - - stopAllGrids(); - } - - /** - * Test custom factory. - * - * @throws Exception If failed. - */ - @SuppressWarnings("ThrowableResultOfMethodCallIgnored") - public void testCustomFactory() throws Exception { - assert START_CNT.get() == 1; - assert STOP_CNT.get() == 0; - - // Use IGFS directly. - primary.mkdirs(IGFS_PATH_DUAL); - - assert primary.exists(IGFS_PATH_DUAL); - assert secondary.exists(IGFS_PATH_DUAL); - - // Create remote instance. - FileSystem fs = FileSystem.get(URI.create("igfs://primary:primary@127.0.0.1:10500/"), baseConfiguration()); - - // Ensure lifecycle callback was invoked. - assert START_CNT.get() == 2; - assert STOP_CNT.get() == 0; - - // Check file system operations. - assert fs.exists(PATH_DUAL); - - assert fs.delete(PATH_DUAL, true); - assert !primary.exists(IGFS_PATH_DUAL); - assert !secondary.exists(IGFS_PATH_DUAL); - assert !fs.exists(PATH_DUAL); - - assert fs.mkdirs(PATH_DUAL); - assert primary.exists(IGFS_PATH_DUAL); - assert secondary.exists(IGFS_PATH_DUAL); - assert fs.exists(PATH_DUAL); - - assert fs.mkdirs(PATH_PROXY); - assert secondary.exists(IGFS_PATH_PROXY); - assert fs.exists(PATH_PROXY); - - // Close file system and ensure that associated factory was notified. - fs.close(); - - assert START_CNT.get() == 2; - assert STOP_CNT.get() == 1; - - // Stop primary node and ensure that base factory was notified. - G.stop(primary.context().kernalContext().grid().name(), true); - - assert START_CNT.get() == 2; - assert STOP_CNT.get() == 2; - } - - /** - * Start secondary IGFS. - * - * @return IGFS. - * @throws Exception If failed. - */ - private static IgfsEx startSecondary() throws Exception { - return start("secondary", 11500, IgfsMode.PRIMARY, null); - } - - /** - * Start primary IGFS. - * - * @return IGFS. - * @throws Exception If failed. - */ - private static IgfsEx startPrimary() throws Exception { - // Prepare configuration. - Configuration conf = baseConfiguration(); - - conf.set("fs.defaultFS", "igfs://secondary:secondary@127.0.0.1:11500/"); - - writeConfigurationToFile(conf); - - // Get file system instance to be used. - CachingHadoopFileSystemFactory delegate = new CachingHadoopFileSystemFactory(); - - delegate.setUri("igfs://secondary:secondary@127.0.0.1:11500/"); - delegate.setConfigPaths(SECONDARY_CFG_PATH); - - // Configure factory. - TestFactory factory = new TestFactory(delegate); - - // Configure file system. - IgniteHadoopIgfsSecondaryFileSystem secondaryFs = new IgniteHadoopIgfsSecondaryFileSystem(); - - secondaryFs.setFileSystemFactory(factory); - - // Start. - return start("primary", 10500, IgfsMode.DUAL_ASYNC, secondaryFs); - } - - /** - * Start Ignite node with IGFS instance. - * - * @param name Node and IGFS name. - * @param endpointPort Endpoint port. - * @param dfltMode Default path mode. - * @param secondaryFs Secondary file system. - * @return Igfs instance. - */ - private static IgfsEx start(String name, int endpointPort, IgfsMode dfltMode, - @Nullable IgfsSecondaryFileSystem secondaryFs) { - IgfsIpcEndpointConfiguration endpointCfg = new IgfsIpcEndpointConfiguration(); - - endpointCfg.setType(IgfsIpcEndpointType.TCP); - endpointCfg.setHost("127.0.0.1"); - endpointCfg.setPort(endpointPort); - - FileSystemConfiguration igfsCfg = new FileSystemConfiguration(); - - igfsCfg.setDataCacheName("dataCache"); - igfsCfg.setMetaCacheName("metaCache"); - igfsCfg.setName(name); - igfsCfg.setDefaultMode(dfltMode); - igfsCfg.setIpcEndpointConfiguration(endpointCfg); - igfsCfg.setSecondaryFileSystem(secondaryFs); - igfsCfg.setInitializeDefaultPathModes(true); - - CacheConfiguration dataCacheCfg = defaultCacheConfiguration(); - - dataCacheCfg.setName("dataCache"); - dataCacheCfg.setCacheMode(PARTITIONED); - dataCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); - dataCacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(2)); - dataCacheCfg.setBackups(0); - dataCacheCfg.setAtomicityMode(TRANSACTIONAL); - dataCacheCfg.setOffHeapMaxMemory(0); - - CacheConfiguration metaCacheCfg = defaultCacheConfiguration(); - - metaCacheCfg.setName("metaCache"); - metaCacheCfg.setCacheMode(REPLICATED); - metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); - metaCacheCfg.setAtomicityMode(TRANSACTIONAL); - - IgniteConfiguration cfg = new IgniteConfiguration(); - - cfg.setGridName(name); - - TcpDiscoverySpi discoSpi = new TcpDiscoverySpi(); - - discoSpi.setIpFinder(new TcpDiscoveryVmIpFinder(true)); - - cfg.setDiscoverySpi(discoSpi); - cfg.setCacheConfiguration(dataCacheCfg, metaCacheCfg); - cfg.setFileSystemConfiguration(igfsCfg); - - cfg.setLocalHost("127.0.0.1"); - cfg.setConnectorConfiguration(null); - - return (IgfsEx)G.start(cfg).fileSystem(name); - } - - /** - * Create base FileSystem configuration. - * - * @return Configuration. - */ - private static Configuration baseConfiguration() { - Configuration conf = new Configuration(); - - conf.set("fs.igfs.impl", IgniteHadoopFileSystem.class.getName()); - - return conf; - } - - /** - * Write configuration to file. - * - * @param conf Configuration. - * @throws Exception If failed. - */ - @SuppressWarnings("ResultOfMethodCallIgnored") - private static void writeConfigurationToFile(Configuration conf) throws Exception { - final String path = U.getIgniteHome() + SECONDARY_CFG_PATH; - - File file = new File(path); - - file.delete(); - - assertFalse(file.exists()); - - try (FileOutputStream fos = new FileOutputStream(file)) { - conf.writeXml(fos); - } - - assertTrue(file.exists()); - } - - /** - * Test factory. - */ - private static class TestFactory implements HadoopFileSystemFactory, LifecycleAware { - /** */ - private static final long serialVersionUID = 0L; - - /** File system factory. */ - private CachingHadoopFileSystemFactory factory; - - /** File system. */ - private transient HadoopFileSystemFactoryDelegate delegate; - - /** - * Constructor. - * - * @param factory File system factory. - */ - public TestFactory(CachingHadoopFileSystemFactory factory) { - this.factory = factory; - } - - /** {@inheritDoc} */ - @Override public Object get(String usrName) throws IOException { - return delegate.get(usrName); - } - - /** {@inheritDoc} */ - @Override public void start() throws IgniteException { - delegate = HadoopDelegateUtils.fileSystemFactoryDelegate(factory); - - delegate.start(); - - START_CNT.incrementAndGet(); - } - - /** {@inheritDoc} */ - @Override public void stop() throws IgniteException { - STOP_CNT.incrementAndGet(); - } - } -}