ignite-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sboi...@apache.org
Subject [10/31] incubator-ignite git commit: ignite-790: several times performance improvements
Date Wed, 29 Apr 2015 13:21:50 GMT
ignite-790: several times performance improvements


Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/26fda31b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/26fda31b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/26fda31b

Branch: refs/heads/ignite-286
Commit: 26fda31b70fdbbb5c3d09b4d3a04d6e0fb48f6d0
Parents: 4ee6dd9
Author: Denis Magda <dmagda@gridgain.com>
Authored: Mon Apr 27 20:15:56 2015 +0300
Committer: Denis Magda <dmagda@gridgain.com>
Committed: Mon Apr 27 20:15:56 2015 +0300

----------------------------------------------------------------------
 modules/cloud/configs/example-aws-config.xml    |  15 ++
 modules/cloud/configs/example-gce-config.xml    |   9 ++
 .../cloud/configs/example-rackspace-config.xml  |   8 ++
 modules/cloud/pom.xml                           |  12 --
 .../cloud/TcpDiscoveryCloudNodesIpFinder.java   | 137 ++++++++++++++++++-
 .../TcpDiscoveryCloudNodesIpFinderSelfTest.java |   2 +
 .../ignite/testsuites/IgniteCloudTestSuite.java |  43 ++++++
 7 files changed, 208 insertions(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/26fda31b/modules/cloud/configs/example-aws-config.xml
----------------------------------------------------------------------
diff --git a/modules/cloud/configs/example-aws-config.xml b/modules/cloud/configs/example-aws-config.xml
index 018c458..ba3a28e 100644
--- a/modules/cloud/configs/example-aws-config.xml
+++ b/modules/cloud/configs/example-aws-config.xml
@@ -72,12 +72,27 @@
                         <property name="identity" value="Ahjdns2323_TEST_ID"/>
                         <property name="credential" value="Jkndwqo89834Jkljdlsd_TEST_KEY"/>
                         <property name="discoveryPort" value="45712"/>
+
+                        <property name="regions">
+                            <list>
+                                <value>us-east-1</value>
+                            </list>
+                        </property>
+
+                        <property name="zones">
+                            <list>
+                                <value>us-east-1b</value>
+                                <value>us-east-1e</value>
+                            </list>
+                        </property>
                     </bean>
                 </property>
                 
                 <!-- All nodes must be started on the same port. So localPort must be
equal to
                     TcpDiscoveryCloudNodesIpFinder.discoveryPort -->
                 <property name="localPort" value="45712"/>
+
+                <property name="socketTimeout" value="400"/>
             </bean>
         </property>
     </bean>

http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/26fda31b/modules/cloud/configs/example-gce-config.xml
----------------------------------------------------------------------
diff --git a/modules/cloud/configs/example-gce-config.xml b/modules/cloud/configs/example-gce-config.xml
index ff7b826..a3a912a 100644
--- a/modules/cloud/configs/example-gce-config.xml
+++ b/modules/cloud/configs/example-gce-config.xml
@@ -72,12 +72,21 @@
                         <property name="identity" value="209979073-test6@developer.gserviceaccount.com"/>
                         <property name="credentialPath" value="\var\credential\key.pem"/>
                         <property name="discoveryPort" value="45712"/>
+                        <property name="zones">
+                            <list>
+                                <value>us-central1-a</value>
+                                <value>asia-east1-a</value>
+                                <value>europe-west1-b</value>
+                            </list>
+                        </property>
                     </bean>
                 </property>
 
                 <!-- All nodes must be started on the same port. So localPort must be
equal to
                     TcpDiscoveryCloudNodesIpFinder.discoveryPort -->
                 <property name="localPort" value="45712"/>
+
+                <property name="socketTimeout" value="400"/>
             </bean>
         </property>
     </bean>

http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/26fda31b/modules/cloud/configs/example-rackspace-config.xml
----------------------------------------------------------------------
diff --git a/modules/cloud/configs/example-rackspace-config.xml b/modules/cloud/configs/example-rackspace-config.xml
index b3d8ec9..227bcbc 100644
--- a/modules/cloud/configs/example-rackspace-config.xml
+++ b/modules/cloud/configs/example-rackspace-config.xml
@@ -72,6 +72,12 @@
                         <property name="identity" value="username"/>
                         <property name="credential" value="7878a5a5b1aba2c4626TEST_KEY"/>
                         <property name="discoveryPort" value="45712"/>
+                        <property name="regions">
+                            <list>
+                                <value>IAD</value>
+                                <value>HKG</value>
+                            </list>
+                        </property>
                     </bean>
                 </property>
 
@@ -80,6 +86,8 @@
                     TcpDiscoveryCloudNodesIpFinder.discoveryPort
                 -->
                 <property name="localPort" value="45712"/>
+
+                <property name="socketTimeout" value="400"/>
             </bean>
         </property>
     </bean>

http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/26fda31b/modules/cloud/pom.xml
----------------------------------------------------------------------
diff --git a/modules/cloud/pom.xml b/modules/cloud/pom.xml
index a555dce..38c126f 100644
--- a/modules/cloud/pom.xml
+++ b/modules/cloud/pom.xml
@@ -69,18 +69,6 @@
         </dependency>
 
         <dependency>
-            <groupId>org.apache.jclouds.driver</groupId>
-            <artifactId>jclouds-sshj</artifactId>
-            <version>${jcloud.version}</version>
-        </dependency>
-
-        <dependency>
-            <groupId>org.apache.jclouds.driver</groupId>
-            <artifactId>jclouds-log4j</artifactId>
-            <version>${jcloud.version}</version>
-        </dependency>
-
-        <dependency>
             <groupId>org.apache.ignite</groupId>
             <artifactId>ignite-core</artifactId>
             <version>${project.version}</version>

http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/26fda31b/modules/cloud/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/cloud/TcpDiscoveryCloudNodesIpFinder.java
----------------------------------------------------------------------
diff --git a/modules/cloud/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/cloud/TcpDiscoveryCloudNodesIpFinder.java
b/modules/cloud/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/cloud/TcpDiscoveryCloudNodesIpFinder.java
index 72b23d0..4ef0a0c 100644
--- a/modules/cloud/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/cloud/TcpDiscoveryCloudNodesIpFinder.java
+++ b/modules/cloud/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/cloud/TcpDiscoveryCloudNodesIpFinder.java
@@ -25,8 +25,8 @@ import com.google.inject.*;
 import org.jclouds.*;
 import org.jclouds.compute.*;
 import org.jclouds.compute.domain.*;
-import org.jclouds.sshj.config.*;
-import org.jclouds.logging.log4j.config.*;
+import org.jclouds.domain.*;
+import org.jclouds.location.reference.LocationConstants;
 
 import org.apache.ignite.internal.*;
 import org.apache.ignite.internal.util.tostring.*;
@@ -34,6 +34,7 @@ import org.apache.ignite.internal.util.typedef.internal.*;
 import org.apache.ignite.spi.*;
 import org.apache.ignite.spi.discovery.tcp.*;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.*;
+import org.apache.ignite.internal.util.typedef.*;
 
 import java.io.*;
 import java.net.*;
@@ -70,6 +71,9 @@ import java.util.concurrent.atomic.*;
  * </p>
  */
 public class TcpDiscoveryCloudNodesIpFinder extends TcpDiscoveryIpFinderAdapter {
+    /* JCloud default connection timeout. */
+    private final static String JCLOUD_CONNECTION_TIMEOUT = "10000"; //10 secs
+
     /* Cloud provider. */
     private String provider;
 
@@ -84,6 +88,15 @@ public class TcpDiscoveryCloudNodesIpFinder extends TcpDiscoveryIpFinderAdapter
     @GridToStringExclude
     private String credentialPath;
 
+    /* Regions where VMs are located. */
+    private TreeSet<String> regions;
+
+    /* Zones where VMs are located. */
+    private TreeSet<String> zones;
+
+    /* Nodes filter by regions and zones. */
+    private Predicate<ComputeMetadata> nodesFilter;
+
     /* Port to use to connect to nodes across a cluster. */
     private Integer discoveryPort;
 
@@ -112,9 +125,18 @@ public class TcpDiscoveryCloudNodesIpFinder extends TcpDiscoveryIpFinderAdapter
         Collection<InetSocketAddress> addresses = new LinkedList<>();
 
         try {
-            for (ComputeMetadata node : computeService.listNodes()) {
-                NodeMetadata metadata = computeService.getNodeMetadata(node.getId());
+            Set<NodeMetadata> nodes;
+
+            if (nodesFilter != null)
+                nodes = (Set<NodeMetadata>)computeService.listNodesDetailsMatching(nodesFilter);
+            else {
+                nodes = new HashSet<>();
+
+                for (ComputeMetadata metadata : computeService.listNodes())
+                    nodes.add(computeService.getNodeMetadata(metadata.getId()));
+            }
 
+            for (NodeMetadata metadata : nodes) {
                 for (String addr : metadata.getPrivateAddresses())
                     addresses.add(new InetSocketAddress(addr, discoveryPort));
 
@@ -123,7 +145,7 @@ public class TcpDiscoveryCloudNodesIpFinder extends TcpDiscoveryIpFinderAdapter
             }
         }
         catch (Exception e) {
-            throw new IgniteSpiException("Failed to get registered addresses for the provider:
" + provider);
+            throw new IgniteSpiException("Failed to get registered addresses for the provider:
" + provider, e);
         }
 
         return addresses;
@@ -212,6 +234,44 @@ public class TcpDiscoveryCloudNodesIpFinder extends TcpDiscoveryIpFinderAdapter
     }
 
     /**
+     * Sets list of zones where VMs are located.
+     *
+     * If the zones are not set then every zone from regions, set by {@link #setRegions(Collection)}},
will be
+     * taken into account.
+     *
+     * Note, that some cloud providers, like Rackspace, doesn't have a notion of a zone.
For such
+     * providers a call to this method is redundant.
+     *
+     * @param zones Zones where VMs are located or null if to take every zone into account.
+     */
+    @IgniteSpiConfiguration(optional = true)
+    public void setZones(Collection<String> zones) {
+        if (F.isEmpty(zones))
+            return;
+
+        this.zones = new TreeSet<>(zones);
+    }
+
+    /**
+     * Sets list of regions where VMs are located.
+     *
+     * If the regions are not set then every region, that a cloud provider has, will be investigated.
This could lead
+     * to significant performance degradation.
+     *
+     * Note, that some cloud providers, like Google Compute Engine, doesn't have a notion
of a region. For such
+     * providers a call to this method is redundant.
+     *
+     * @param regions Regions where VMs are located or null if to check every region a provider
has.
+     */
+    @IgniteSpiConfiguration(optional = true)
+    public void setRegions(Collection<String> regions) {
+        if (F.isEmpty(regions))
+            return;
+
+        this.regions = new TreeSet<>(regions);
+    }
+
+    /**
      * Initializes Apache jclouds compute service.
      */
     private void initComputeService() {
@@ -231,10 +291,55 @@ public class TcpDiscoveryCloudNodesIpFinder extends TcpDiscoveryIpFinderAdapter
 
                 try {
                     ContextBuilder ctxBuilder = ContextBuilder.newBuilder(provider);
+
                     ctxBuilder.credentials(identity, credential);
-                    ctxBuilder.modules(ImmutableSet.<Module>of(new Log4JLoggingModule(),
new SshjSshClientModule()));
+
+                    Properties properties = new Properties();
+                    properties.setProperty(Constants.PROPERTY_SO_TIMEOUT, JCLOUD_CONNECTION_TIMEOUT);
+                    properties.setProperty(Constants.PROPERTY_CONNECTION_TIMEOUT, JCLOUD_CONNECTION_TIMEOUT);
+
+                    if (!F.isEmpty(regions))
+                        properties.setProperty(LocationConstants.PROPERTY_REGIONS, keysSetToStr(regions));
+
+                    if (!F.isEmpty(zones))
+                        properties.setProperty(LocationConstants.PROPERTY_ZONES, keysSetToStr(zones));
+
+                    ctxBuilder.overrides(properties);
 
                     computeService = ctxBuilder.buildView(ComputeServiceContext.class).getComputeService();
+
+                    if (!F.isEmpty(zones) || !F.isEmpty(regions)) {
+                        nodesFilter = new Predicate<ComputeMetadata>() {
+                            @Override public boolean apply(ComputeMetadata computeMetadata)
{
+                                String region = null;
+                                String zone = null;
+
+                                Location location = computeMetadata.getLocation();
+
+                                while (location != null) {
+                                    switch (location.getScope()) {
+                                        case ZONE:
+                                            zone = location.getId();
+                                            break;
+
+                                        case REGION:
+                                            region = location.getId();
+                                            break;
+                                    }
+
+                                    location = location.getParent();
+                                }
+
+                                if (regions != null && region != null &&
!regions.contains(region))
+                                    return false;
+
+                                if (zones != null && zone != null && !zones.contains(zone))
+                                    return false;
+
+                                return true;
+                            }
+                        };
+                    }
                 }
                 catch (Exception e) {
                     throw new IgniteSpiException("Failed to connect to the provider: " +
provider, e);
@@ -273,4 +378,24 @@ public class TcpDiscoveryCloudNodesIpFinder extends TcpDiscoveryIpFinderAdapter
             throw new IgniteSpiException("Failed to retrieve the private key from the file:
" + credentialPath, e);
         }
     }
+
+    /**
+     * Converts set keys to string.
+     *
+     * @param set Set.
+     * @return String where keys delimited by ','.
+     */
+    private String keysSetToStr(Set<String> set) {
+        Iterator<String> iter = set.iterator();
+        StringBuilder builder = new StringBuilder();
+
+        while (iter.hasNext()) {
+            builder.append(iter.next());
+
+            if (iter.hasNext())
+                builder.append(',');
+        }
+
+        return builder.toString();
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/26fda31b/modules/cloud/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/cloud/TcpDiscoveryCloudNodesIpFinderSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/cloud/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/cloud/TcpDiscoveryCloudNodesIpFinderSelfTest.java
b/modules/cloud/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/cloud/TcpDiscoveryCloudNodesIpFinderSelfTest.java
index e63d73e..bf0e93d 100644
--- a/modules/cloud/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/cloud/TcpDiscoveryCloudNodesIpFinderSelfTest.java
+++ b/modules/cloud/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/cloud/TcpDiscoveryCloudNodesIpFinderSelfTest.java
@@ -63,6 +63,8 @@ public class TcpDiscoveryCloudNodesIpFinderSelfTest extends
 
             ipFinder.setProvider(provider);
             ipFinder.setIdentity(IgniteCloudTestSuite.getAccessKey(provider));
+            ipFinder.setRegions(IgniteCloudTestSuite.getRegions(provider));
+            ipFinder.setZones(IgniteCloudTestSuite.getZones(provider));
 
             if (provider.equals("google-compute-engine"))
                 ipFinder.setCredentialPath(IgniteCloudTestSuite.getSecretKey(provider));

http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/26fda31b/modules/cloud/src/test/java/org/apache/ignite/testsuites/IgniteCloudTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/cloud/src/test/java/org/apache/ignite/testsuites/IgniteCloudTestSuite.java
b/modules/cloud/src/test/java/org/apache/ignite/testsuites/IgniteCloudTestSuite.java
index e838540..b542c27 100644
--- a/modules/cloud/src/test/java/org/apache/ignite/testsuites/IgniteCloudTestSuite.java
+++ b/modules/cloud/src/test/java/org/apache/ignite/testsuites/IgniteCloudTestSuite.java
@@ -19,6 +19,7 @@ package org.apache.ignite.testsuites;
 
 import junit.framework.*;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.cloud.*;
+import java.util.*;
 
 /**
  * Ignite Cloud integration test.
@@ -66,4 +67,46 @@ public class IgniteCloudTestSuite extends TestSuite {
 
         return key;
     }
+
+    /**
+     * Zones where VMs are located.
+     *
+     * @return Zones list or null.
+     */
+    public static Collection<String> getZones(String provider) {
+        String zonesStr = System.getenv("test." + provider + ".zones.list");
+
+        if (zonesStr == null)
+            return null;
+
+        String[] zonesArr = zonesStr.split(",");
+
+        LinkedList<String> list = new LinkedList<>();
+
+        for (String zone : zonesArr)
+            list.add(zone.trim());
+
+        return list;
+    }
+
+    /**
+     * Regions where VMs are located.
+     *
+     * @return Zones list or null.
+     */
+    public static Collection<String> getRegions(String provider) {
+        String regionStr = System.getenv("test." + provider + ".regions.list");
+
+        if (regionStr == null)
+            return null;
+
+        String[] zonesArr = regionStr.split(",");
+
+        LinkedList<String> list = new LinkedList<>();
+
+        for (String zone : zonesArr)
+            list.add(zone.trim());
+
+        return list;
+    }
 }


Mime
View raw message