Return-Path: X-Original-To: apmail-ignite-commits-archive@minotaur.apache.org Delivered-To: apmail-ignite-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id CDAD41843E for ; Wed, 29 Apr 2015 10:46:16 +0000 (UTC) Received: (qmail 96878 invoked by uid 500); 29 Apr 2015 10:46:16 -0000 Delivered-To: apmail-ignite-commits-archive@ignite.apache.org Received: (qmail 96850 invoked by uid 500); 29 Apr 2015 10:46:16 -0000 Mailing-List: contact commits-help@ignite.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@ignite.incubator.apache.org Delivered-To: mailing list commits@ignite.incubator.apache.org Received: (qmail 96840 invoked by uid 99); 29 Apr 2015 10:46:16 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 29 Apr 2015 10:46:16 +0000 X-ASF-Spam-Status: No, hits=-0.0 required=5.0 tests=SPF_PASS X-Spam-Check-By: apache.org Received-SPF: pass (nike.apache.org: domain of root@apache.org designates 54.76.25.247 as permitted sender) Received: from [54.76.25.247] (HELO mx1-eu-west.apache.org) (54.76.25.247) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 29 Apr 2015 10:45:51 +0000 Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx1-eu-west.apache.org (ASF Mail Server at mx1-eu-west.apache.org) with SMTP id 282952AAED for ; Wed, 29 Apr 2015 10:45:33 +0000 (UTC) Received: (qmail 94937 invoked by uid 99); 29 Apr 2015 10:45:32 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 29 Apr 2015 10:45:32 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 4AD03E083A; Wed, 29 Apr 2015 10:45:32 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: sboikov@apache.org To: commits@ignite.incubator.apache.org Date: Wed, 29 Apr 2015 10:45:34 -0000 Message-Id: In-Reply-To: <79d280eb355b4240a89910bcfae18ac7@git.apache.org> References: <79d280eb355b4240a89910bcfae18ac7@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [3/7] incubator-ignite git commit: ignite-734: google cloud storage ipfinder is fully implemented X-Virus-Checked: Checked by ClamAV on apache.org ignite-734: google cloud storage ipfinder is fully implemented Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/b98da84a Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/b98da84a Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/b98da84a Branch: refs/heads/ignite-sprint-4 Commit: b98da84a598e50d18134060cd4ff885563a62941 Parents: 546ec8e Author: Denis Magda Authored: Wed Apr 22 17:11:07 2015 +0300 Committer: Denis Magda Committed: Wed Apr 22 17:11:07 2015 +0300 ---------------------------------------------------------------------- .../google/TcpDiscoveryGoogleCloudIpFinder.java | 185 ++++++++++++------- 1 file changed, 123 insertions(+), 62 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/b98da84a/modules/gce/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/google/TcpDiscoveryGoogleCloudIpFinder.java ---------------------------------------------------------------------- diff --git a/modules/gce/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/google/TcpDiscoveryGoogleCloudIpFinder.java b/modules/gce/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/google/TcpDiscoveryGoogleCloudIpFinder.java index 3bfd8f3..4f7529f 100644 --- a/modules/gce/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/google/TcpDiscoveryGoogleCloudIpFinder.java +++ b/modules/gce/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/google/TcpDiscoveryGoogleCloudIpFinder.java @@ -18,15 +18,18 @@ package org.apache.ignite.spi.discovery.tcp.ipfinder.google; import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; +import com.google.api.client.googleapis.json.GoogleJsonResponseException; import com.google.api.client.http.InputStreamContent; import com.google.api.client.http.javanet.NetHttpTransport; import com.google.api.services.storage.Storage; import com.google.api.services.storage.StorageScopes; import com.google.api.services.storage.model.*; import com.google.common.collect.ImmutableMap; +import org.apache.ignite.IgniteLogger; import org.apache.ignite.internal.IgniteInterruptedCheckedException; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.resources.LoggerResource; import org.apache.ignite.spi.*; import org.apache.ignite.spi.discovery.tcp.ipfinder.*; @@ -42,17 +45,42 @@ import java.util.concurrent.atomic.AtomicBoolean; /** * Google Cloud Storage based IP finder. - * - * TODO: complete + *

+ * For information about Cloud Storage visit cloud.google.com. + *

Configuration

+ *

Mandatory

+ *
    + *
  • Service Account Id (see {@link #setServiceAccountId(String)})
    • + *
  • Service Account P12 key file path (see {@link #setServiceAccountP12FilePath(String)})
    • + *
  • Google Platform project name (see {@link #setProjectName(String)})
    • + *
  • Google Storage bucket name (see {@link #setBucketName(String)})
    • + *
+ *

Optional

+ *
    + *
  • Shared flag (see {@link #setShared(boolean)})
    • + *
+ *

+ * The finder will create a bucket with the provided name. The bucket will contain entries named + * like the following: {@code 192.168.1.136#1001}. + *

+ * Note that storing data in Google Cloud Storage service will result in charges to your Google Cloud Platform account. + * Choose another implementation of {@link org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder} for local + * or home network tests. + *

+ * Note that this finder is shared by default (see {@link org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder#isShared()}. */ public class TcpDiscoveryGoogleCloudIpFinder extends TcpDiscoveryIpFinderAdapter { /* Default object's content. */ - private final static ByteArrayInputStream OBJECT_CONTENT = new ByteArrayInputStream(new byte[1]); + private final static ByteArrayInputStream OBJECT_CONTENT = new ByteArrayInputStream(new byte[0]); + + /** Grid logger. */ + @LoggerResource + private IgniteLogger log; /* Google Cloud Platform's project name.*/ private String projectName; - /* Google Cloud Platform's bucket name. */ + /* Google Storage bucket name. */ private String bucketName; /* Service account p12 private key file name. */ @@ -146,7 +174,8 @@ public class TcpDiscoveryGoogleCloudIpFinder extends TcpDiscoveryIpFinderAdapter Storage.Objects.Delete deleteObject = storage.objects().delete(bucketName, key); deleteObject.execute(); - } catch (Exception e) { + } + catch (Exception e) { throw new IgniteSpiException("Failed to delete entry [bucketName=" + bucketName + ", entry=" + key + ']', e); } @@ -155,7 +184,8 @@ public class TcpDiscoveryGoogleCloudIpFinder extends TcpDiscoveryIpFinderAdapter /** * Sets Google Cloud Platforms project name. - * The project name is the one which your Google VM instances, Cloud Storage, etc. belong to. + * Usually this is an auto generated project number (ex. 208709979073) that can be found in "Overview" section + * of Google Developer Console. *

* For details refer to Google Cloud Platform API reference. * @@ -167,7 +197,10 @@ public class TcpDiscoveryGoogleCloudIpFinder extends TcpDiscoveryIpFinderAdapter } /** - * Sets Google Cloud Platforms bucket name. + * Sets Google Cloud Storage bucket name. + * If the bucket doesn't exist Ignite will automatically create it. However the name must be unique across whole + * Google Cloud Storage and Service Account Id (see {@link #setServiceAccountId(String)}) must be authorized to + * perform this operation. * * @param bucketName Bucket name. */ @@ -176,18 +209,37 @@ public class TcpDiscoveryGoogleCloudIpFinder extends TcpDiscoveryIpFinderAdapter this.bucketName = bucketName; } + + /** + * Sets a full path to the private key in PKCS12 format of the Service Account. + *

+ * For more information refer to + * Service Account Authentication. + * + * @param p12FileName Private key file full path. + */ @IgniteSpiConfiguration(optional = false) public void setServiceAccountP12FilePath(String p12FileName) { this.serviceAccountP12FilePath = p12FileName; } + /** + * Sets the service account ID (typically an e-mail address). + *

+ * For more information refer to + * Service Account Authentication. + * + * @param id + */ @IgniteSpiConfiguration(optional = false) public void setServiceAccountId(String id) { this.serviceAccountId = id; } /** + * Google Cloud Storage initialization. * + * @throws IgniteSpiException In case of error. */ private void init() throws IgniteSpiException { if (initGuard.compareAndSet(false, true)) { @@ -196,7 +248,8 @@ public class TcpDiscoveryGoogleCloudIpFinder extends TcpDiscoveryIpFinderAdapter try { httpTransport = GoogleNetHttpTransport.newTrustedTransport(); - } catch (GeneralSecurityException | IOException e) { + } + catch (GeneralSecurityException | IOException e) { throw new IgniteSpiException(e); } @@ -206,16 +259,55 @@ public class TcpDiscoveryGoogleCloudIpFinder extends TcpDiscoveryIpFinderAdapter credential = new GoogleCredential.Builder().setTransport(httpTransport) .setJsonFactory(JacksonFactory.getDefaultInstance()).setServiceAccountId(serviceAccountId) .setServiceAccountPrivateKeyFromP12File(new File(serviceAccountP12FilePath)) - .setServiceAccountScopes(Collections.singleton(StorageScopes.DEVSTORAGE_READ_WRITE)).build(); - } catch (Exception e) { + .setServiceAccountScopes(Collections.singleton(StorageScopes.DEVSTORAGE_FULL_CONTROL)).build(); + + } + catch (Exception e) { throw new IgniteSpiException("Failed to authenticate on Google Cloud Platform", e); } try { storage = new Storage.Builder(httpTransport, JacksonFactory.getDefaultInstance(), credential) .setApplicationName(projectName).build(); - } catch (Exception e) { - throw new IgniteSpiException("Failed to open a storage for given project name: " + projectName); + } + catch (Exception e) { + throw new IgniteSpiException("Failed to open a storage for given project name: " + projectName, e); + } + + boolean createBucket = false; + + try { + Storage.Buckets.Get getBucket = storage.buckets().get(bucketName); + getBucket.setProjection("full"); + + getBucket.execute(); + } + catch (GoogleJsonResponseException e) { + if (e.getStatusCode() == 404) { + U.warn(log, "Bucket doesn't exist, will create it [bucketName=" + bucketName + "]"); + createBucket = true; + } + else + throw new IgniteSpiException("Failed to open the bucket: " + bucketName, e); + } + catch (Exception e) { + throw new IgniteSpiException("Failed to open the bucket: " + bucketName, e); + } + + if (createBucket) { + Bucket newBucket = new Bucket(); + newBucket.setName(bucketName); + + try { + Storage.Buckets.Insert insertBucket = storage.buckets().insert(projectName, newBucket); + insertBucket.setProjection("full"); + insertBucket.setPredefinedDefaultObjectAcl("projectPrivate"); + + insertBucket.execute(); + } + catch (Exception e) { + throw new IgniteSpiException("Failed to create the bucket: " + bucketName, e); + } } } finally { @@ -225,7 +317,8 @@ public class TcpDiscoveryGoogleCloudIpFinder extends TcpDiscoveryIpFinderAdapter else { try { U.await(initLatch); - } catch (IgniteInterruptedCheckedException e) { + } + catch (IgniteInterruptedCheckedException e) { throw new IgniteSpiException("Thread has been interrupted.", e); } @@ -234,15 +327,28 @@ public class TcpDiscoveryGoogleCloudIpFinder extends TcpDiscoveryIpFinderAdapter } } + /** + * Constructs bucket's key from an address. + * + * @param addr Node address. + * @return Bucket key. + */ private String keyFromAddr(InetSocketAddress addr) { - return addr.getAddress().getHostAddress() + ":" + addr.getPort(); + return addr.getAddress().getHostAddress() + "#" + addr.getPort(); } - private InetSocketAddress addrFromString(String address) throws IgniteSpiException { - String[] res = address.split(":"); + /** + * Constructs a node address from bucket's key. + * + * @param key Bucket key. + * @return Node address. + * @throws IgniteSpiException In case of error. + */ + private InetSocketAddress addrFromString(String key) throws IgniteSpiException { + String[] res = key.split("#"); if (res.length != 2) - throw new IgniteSpiException("Invalid address string: " + address); + throw new IgniteSpiException("Invalid address string: " + key); int port; @@ -255,49 +361,4 @@ public class TcpDiscoveryGoogleCloudIpFinder extends TcpDiscoveryIpFinderAdapter return new InetSocketAddress(res[0], port); } - - - - public static void main(String args[]) { - TcpDiscoveryGoogleCloudIpFinder ipFinder = new TcpDiscoveryGoogleCloudIpFinder(); - - String bucketName = "grid-gain-test-bucket1"; - - ipFinder.setBucketName(bucketName); - ipFinder.setProjectName("gridgain"); - ipFinder.setServiceAccountId("208709979073-v0mn6ttpd3mqu2b5lbhh1mvdet7os3n6@developer.gserviceaccount.com"); - ipFinder.setServiceAccountP12FilePath("C:\\ignite\\GCE\\gridgain-0889e44b58b7.p12"); - - List addresses = new LinkedList<>(); - addresses.add(new InetSocketAddress("192.168.0.1", 23)); - addresses.add(new InetSocketAddress("192.168.0.1", 89)); - addresses.add(new InetSocketAddress("92.68.0.1", 1223)); - - System.out.println("PUT ADDR"); - ipFinder.registerAddresses(addresses); - - Collection result = ipFinder.getRegisteredAddresses(); - System.out.println("GET ADDR"); - - for (InetSocketAddress add: result) { - System.out.println(add.getAddress().getHostAddress() + ":" + add.getPort()); - } - - System.out.println("REMOVE"); - ipFinder.unregisterAddresses(addresses); - - - result = ipFinder.getRegisteredAddresses(); - System.out.println("GET ADDR 2"); - - for (InetSocketAddress add: result) { - System.out.println(add.getAddress().getHostAddress() + ":" + add.getPort()); - } - - List addresses2 = new LinkedList<>(); - addresses.add(new InetSocketAddress("192.1638.02.1", 23)); - ipFinder.unregisterAddresses(addresses2); - } - - }