beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From lc...@apache.org
Subject [1/2] beam git commit: [BEAM-1871] Hide CloudResourceManager client in sdks/java/extensions/gcp-core
Date Mon, 01 May 2017 22:05:12 GMT
Repository: beam
Updated Branches:
  refs/heads/master c2a555783 -> ba539b6ce


[BEAM-1871] Hide CloudResourceManager client in sdks/java/extensions/gcp-core


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/0f010009
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/0f010009
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/0f010009

Branch: refs/heads/master
Commit: 0f0100098f7a86fa0b34daaba03495a8110b6811
Parents: c2a5557
Author: Luke Cwik <lcwik@google.com>
Authored: Mon May 1 13:30:36 2017 -0700
Committer: Luke Cwik <lcwik@google.com>
Committed: Mon May 1 15:04:35 2017 -0700

----------------------------------------------------------------------
 .../sdk/extensions/gcp/options/GcpOptions.java  | 42 +++++++++++++++++++-
 .../org/apache/beam/sdk/util/Transport.java     | 21 ----------
 .../extensions/gcp/GcpCoreApiSurfaceTest.java   |  1 -
 .../gcp/options/GoogleApiDebugOptionsTest.java  |  6 +--
 4 files changed, 43 insertions(+), 27 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/beam/blob/0f010009/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java
----------------------------------------------------------------------
diff --git a/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java
b/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java
index b2a83e9..445f00f 100644
--- a/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java
+++ b/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java
@@ -21,15 +21,19 @@ import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Strings.isNullOrEmpty;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.google.api.client.http.HttpRequestInitializer;
 import com.google.api.client.util.BackOff;
 import com.google.api.client.util.Sleeper;
 import com.google.api.services.cloudresourcemanager.CloudResourceManager;
 import com.google.api.services.cloudresourcemanager.model.Project;
 import com.google.api.services.storage.model.Bucket;
 import com.google.auth.Credentials;
+import com.google.auth.http.HttpCredentialsAdapter;
+import com.google.cloud.hadoop.util.ChainingHttpRequestInitializer;
 import com.google.cloud.hadoop.util.ResilientOperation;
 import com.google.cloud.hadoop.util.RetryDeterminer;
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
 import com.google.common.io.Files;
 import java.io.File;
 import java.io.IOException;
@@ -43,6 +47,7 @@ import java.util.regex.Pattern;
 import javax.annotation.Nullable;
 import org.apache.beam.sdk.extensions.gcp.auth.CredentialFactory;
 import org.apache.beam.sdk.extensions.gcp.auth.GcpCredentialFactory;
+import org.apache.beam.sdk.extensions.gcp.auth.NullCredentialInitializer;
 import org.apache.beam.sdk.options.Default;
 import org.apache.beam.sdk.options.DefaultValueFactory;
 import org.apache.beam.sdk.options.Description;
@@ -50,6 +55,7 @@ import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.util.FluentBackoff;
 import org.apache.beam.sdk.util.InstanceBuilder;
 import org.apache.beam.sdk.util.PathValidator;
+import org.apache.beam.sdk.util.RetryHttpRequestInitializer;
 import org.apache.beam.sdk.util.Transport;
 import org.apache.beam.sdk.util.gcsfs.GcsPath;
 import org.joda.time.Duration;
@@ -229,8 +235,8 @@ public interface GcpOptions extends GoogleApiDebugOptions, PipelineOptions
{
     public String create(PipelineOptions options) {
       String tempLocation = options.getTempLocation();
       if (isNullOrEmpty(tempLocation)) {
-        tempLocation = tryCreateDefaultBucket(options, Transport.newCloudResourceManagerClient(
-            options.as(CloudResourceManagerOptions.class)).build());
+        tempLocation = tryCreateDefaultBucket(options,
+            newCloudResourceManagerClient(options.as(CloudResourceManagerOptions.class)).build());
         options.setTempLocation(tempLocation);
       } else {
         try {
@@ -347,5 +353,37 @@ public interface GcpOptions extends GoogleApiDebugOptions, PipelineOptions
{
       checkArgument(zoneParts.length >= 2, "Invalid zone provided: %s", zone);
       return zoneParts[0] + "-" + zoneParts[1];
     }
+
+    /**
+     * Returns a CloudResourceManager client builder using the specified
+     * {@link CloudResourceManagerOptions}.
+     */
+    @VisibleForTesting
+    static CloudResourceManager.Builder newCloudResourceManagerClient(
+        CloudResourceManagerOptions options) {
+      Credentials credentials = options.getGcpCredential();
+      if (credentials == null) {
+        NullCredentialInitializer.throwNullCredentialException();
+      }
+      return new CloudResourceManager.Builder(Transport.getTransport(), Transport.getJsonFactory(),
+          chainHttpRequestInitializer(
+              credentials,
+              // Do not log 404. It clutters the output and is possibly even required by
the caller.
+              new RetryHttpRequestInitializer(ImmutableList.of(404))))
+          .setApplicationName(options.getAppName())
+          .setGoogleClientRequestInitializer(options.getGoogleApiTrace());
+    }
+
+    private static HttpRequestInitializer chainHttpRequestInitializer(
+        Credentials credential, HttpRequestInitializer httpRequestInitializer) {
+      if (credential == null) {
+        return new ChainingHttpRequestInitializer(
+            new NullCredentialInitializer(), httpRequestInitializer);
+      } else {
+        return new ChainingHttpRequestInitializer(
+            new HttpCredentialsAdapter(credential),
+            httpRequestInitializer);
+      }
+    }
   }
 }

http://git-wip-us.apache.org/repos/asf/beam/blob/0f010009/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/Transport.java
----------------------------------------------------------------------
diff --git a/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/Transport.java
b/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/Transport.java
index 7903011..b8474bb 100644
--- a/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/Transport.java
+++ b/sdks/java/extensions/gcp-core/src/main/java/org/apache/beam/sdk/util/Transport.java
@@ -22,7 +22,6 @@ import com.google.api.client.http.HttpRequestInitializer;
 import com.google.api.client.http.HttpTransport;
 import com.google.api.client.json.JsonFactory;
 import com.google.api.client.json.jackson2.JacksonFactory;
-import com.google.api.services.cloudresourcemanager.CloudResourceManager;
 import com.google.api.services.storage.Storage;
 import com.google.auth.Credentials;
 import com.google.auth.http.HttpCredentialsAdapter;
@@ -33,7 +32,6 @@ import java.net.MalformedURLException;
 import java.net.URL;
 import java.security.GeneralSecurityException;
 import org.apache.beam.sdk.extensions.gcp.auth.NullCredentialInitializer;
-import org.apache.beam.sdk.extensions.gcp.options.CloudResourceManagerOptions;
 import org.apache.beam.sdk.extensions.gcp.options.GcsOptions;
 
 /**
@@ -88,25 +86,6 @@ public class Transport {
   }
 
   /**
-   * Returns a CloudResourceManager client builder using the specified
-   * {@link CloudResourceManagerOptions}.
-   */
-  public static CloudResourceManager.Builder
-      newCloudResourceManagerClient(CloudResourceManagerOptions options) {
-    Credentials credentials = options.getGcpCredential();
-    if (credentials == null) {
-      NullCredentialInitializer.throwNullCredentialException();
-    }
-    return new CloudResourceManager.Builder(getTransport(), getJsonFactory(),
-        chainHttpRequestInitializer(
-            credentials,
-            // Do not log 404. It clutters the output and is possibly even required by the
caller.
-            new RetryHttpRequestInitializer(ImmutableList.of(404))))
-        .setApplicationName(options.getAppName())
-        .setGoogleClientRequestInitializer(options.getGoogleApiTrace());
-  }
-
-  /**
    * Returns a Cloud Storage client builder using the specified {@link GcsOptions}.
    */
   public static Storage.Builder

http://git-wip-us.apache.org/repos/asf/beam/blob/0f010009/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/extensions/gcp/GcpCoreApiSurfaceTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/extensions/gcp/GcpCoreApiSurfaceTest.java
b/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/extensions/gcp/GcpCoreApiSurfaceTest.java
index 215b0f4..a8772c3 100644
--- a/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/extensions/gcp/GcpCoreApiSurfaceTest.java
+++ b/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/extensions/gcp/GcpCoreApiSurfaceTest.java
@@ -39,7 +39,6 @@ public class GcpCoreApiSurfaceTest {
         ImmutableSet.of(
             "org.apache.beam",
             "com.google.api.client",
-            "com.google.api.services.cloudresourcemanager",
             "com.google.api.services.storage",
             "com.google.auth",
             "com.fasterxml.jackson.annotation",

http://git-wip-us.apache.org/repos/asf/beam/blob/0f010009/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GoogleApiDebugOptionsTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GoogleApiDebugOptionsTest.java
b/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GoogleApiDebugOptionsTest.java
index 79d24f6..67d5880 100644
--- a/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GoogleApiDebugOptionsTest.java
+++ b/sdks/java/extensions/gcp-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GoogleApiDebugOptionsTest.java
@@ -113,9 +113,9 @@ public class GoogleApiDebugOptionsTest {
         Transport.newStorageClient(options).build().objects().get("testBucketId", "testObjectId");
     assertEquals("TraceDestination", getRequest.get("$trace"));
 
-    Delete deleteRequest =
-        Transport.newCloudResourceManagerClient(options.as(CloudResourceManagerOptions.class))
-            .build().projects().delete("testProjectId");
+    Delete deleteRequest = GcpOptions.GcpTempLocationFactory.newCloudResourceManagerClient(
+        options.as(CloudResourceManagerOptions.class))
+        .build().projects().delete("testProjectId");
     assertNull(deleteRequest.get("$trace"));
   }
 


Mime
View raw message