beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject [1/2] beam git commit: Fix default temp location for DataflowRunner
Date Fri, 05 May 2017 23:20:33 GMT
Repository: beam
Updated Branches:
  refs/heads/master 060889701 -> e9190834b


Fix default temp location for DataflowRunner


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/5fdb4055
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/5fdb4055
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/5fdb4055

Branch: refs/heads/master
Commit: 5fdb40554c7625a6e1ecfc5a8516885d112ae1d0
Parents: 0608897
Author: Vikas Kedigehalli <vikasrk@google.com>
Authored: Thu May 4 18:30:35 2017 -0700
Committer: Davor Bonaci <davor@google.com>
Committed: Fri May 5 16:20:17 2017 -0700

----------------------------------------------------------------------
 .../beam/runners/dataflow/options/DataflowPipelineOptions.java   | 4 ++++
 .../org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java   | 4 ++--
 .../apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java   | 2 +-
 3 files changed, 7 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/beam/blob/5fdb4055/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java
b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java
index 11618af..4af420b 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java
@@ -33,6 +33,8 @@ import org.apache.beam.sdk.options.Hidden;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.StreamingOptions;
 import org.apache.beam.sdk.options.Validation;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Options that can be used to configure the {@link DataflowRunner}.
@@ -117,10 +119,12 @@ public interface DataflowPipelineOptions
    * Returns a default staging location under {@link GcpOptions#getGcpTempLocation}.
    */
   class StagingLocationFactory implements DefaultValueFactory<String> {
+    private static final Logger LOG = LoggerFactory.getLogger(StagingLocationFactory.class);
 
     @Override
     public String create(PipelineOptions options) {
       GcsOptions gcsOptions = options.as(GcsOptions.class);
+      LOG.info("No stagingLocation provided, falling back to gcpTempLocation");
       String gcpTempLocation;
       try {
         gcpTempLocation = gcsOptions.getGcpTempLocation();

http://git-wip-us.apache.org/repos/asf/beam/blob/5fdb4055/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java
----------------------------------------------------------------------
diff --git a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java
b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java
index 126b795..a4128e8 100644
--- a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java
+++ b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java
@@ -278,7 +278,7 @@ public interface GcpOptions extends GoogleApiDebugOptions, PipelineOptions
{
       }
       final String bucketName =
           "dataflow-staging-" + region + "-" + projectNumber;
-      LOG.info("No staging location provided, attempting to use default bucket: {}",
+      LOG.info("No tempLocation specified, attempting to use default bucket: {}",
           bucketName);
       Bucket bucket = new Bucket()
           .setName(bucketName)
@@ -306,7 +306,7 @@ public interface GcpOptions extends GoogleApiDebugOptions, PipelineOptions
{
         throw new RuntimeException(
             "Unable to determine the owner of the default bucket at gs://" + bucketName,
e);
       }
-      return "gs://" + bucketName;
+      return "gs://" + bucketName + "/temp/";
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/beam/blob/5fdb4055/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java
b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java
index 68b3818..3cf6980 100644
--- a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java
+++ b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java
@@ -220,7 +220,7 @@ public class GcpOptionsTest {
       when(mockGcsUtil.bucketOwner(any(GcsPath.class))).thenReturn(1L);
 
       String bucket = GcpTempLocationFactory.tryCreateDefaultBucket(options, mockCrmClient);
-      assertEquals("gs://dataflow-staging-us-north1-1", bucket);
+      assertEquals("gs://dataflow-staging-us-north1-1/temp/", bucket);
     }
 
     @Test


Mime
View raw message