Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id F2A0E200CA5 for ; Sat, 6 May 2017 02:26:53 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id EFCA2160BBF; Sat, 6 May 2017 00:26:53 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 488EB160BAF for ; Sat, 6 May 2017 02:26:53 +0200 (CEST) Received: (qmail 1609 invoked by uid 500); 6 May 2017 00:26:52 -0000 Mailing-List: contact commits-help@beam.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@beam.apache.org Delivered-To: mailing list commits@beam.apache.org Received: (qmail 1599 invoked by uid 99); 6 May 2017 00:26:52 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 06 May 2017 00:26:52 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 4B9ADE00C4; Sat, 6 May 2017 00:26:52 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: dhalperi@apache.org To: commits@beam.apache.org Date: Sat, 06 May 2017 00:26:52 -0000 Message-Id: <157188eafa3c45f29c6c78ef1a0c802a@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [1/3] beam git commit: Cherry-pick pull request #2907 into release-2.0.0 branch archived-at: Sat, 06 May 2017 00:26:54 -0000 Repository: beam Updated Branches: refs/heads/release-2.0.0 b9c8cfe22 -> 96aeb97cc Cherry-pick pull request #2907 into release-2.0.0 branch Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/67ea7ae4 Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/67ea7ae4 Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/67ea7ae4 Branch: refs/heads/release-2.0.0 Commit: 67ea7ae4d2144525582c7de03b17d06daa9f35bb Parents: d2fa51b Author: Davor Bonaci Authored: Fri May 5 16:20:28 2017 -0700 Committer: Dan Halperin Committed: Fri May 5 17:26:24 2017 -0700 ---------------------------------------------------------------------- .../beam/runners/dataflow/options/DataflowPipelineOptions.java | 4 ++++ .../org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java | 4 ++-- .../apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/beam/blob/67ea7ae4/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java ---------------------------------------------------------------------- diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java index 11618af..4af420b 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java @@ -33,6 +33,8 @@ import org.apache.beam.sdk.options.Hidden; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.options.StreamingOptions; import org.apache.beam.sdk.options.Validation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Options that can be used to configure the {@link DataflowRunner}. @@ -117,10 +119,12 @@ public interface DataflowPipelineOptions * Returns a default staging location under {@link GcpOptions#getGcpTempLocation}. */ class StagingLocationFactory implements DefaultValueFactory { + private static final Logger LOG = LoggerFactory.getLogger(StagingLocationFactory.class); @Override public String create(PipelineOptions options) { GcsOptions gcsOptions = options.as(GcsOptions.class); + LOG.info("No stagingLocation provided, falling back to gcpTempLocation"); String gcpTempLocation; try { gcpTempLocation = gcsOptions.getGcpTempLocation(); http://git-wip-us.apache.org/repos/asf/beam/blob/67ea7ae4/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java ---------------------------------------------------------------------- diff --git a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java index 126b795..a4128e8 100644 --- a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java +++ b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptions.java @@ -278,7 +278,7 @@ public interface GcpOptions extends GoogleApiDebugOptions, PipelineOptions { } final String bucketName = "dataflow-staging-" + region + "-" + projectNumber; - LOG.info("No staging location provided, attempting to use default bucket: {}", + LOG.info("No tempLocation specified, attempting to use default bucket: {}", bucketName); Bucket bucket = new Bucket() .setName(bucketName) @@ -306,7 +306,7 @@ public interface GcpOptions extends GoogleApiDebugOptions, PipelineOptions { throw new RuntimeException( "Unable to determine the owner of the default bucket at gs://" + bucketName, e); } - return "gs://" + bucketName; + return "gs://" + bucketName + "/temp/"; } /** http://git-wip-us.apache.org/repos/asf/beam/blob/67ea7ae4/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java ---------------------------------------------------------------------- diff --git a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java index 68b3818..3cf6980 100644 --- a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java +++ b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/options/GcpOptionsTest.java @@ -220,7 +220,7 @@ public class GcpOptionsTest { when(mockGcsUtil.bucketOwner(any(GcsPath.class))).thenReturn(1L); String bucket = GcpTempLocationFactory.tryCreateDefaultBucket(options, mockCrmClient); - assertEquals("gs://dataflow-staging-us-north1-1", bucket); + assertEquals("gs://dataflow-staging-us-north1-1/temp/", bucket); } @Test