beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From j...@apache.org
Subject [1/2] beam git commit: Ignore HTTP 404 when deleting GCS files
Date Wed, 06 Sep 2017 03:06:45 GMT
Repository: beam
Updated Branches:
  refs/heads/master 7d6f28633 -> 2e9b728a8


Ignore HTTP 404 when deleting GCS files

You can get a 404 if a previous delete RPC actually succeeded
but returned a failed response due to some network issues,
so the next attempt gets a 404.


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/17e346ee
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/17e346ee
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/17e346ee

Branch: refs/heads/master
Commit: 17e346ee342df11c12db5ba856c2004880cfa40e
Parents: 7d6f286
Author: Eugene Kirpichov <kirpichov@google.com>
Authored: Tue Sep 5 15:55:08 2017 -0700
Committer: Eugene Kirpichov <kirpichov@google.com>
Committed: Tue Sep 5 19:50:20 2017 -0700

----------------------------------------------------------------------
 .../java/org/apache/beam/sdk/util/GcsUtil.java  |  6 ++-
 .../org/apache/beam/sdk/util/GcsUtilTest.java   | 45 ++++++++++++++++++++
 2 files changed, 50 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/beam/blob/17e346ee/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/GcsUtil.java
----------------------------------------------------------------------
diff --git a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/GcsUtil.java
b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/GcsUtil.java
index d7205bf..da4a1e0 100644
--- a/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/GcsUtil.java
+++ b/sdks/java/extensions/google-cloud-platform-core/src/main/java/org/apache/beam/sdk/util/GcsUtil.java
@@ -767,7 +767,11 @@ public class GcsUtil {
 
       @Override
       public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) throws IOException
{
-        throw new IOException(String.format("Error trying to delete %s: %s", file, e));
+        if (e.getCode() == 404) {
+          LOG.info("Ignoring failed deletion of file {} which already does not exist: {}",
file, e);
+        } else {
+          throw new IOException(String.format("Error trying to delete %s: %s", file, e));
+        }
       }
     });
   }

http://git-wip-us.apache.org/repos/asf/beam/blob/17e346ee/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util/GcsUtilTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util/GcsUtilTest.java
b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util/GcsUtilTest.java
index 2164e75..897cd53 100644
--- a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util/GcsUtilTest.java
+++ b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/util/GcsUtilTest.java
@@ -524,6 +524,51 @@ public class GcsUtilTest {
   }
 
   @Test
+  public void testRemoveWhenFileNotFound() throws Exception {
+    JsonFactory jsonFactory = new JacksonFactory();
+
+    String contentBoundary = "batch_foobarbaz";
+    String contentBoundaryLine = "--" + contentBoundary;
+    String endOfContentBoundaryLine = "--" + contentBoundary + "--";
+
+    GenericJson error = new GenericJson()
+        .set("error", new GenericJson().set("code", 404));
+    error.setFactory(jsonFactory);
+
+    String content = contentBoundaryLine + "\n"
+        + "Content-Type: application/http\n"
+        + "\n"
+        + "HTTP/1.1 404 Not Found\n"
+        + "Content-Length: -1\n"
+        + "\n"
+        + error.toString()
+        + "\n"
+        + "\n"
+        + endOfContentBoundaryLine
+        + "\n";
+
+    final LowLevelHttpResponse mockResponse = Mockito.mock(LowLevelHttpResponse.class);
+    when(mockResponse.getContentType()).thenReturn("multipart/mixed; boundary=" + contentBoundary);
+    when(mockResponse.getStatusCode()).thenReturn(200);
+    when(mockResponse.getContent()).thenReturn(toStream(content));
+
+    // A mock transport that lets us mock the API responses.
+    MockLowLevelHttpRequest request = new MockLowLevelHttpRequest() {
+      @Override
+      public LowLevelHttpResponse execute() throws IOException {
+        return mockResponse;
+      }
+    };
+    MockHttpTransport mockTransport =
+        new MockHttpTransport.Builder().setLowLevelHttpRequest(request).build();
+
+    GcsUtil gcsUtil = gcsOptionsWithTestCredential().getGcsUtil();
+    gcsUtil.setStorageClient(
+        new Storage(mockTransport, Transport.getJsonFactory(), new RetryHttpRequestInitializer()));
+    gcsUtil.remove(Arrays.asList("gs://some-bucket/already-deleted"));
+  }
+
+  @Test
   public void testCreateBucket() throws IOException {
     GcsOptions pipelineOptions = gcsOptionsWithTestCredential();
     GcsUtil gcsUtil = pipelineOptions.getGcsUtil();


Mime
View raw message