hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From lium...@apache.org
Subject hadoop git commit: HADOOP-14107. S3Guard: TestS3GuardTool hangs/fails when offline: it's an IT test. Contributed by Mingliang Liu
Date Wed, 12 Apr 2017 22:55:20 GMT
Repository: hadoop
Updated Branches:
  refs/heads/HADOOP-13345 c41f9f3b7 -> c6dc3e6bd


HADOOP-14107. S3Guard: TestS3GuardTool hangs/fails when offline: it's an IT test. Contributed
by Mingliang Liu


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c6dc3e6b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c6dc3e6b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c6dc3e6b

Branch: refs/heads/HADOOP-13345
Commit: c6dc3e6bd85791636ba18bb647bae13fc47958e4
Parents: c41f9f3
Author: Mingliang Liu <liuml07@apache.org>
Authored: Tue Apr 4 12:29:18 2017 -0700
Committer: Mingliang Liu <liuml07@apache.org>
Committed: Wed Apr 12 15:55:12 2017 -0700

----------------------------------------------------------------------
 .../fs/s3a/s3guard/ITestS3GuardToolLocal.java   | 150 +++++++++++++++++++
 .../hadoop/fs/s3a/s3guard/TestS3GuardTool.java  | 150 -------------------
 2 files changed, 150 insertions(+), 150 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c6dc3e6b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java
new file mode 100644
index 0000000..d9d6a42
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java
@@ -0,0 +1,150 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.s3guard;
+
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.Diff;
+import org.junit.Test;
+
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.PrintStream;
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.SUCCESS;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+/**
+ * Test S3Guard related CLI commands against a LocalMetadataStore.
+ */
+public class ITestS3GuardToolLocal extends S3GuardToolTestBase {
+
+  @Override
+  protected MetadataStore newMetadataStore() {
+    return new LocalMetadataStore();
+  }
+
+  @Test
+  public void testImportCommand() throws IOException {
+    S3AFileSystem fs = getFs();
+    MetadataStore ms = getMetadataStore();
+    Path parent = path("test-import");
+    fs.mkdirs(parent);
+    Path dir = new Path(parent, "a");
+    fs.mkdirs(dir);
+    Path emptyDir = new Path(parent, "emptyDir");
+    fs.mkdirs(emptyDir);
+    for (int i = 0; i < 10; i++) {
+      String child = String.format("file-%d", i);
+      try (FSDataOutputStream out = fs.create(new Path(dir, child))) {
+        out.write(1);
+      }
+    }
+
+    S3GuardTool.Import cmd = new S3GuardTool.Import(fs.getConf());
+    cmd.setMetadataStore(ms);
+
+    assertEquals("Import command did not exit successfully - see output",
+        SUCCESS, cmd.run(new String[]{"import", parent.toString()}));
+
+    DirListingMetadata children =
+        ms.listChildren(dir);
+    assertEquals("Unexpected number of paths imported", 10, children
+        .getListing().size());
+    assertEquals("Expected 2 items: empty directory and a parent directory", 2,
+        ms.listChildren(parent).getListing().size());
+    // assertTrue(children.isAuthoritative());
+  }
+
+  @Test
+  public void testDiffCommand() throws IOException {
+    S3AFileSystem fs = getFs();
+    MetadataStore ms = getMetadataStore();
+    Set<Path> filesOnS3 = new HashSet<>(); // files on S3.
+    Set<Path> filesOnMS = new HashSet<>(); // files on metadata store.
+
+    Path testPath = path("test-diff");
+    mkdirs(testPath, true, true);
+
+    Path msOnlyPath = new Path(testPath, "ms_only");
+    mkdirs(msOnlyPath, false, true);
+    filesOnMS.add(msOnlyPath);
+    for (int i = 0; i < 5; i++) {
+      Path file = new Path(msOnlyPath, String.format("file-%d", i));
+      createFile(file, false, true);
+      filesOnMS.add(file);
+    }
+
+    Path s3OnlyPath = new Path(testPath, "s3_only");
+    mkdirs(s3OnlyPath, true, false);
+    filesOnS3.add(s3OnlyPath);
+    for (int i = 0; i < 5; i++) {
+      Path file = new Path(s3OnlyPath, String.format("file-%d", i));
+      createFile(file, true, false);
+      filesOnS3.add(file);
+    }
+
+    ByteArrayOutputStream buf = new ByteArrayOutputStream();
+    PrintStream out = new PrintStream(buf);
+    Diff cmd = new Diff(fs.getConf());
+    cmd.setMetadataStore(ms);
+    assertEquals("Diff command did not exit successfully - see output", SUCCESS,
+        cmd.run(new String[]{"diff", "-meta", "local://metadata",
+            testPath.toString()}, out));
+
+    Set<Path> actualOnS3 = new HashSet<>();
+    Set<Path> actualOnMS = new HashSet<>();
+    boolean duplicates = false;
+    try (ByteArrayInputStream in =
+             new ByteArrayInputStream(buf.toByteArray())) {
+      try (BufferedReader reader =
+               new BufferedReader(new InputStreamReader(in))) {
+        String line;
+        while ((line = reader.readLine()) != null) {
+          String[] fields = line.split("\\s");
+          assertEquals("[" + line + "] does not have enough fields",
+              4, fields.length);
+          String where = fields[0];
+          Path path = new Path(fields[3]);
+          if (Diff.S3_PREFIX.equals(where)) {
+            duplicates = duplicates || actualOnS3.contains(path);
+            actualOnS3.add(path);
+          } else if (Diff.MS_PREFIX.equals(where)) {
+            duplicates = duplicates || actualOnMS.contains(path);
+            actualOnMS.add(path);
+          } else {
+            fail("Unknown prefix: " + where);
+          }
+        }
+      }
+    }
+    String actualOut = out.toString();
+    assertEquals("Mismatched metadata store outputs: " + actualOut,
+        filesOnMS, actualOnMS);
+    assertEquals("Mismatched s3 outputs: " + actualOut, filesOnS3, actualOnS3);
+    assertFalse("Diff contained duplicates", duplicates);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c6dc3e6b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestS3GuardTool.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestS3GuardTool.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestS3GuardTool.java
deleted file mode 100644
index 288daa0..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestS3GuardTool.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.s3guard;
-
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3a.S3AFileSystem;
-import org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.Diff;
-import org.junit.Test;
-
-import java.io.BufferedReader;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.PrintStream;
-import java.util.HashSet;
-import java.util.Set;
-
-import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.SUCCESS;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-/**
- * Test S3Guard related CLI commands against a LocalMetadataStore.
- */
-public class TestS3GuardTool extends S3GuardToolTestBase {
-
-  @Override
-  protected MetadataStore newMetadataStore() {
-    return new LocalMetadataStore();
-  }
-
-  @Test
-  public void testImportCommand() throws IOException {
-    S3AFileSystem fs = getFs();
-    MetadataStore ms = getMetadataStore();
-    Path parent = path("test-import");
-    fs.mkdirs(parent);
-    Path dir = new Path(parent, "a");
-    fs.mkdirs(dir);
-    Path emptyDir = new Path(parent, "emptyDir");
-    fs.mkdirs(emptyDir);
-    for (int i = 0; i < 10; i++) {
-      String child = String.format("file-%d", i);
-      try (FSDataOutputStream out = fs.create(new Path(dir, child))) {
-        out.write(1);
-      }
-    }
-
-    S3GuardTool.Import cmd = new S3GuardTool.Import(fs.getConf());
-    cmd.setMetadataStore(ms);
-
-    assertEquals("Import command did not exit successfully - see output",
-        SUCCESS, cmd.run(new String[]{"import", parent.toString()}));
-
-    DirListingMetadata children =
-        ms.listChildren(dir);
-    assertEquals("Unexpected number of paths imported", 10, children
-        .getListing().size());
-    assertEquals("Expected 2 items: empty directory and a parent directory", 2,
-        ms.listChildren(parent).getListing().size());
-    // assertTrue(children.isAuthoritative());
-  }
-
-  @Test
-  public void testDiffCommand() throws IOException {
-    S3AFileSystem fs = getFs();
-    MetadataStore ms = getMetadataStore();
-    Set<Path> filesOnS3 = new HashSet<>(); // files on S3.
-    Set<Path> filesOnMS = new HashSet<>(); // files on metadata store.
-
-    Path testPath = path("test-diff");
-    mkdirs(testPath, true, true);
-
-    Path msOnlyPath = new Path(testPath, "ms_only");
-    mkdirs(msOnlyPath, false, true);
-    filesOnMS.add(msOnlyPath);
-    for (int i = 0; i < 5; i++) {
-      Path file = new Path(msOnlyPath, String.format("file-%d", i));
-      createFile(file, false, true);
-      filesOnMS.add(file);
-    }
-
-    Path s3OnlyPath = new Path(testPath, "s3_only");
-    mkdirs(s3OnlyPath, true, false);
-    filesOnS3.add(s3OnlyPath);
-    for (int i = 0; i < 5; i++) {
-      Path file = new Path(s3OnlyPath, String.format("file-%d", i));
-      createFile(file, true, false);
-      filesOnS3.add(file);
-    }
-
-    ByteArrayOutputStream buf = new ByteArrayOutputStream();
-    PrintStream out = new PrintStream(buf);
-    Diff cmd = new Diff(fs.getConf());
-    cmd.setMetadataStore(ms);
-    assertEquals("Diff command did not exit successfully - see output", SUCCESS,
-        cmd.run(new String[]{"diff", "-meta", "local://metadata",
-            testPath.toString()}, out));
-
-    Set<Path> actualOnS3 = new HashSet<>();
-    Set<Path> actualOnMS = new HashSet<>();
-    boolean duplicates = false;
-    try (ByteArrayInputStream in =
-             new ByteArrayInputStream(buf.toByteArray())) {
-      try (BufferedReader reader =
-               new BufferedReader(new InputStreamReader(in))) {
-        String line;
-        while ((line = reader.readLine()) != null) {
-          String[] fields = line.split("\\s");
-          assertEquals("[" + line + "] does not have enough fields",
-              4, fields.length);
-          String where = fields[0];
-          Path path = new Path(fields[3]);
-          if (Diff.S3_PREFIX.equals(where)) {
-            duplicates = duplicates || actualOnS3.contains(path);
-            actualOnS3.add(path);
-          } else if (Diff.MS_PREFIX.equals(where)) {
-            duplicates = duplicates || actualOnMS.contains(path);
-            actualOnMS.add(path);
-          } else {
-            fail("Unknown prefix: " + where);
-          }
-        }
-      }
-    }
-    String actualOut = out.toString();
-    assertEquals("Mismatched metadata store outputs: " + actualOut,
-        filesOnMS, actualOnMS);
-    assertEquals("Mismatched s3 outputs: " + actualOut, filesOnS3, actualOnS3);
-    assertFalse("Diff contained duplicates", duplicates);
-  }
-}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message