hudi-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From GitBox <...@apache.org>
Subject [GitHub] [incubator-hudi] leesf commented on a change in pull request #1436: [HUDI-711] Refactor exporter main logic
Date Wed, 25 Mar 2020 03:09:03 GMT
leesf commented on a change in pull request #1436: [HUDI-711] Refactor exporter main logic
URL: https://github.com/apache/incubator-hudi/pull/1436#discussion_r397585126
 
 

 ##########
 File path: hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieSnapshotExporter.java
 ##########
 @@ -159,18 +161,85 @@ public void testExportAsHudi() throws IOException {
       assertTrue(dfs.exists(new Path(partition + "/.hoodie_partition_metadata")));
       assertTrue(dfs.exists(new Path(targetPath + "/_SUCCESS")));
     }
+  }
+
+  public static class TestHoodieSnapshotExporterForEarlyAbort extends ExporterTestHarness
{
+
+    private HoodieSnapshotExporter.Config cfg;
+
+    @Before
+    public void setUp() throws Exception {
+      super.setUp();
+      cfg = new Config();
+      cfg.sourceBasePath = sourcePath;
+      cfg.targetOutputPath = targetPath;
+      cfg.outputFormat = OutputFormatValidator.HUDI;
+    }
 
     @Test
-    public void testExportEmptyDataset() throws IOException {
+    public void testExportWhenTargetPathExists() throws IOException {
+      // make target output path present
+      dfs.mkdirs(new Path(targetPath));
+
+      // export
+      Throwable t = null;
+      try {
+        new HoodieSnapshotExporter().export(jsc, cfg);
+      } catch (Exception e) {
+        t = e;
+      } finally {
+        assertNotNull(t);
+        assertTrue(t instanceof HoodieSnapshotExporterException);
+        assertEquals("The target output path already exists.", t.getMessage());
+      }
+    }
+
+    @Test
+    public void testExportDatasetWithNoCommit() throws IOException {
+      // delete commit files
+      List<Path> commitFiles = Arrays.stream(dfs.listStatus(new Path(sourcePath + "/.hoodie")))
+          .map(FileStatus::getPath)
+          .filter(filePath -> filePath.getName().endsWith(".commit"))
+          .collect(Collectors.toList());
+      for (Path p : commitFiles) {
+        dfs.delete(p, false);
+      }
+
+      // export
+      Throwable t = null;
+      try {
+        new HoodieSnapshotExporter().export(jsc, cfg);
+      } catch (Exception e) {
+        t = e;
+      } finally {
+        assertNotNull(t);
+        assertTrue(t instanceof HoodieSnapshotExporterException);
+        assertEquals("No commits present. Nothing to snapshot.", t.getMessage());
+      }
+
+      // Check results
+      assertFalse(dfs.exists(new Path(targetPath)));
+    }
+
+    @Test
+    public void testExportDatasetWithNoPartition() throws IOException {
       // delete all source data
       dfs.delete(new Path(sourcePath + "/" + PARTITION_PATH), true);
 
       // export
-      new HoodieSnapshotExporter().export(SparkSession.builder().config(jsc.getConf()).getOrCreate(),
cfg);
+      Throwable t = null;
+      try {
+        new HoodieSnapshotExporter().export(jsc, cfg);
+      } catch (Exception e) {
+        t = e;
+      } finally {
+        assertNotNull(t);
+        assertTrue(t instanceof HoodieSnapshotExporterException);
+        assertEquals("The source dataset has 0 partition to snapshot.", t.getMessage());
 
 Review comment:
   ditto

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services

Mime
View raw message