hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From lium...@apache.org
Subject [16/40] hadoop git commit: HDFS-11420. Edit file should not be processed by the same type processor in OfflineEditsViewer. Contributed by Yiqun Lin.
Date Mon, 20 Mar 2017 23:18:27 GMT
HDFS-11420. Edit file should not be processed by the same type processor in OfflineEditsViewer.
Contributed by Yiqun Lin.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4416a07c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4416a07c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4416a07c

Branch: refs/heads/HADOOP-13345
Commit: 4416a07c9c57af01ffbd2e43afcde2a1ba94c736
Parents: 2d0e24e
Author: Yiqun Lin <yqlin@apache.org>
Authored: Wed Mar 15 18:05:03 2017 +0800
Committer: Yiqun Lin <yqlin@apache.org>
Committed: Wed Mar 15 18:05:03 2017 +0800

----------------------------------------------------------------------
 .../offlineEditsViewer/OfflineEditsViewer.java   | 19 +++++++++++++++++--
 .../src/site/markdown/HdfsEditsViewer.md         |  2 ++
 .../TestOfflineEditsViewer.java                  | 18 ++++++++++++++++++
 3 files changed, 37 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4416a07c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsViewer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsViewer.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsViewer.java
index 107881f..f075ed2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsViewer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsViewer.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.classification.InterfaceStability;
 
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.hdfs.tools.offlineEditsViewer.OfflineEditsLoader.OfflineEditsLoaderFactory;
+import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
 
@@ -55,7 +56,9 @@ public class OfflineEditsViewer extends Configured implements Tool {
       "Required command line arguments:\n" +
       "-i,--inputFile <arg>   edits file to process, xml (case\n" +
       "                       insensitive) extension means XML format,\n" +
-      "                       any other filename means binary format\n" +
+      "                       any other filename means binary format.\n" +
+      "                       XML/Binary format input file is not allowed\n" +
+      "                       to be processed by the same type processor.\n" +
       "-o,--outputFile <arg>  Name of output file. If the specified\n" +
       "                       file exists, it will be overwritten,\n" +
       "                       format of the file is determined\n" +
@@ -132,12 +135,24 @@ public class OfflineEditsViewer extends Configured implements Tool {
       System.out.println("input  [" + inputFileName  + "]");
       System.out.println("output [" + outputFileName + "]");
     }
+
+    boolean xmlInput = StringUtils.toLowerCase(inputFileName).endsWith(".xml");
+    if (xmlInput && StringUtils.equalsIgnoreCase("xml", processor)) {
+      System.err.println("XML format input file is not allowed"
+          + " to be processed by XML processor.");
+      return -1;
+    } else if(!xmlInput && StringUtils.equalsIgnoreCase("binary", processor)) {
+      System.err.println("Binary format input file is not allowed"
+          + " to be processed by Binary processor.");
+      return -1;
+    }
+
     try {
       if (visitor == null) {
         visitor = OfflineEditsVisitorFactory.getEditsVisitor(
             outputFileName, processor, flags.getPrintToScreen());
       }
-      boolean xmlInput = inputFileName.toLowerCase().endsWith(".xml");
+
       OfflineEditsLoader loader = OfflineEditsLoaderFactory.
           createLoader(visitor, inputFileName, xmlInput, flags);
       loader.loadEdits();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4416a07c/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsEditsViewer.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsEditsViewer.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsEditsViewer.md
index 5e069bb..4ab07ce 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsEditsViewer.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsEditsViewer.md
@@ -30,6 +30,8 @@ Input formats supported:
 2.  **xml**: XML format, as produced by xml processor, used if filename
     has `.xml` (case insensitive) extension
 
+Note: XML/Binary format input file is not allowed to be processed by the same type processor.
+
 The Offline Edits Viewer provides several output processors (unless stated otherwise the
output of the processor can be converted back to original edits file):
 
 1.  **binary**: native binary format that Hadoop uses internally

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4416a07c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java
index a21bc8f..bbad73c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java
@@ -338,4 +338,22 @@ public class TestOfflineEditsViewer {
       }
     }
   }
+
+  @Test
+  public void testProcessorWithSameTypeFormatFile() throws IOException {
+    String edits = nnHelper.generateEdits();
+    LOG.info("Generated edits=" + edits);
+    String binaryEdits = folder.newFile("binaryEdits").getAbsolutePath();
+    String editsParsedXml = folder.newFile("editsParsed.xml").getAbsolutePath();
+    String editsReparsedXml = folder.newFile("editsReparsed.xml")
+        .getAbsolutePath();
+
+    // Binary format input file is not allowed to be processed
+    // by Binary processor.
+    assertEquals(-1, runOev(edits, binaryEdits, "binary", false));
+    // parse to XML then back to XML
+    assertEquals(0, runOev(edits, editsParsedXml, "xml", false));
+    // XML format input file is not allowed to be processed by XML processor.
+    assertEquals(-1, runOev(editsParsedXml, editsReparsedXml, "xml", false));
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message