avro-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r1173848 - in /avro/trunk: ./ lang/java/tools/src/main/java/org/apache/avro/tool/ lang/java/tools/src/test/java/org/apache/avro/tool/
Date Wed, 21 Sep 2011 20:57:09 GMT
Author: cutting
Date: Wed Sep 21 20:57:09 2011
New Revision: 1173848

URL: http://svn.apache.org/viewvc?rev=1173848&view=rev
Log:
AVRO-881. Java: Add a 'getmeta' tool that lists a file's metadata.  Contributed by Tom White.

Added:
    avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/DataFileGetMetaTool.java
Modified:
    avro/trunk/CHANGES.txt
    avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/Main.java
    avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestDataFileTools.java

Modified: avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1173848&r1=1173847&r2=1173848&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Wed Sep 21 20:57:09 2011
@@ -17,6 +17,9 @@ Avro 1.6.0 (unreleased)
     protobuf-defined data structures to be written and read from
     Avro-format data files.  (cutting)
 
+    AVRO-881. Java: Add a 'getmeta' tool that lists a file's metadata.
+    (Tom White via cutting)
+
   OPTIMIZATIONS
 
     AVRO-853: Java: Cache Schema hash codes. (cutting)

Added: avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/DataFileGetMetaTool.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/DataFileGetMetaTool.java?rev=1173848&view=auto
==============================================================================
--- avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/DataFileGetMetaTool.java
(added)
+++ avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/DataFileGetMetaTool.java
Wed Sep 21 20:57:09 2011
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.avro.tool;
+
+import java.io.File;
+import java.io.InputStream;
+import java.io.PrintStream;
+import java.util.List;
+
+import joptsimple.OptionParser;
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
+
+import org.apache.avro.file.DataFileReader;
+import org.apache.avro.generic.GenericDatumReader;
+
+/** Reads a data file to get its metadata. */
+public class DataFileGetMetaTool implements Tool {
+
+  @Override
+  public String getName() {
+    return "getmeta";
+  }
+
+  @Override
+  public String getShortDescription() {
+    return "Prints out the metadata of an Avro data file.";
+  }
+
+  @Override
+  public int run(InputStream stdin, PrintStream out, PrintStream err,
+      List<String> args) throws Exception {
+    
+    OptionParser p = new OptionParser();
+    OptionSpec<String> keyOption =
+        p.accepts("key", "Metadata key")
+        .withOptionalArg()
+        .ofType(String.class);
+    OptionSet opts = p.parse(args.toArray(new String[0]));
+    String keyName = keyOption.value(opts);
+    
+    List<String> nargs = opts.nonOptionArguments();
+    if (nargs.size() != 1) {
+      err.println("Expected 1 arg: input_file");
+      p.printHelpOn(err);
+      return 1;
+    }
+    DataFileReader<Void> reader =
+      new DataFileReader<Void>(new File(args.get(0)),
+                               new GenericDatumReader<Void>());
+    if (keyName != null) {
+      byte[] value = reader.getMeta(keyName);
+      if (value != null) {
+        out.write(value, 0, value.length);
+        out.println();
+      }
+    } else {
+      List<String> keys = reader.getMetaKeys();
+      for (String key : keys) {
+        out.print(escapeKey(key));
+        out.print('\t');
+        byte[] value = reader.getMeta(key);
+        out.write(value, 0, value.length);
+        out.println();
+      }
+    }
+    return 0;
+  }
+  
+  // escape TAB, NL and CR in keys, so that output can be reliably parsed
+  static String escapeKey(String key) {
+    key = key.replace("\\","\\\\");               // escape backslashes first
+    key = key.replace("\t","\\t");                // TAB
+    key = key.replace("\n","\\n");                // NL
+    key = key.replace("\r","\\r");                // CR
+    return key;
+  }
+
+}

Modified: avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/Main.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/Main.java?rev=1173848&r1=1173847&r2=1173848&view=diff
==============================================================================
--- avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/Main.java (original)
+++ avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/Main.java Wed Sep 21 20:57:09
2011
@@ -41,6 +41,7 @@ public class Main {
         new BinaryFragmentToJsonTool(),
         new DataFileReadTool(),
         new DataFileWriteTool(),
+        new DataFileGetMetaTool(),
         new DataFileGetSchemaTool(),
         new IdlTool(),
         new RecodecTool(),

Modified: avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestDataFileTools.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestDataFileTools.java?rev=1173848&r1=1173847&r2=1173848&view=diff
==============================================================================
--- avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestDataFileTools.java (original)
+++ avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestDataFileTools.java Wed
Sep 21 20:57:09 2011
@@ -18,6 +18,7 @@
 package org.apache.avro.tool;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 import java.io.ByteArrayOutputStream;
@@ -50,6 +51,9 @@ public class TestDataFileTools {
   static Schema schema;
   static File schemaFile;
   
+  private static final String KEY_NEEDING_ESCAPES = "trn\\\r\t\n";
+  private static final String ESCAPED_KEY = "trn\\\\\\r\\t\\n";
+
   @BeforeClass
   public static void writeSampleFile() throws IOException {
     sampleFile = AvroTestUtil.tempFile(
@@ -62,6 +66,7 @@ public class TestDataFileTools {
     
     DataFileWriter<Object> writer
       = new DataFileWriter<Object>(new GenericDatumWriter<Object>(schema))
+      .setMeta(KEY_NEEDING_ESCAPES, "")
       .create(schema, sampleFile);
     StringBuilder builder = new StringBuilder();
 
@@ -77,30 +82,41 @@ public class TestDataFileTools {
     jsonData = builder.toString();
   }
   
-  @Test
-  public void testRead() throws Exception {
+  private String run(Tool tool, String... args) throws Exception {
     ByteArrayOutputStream baos = new ByteArrayOutputStream();
     PrintStream p = new PrintStream(baos);
-    new DataFileReadTool().run(
+    tool.run(
         null, // stdin
         p, // stdout
         null, // stderr
-        Arrays.asList(sampleFile.getPath()));
-    assertEquals(jsonData.toString(), baos.toString("UTF-8").
-        replace("\r", ""));
+        Arrays.asList(args));
+    return baos.toString("UTF-8").replace("\r", "");
+  }
+  
+  @Test
+  public void testRead() throws Exception {
+    assertEquals(jsonData.toString(),
+        run(new DataFileReadTool(), sampleFile.getPath()));
+  }
+  
+  @Test
+  public void testGetMeta() throws Exception {
+    String output = run(new DataFileGetMetaTool(), sampleFile.getPath());
+    assertTrue(output, output.contains("avro.schema\t"+schema.toString()+"\n"));
+    assertTrue(output, output.contains(ESCAPED_KEY+"\t\n"));
+  }
+  
+  @Test
+  public void testGetMetaForSingleKey() throws Exception {
+    assertEquals(schema.toString() + "\n",
+        run(new DataFileGetMetaTool(), sampleFile.getPath(), "--key",
+            "avro.schema"));
   }
   
   @Test
   public void testGetSchema() throws Exception {
-    ByteArrayOutputStream baos = new ByteArrayOutputStream();
-    PrintStream p = new PrintStream(baos);
-    new DataFileGetSchemaTool().run(
-        null, // stdin
-        p, // stdout
-        null, // stderr
-        Arrays.asList(sampleFile.getPath()));
     assertEquals(schema.toString() + "\n",
-        baos.toString("UTF-8").replace("\r", ""));
+        run(new DataFileGetSchemaTool(), sampleFile.getPath()));
   }
   
   @Test



Mime
View raw message