avro-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ph...@apache.org
Subject svn commit: r903362 - in /hadoop/avro/trunk: ./ doc/src/content/xdocs/ lang/java/src/java/org/apache/avro/file/ lang/java/src/test/java/org/apache/avro/ lang/java/src/test/java/org/apache/avro/tool/
Date Tue, 26 Jan 2010 18:51:55 GMT
Author: philz
Date: Tue Jan 26 18:51:54 2010
New Revision: 903362

URL: http://svn.apache.org/viewvc?rev=903362&view=rev
Log:
AVRO-368. Reserve avro.* in object container files, and 
rename existing reserved words.


Added:
    hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/TestDataFileMeta.java
Modified:
    hadoop/avro/trunk/CHANGES.txt
    hadoop/avro/trunk/doc/src/content/xdocs/spec.xml
    hadoop/avro/trunk/lang/java/src/java/org/apache/avro/file/DataFileConstants.java
    hadoop/avro/trunk/lang/java/src/java/org/apache/avro/file/DataFileWriter.java
    hadoop/avro/trunk/lang/java/src/java/org/apache/avro/file/DeflateCodec.java
    hadoop/avro/trunk/lang/java/src/java/org/apache/avro/file/NullCodec.java
    hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/tool/TestDataFileTools.java

Modified: hadoop/avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/CHANGES.txt?rev=903362&r1=903361&r2=903362&view=diff
==============================================================================
--- hadoop/avro/trunk/CHANGES.txt (original)
+++ hadoop/avro/trunk/CHANGES.txt Tue Jan 26 18:51:54 2010
@@ -74,6 +74,9 @@
     
     AVRO-135. Add compression to data files. (philz)
 
+    AVRO-368. Reserve avro.* in object container files, and 
+    rename existing reserved words. (philz)
+
   IMPROVEMENTS
 
     AVRO-157. Changes from code review comments for C++. (sbanacho)

Modified: hadoop/avro/trunk/doc/src/content/xdocs/spec.xml
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/doc/src/content/xdocs/spec.xml?rev=903362&r1=903361&r2=903362&view=diff
==============================================================================
--- hadoop/avro/trunk/doc/src/content/xdocs/spec.xml (original)
+++ hadoop/avro/trunk/doc/src/content/xdocs/spec.xml Tue Jan 26 18:51:54 2010
@@ -619,11 +619,12 @@
 	<li>For each pair, a string key and bytes value.</li>
       </ul>
 
-      <p>The following file metadata properties are reserved:</p>
+      <p>All metadata properties that start with "avro." are reserved.
+      The following file metadata properties are currently used:</p>
       <ul>
-	<li><strong>schema</strong> contains the schema of objects
+	<li><strong>avro.schema</strong> contains the schema of objects
 	stored in the file, as JSON data (required).</li>
-	<li><strong>codec</strong> the name of the compression codec
+	<li><strong>avro.codec</strong> the name of the compression codec
 	used to compress blocks, as a string.  Implementations
         are required to support the following codecs: "null" and "deflate".  
         If codec is absent, it is assumed to be "null".  The codecs

Modified: hadoop/avro/trunk/lang/java/src/java/org/apache/avro/file/DataFileConstants.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/java/src/java/org/apache/avro/file/DataFileConstants.java?rev=903362&r1=903361&r2=903362&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/java/src/java/org/apache/avro/file/DataFileConstants.java (original)
+++ hadoop/avro/trunk/lang/java/src/java/org/apache/avro/file/DataFileConstants.java Tue Jan
26 18:51:54 2010
@@ -30,8 +30,9 @@
   public static final int SYNC_SIZE = 16;
   public static final int DEFAULT_SYNC_INTERVAL = 1000*SYNC_SIZE; 
 
-  public static final String SCHEMA = "schema";
-  public static final String COUNT = "count";
-  public static final String CODEC = "codec";
+  public static final String SCHEMA = "avro.schema";
+  public static final String CODEC = "avro.codec";
   public static final String NULL_CODEC = "null";
+  public static final String DEFLATE_CODEC = "deflate";
+
 }

Modified: hadoop/avro/trunk/lang/java/src/java/org/apache/avro/file/DataFileWriter.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/java/src/java/org/apache/avro/file/DataFileWriter.java?rev=903362&r1=903361&r2=903362&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/java/src/java/org/apache/avro/file/DataFileWriter.java (original)
+++ hadoop/avro/trunk/lang/java/src/java/org/apache/avro/file/DataFileWriter.java Tue Jan
26 18:51:54 2010
@@ -35,7 +35,6 @@
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.Map;
 
 import org.apache.avro.AvroRuntimeException;
@@ -72,12 +71,6 @@
 
   private boolean isOpen;
   private Codec codec;
-  
-  private static final HashSet<String> RESERVED_META = new HashSet<String>();
-  static {
-    RESERVED_META.add("codec");
-    RESERVED_META.add("schema");
-  }
 
   /** Construct a writer, not yet open. */
   public DataFileWriter(DatumWriter<D> dout) {
@@ -98,7 +91,7 @@
   public DataFileWriter<D> setCodec(CodecFactory c) {
     assertNotOpen();
     this.codec = c.createInstance();
-    setMetaInternal("codec", codec.getName());
+    setMetaInternal(DataFileConstants.CODEC, codec.getName());
     return this;
   }
 
@@ -153,7 +146,7 @@
     this.schema = reader.getSchema();
     this.sync = reader.sync;
     this.meta.putAll(reader.meta);
-    byte[] codecBytes = this.meta.get("codec");
+    byte[] codecBytes = this.meta.get(DataFileConstants.CODEC);
     if (codecBytes != null) {
       String strCodec = new String(codecBytes, "UTF-8");
       this.codec = CodecFactory.fromString(strCodec).createInstance();
@@ -208,12 +201,16 @@
 
   /** Set a metadata property. */
   public DataFileWriter<D> setMeta(String key, byte[] value) {
-    if (RESERVED_META.contains(key)) {
+    if (isReserved(key)) {
       throw new AvroRuntimeException("Cannot set reserved meta key: " + key);
     }
     return setMetaInternal(key, value);
   }
   
+  private boolean isReserved(String key) {
+    return key.startsWith("avro.");
+  }
+
   /** Set a metadata property. */
   public DataFileWriter<D> setMeta(String key, String value) {
     try {

Modified: hadoop/avro/trunk/lang/java/src/java/org/apache/avro/file/DeflateCodec.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/java/src/java/org/apache/avro/file/DeflateCodec.java?rev=903362&r1=903361&r2=903362&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/java/src/java/org/apache/avro/file/DeflateCodec.java (original)
+++ hadoop/avro/trunk/lang/java/src/java/org/apache/avro/file/DeflateCodec.java Tue Jan 26
18:51:54 2010
@@ -66,7 +66,7 @@
 
   @Override
   String getName() {
-    return "deflate";
+    return DataFileConstants.DEFLATE_CODEC;
   }
 
   @Override

Modified: hadoop/avro/trunk/lang/java/src/java/org/apache/avro/file/NullCodec.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/java/src/java/org/apache/avro/file/NullCodec.java?rev=903362&r1=903361&r2=903362&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/java/src/java/org/apache/avro/file/NullCodec.java (original)
+++ hadoop/avro/trunk/lang/java/src/java/org/apache/avro/file/NullCodec.java Tue Jan 26 18:51:54
2010
@@ -42,7 +42,7 @@
 
   @Override
   String getName() {
-    return "null";
+    return DataFileConstants.NULL_CODEC;
   }
 
   @Override

Added: hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/TestDataFileMeta.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/TestDataFileMeta.java?rev=903362&view=auto
==============================================================================
--- hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/TestDataFileMeta.java (added)
+++ hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/TestDataFileMeta.java Tue Jan
26 18:51:54 2010
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.avro;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+
+import org.apache.avro.Schema.Type;
+import org.apache.avro.file.DataFileStream;
+import org.apache.avro.file.DataFileWriter;
+import org.apache.avro.generic.GenericDatumReader;
+import org.apache.avro.generic.GenericDatumWriter;
+import org.junit.Test;
+
+public class TestDataFileMeta {
+  @Test(expected=AvroRuntimeException.class)
+  public void testUseReservedMeta() {
+    DataFileWriter<?> w = new DataFileWriter<Object>(new GenericDatumWriter<Object>());
+    w.setMeta("avro.foo", "bar");
+  }
+
+  @Test()
+  public void testUseMeta() throws IOException {
+    DataFileWriter<?> w = new DataFileWriter<Object>(new GenericDatumWriter<Object>());
+    File f = AvroTestUtil.tempFile("testDataFileMeta.avro");
+    w.setMeta("hello", "bar");
+    w.create(Schema.create(Type.NULL), f);
+    w.close();
+
+    DataFileStream<Void> r = new DataFileStream<Void>(new FileInputStream(f),
new GenericDatumReader<Void>());
+    assertEquals("bar", r.getMetaString("hello"));
+  }
+
+  @Test(expected=AvroRuntimeException.class)
+  public void testUseMetaAfterCreate() throws IOException {
+    DataFileWriter<?> w = new DataFileWriter<Object>(new GenericDatumWriter<Object>());
+    w.create(Schema.create(Type.NULL), new ByteArrayOutputStream());
+    w.setMeta("foo", "bar");
+  }
+
+
+}

Modified: hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/tool/TestDataFileTools.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/tool/TestDataFileTools.java?rev=903362&r1=903361&r2=903362&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/tool/TestDataFileTools.java
(original)
+++ hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/tool/TestDataFileTools.java
Tue Jan 26 18:51:54 2010
@@ -49,7 +49,8 @@
   
   @BeforeClass
   public static void writeSampleFile() throws IOException {
-    sampleFile = AvroTestUtil.tempFile(TestDataFileTools.class + ".avro");
+    sampleFile = AvroTestUtil.tempFile(
+      TestDataFileTools.class.getName() + ".avro");
     schema = Schema.create(Type.INT);
     
     DataFileWriter<Object> writer
@@ -133,7 +134,7 @@
     }
     assertEquals(COUNT, i);
     assertEquals(schema, fileReader.getSchema());
-    assertEquals(expectedCodec, fileReader.getMetaString("codec"));
+    assertEquals(expectedCodec, fileReader.getMetaString("avro.codec"));
   }
   
   @Test



Mime
View raw message