Return-Path: 
 <commits-return-861-apmail-gora-commits-archive=gora.apache.org@gora.apache.org>
X-Original-To: apmail-gora-commits-archive@www.apache.org
Delivered-To: apmail-gora-commits-archive@www.apache.org
Received: from mail.apache.org (hermes.apache.org [140.211.11.3])
	by minotaur.apache.org (Postfix) with SMTP id C6BD310905
	for <apmail-gora-commits-archive@www.apache.org>;
 Sat, 12 Apr 2014 19:22:50 +0000 (UTC)
Received: (qmail 44387 invoked by uid 500); 12 Apr 2014 19:22:50 -0000
Delivered-To: apmail-gora-commits-archive@gora.apache.org
Received: (qmail 44355 invoked by uid 500); 12 Apr 2014 19:22:49 -0000
Mailing-List: contact commits-help@gora.apache.org; run by ezmlm
Precedence: bulk
List-Help: <mailto:commits-help@gora.apache.org>
List-Unsubscribe: <mailto:commits-unsubscribe@gora.apache.org>
List-Post: <mailto:commits@gora.apache.org>
List-Id: <commits.gora.apache.org>
Reply-To: dev@gora.apache.org
Delivered-To: mailing list commits@gora.apache.org
Received: (qmail 44348 invoked by uid 99); 12 Apr 2014 19:22:49 -0000
Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136)
    by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 12 Apr 2014 19:22:49 +0000
X-ASF-Spam-Status: No, hits=-1996.5 required=5.0
	tests=ALL_TRUSTED,URIBL_BLACK
X-Spam-Check-By: apache.org
Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4)
    by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 12 Apr 2014 19:22:46 +0000
Received: from eris.apache.org (localhost [127.0.0.1])
	by eris.apache.org (Postfix) with ESMTP id 58C3A2388BBD;
	Sat, 12 Apr 2014 19:22:02 +0000 (UTC)
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r1586888 [7/10] - in /gora/trunk: ./ bin/ gora-accumulo/
 gora-accumulo/src/main/java/org/apache/gora/accumulo/encoders/
 gora-accumulo/src/main/java/org/apache/gora/accumulo/query/
 gora-accumulo/src/main/java/org/apache/gora/accumulo/store/ ...
Date: Sat, 12 Apr 2014 19:21:56 -0000
To: commits@gora.apache.org
From: rmarroquin@apache.org
X-Mailer: svnmailer-1.0.9
Message-Id: <20140412192202.58C3A2388BBD@eris.apache.org>
X-Virus-Checked: Checked by ClamAV on apache.org

Modified: gora/trunk/gora-core/src/main/java/org/apache/gora/store/impl/DataStoreBase.java
URL: http://svn.apache.org/viewvc/gora/trunk/gora-core/src/main/java/org/apache/gora/store/impl/DataStoreBase.java?rev=1586888&r1=1586887&r2=1586888&view=diff
==============================================================================
--- gora/trunk/gora-core/src/main/java/org/apache/gora/store/impl/DataStoreBase.java (original)
+++ gora/trunk/gora-core/src/main/java/org/apache/gora/store/impl/DataStoreBase.java Sat Apr 12 19:21:53 2014
@@ -22,16 +22,19 @@ import java.io.Closeable;
 import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 
 import org.apache.avro.Schema;
 import org.apache.avro.Schema.Field;
+import org.apache.avro.specific.SpecificDatumReader;
+import org.apache.avro.specific.SpecificDatumWriter; 
 import org.apache.commons.lang.builder.EqualsBuilder;
-import org.apache.gora.avro.PersistentDatumReader;
-import org.apache.gora.avro.PersistentDatumWriter;
 import org.apache.gora.avro.store.AvroStore;
 import org.apache.gora.persistency.BeanFactory;
+import org.apache.gora.persistency.Persistent;
 import org.apache.gora.persistency.impl.BeanFactoryImpl;
 import org.apache.gora.persistency.impl.PersistentBase;
 import org.apache.gora.store.DataStore;
@@ -70,9 +73,9 @@ public abstract class DataStoreBase<K, T
 
   protected Properties properties;
 
-  protected PersistentDatumReader<T> datumReader;
+  protected SpecificDatumReader<T> datumReader;
 
-  protected PersistentDatumWriter<T> datumWriter;
+  protected SpecificDatumWriter<T> datumWriter;
   
   public static final Logger LOG = LoggerFactory.getLogger(AvroStore.class);
 
@@ -93,8 +96,8 @@ public abstract class DataStoreBase<K, T
     autoCreateSchema = DataStoreFactory.getAutoCreateSchema(properties, this);
     this.properties = properties;
 
-    datumReader = new PersistentDatumReader<T>(schema, false);
-    datumWriter = new PersistentDatumWriter<T>(schema, false);
+    datumReader = new SpecificDatumReader<T>(schema);
+    datumWriter = new SpecificDatumWriter<T>(schema);
   }
 
   @Override
@@ -164,7 +167,26 @@ public abstract class DataStoreBase<K, T
     if(fields != null) {
       return fields;
     }
-    return beanFactory.getCachedPersistent().getFields();
+    return getFields();
+  }
+  
+  protected String[] getFields() {
+    List<Field> schemaFields = beanFactory.getCachedPersistent().getSchema().getFields();
+    
+    List<Field> list = new ArrayList<Field>();
+    for (Field field : schemaFields) {
+      if (!Persistent.DIRTY_BYTES_FIELD_NAME.equalsIgnoreCase(field.name())) {
+        list.add(field);
+      }
+    }
+    schemaFields = list;
+    
+    String[] fieldNames = new String[schemaFields.size()];
+    for(int i = 0; i<fieldNames.length; i++ ){
+      fieldNames[i] = schemaFields.get(i).name();
+    }
+    
+    return fieldNames;
   }
 
   @Override
@@ -234,25 +256,28 @@ public abstract class DataStoreBase<K, T
   /**
    * Returns the name of the schema to use for the persistent class. 
    * 
-   * The schema name is prefixed with schema.prefix from {@link Configuration}.
-   * The schema name in the defined properties is returned. If null then
+   * First the schema name in the {@link Configuration} is used. If null,
+   * the schema name in the defined properties is returned. If null then
    * the provided mappingSchemaName is returned. If this is null too,
    * the class name, without the package, of the persistent class is returned.
    * @param mappingSchemaName the name of the schema as read from the mapping file
    * @param persistentClass persistent class
    */
   protected String getSchemaName(String mappingSchemaName, Class<?> persistentClass) {
-    String prefix = getOrCreateConf().get("schema.prefix","");
+    String confSchemaName = getOrCreateConf().get("preferred.schema.name");
+    if (confSchemaName != null) {
+      return confSchemaName;
+    }
     
     String schemaName = DataStoreFactory.getDefaultSchemaName(properties, this);
     if(schemaName != null) {
-      return prefix+schemaName;
+      return schemaName;
     }
 
     if(mappingSchemaName != null) {
-      return prefix+mappingSchemaName;
+      return mappingSchemaName;
     }
 
-    return prefix+StringUtils.getClassname(persistentClass);
+    return StringUtils.getClassname(persistentClass);
   }
 }

Modified: gora/trunk/gora-core/src/main/java/org/apache/gora/util/AvroUtils.java
URL: http://svn.apache.org/viewvc/gora/trunk/gora-core/src/main/java/org/apache/gora/util/AvroUtils.java?rev=1586888&r1=1586887&r2=1586888&view=diff
==============================================================================
--- gora/trunk/gora-core/src/main/java/org/apache/gora/util/AvroUtils.java (original)
+++ gora/trunk/gora-core/src/main/java/org/apache/gora/util/AvroUtils.java Sat Apr 12 19:21:53 2014
@@ -18,17 +18,25 @@
 
 package org.apache.gora.util;
 
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
 import org.apache.avro.Schema;
 import org.apache.avro.Schema.Field;
+import org.apache.avro.io.BinaryEncoder;
+import org.apache.avro.io.Decoder;
+import org.apache.avro.io.DecoderFactory;
+import org.apache.avro.io.EncoderFactory;
 import org.apache.avro.reflect.ReflectData;
+import org.apache.avro.specific.SpecificDatumReader;
+import org.apache.avro.specific.SpecificDatumWriter;
 import org.apache.gora.persistency.Persistent;
 
 /**
- * An utility class for Avro related tasks 
+ * An utility class for Avro related tasks
  */
 public class AvroUtils {
 
@@ -38,31 +46,87 @@ public class AvroUtils {
   public static Map<String, Field> getFieldMap(Schema schema) {
     List<Field> fields = schema.getFields();
     HashMap<String, Field> fieldMap = new HashMap<String, Field>(fields.size());
-    for(Field field: fields) {
+    for (Field field : fields) {
       fieldMap.put(field.name(), field);
     }
     return fieldMap;
   }
-  
+
   @SuppressWarnings("unchecked")
   public static Object getEnumValue(Schema schema, String symbol) {
     return Enum.valueOf(ReflectData.get().getClass(schema), symbol);
   }
-  
+
   public static Object getEnumValue(Schema schema, int enumOrdinal) {
     String symbol = schema.getEnumSymbols().get(enumOrdinal);
     return getEnumValue(schema, symbol);
   }
-  
+
   /**
    * Returns the schema of the class
    */
-  public static Schema getSchema(Class<? extends Persistent> clazz) 
-    throws SecurityException, NoSuchFieldException
-    , IllegalArgumentException, IllegalAccessException {
-    
-    java.lang.reflect.Field field = clazz.getDeclaredField("_SCHEMA");
+  public static Schema getSchema(Class<? extends Persistent> clazz)
+      throws SecurityException, NoSuchFieldException, IllegalArgumentException,
+      IllegalAccessException {
+
+    java.lang.reflect.Field field = clazz.getDeclaredField("SCHEMA$");
     return (Schema) field.get(null);
   }
-  
+
+  /**
+   * Return the field names from a persistent object
+   * 
+   * @param persistent
+   *          the persistent object to get the fields names from
+   * @return the field names
+   */
+  public static String[] getPersistentFieldNames(Persistent persistent) {
+    return getSchemaFieldNames(persistent.getSchema());
+  }
+
+  /**
+   * Return the field names from a schema object
+   * 
+   * @param persistent
+   *          the persistent object to get the fields names from
+   * @return the field names
+   */
+  public static String[] getSchemaFieldNames(Schema schema) {
+    List<Field> fields = schema.getFields();
+    String[] fieldNames = new String[fields.size() - 1];
+    for (int i = 0; i < fieldNames.length; i++) {
+      fieldNames[i] = fields.get(i + 1).name();
+    }
+    return fieldNames;
+  }
+
+  public static <T extends Persistent> T deepClonePersistent(T persistent) {
+    ByteArrayOutputStream bos = new ByteArrayOutputStream();
+    BinaryEncoder enc = EncoderFactory.get().binaryEncoder(bos, null);
+    SpecificDatumWriter<Persistent> writer = new SpecificDatumWriter<Persistent>(
+        persistent.getSchema());
+    try {
+      writer.write(persistent, enc);
+    } catch (IOException e) {
+      throw new RuntimeException(
+          "Unable to serialize avro object to byte buffer - "
+              + "please report this issue to the Gora bugtracker "
+              + "or your administrator.");
+    }
+    byte[] value = bos.toByteArray();
+    Decoder dec = DecoderFactory.get().binaryDecoder(value, null);
+    @SuppressWarnings("unchecked")
+    SpecificDatumReader<T> reader = new SpecificDatumReader<T>(
+        (Class<T>) persistent.getClass());
+    try {
+      return reader.read(null, dec);
+    } catch (IOException e) {
+      throw new RuntimeException(
+          "Unable to deserialize avro object from byte buffer - "
+              + "please report this issue to the Gora bugtracker "
+              + "or your administrator.");
+    }
+
+  }
+
 }

Modified: gora/trunk/gora-core/src/main/java/org/apache/gora/util/ByteUtils.java
URL: http://svn.apache.org/viewvc/gora/trunk/gora-core/src/main/java/org/apache/gora/util/ByteUtils.java?rev=1586888&r1=1586887&r2=1586888&view=diff
==============================================================================
--- gora/trunk/gora-core/src/main/java/org/apache/gora/util/ByteUtils.java (original)
+++ gora/trunk/gora-core/src/main/java/org/apache/gora/util/ByteUtils.java Sat Apr 12 19:21:53 2014
@@ -43,9 +43,10 @@ import java.nio.ByteBuffer;
 import org.apache.avro.Schema;
 import org.apache.avro.Schema.Type;
 import org.apache.avro.reflect.ReflectData;
+import org.apache.avro.specific.SpecificDatumReader;
+import org.apache.avro.specific.SpecificDatumWriter;
+import org.apache.avro.specific.SpecificRecord;
 import org.apache.avro.util.Utf8;
-import org.apache.gora.avro.PersistentDatumReader;
-import org.apache.gora.avro.PersistentDatumWriter;
 import org.apache.hadoop.io.WritableUtils;
 
 //  This code is copied almost directly from HBase project's Bytes class.
@@ -677,30 +678,31 @@ public class ByteUtils {
   }
 
   @SuppressWarnings("unchecked")
-  public static Object fromBytes( byte[] val, Schema schema
-      , PersistentDatumReader<?> datumReader, Object object)
+  public static <T> T fromBytes( byte[] val, Schema schema
+      , SpecificDatumReader<T> datumReader, T object)
   throws IOException {
     Type type = schema.getType();
     switch (type) {
     case ENUM:
       String symbol = schema.getEnumSymbols().get(val[0]);
-      return Enum.valueOf(ReflectData.get().getClass(schema), symbol);
-    case STRING:  return new Utf8(toString(val));
-    case BYTES:   return ByteBuffer.wrap(val);
-    case INT:     return bytesToVint(val);
-    case LONG:    return bytesToVlong(val);
-    case FLOAT:   return toFloat(val);
-    case DOUBLE:  return toDouble(val);
-    case BOOLEAN: return val[0] != 0;
+      return (T)Enum.valueOf(ReflectData.get().getClass(schema), symbol);
+    case STRING:  return (T)new Utf8(toString(val));
+    case BYTES:   return (T)ByteBuffer.wrap(val);
+    case INT:     return (T)Integer.valueOf(bytesToVint(val));
+    case LONG:    return (T)Long.valueOf(bytesToVlong(val));
+    case FLOAT:   return (T)Float.valueOf(toFloat(val));
+    case DOUBLE:  return (T)Double.valueOf(toDouble(val));
+    case BOOLEAN: return (T)Boolean.valueOf(val[0] != 0);
     case RECORD:  //fall
     case MAP:
-    case ARRAY:   return IOUtils.deserialize(val, datumReader, schema, object);
+    case ARRAY:   return (T)IOUtils.deserialize(val, (SpecificDatumReader<SpecificRecord>)datumReader, schema, (SpecificRecord)object);
     default: throw new RuntimeException("Unknown type: "+type);
     }
   }
 
-  public static byte[] toBytes(Object o, Schema schema
-      , PersistentDatumWriter<?> datumWriter)
+  @SuppressWarnings("unchecked")
+  public static <T> byte[] toBytes(T o, Schema schema
+      , SpecificDatumWriter<T> datumWriter)
   throws IOException {
     Type type = schema.getType();
     switch (type) {
@@ -714,7 +716,7 @@ public class ByteUtils {
     case ENUM:    return new byte[] { (byte)((Enum<?>) o).ordinal() };
     case RECORD:  //fall
     case MAP:
-    case ARRAY:   return IOUtils.serialize(datumWriter, schema, o);
+    case ARRAY:   return IOUtils.serialize((SpecificDatumWriter<SpecificRecord>)datumWriter, schema, (SpecificRecord)o);
     default: throw new RuntimeException("Unknown type: "+type);
     }
   }

Modified: gora/trunk/gora-core/src/main/java/org/apache/gora/util/IOUtils.java
URL: http://svn.apache.org/viewvc/gora/trunk/gora-core/src/main/java/org/apache/gora/util/IOUtils.java?rev=1586888&r1=1586887&r2=1586888&view=diff
==============================================================================
--- gora/trunk/gora-core/src/main/java/org/apache/gora/util/IOUtils.java (original)
+++ gora/trunk/gora-core/src/main/java/org/apache/gora/util/IOUtils.java Sat Apr 12 19:21:53 2014
@@ -38,11 +38,12 @@ import org.apache.avro.io.BinaryEncoder;
 import org.apache.avro.io.Decoder;
 import org.apache.avro.io.DecoderFactory;
 import org.apache.avro.io.Encoder;
-import org.apache.avro.ipc.ByteBufferInputStream;
-import org.apache.avro.ipc.ByteBufferOutputStream;
-import org.apache.gora.avro.PersistentDatumReader;
-import org.apache.gora.avro.PersistentDatumWriter;
-import org.apache.gora.persistency.impl.PersistentBase;
+import org.apache.avro.io.EncoderFactory;
+import org.apache.avro.specific.SpecificDatumReader;
+import org.apache.avro.specific.SpecificDatumWriter; 
+import org.apache.avro.specific.SpecificRecord;
+import org.apache.avro.util.ByteBufferInputStream;
+import org.apache.avro.util.ByteBufferOutputStream;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.DataInputBuffer;
 import org.apache.hadoop.io.DataOutputBuffer;
@@ -152,25 +153,47 @@ public class IOUtils {
   /**
    * Serializes the field object using the datumWriter.
    */
-  public static<T extends PersistentBase> void serialize(OutputStream os,
-      PersistentDatumWriter<T> datumWriter, Schema schema, Object object)
+  public static<T extends SpecificRecord> void serialize(OutputStream os,
+      SpecificDatumWriter<T> datumWriter, Schema schema, T object)
       throws IOException {
 
-    BinaryEncoder encoder = new BinaryEncoder(os);
-    datumWriter.write(schema, object, encoder);
+    BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(os, null);
+    datumWriter.write(object, encoder);
     encoder.flush();
   }
 
   /**
    * Serializes the field object using the datumWriter.
    */
-  public static<T extends PersistentBase> byte[] serialize(PersistentDatumWriter<T> datumWriter
-      , Schema schema, Object object) throws IOException {
+  public static<T> void serialize(OutputStream os,
+      SpecificDatumWriter<T> datumWriter, Schema schema, T object)
+      throws IOException {
+
+    BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(os, null);
+    datumWriter.write(object, encoder);
+    encoder.flush();
+  }
+  
+  /**
+   * Serializes the field object using the datumWriter.
+   */
+  public static<T extends SpecificRecord> byte[] serialize(SpecificDatumWriter<T> datumWriter
+      , Schema schema, T object) throws IOException {
     ByteArrayOutputStream os = new ByteArrayOutputStream();
     serialize(os, datumWriter, schema, object);
     return os.toByteArray();
   }
 
+  /**
+   * Serializes the field object using the datumWriter.
+   */
+  public static<T> byte[] serialize(SpecificDatumWriter<T> datumWriter
+      , Schema schema, T object) throws IOException {
+    ByteArrayOutputStream os = new ByteArrayOutputStream();
+    serialize(os, datumWriter, schema, object);
+    return os.toByteArray();
+  }
+  
   /** Deserializes the object in the given datainput using
    * available Hadoop serializations.
    * @throws IOException
@@ -239,36 +262,33 @@ public class IOUtils {
   /**
    * Deserializes the field object using the datumReader.
    */
-  @SuppressWarnings("unchecked")
-  public static<K, T extends PersistentBase> K deserialize(InputStream is,
-      PersistentDatumReader<T> datumReader, Schema schema, K object)
+  public static<K, T extends SpecificRecord> T deserialize(InputStream is,
+      SpecificDatumReader<T> datumReader, Schema schema, T object)
       throws IOException {
-    decoder = DecoderFactory.defaultFactory().createBinaryDecoder(is, decoder);
-    return (K)datumReader.read(object, schema, decoder);
+    decoder = DecoderFactory.get().binaryDecoder(is, decoder);
+    return (T)datumReader.read(object, decoder);
   }
 
   /**
    * Deserializes the field object using the datumReader.
    */
-  @SuppressWarnings("unchecked")
-  public static<K, T extends PersistentBase> K deserialize(byte[] bytes,
-      PersistentDatumReader<T> datumReader, Schema schema, K object)
+  public static<K, T extends SpecificRecord> T deserialize(byte[] bytes,
+      SpecificDatumReader<T> datumReader, Schema schema, T object)
       throws IOException {
-    decoder = DecoderFactory.defaultFactory().createBinaryDecoder(bytes, decoder);
-    return (K)datumReader.read(object, schema, decoder);
+    decoder = DecoderFactory.get().binaryDecoder(bytes, decoder);
+    return (T)datumReader.read(object, decoder);
   }
 
-
   /**
-   * Serializes the field object using the datumWriter.
+   * Deserializes the field object using the datumReader.
    */
-  public static<T extends PersistentBase> byte[] deserialize(PersistentDatumWriter<T> datumWriter
-      , Schema schema, Object object) throws IOException {
-    ByteArrayOutputStream os = new ByteArrayOutputStream();
-    serialize(os, datumWriter, schema, object);
-    return os.toByteArray();
+  public static<K, T> T deserialize(byte[] bytes,
+      SpecificDatumReader<T> datumReader, Schema schema, T object)
+      throws IOException {
+    decoder = DecoderFactory.get().binaryDecoder(bytes, decoder);
+    return (T)datumReader.read(object, decoder);
   }
-
+  
   /**
    * Writes a byte[] to the output, representing whether each given field is null
    * or not. A Vint and ceil( fields.length / 8 ) bytes are written to the output.

Modified: gora/trunk/gora-core/src/main/java/org/apache/gora/util/ReflectionUtils.java
URL: http://svn.apache.org/viewvc/gora/trunk/gora-core/src/main/java/org/apache/gora/util/ReflectionUtils.java?rev=1586888&r1=1586887&r2=1586888&view=diff
==============================================================================
--- gora/trunk/gora-core/src/main/java/org/apache/gora/util/ReflectionUtils.java (original)
+++ gora/trunk/gora-core/src/main/java/org/apache/gora/util/ReflectionUtils.java Sat Apr 12 19:21:53 2014
@@ -20,6 +20,10 @@ package org.apache.gora.util;
 
 import java.lang.reflect.Constructor;
 import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+
+import org.apache.avro.specific.SpecificRecordBuilderBase;
+import org.apache.gora.persistency.Persistent;
 
 /**
  * Utility methods related to reflection
@@ -100,4 +104,10 @@ public class ReflectionUtils {
     
     return clazz.getField(fieldName).get(null);
   }
+  
+  public static <T extends Persistent> SpecificRecordBuilderBase<T> classBuilder(Class<T> clazz) throws SecurityException
+    , NoSuchMethodException, IllegalArgumentException, IllegalAccessException, InvocationTargetException {
+    return (SpecificRecordBuilderBase<T>) clazz.getMethod("newBuilder").invoke(null);
+  }
+  
 }

Added: gora/trunk/gora-core/src/test/conf/hadoop-metrics2.properties
URL: http://svn.apache.org/viewvc/gora/trunk/gora-core/src/test/conf/hadoop-metrics2.properties?rev=1586888&view=auto
==============================================================================
--- gora/trunk/gora-core/src/test/conf/hadoop-metrics2.properties (added)
+++ gora/trunk/gora-core/src/test/conf/hadoop-metrics2.properties Sat Apr 12 19:21:53 2014
@@ -0,0 +1,8 @@
+# Configuration of the "dfs" context for null
+dfs.class=org.apache.hadoop.metrics.spi.NullContext
+
+# Configuration of the "mapred" context for null
+mapred.class=org.apache.hadoop.metrics.spi.NullContext
+
+# Configuration of the "jvm" context for null
+jvm.class=org.apache.hadoop.metrics.spi.NullContext

Propchange: gora/trunk/gora-core/src/test/conf/hadoop-metrics2.properties
------------------------------------------------------------------------------
    svn:mime-type = text/plain

Modified: gora/trunk/gora-core/src/test/java/org/apache/gora/GoraTestDriver.java
URL: http://svn.apache.org/viewvc/gora/trunk/gora-core/src/test/java/org/apache/gora/GoraTestDriver.java?rev=1586888&r1=1586887&r2=1586888&view=diff
==============================================================================
--- gora/trunk/gora-core/src/test/java/org/apache/gora/GoraTestDriver.java (original)
+++ gora/trunk/gora-core/src/test/java/org/apache/gora/GoraTestDriver.java Sat Apr 12 19:21:53 2014
@@ -18,7 +18,6 @@
 
 package org.apache.gora;
 
-import java.io.IOException;
 import java.util.HashSet;
 import java.util.Properties;
 
@@ -40,15 +39,15 @@ public class GoraTestDriver {
 
   protected static final Logger log = LoggerFactory.getLogger(GoraTestDriver.class);
 
-  protected Class<? extends DataStore> dataStoreClass;
+  protected Class<? extends DataStore<?, ?>> dataStoreClass;
   protected Configuration conf = new Configuration();
 
   @SuppressWarnings("rawtypes")
   protected HashSet<DataStore> dataStores;
 
-  @SuppressWarnings("rawtypes")
+  @SuppressWarnings({ "rawtypes", "unchecked" })
   protected GoraTestDriver(Class<? extends DataStore> dataStoreClass) {
-    this.dataStoreClass = dataStoreClass;
+    this.dataStoreClass = (Class<? extends DataStore<?, ?>>) dataStoreClass;
     this.dataStores = new HashSet<DataStore>();
   }
 
@@ -71,7 +70,7 @@ public class GoraTestDriver {
    */
   public void setUp() throws Exception {
     log.info("setting up test");
-    for(DataStore store : dataStores) {
+    for(DataStore<?, ?> store : dataStores) {
       store.truncateSchema();
     }
   }
@@ -112,4 +111,8 @@ public class GoraTestDriver {
   public Class<?> getDataStoreClass() {
     return dataStoreClass;
   }
+  
+  public Configuration getConfiguration() {
+    return this.conf;
+  }
 }

Modified: gora/trunk/gora-core/src/test/java/org/apache/gora/avro/TestPersistentDatumReader.java
URL: http://svn.apache.org/viewvc/gora/trunk/gora-core/src/test/java/org/apache/gora/avro/TestPersistentDatumReader.java?rev=1586888&r1=1586887&r2=1586888&view=diff
==============================================================================
--- gora/trunk/gora-core/src/test/java/org/apache/gora/avro/TestPersistentDatumReader.java (original)
+++ gora/trunk/gora-core/src/test/java/org/apache/gora/avro/TestPersistentDatumReader.java Sat Apr 12 19:21:53 2014
@@ -1,104 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.gora.avro;
-
-import java.io.IOException;
-
-import org.apache.avro.util.Utf8;
-import org.apache.gora.examples.WebPageDataCreator;
-import org.apache.gora.examples.generated.Employee;
-import org.apache.gora.examples.generated.WebPage;
-import org.apache.gora.memory.store.MemStore;
-import org.apache.gora.persistency.impl.PersistentBase;
-import org.apache.gora.query.Query;
-import org.apache.gora.query.Result;
-import org.apache.gora.store.DataStore;
-import org.apache.gora.store.DataStoreFactory;
-import org.apache.gora.store.DataStoreTestUtil;
-import org.apache.hadoop.conf.Configuration;
-import org.junit.Test;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertEquals;
-
-/**
- * Test case for {@link PersistentDatumReader}.
- */
-public class TestPersistentDatumReader {
-
-  private PersistentDatumReader<WebPage> webPageDatumReader 
-    = new PersistentDatumReader<WebPage>();
-  private Configuration conf = new Configuration();
-  
-  private void testClone(PersistentBase persistent) throws IOException {
-    PersistentBase cloned = ((PersistentBase)webPageDatumReader.clone(persistent, persistent.getSchema()));
-    assertClone(persistent, cloned);
-  }
-  
-  private void assertClone(PersistentBase persistent, PersistentBase cloned) {
-    assertNotNull("cloned object is null", cloned);
-    assertEquals("cloned object is not equal to original object", persistent, cloned);
-  }
-  
-  @Test
-  public void testCloneEmployee() throws Exception {
-    @SuppressWarnings("unchecked")
-    MemStore<String, Employee> store = DataStoreFactory.getDataStore(
-        MemStore.class, String.class, Employee.class, conf);
-
-    Employee employee = DataStoreTestUtil.createEmployee(store);
-    
-    testClone(employee);
-  }
-  
-  @Test
-  public void testCloneEmployeeOneField() throws Exception {
-    Employee employee = new Employee();
-    employee.setSsn(new Utf8("11111"));
-
-    testClone(employee);
-  }
-
-  @Test
-  public void testCloneEmployeeTwoFields() throws Exception {
-    Employee employee = new Employee();
-    employee.setSsn(new Utf8("11111"));
-    employee.setSalary(100);
-
-    testClone(employee);
-  }
-
-  @Test
-  public void testCloneWebPage() throws Exception {
-    @SuppressWarnings("unchecked")
-    DataStore<String, WebPage> store = DataStoreFactory.createDataStore(
-        MemStore.class, String.class, WebPage.class, conf);
-    WebPageDataCreator.createWebPageData(store);
-
-    Query<String, WebPage> query = store.newQuery();
-    Result<String, WebPage> result = query.execute();
-    
-    int tested = 0;
-    while(result.next()) {
-      WebPage page = result.get();
-      testClone(page);
-      tested++;
-    }
-    assertEquals(WebPageDataCreator.URLS.length, tested);
-  }
-}

Added: gora/trunk/gora-core/src/test/java/org/apache/gora/examples/TestWebPageDataCreator.java
URL: http://svn.apache.org/viewvc/gora/trunk/gora-core/src/test/java/org/apache/gora/examples/TestWebPageDataCreator.java?rev=1586888&view=auto
==============================================================================
--- gora/trunk/gora-core/src/test/java/org/apache/gora/examples/TestWebPageDataCreator.java (added)
+++ gora/trunk/gora-core/src/test/java/org/apache/gora/examples/TestWebPageDataCreator.java Sat Apr 12 19:21:53 2014
@@ -0,0 +1,19 @@
+package org.apache.gora.examples;
+
+import java.io.IOException;
+
+import org.apache.gora.examples.generated.WebPage;
+import org.apache.gora.memory.store.MemStore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+@RunWith(JUnit4.class)
+public class TestWebPageDataCreator {
+
+  @Test public void testCreatesData() throws IOException{
+    MemStore<String, WebPage> dataStore = new MemStore<String, WebPage>();
+    WebPageDataCreator.createWebPageData(dataStore);
+  }
+  
+}

Propchange: gora/trunk/gora-core/src/test/java/org/apache/gora/examples/TestWebPageDataCreator.java
------------------------------------------------------------------------------
    svn:mime-type = text/plain

Modified: gora/trunk/gora-core/src/test/java/org/apache/gora/filter/TestMapFieldValueFilter.java
URL: http://svn.apache.org/viewvc/gora/trunk/gora-core/src/test/java/org/apache/gora/filter/TestMapFieldValueFilter.java?rev=1586888&r1=1586887&r2=1586888&view=diff
==============================================================================
--- gora/trunk/gora-core/src/test/java/org/apache/gora/filter/TestMapFieldValueFilter.java (original)
+++ gora/trunk/gora-core/src/test/java/org/apache/gora/filter/TestMapFieldValueFilter.java Sat Apr 12 19:21:53 2014
@@ -32,6 +32,7 @@ public class TestMapFieldValueFilter {
   @Test
   public void testSerialization() throws IOException {
     MapFieldValueFilter<String, WebPage> filter = new MapFieldValueFilter<String, WebPage>();
+    //set filter field name as metadata
     filter.setFieldName(WebPage.Field.METADATA.toString());
     filter.setMapKey(new Utf8("fetchTime"));
     filter.setFilterOp(FilterOp.EQUALS);
@@ -50,16 +51,17 @@ public class TestMapFieldValueFilter {
   @Test
   public void testFilterBasics() {
     MapFieldValueFilter<String, WebPage> filter = new MapFieldValueFilter<String, WebPage>();
+    //set filter field name as outlinks
     filter.setFieldName(WebPage.Field.OUTLINKS.toString());
     filter.setMapKey(new Utf8("example"));
     filter.setFilterOp(FilterOp.EQUALS);
     filter.setFilterIfMissing(true);
     filter.getOperands().add(new Utf8("http://example.org"));
     
-    WebPage page = new WebPage();
-    page.putToOutlinks(new Utf8("example"), new Utf8("http://example.org"));
+    WebPage page = WebPage.newBuilder().build();
+    page.getOutlinks().put(new Utf8("example"), new Utf8("http://example.org"));
     assertFalse(filter.filter("irrelevant", page));
-    page.putToOutlinks(new Utf8("example"), new Utf8("http://example2.com"));
+    page.getOutlinks().put(new Utf8("example"), new Utf8("http://example2.com"));
     assertTrue(filter.filter("irrelevant", page));
     page = new WebPage();
     assertTrue(filter.filter("irrelevant", page));
@@ -71,17 +73,18 @@ public class TestMapFieldValueFilter {
   @Test
   public void testFilterEntryInMap() {
     MapFieldValueFilter<String, WebPage> filter = new MapFieldValueFilter<String, WebPage>();
+    //set filter field name as outlinks
     filter.setFieldName(WebPage.Field.OUTLINKS.toString());
     filter.setMapKey(new Utf8("foobar.whatever"));
     filter.setFilterOp(FilterOp.EQUALS);
     filter.setFilterIfMissing(true);
     filter.getOperands().add(new Utf8("Click here for foobar!"));
     
-    WebPage page = new WebPage();
+    WebPage page = WebPage.newBuilder().build();
     assertTrue(filter.filter("irrelevant", page));
-    page.putToOutlinks(new Utf8("foobar.whatever"), new Utf8("Mismatch!"));
+    page.getOutlinks().put(new Utf8("foobar.whatever"), new Utf8("Mismatch!"));
     assertTrue(filter.filter("irrelevant", page));
-    page.putToOutlinks(new Utf8("foobar.whatever"), new Utf8("Click here for foobar!"));
+    page.getOutlinks().put(new Utf8("foobar.whatever"), new Utf8("Click here for foobar!"));
     assertFalse(filter.filter("irrelevant", page));
   }
 

Modified: gora/trunk/gora-core/src/test/java/org/apache/gora/filter/TestSingleFieldValueFilter.java
URL: http://svn.apache.org/viewvc/gora/trunk/gora-core/src/test/java/org/apache/gora/filter/TestSingleFieldValueFilter.java?rev=1586888&r1=1586887&r2=1586888&view=diff
==============================================================================
--- gora/trunk/gora-core/src/test/java/org/apache/gora/filter/TestSingleFieldValueFilter.java (original)
+++ gora/trunk/gora-core/src/test/java/org/apache/gora/filter/TestSingleFieldValueFilter.java Sat Apr 12 19:21:53 2014
@@ -33,6 +33,7 @@ public class TestSingleFieldValueFilter 
   @Test
   public void testSerialization() throws IOException {
     SingleFieldValueFilter<String, WebPage> filter = new SingleFieldValueFilter<String, WebPage>();
+    //set filter field to url
     filter.setFieldName(WebPage.Field.URL.toString());
     filter.setFilterOp(FilterOp.EQUALS);
     filter.setFilterIfMissing(true);
@@ -49,12 +50,13 @@ public class TestSingleFieldValueFilter 
   @Test
   public void testFilterBasics() {
     SingleFieldValueFilter<String, WebPage> filter = new SingleFieldValueFilter<String, WebPage>();
+    //set filter field to url
     filter.setFieldName(WebPage.Field.URL.toString());
     filter.setFilterOp(FilterOp.EQUALS);
     filter.setFilterIfMissing(true);
     filter.getOperands().add(new Utf8("example.org"));
     
-    WebPage page = new WebPage();
+    WebPage page = WebPage.newBuilder().build();
     page.setUrl(new Utf8("example.org"));
     assertFalse(filter.filter("irrelevant", page));
     page.setUrl(new Utf8("mismatch.whatever"));
@@ -70,12 +72,13 @@ public class TestSingleFieldValueFilter 
   @Test
   public void testFilterInequals() {
     SingleFieldValueFilter<String, WebPage> filter = new SingleFieldValueFilter<String, WebPage>();
+    //set filter field to url
     filter.setFieldName(WebPage.Field.URL.toString());
     filter.setFilterOp(FilterOp.NOT_EQUALS);
     filter.setFilterIfMissing(true);
     filter.getOperands().add(new Utf8("example.org"));
     
-    WebPage page = new WebPage();
+    WebPage page = WebPage.newBuilder().build();
     page.setUrl(new Utf8("example.org"));
     assertTrue(filter.filter("irrelevant", page));
     

Modified: gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/DataStoreMapReduceTestBase.java
URL: http://svn.apache.org/viewvc/gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/DataStoreMapReduceTestBase.java?rev=1586888&r1=1586887&r2=1586888&view=diff
==============================================================================
--- gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/DataStoreMapReduceTestBase.java (original)
+++ gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/DataStoreMapReduceTestBase.java Sat Apr 12 19:21:53 2014
@@ -37,7 +37,6 @@ import org.slf4j.LoggerFactory;
  * class, which actually only uses {@link MapReduceTestUtils} methods to
  * run the tests.
  */
-@SuppressWarnings("deprecation")
 public abstract class DataStoreMapReduceTestBase extends HadoopTestCase {
   public static final Logger LOG = LoggerFactory.getLogger(DataStoreMapReduceTestBase.class);
 
@@ -71,8 +70,8 @@ public abstract class DataStoreMapReduce
   @Override
   public void tearDown() throws Exception {
     LOG.info("Tearing down Hadoop Test Case...");
-    super.tearDown();
     webPageStore.close();
+    super.tearDown();
   }
 
   protected abstract DataStore<String, WebPage> createWebPageDataStore()

Modified: gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/MapReduceTestUtils.java
URL: http://svn.apache.org/viewvc/gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/MapReduceTestUtils.java?rev=1586888&r1=1586887&r2=1586888&view=diff
==============================================================================
--- gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/MapReduceTestUtils.java (original)
+++ gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/MapReduceTestUtils.java Sat Apr 12 19:21:53 2014
@@ -19,10 +19,10 @@
 package org.apache.gora.mapreduce;
 
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.avro.Schema.Field;
 import org.apache.gora.examples.WebPageDataCreator;
 import org.apache.gora.examples.generated.TokenDatum;
 import org.apache.gora.examples.generated.WebPage;
@@ -32,14 +32,20 @@ import org.apache.gora.query.Query;
 import org.apache.gora.store.DataStore;
 import org.apache.gora.store.impl.DataStoreBase;
 import org.apache.hadoop.conf.Configuration;
+
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class MapReduceTestUtils {
 
   private static final Logger log = LoggerFactory.getLogger(MapReduceTestUtils.class);
   
-  /** Tests by running the {@link QueryCounter} mapreduce job */
+  /** 
+   * Tests by running the {@link org.apache.gora.examples.mapreduce.QueryCounter} 
+   * mapreduce job 
+   */
   public static void testCountQuery(DataStore<String, WebPage> dataStore, Configuration conf)
       throws Exception {
 
@@ -50,7 +56,12 @@ public class MapReduceTestUtils {
 
     QueryCounter<String,WebPage> counter = new QueryCounter<String,WebPage>(conf);
     Query<String,WebPage> query = dataStore.newQuery();
-    query.setFields(WebPage._ALL_FIELDS);
+    List<Field> fields = WebPage.SCHEMA$.getFields();
+    String[] fieldNames = new String[fields.size() - 1];
+    for(int i = 0; i< fieldNames.length; i++){
+      fieldNames[i] = fields.get(i+1).name();
+    }
+    query.setFields(fieldNames);
     
     dataStore.close();
 
@@ -65,7 +76,7 @@ public class MapReduceTestUtils {
  
   public static void testWordCount(Configuration conf, DataStore<String,WebPage> inStore, DataStore<String,
       TokenDatum> outStore) throws Exception {
-	  //Datastore now has to be a Hadoop based datastore
+    //Datastore now has to be a Hadoop based datastore
     ((DataStoreBase<String,WebPage>)inStore).setConf(conf);
     ((DataStoreBase<String,TokenDatum>)outStore).setConf(conf);
     
@@ -97,6 +108,6 @@ public class MapReduceTestUtils {
       String token, int count) throws Exception {
     TokenDatum datum = outStore.get(token, null);
     assertNotNull("token:" + token + " cannot be found in datastore", datum);
-    assertEquals("count for token:" + token + " is wrong", count, datum.getCount());
+    assertEquals("count for token:" + token + " is wrong", count, datum.getCount().intValue());
   }
 }

Modified: gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/TestGoraInputFormat.java
URL: http://svn.apache.org/viewvc/gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/TestGoraInputFormat.java?rev=1586888&r1=1586887&r2=1586888&view=diff
==============================================================================
--- gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/TestGoraInputFormat.java (original)
+++ gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/TestGoraInputFormat.java Sat Apr 12 19:21:53 2014
@@ -22,6 +22,7 @@ import java.io.IOException;
 import java.util.Arrays;
 import java.util.List;
 
+import org.apache.avro.Schema.Field;
 import org.apache.gora.examples.generated.Employee;
 import org.apache.gora.mock.persistency.MockPersistent;
 import org.apache.gora.mock.query.MockQuery;
@@ -41,7 +42,8 @@ public class TestGoraInputFormat {
     MockDataStore store = MockDataStore.get();
 
     MockQuery query = store.newQuery();
-    query.setFields(Employee._ALL_FIELDS);
+    
+    query.setFields(getEmployeeFieldNames());
     GoraInputFormat.setInput(job, query, false);
 
     GoraInputFormat<String, MockPersistent> inputFormat
@@ -52,6 +54,15 @@ public class TestGoraInputFormat {
     return inputFormat.getSplits(job);
   }
 
+  /**
+   * First, asserts that the attempt to obtain splits results in 
+   * greater than 0 splits which can be used for computation.
+   * We then check that the partition query (obtained by using the 
+   * splits) has the same fields as we would expect by directly 
+   * accessing the fields of an Employee object.
+   * @throws IOException
+   * @throws InterruptedException
+   */
   @Test
   @SuppressWarnings("rawtypes")
   public void testGetSplits() throws IOException, InterruptedException {
@@ -61,7 +72,16 @@ public class TestGoraInputFormat {
 
     InputSplit split = splits.get(0);
     PartitionQuery query = ((GoraInputSplit)split).getQuery();
-    assertTrue(Arrays.equals(Employee._ALL_FIELDS, query.getFields()));
+    assertTrue(Arrays.equals(getEmployeeFieldNames(), query.getFields()));
+  }
+  
+  private static String[] getEmployeeFieldNames(){
+    List<Field> fields = Employee.SCHEMA$.getFields();
+    String[] fieldNames = new String[fields.size()];
+    for(int i = 0; i< fieldNames.length; i++){
+      fieldNames[i] = fields.get(i).name();
+    }
+    return fieldNames;
   }
 
 }
\ No newline at end of file

Modified: gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/TestPersistentSerialization.java
URL: http://svn.apache.org/viewvc/gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/TestPersistentSerialization.java?rev=1586888&r1=1586887&r2=1586888&view=diff
==============================================================================
--- gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/TestPersistentSerialization.java (original)
+++ gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/TestPersistentSerialization.java Sat Apr 12 19:21:53 2014
@@ -18,6 +18,9 @@
 
 package org.apache.gora.mapreduce;
 
+import java.util.ArrayList;
+import java.util.HashMap;
+
 import org.apache.avro.util.Utf8;
 import org.apache.gora.examples.WebPageDataCreator;
 import org.apache.gora.examples.generated.Employee;
@@ -31,11 +34,19 @@ import org.apache.hadoop.conf.Configurat
 import org.junit.Test;
 import static org.junit.Assert.assertEquals;
 
-/** Test class for {@link PersistentSerialization}, {@link PersistentSerializer}
- *  and {@link PersistentDeserializer}
+/**
+ * Test class for {@link PersistentSerialization}, {@link PersistentSerializer}
+ * and {@link PersistentDeserializer}
  */
 public class TestPersistentSerialization {
 
+  /**
+   * Creates an Employee object in-memory setting several fields to dirty. 
+   * Asserts that it can be serialized and 
+   * deserialzed without loosing data. We do this by asserting
+   * what we get 'before' and 'after' (de)serialization processes.
+   * @throws Exception
+   */
   @SuppressWarnings("unchecked")
   @Test
   public void testSerdeEmployee() throws Exception {
@@ -48,23 +59,45 @@ public class TestPersistentSerialization
     TestIOUtils.testSerializeDeserialize(employee);
   }
 
+  /**
+   * Creates an Employee object but only sets one field as dirty.
+   * We then do (de)serialization and check 'before' and 'after'
+   * states. 
+   * @throws Exception
+   */
   @Test
   public void testSerdeEmployeeOneField() throws Exception {
-    Employee employee = new Employee();
+    Employee employee = Employee.newBuilder().build();
     employee.setSsn(new Utf8("11111"));
 
     TestIOUtils.testSerializeDeserialize(employee);
   }
 
+  /**
+   * Creates an Employee object setting only two fields as dirty.
+   * We then do (de)serialization and check 'before' and 'after'
+   * states. 
+   * @throws Exception
+   */
   @Test
   public void testSerdeEmployeeTwoFields() throws Exception {
-    Employee employee = new Employee();
+    Employee employee = Employee.newBuilder().build();
     employee.setSsn(new Utf8("11111"));
     employee.setSalary(100);
 
     TestIOUtils.testSerializeDeserialize(employee);
   }
 
+  /**
+   * Creates an WebPage object in-memory setting several fields to dirty. 
+   * Run a query over the persistent data.
+   * Asserts that the results can be serialized and 
+   * deserialzed without loosing data. We do this by asserting
+   * what we get 'before' and 'after' (de)serialization processes.
+   * Also simple assertion for equal number of URL's in WebPage 
+   * and results.
+   * @throws Exception
+   */
   @SuppressWarnings("unchecked")
   @Test
   public void testSerdeWebPage() throws Exception {
@@ -75,8 +108,8 @@ public class TestPersistentSerialization
 
     Result<String, WebPage> result = store.newQuery().execute();
 
-    int i=0;
-    while(result.next()) {
+    int i = 0;
+    while (result.next()) {
       WebPage page = result.get();
       TestIOUtils.testSerializeDeserialize(page);
       i++;
@@ -84,19 +117,26 @@ public class TestPersistentSerialization
     assertEquals(WebPageDataCreator.URLS.length, i);
   }
 
+  /**
+   * Creates multiple WebPage objects setting several fields to dirty. 
+   * Asserts that the data can be serialized and 
+   * deserialzed without loosing data. We do this by asserting
+   * what we get 'before' and 'after' (de)serialization processes.
+   * @throws Exception
+   */
   @Test
   public void testSerdeMultipleWebPages() throws Exception {
-    WebPage page1 = new WebPage();
-    WebPage page2 = new WebPage();
-    WebPage page3 = new WebPage();
+    WebPage page1 = WebPage.newBuilder().build();
+    WebPage page2 = WebPage.newBuilder().build();
+    WebPage page3 = WebPage.newBuilder().build();
 
     page1.setUrl(new Utf8("foo"));
     page2.setUrl(new Utf8("baz"));
     page3.setUrl(new Utf8("bar"));
-
-    page1.addToParsedContent(new Utf8("coo"));
-
-    page2.putToOutlinks(new Utf8("a"), new Utf8("b"));
+    page1.setParsedContent(new ArrayList<CharSequence>());
+    page1.getParsedContent().add(new Utf8("coo"));
+    page2.setOutlinks(new HashMap<CharSequence, CharSequence>());
+    page2.getOutlinks().put(new Utf8("a"), new Utf8("b"));
 
     TestIOUtils.testSerializeDeserialize(page1, page2, page3);
   }

Modified: gora/trunk/gora-core/src/test/java/org/apache/gora/mock/persistency/MockPersistent.java
URL: http://svn.apache.org/viewvc/gora/trunk/gora-core/src/test/java/org/apache/gora/mock/persistency/MockPersistent.java?rev=1586888&r1=1586887&r2=1586888&view=diff
==============================================================================
--- gora/trunk/gora-core/src/test/java/org/apache/gora/mock/persistency/MockPersistent.java (original)
+++ gora/trunk/gora-core/src/test/java/org/apache/gora/mock/persistency/MockPersistent.java Sat Apr 12 19:21:53 2014
@@ -20,7 +20,7 @@ package org.apache.gora.mock.persistency
 
 import org.apache.avro.Schema;
 import org.apache.gora.persistency.Persistent;
-import org.apache.gora.persistency.StateManager;
+import org.apache.gora.persistency.Tombstone;
 import org.apache.gora.persistency.impl.PersistentBase;
 
 public class MockPersistent extends PersistentBase {
@@ -36,9 +36,6 @@ public class MockPersistent extends Pers
   public MockPersistent() {
   }
   
-  public MockPersistent(StateManager stateManager) {
-    super(stateManager);
-  }
   
   @Override
   public Object get(int field) {
@@ -79,22 +76,13 @@ public class MockPersistent extends Pers
   }
 
   @Override
-  public String getField(int index) {
-    return null;
-  }
-
-  @Override
-  public int getFieldIndex(String field) {
-    return 0;
+  public Tombstone getTombstone() {
+    return new Tombstone(){};
   }
 
   @Override
-  public String[] getFields() {
-    return null;
+  public Persistent newInstance() {
+    return new MockPersistent();
   }
 
-  @Override
-  public Persistent newInstance(StateManager stateManager) {
-    return new MockPersistent(stateManager);
-  }
 }

Modified: gora/trunk/gora-core/src/test/java/org/apache/gora/persistency/TestListGenericArray.java
URL: http://svn.apache.org/viewvc/gora/trunk/gora-core/src/test/java/org/apache/gora/persistency/TestListGenericArray.java?rev=1586888&r1=1586887&r2=1586888&view=diff
==============================================================================
--- gora/trunk/gora-core/src/test/java/org/apache/gora/persistency/TestListGenericArray.java (original)
+++ gora/trunk/gora-core/src/test/java/org/apache/gora/persistency/TestListGenericArray.java Sat Apr 12 19:21:53 2014
@@ -1,57 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.gora.persistency;
-
-import org.apache.avro.Schema;
-import org.apache.avro.util.Utf8;
-import static org.junit.Assert.assertFalse;
-import org.junit.Test;
-
-/**
- * Testcase for ListGenericArray class
- */
-public class TestListGenericArray {
-  
-  @Test
-  public void testHashCode() {
-    ListGenericArray array = new ListGenericArray(Schema.create(Schema.Type.STRING)); 
-    boolean stackOverflowError = false;
-    array.add(new Utf8("array test")); 
-    try {
-      int hashCode = array.hashCode();
-    }
-    catch (StackOverflowError e) {
-      stackOverflowError = true;
-    }
-    assertFalse(stackOverflowError);
-  }
-  
-  @Test
-  public void testCompareTo() {
-    ListGenericArray array = new ListGenericArray(Schema.create(Schema.Type.STRING));
-    boolean stackOverflowError = false;
-    array.add(new Utf8("array comparison test"));
-    try {
-      int compareTo = array.compareTo(array);
-    } catch (StackOverflowError e) {
-      stackOverflowError = true;
-    }
-    assertFalse(stackOverflowError);
-  }
-}

Modified: gora/trunk/gora-core/src/test/java/org/apache/gora/persistency/impl/TestPersistentBase.java
URL: http://svn.apache.org/viewvc/gora/trunk/gora-core/src/test/java/org/apache/gora/persistency/impl/TestPersistentBase.java?rev=1586888&r1=1586887&r2=1586888&view=diff
==============================================================================
--- gora/trunk/gora-core/src/test/java/org/apache/gora/persistency/impl/TestPersistentBase.java (original)
+++ gora/trunk/gora-core/src/test/java/org/apache/gora/persistency/impl/TestPersistentBase.java Sat Apr 12 19:21:53 2014
@@ -20,7 +20,10 @@ package org.apache.gora.persistency.impl
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
-
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import org.apache.avro.Schema.Field;
 import org.apache.avro.util.Utf8;
 import org.apache.gora.examples.generated.Employee;
 import org.apache.gora.examples.generated.WebPage;
@@ -28,10 +31,8 @@ import org.apache.gora.memory.store.MemS
 import org.apache.gora.store.DataStoreFactory;
 import org.apache.gora.store.DataStoreTestUtil;
 import org.apache.hadoop.conf.Configuration;
-import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
-
 import org.junit.Test;
 
 /**
@@ -39,53 +40,83 @@ import org.junit.Test;
  */
 public class TestPersistentBase {
   
+  /**
+   * Assert that the list of fields from the WebPage Schema
+   * are as we expect. This is done by creating and accessing 
+   * a WebPage object, then comparing the results against 
+   * static fields of the WebPage.SCHEMA$.
+   */
   @Test
   public void testGetFields() {
-    WebPage page = new WebPage();
-    String[] fields = page.getFields();
-    assertArrayEquals(WebPage._ALL_FIELDS, fields);
+    WebPage page = WebPage.newBuilder().build();
+    List<Field> fields = page.getSchema().getFields();
+    assertEquals(WebPage.SCHEMA$.getFields(), fields);
   }
   
+  /**
+   * Assert that individual field values are as we would
+   * expect from directly accessing WebPage.SCHEMA$ values.
+   */
   @Test
   public void testGetField() {
-    WebPage page = new WebPage();
-    for(int i=0; i<WebPage._ALL_FIELDS.length; i++) {
-      String field = page.getField(i);
-      assertEquals(WebPage._ALL_FIELDS[i], field);
+    WebPage page = WebPage.newBuilder().build();
+    for(int i=0; i<WebPage.SCHEMA$.getFields().toArray().length; i++) {
+      Field field = page.getSchema().getFields().get(i);
+      assertEquals(WebPage.SCHEMA$.getFields().get(i), field);
     }
   }
   
+  /**
+   * Assert that field positions as found within the SCHEMA array
+   * are as we would expect by accessing them directly. 
+   */
   @Test
   public void testGetFieldIndex() {
-    WebPage page = new WebPage();
-    for(int i=0; i<WebPage._ALL_FIELDS.length; i++) {
-      int index = page.getFieldIndex(WebPage._ALL_FIELDS[i]);
+    WebPage page = WebPage.newBuilder().build();
+    for(int i=0; i<WebPage.SCHEMA$.getFields().toArray().length; i++) {
+      int index = page.getSchema().getFields().get(i).pos();
       assertEquals(i, index);
     }
   }
   
+  /**
+   * Assert that field positions as found within the SCHEMA array
+   * are as we would expect by accessing them directly. 
+   * This tests for both WebPage and Employee data beans.
+   */
   @Test
   public void testFieldsWithTwoClasses() {
-    WebPage page = new WebPage();
-    for(int i=0; i<WebPage._ALL_FIELDS.length; i++) {
-      int index = page.getFieldIndex(WebPage._ALL_FIELDS[i]);
+    WebPage page = WebPage.newBuilder().build();
+    for(int i=0; i<WebPage.SCHEMA$.getFields().toArray().length; i++) {
+      int index = page.getSchema().getFields().get(i).pos();
       assertEquals(i, index);
     }
-    Employee employee = new Employee();
-    for(int i=0; i<Employee._ALL_FIELDS.length; i++) {
-      int index = employee.getFieldIndex(Employee._ALL_FIELDS[i]);
+    Employee employee = Employee.newBuilder().build();
+    for(int i=0; i<Employee.SCHEMA$.getFields().toArray().length; i++) {
+      int index = employee.getSchema().getFields().get(i).pos();
       assertEquals(i, index);
     }
   }
   
+  /**
+   * First we create a new WebPage object, to which we add some
+   * field values. This makes the fields dirty as we have not 
+   * flushed them to the datastore. We then clear the dirty
+   * fields and assert that the values DO NOT exist for the 
+   * field we previously made dirty.
+   * We then set new values for fields, consequently making them 
+   * dirty, before testing the clearing of an entirely new object
+   * has all fields as null as they should be clean.
+   */
   @Test
   public void testClear() {
     
     //test clear all fields
-    WebPage page = new WebPage();
+    WebPage page = WebPage.newBuilder().build();
+   
     page.setUrl(new Utf8("http://foo.com"));
-    page.addToParsedContent(new Utf8("foo"));
-    page.putToOutlinks(new Utf8("foo"), new Utf8("bar"));
+    page.getParsedContent().add(new Utf8("foo"));
+    page.getOutlinks().put(new Utf8("foo"), new Utf8("bar"));
     page.setContent(ByteBuffer.wrap("foo baz bar".getBytes()));
     
     page.clear();
@@ -97,19 +128,21 @@ public class TestPersistentBase {
     
     //set fields again
     page.setUrl(new Utf8("http://bar.com"));
-    page.addToParsedContent(new Utf8("bar"));
-    page.putToOutlinks(new Utf8("bar"), new Utf8("baz"));
+    page.getParsedContent().add(new Utf8("bar"));
+    page.getOutlinks().put(new Utf8("bar"), new Utf8("baz"));
     page.setContent(ByteBuffer.wrap("foo baz bar barbaz".getBytes()));
     
     //test clear new object
-    page = new WebPage();
+    page = WebPage.newBuilder().build();
     page.clear();
-    
-    //test primitive fields
-    Employee employee = new Employee();
-    employee.clear();
   }
   
+  /**
+   * Tests and asserts that an in-memory representation of the 
+   * Employee object is Equal to a clone of the same object.
+   * @throws IOException
+   * @throws Exception
+   */
   @Test
   public void testClone() throws IOException, Exception {
     //more tests for clone are in TestPersistentDatumReader
@@ -119,6 +152,6 @@ public class TestPersistentBase {
 
     Employee employee = DataStoreTestUtil.createEmployee(store);
     
-    assertEquals(employee, employee.clone());
+    assertEquals(employee, Employee.newBuilder(employee).build());
   }
 }

Modified: gora/trunk/gora-core/src/test/java/org/apache/gora/persistency/impl/TestStateManagerImpl.java
URL: http://svn.apache.org/viewvc/gora/trunk/gora-core/src/test/java/org/apache/gora/persistency/impl/TestStateManagerImpl.java?rev=1586888&r1=1586887&r2=1586888&view=diff
==============================================================================
--- gora/trunk/gora-core/src/test/java/org/apache/gora/persistency/impl/TestStateManagerImpl.java (original)
+++ gora/trunk/gora-core/src/test/java/org/apache/gora/persistency/impl/TestStateManagerImpl.java Sat Apr 12 19:21:53 2014
@@ -1,115 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.gora.persistency.impl;
-
-import java.io.IOException;
-
-import org.apache.avro.util.Utf8;
-import org.apache.gora.examples.generated.Employee;
-import org.apache.gora.mock.persistency.MockPersistent;
-import org.apache.gora.persistency.impl.StateManagerImpl;
-import org.junit.Before;
-import org.junit.Test;
-import static junit.framework.Assert.assertFalse;
-import static junit.framework.Assert.assertTrue;
-
-/**
- * Test case for {@link StateManagerImpl}
- */
-public class TestStateManagerImpl {
-
-  private StateManagerImpl stateManager;
-  private MockPersistent persistent;
-  
-  @Before
-  public void setUp() {
-    this.stateManager = new StateManagerImpl();
-    this.persistent = new MockPersistent(stateManager);
-  }
-  
-  @Test
-  public void testDirty() {
-    assertFalse(stateManager.isDirty(persistent));
-    stateManager.setDirty(persistent);
-    assertTrue(stateManager.isDirty(persistent));
-  }
-  
-  @Test
-  public void testDirty2() {
-    assertFalse(stateManager.isDirty(persistent, 0));
-    assertFalse(stateManager.isDirty(persistent, 1));
-    stateManager.setDirty(persistent, 0);
-    assertTrue(stateManager.isDirty(persistent, 0));
-    assertFalse(stateManager.isDirty(persistent, 1));
-  }
-  
-  @Test
-  public void testClearDirty() {
-    assertFalse(stateManager.isDirty(persistent));
-    stateManager.setDirty(persistent, 0);
-    stateManager.clearDirty(persistent);
-    assertFalse(this.stateManager.isDirty(persistent));
-  }
-  
-  @Test
-  public void testReadable() throws IOException {
-    assertFalse(stateManager.isReadable(persistent, 0));
-    assertFalse(stateManager.isReadable(persistent, 1));
-    stateManager.setReadable(persistent, 0);
-    assertTrue(stateManager.isReadable(persistent, 0));
-    assertFalse(stateManager.isReadable(persistent, 1));
-  }
-
-  @Test
-  public void testReadable2() {
-    stateManager = new StateManagerImpl();
-    Employee employee = new Employee(stateManager);
-    assertFalse(stateManager.isReadable(employee, 0));
-    assertFalse(stateManager.isReadable(employee, 1));
-    employee.setName(new Utf8("foo"));
-    assertTrue(stateManager.isReadable(employee, 0));
-    assertFalse(stateManager.isReadable(employee, 1));
-  }
-  
-  @Test
-  public void testClearReadable() {
-    stateManager.setReadable(persistent, 0);
-    stateManager.clearReadable(persistent);
-    assertFalse(stateManager.isReadable(persistent, 0));
-  }
-  
-  @Test
-  public void testIsNew() {
-    //newly created objects should be new
-    assertTrue(persistent.isNew());
-  }
-  
-  @Test
-  public void testNew() {
-    stateManager.setNew(persistent);
-    assertTrue(persistent.isNew());
-  }
-  
-  @Test
-  public void testClearNew() {
-    stateManager.clearNew(persistent);
-    assertFalse(persistent.isNew());
-  }
-  
-}

Modified: gora/trunk/gora-core/src/test/java/org/apache/gora/store/DataStoreTestBase.java
URL: http://svn.apache.org/viewvc/gora/trunk/gora-core/src/test/java/org/apache/gora/store/DataStoreTestBase.java?rev=1586888&r1=1586887&r2=1586888&view=diff
==============================================================================
--- gora/trunk/gora-core/src/test/java/org/apache/gora/store/DataStoreTestBase.java (original)
+++ gora/trunk/gora-core/src/test/java/org/apache/gora/store/DataStoreTestBase.java Sat Apr 12 19:21:53 2014
@@ -19,22 +19,16 @@
 package org.apache.gora.store;
 
 import java.io.IOException;
-import java.nio.ByteBuffer;
-
-import org.apache.avro.util.Utf8;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.gora.GoraTestDriver;
 import org.apache.gora.examples.generated.Employee;
-import org.apache.gora.examples.generated.Metadata;
 import org.apache.gora.examples.generated.WebPage;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
 
 /**
  * A base class for {@link DataStore} tests. This is just a convenience
@@ -216,7 +210,11 @@ public abstract class DataStoreTestBase 
   public void testUpdate() throws IOException, Exception {
     log.info("test method: testUpdate");
     DataStoreTestUtil.testUpdateEmployee(employeeStore);
-    DataStoreTestUtil.testUpdateWebPage(webPageStore);
+    DataStoreTestUtil.testUpdateWebPagePutToArray(webPageStore);
+    DataStoreTestUtil.testUpdateWebPagePutToNotNullableMap(webPageStore);
+    DataStoreTestUtil.testUpdateWebPagePutToNullableMap(webPageStore);
+    DataStoreTestUtil.testUpdateWebPageRemoveMapEntry(webPageStore);
+    DataStoreTestUtil.testUpdateWebPageRemoveField(webPageStore);
   }
 
   public void testEmptyUpdate() throws IOException, Exception {
@@ -243,7 +241,7 @@ public abstract class DataStoreTestBase 
 
   @Test
   /**
-   * Tests put and get a record with a double  nested recursive record
+   * Tests put and get a record with a double nested recursive record
    * Employee with a boss (nested).
    * @throws IOException
    * @throws Exception
@@ -255,8 +253,9 @@ public abstract class DataStoreTestBase 
 
   @Test
   /**
-   * Tests put and get a record with a nested record (not recursive)
-   * The webpage of an Employee
+   * Tests put and get of an {@link org.apache.gora.examples.generated.Employee} 
+   * record with a nested {@link org.apache.gora.examples.generated.WegPage} record (not recursive)
+   * the webpage of an Employee.
    * @throws IOException
    * @throws Exception
    */