gora-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From lewi...@apache.org
Subject [07/50] [abbrv] GORA-321. Merge GORA_94 into Gora trunk
Date Wed, 04 Jun 2014 16:36:24 GMT
http://git-wip-us.apache.org/repos/asf/gora/blob/136fc595/gora-core/src/main/java/org/apache/gora/util/IOUtils.java
----------------------------------------------------------------------
diff --git a/gora-core/src/main/java/org/apache/gora/util/IOUtils.java b/gora-core/src/main/java/org/apache/gora/util/IOUtils.java
index 094262e..a8fa313 100644
--- a/gora-core/src/main/java/org/apache/gora/util/IOUtils.java
+++ b/gora-core/src/main/java/org/apache/gora/util/IOUtils.java
@@ -38,11 +38,12 @@ import org.apache.avro.io.BinaryEncoder;
 import org.apache.avro.io.Decoder;
 import org.apache.avro.io.DecoderFactory;
 import org.apache.avro.io.Encoder;
-import org.apache.avro.ipc.ByteBufferInputStream;
-import org.apache.avro.ipc.ByteBufferOutputStream;
-import org.apache.gora.avro.PersistentDatumReader;
-import org.apache.gora.avro.PersistentDatumWriter;
-import org.apache.gora.persistency.impl.PersistentBase;
+import org.apache.avro.io.EncoderFactory;
+import org.apache.avro.specific.SpecificDatumReader;
+import org.apache.avro.specific.SpecificDatumWriter; 
+import org.apache.avro.specific.SpecificRecord;
+import org.apache.avro.util.ByteBufferInputStream;
+import org.apache.avro.util.ByteBufferOutputStream;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.DataInputBuffer;
 import org.apache.hadoop.io.DataOutputBuffer;
@@ -152,25 +153,47 @@ public class IOUtils {
   /**
    * Serializes the field object using the datumWriter.
    */
-  public static<T extends PersistentBase> void serialize(OutputStream os,
-      PersistentDatumWriter<T> datumWriter, Schema schema, Object object)
+  public static<T extends SpecificRecord> void serialize(OutputStream os,
+      SpecificDatumWriter<T> datumWriter, Schema schema, T object)
       throws IOException {
 
-    BinaryEncoder encoder = new BinaryEncoder(os);
-    datumWriter.write(schema, object, encoder);
+    BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(os, null);
+    datumWriter.write(object, encoder);
     encoder.flush();
   }
 
   /**
    * Serializes the field object using the datumWriter.
    */
-  public static<T extends PersistentBase> byte[] serialize(PersistentDatumWriter<T> datumWriter
-      , Schema schema, Object object) throws IOException {
+  public static<T> void serialize(OutputStream os,
+      SpecificDatumWriter<T> datumWriter, Schema schema, T object)
+      throws IOException {
+
+    BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(os, null);
+    datumWriter.write(object, encoder);
+    encoder.flush();
+  }
+  
+  /**
+   * Serializes the field object using the datumWriter.
+   */
+  public static<T extends SpecificRecord> byte[] serialize(SpecificDatumWriter<T> datumWriter
+      , Schema schema, T object) throws IOException {
     ByteArrayOutputStream os = new ByteArrayOutputStream();
     serialize(os, datumWriter, schema, object);
     return os.toByteArray();
   }
 
+  /**
+   * Serializes the field object using the datumWriter.
+   */
+  public static<T> byte[] serialize(SpecificDatumWriter<T> datumWriter
+      , Schema schema, T object) throws IOException {
+    ByteArrayOutputStream os = new ByteArrayOutputStream();
+    serialize(os, datumWriter, schema, object);
+    return os.toByteArray();
+  }
+  
   /** Deserializes the object in the given datainput using
    * available Hadoop serializations.
    * @throws IOException
@@ -239,36 +262,33 @@ public class IOUtils {
   /**
    * Deserializes the field object using the datumReader.
    */
-  @SuppressWarnings("unchecked")
-  public static<K, T extends PersistentBase> K deserialize(InputStream is,
-      PersistentDatumReader<T> datumReader, Schema schema, K object)
+  public static<K, T extends SpecificRecord> T deserialize(InputStream is,
+      SpecificDatumReader<T> datumReader, Schema schema, T object)
       throws IOException {
-    decoder = DecoderFactory.defaultFactory().createBinaryDecoder(is, decoder);
-    return (K)datumReader.read(object, schema, decoder);
+    decoder = DecoderFactory.get().binaryDecoder(is, decoder);
+    return (T)datumReader.read(object, decoder);
   }
 
   /**
    * Deserializes the field object using the datumReader.
    */
-  @SuppressWarnings("unchecked")
-  public static<K, T extends PersistentBase> K deserialize(byte[] bytes,
-      PersistentDatumReader<T> datumReader, Schema schema, K object)
+  public static<K, T extends SpecificRecord> T deserialize(byte[] bytes,
+      SpecificDatumReader<T> datumReader, Schema schema, T object)
       throws IOException {
-    decoder = DecoderFactory.defaultFactory().createBinaryDecoder(bytes, decoder);
-    return (K)datumReader.read(object, schema, decoder);
+    decoder = DecoderFactory.get().binaryDecoder(bytes, decoder);
+    return (T)datumReader.read(object, decoder);
   }
 
-
   /**
-   * Serializes the field object using the datumWriter.
+   * Deserializes the field object using the datumReader.
    */
-  public static<T extends PersistentBase> byte[] deserialize(PersistentDatumWriter<T> datumWriter
-      , Schema schema, Object object) throws IOException {
-    ByteArrayOutputStream os = new ByteArrayOutputStream();
-    serialize(os, datumWriter, schema, object);
-    return os.toByteArray();
+  public static<K, T> T deserialize(byte[] bytes,
+      SpecificDatumReader<T> datumReader, Schema schema, T object)
+      throws IOException {
+    decoder = DecoderFactory.get().binaryDecoder(bytes, decoder);
+    return (T)datumReader.read(object, decoder);
   }
-
+  
   /**
    * Writes a byte[] to the output, representing whether each given field is null
    * or not. A Vint and ceil( fields.length / 8 ) bytes are written to the output.

http://git-wip-us.apache.org/repos/asf/gora/blob/136fc595/gora-core/src/main/java/org/apache/gora/util/ReflectionUtils.java
----------------------------------------------------------------------
diff --git a/gora-core/src/main/java/org/apache/gora/util/ReflectionUtils.java b/gora-core/src/main/java/org/apache/gora/util/ReflectionUtils.java
index fd8c498..00f94e4 100644
--- a/gora-core/src/main/java/org/apache/gora/util/ReflectionUtils.java
+++ b/gora-core/src/main/java/org/apache/gora/util/ReflectionUtils.java
@@ -20,6 +20,10 @@ package org.apache.gora.util;
 
 import java.lang.reflect.Constructor;
 import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+
+import org.apache.avro.specific.SpecificRecordBuilderBase;
+import org.apache.gora.persistency.Persistent;
 
 /**
  * Utility methods related to reflection
@@ -100,4 +104,10 @@ public class ReflectionUtils {
     
     return clazz.getField(fieldName).get(null);
   }
+  
+  public static <T extends Persistent> SpecificRecordBuilderBase<T> classBuilder(Class<T> clazz) throws SecurityException
+    , NoSuchMethodException, IllegalArgumentException, IllegalAccessException, InvocationTargetException {
+    return (SpecificRecordBuilderBase<T>) clazz.getMethod("newBuilder").invoke(null);
+  }
+  
 }

http://git-wip-us.apache.org/repos/asf/gora/blob/136fc595/gora-core/src/test/conf/hadoop-metrics2.properties
----------------------------------------------------------------------
diff --git a/gora-core/src/test/conf/hadoop-metrics2.properties b/gora-core/src/test/conf/hadoop-metrics2.properties
new file mode 100644
index 0000000..e5a9785
--- /dev/null
+++ b/gora-core/src/test/conf/hadoop-metrics2.properties
@@ -0,0 +1,8 @@
+# Configuration of the "dfs" context for null
+dfs.class=org.apache.hadoop.metrics.spi.NullContext
+
+# Configuration of the "mapred" context for null
+mapred.class=org.apache.hadoop.metrics.spi.NullContext
+
+# Configuration of the "jvm" context for null
+jvm.class=org.apache.hadoop.metrics.spi.NullContext

http://git-wip-us.apache.org/repos/asf/gora/blob/136fc595/gora-core/src/test/java/org/apache/gora/GoraTestDriver.java
----------------------------------------------------------------------
diff --git a/gora-core/src/test/java/org/apache/gora/GoraTestDriver.java b/gora-core/src/test/java/org/apache/gora/GoraTestDriver.java
index e5170c2..817d57b 100644
--- a/gora-core/src/test/java/org/apache/gora/GoraTestDriver.java
+++ b/gora-core/src/test/java/org/apache/gora/GoraTestDriver.java
@@ -18,7 +18,6 @@
 
 package org.apache.gora;
 
-import java.io.IOException;
 import java.util.HashSet;
 import java.util.Properties;
 
@@ -40,15 +39,15 @@ public class GoraTestDriver {
 
   protected static final Logger log = LoggerFactory.getLogger(GoraTestDriver.class);
 
-  protected Class<? extends DataStore> dataStoreClass;
+  protected Class<? extends DataStore<?, ?>> dataStoreClass;
   protected Configuration conf = new Configuration();
 
   @SuppressWarnings("rawtypes")
   protected HashSet<DataStore> dataStores;
 
-  @SuppressWarnings("rawtypes")
+  @SuppressWarnings({ "rawtypes", "unchecked" })
   protected GoraTestDriver(Class<? extends DataStore> dataStoreClass) {
-    this.dataStoreClass = dataStoreClass;
+    this.dataStoreClass = (Class<? extends DataStore<?, ?>>) dataStoreClass;
     this.dataStores = new HashSet<DataStore>();
   }
 
@@ -71,7 +70,7 @@ public class GoraTestDriver {
    */
   public void setUp() throws Exception {
     log.info("setting up test");
-    for(DataStore store : dataStores) {
+    for(DataStore<?, ?> store : dataStores) {
       store.truncateSchema();
     }
   }
@@ -112,4 +111,8 @@ public class GoraTestDriver {
   public Class<?> getDataStoreClass() {
     return dataStoreClass;
   }
+  
+  public Configuration getConfiguration() {
+    return this.conf;
+  }
 }

http://git-wip-us.apache.org/repos/asf/gora/blob/136fc595/gora-core/src/test/java/org/apache/gora/avro/TestPersistentDatumReader.java
----------------------------------------------------------------------
diff --git a/gora-core/src/test/java/org/apache/gora/avro/TestPersistentDatumReader.java b/gora-core/src/test/java/org/apache/gora/avro/TestPersistentDatumReader.java
index a81fd3b..e69de29 100644
--- a/gora-core/src/test/java/org/apache/gora/avro/TestPersistentDatumReader.java
+++ b/gora-core/src/test/java/org/apache/gora/avro/TestPersistentDatumReader.java
@@ -1,104 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.gora.avro;
-
-import java.io.IOException;
-
-import org.apache.avro.util.Utf8;
-import org.apache.gora.examples.WebPageDataCreator;
-import org.apache.gora.examples.generated.Employee;
-import org.apache.gora.examples.generated.WebPage;
-import org.apache.gora.memory.store.MemStore;
-import org.apache.gora.persistency.impl.PersistentBase;
-import org.apache.gora.query.Query;
-import org.apache.gora.query.Result;
-import org.apache.gora.store.DataStore;
-import org.apache.gora.store.DataStoreFactory;
-import org.apache.gora.store.DataStoreTestUtil;
-import org.apache.hadoop.conf.Configuration;
-import org.junit.Test;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertEquals;
-
-/**
- * Test case for {@link PersistentDatumReader}.
- */
-public class TestPersistentDatumReader {
-
-  private PersistentDatumReader<WebPage> webPageDatumReader 
-    = new PersistentDatumReader<WebPage>();
-  private Configuration conf = new Configuration();
-  
-  private void testClone(PersistentBase persistent) throws IOException {
-    PersistentBase cloned = ((PersistentBase)webPageDatumReader.clone(persistent, persistent.getSchema()));
-    assertClone(persistent, cloned);
-  }
-  
-  private void assertClone(PersistentBase persistent, PersistentBase cloned) {
-    assertNotNull("cloned object is null", cloned);
-    assertEquals("cloned object is not equal to original object", persistent, cloned);
-  }
-  
-  @Test
-  public void testCloneEmployee() throws Exception {
-    @SuppressWarnings("unchecked")
-    MemStore<String, Employee> store = DataStoreFactory.getDataStore(
-        MemStore.class, String.class, Employee.class, conf);
-
-    Employee employee = DataStoreTestUtil.createEmployee(store);
-    
-    testClone(employee);
-  }
-  
-  @Test
-  public void testCloneEmployeeOneField() throws Exception {
-    Employee employee = new Employee();
-    employee.setSsn(new Utf8("11111"));
-
-    testClone(employee);
-  }
-
-  @Test
-  public void testCloneEmployeeTwoFields() throws Exception {
-    Employee employee = new Employee();
-    employee.setSsn(new Utf8("11111"));
-    employee.setSalary(100);
-
-    testClone(employee);
-  }
-
-  @Test
-  public void testCloneWebPage() throws Exception {
-    @SuppressWarnings("unchecked")
-    DataStore<String, WebPage> store = DataStoreFactory.createDataStore(
-        MemStore.class, String.class, WebPage.class, conf);
-    WebPageDataCreator.createWebPageData(store);
-
-    Query<String, WebPage> query = store.newQuery();
-    Result<String, WebPage> result = query.execute();
-    
-    int tested = 0;
-    while(result.next()) {
-      WebPage page = result.get();
-      testClone(page);
-      tested++;
-    }
-    assertEquals(WebPageDataCreator.URLS.length, tested);
-  }
-}

http://git-wip-us.apache.org/repos/asf/gora/blob/136fc595/gora-core/src/test/java/org/apache/gora/examples/TestWebPageDataCreator.java
----------------------------------------------------------------------
diff --git a/gora-core/src/test/java/org/apache/gora/examples/TestWebPageDataCreator.java b/gora-core/src/test/java/org/apache/gora/examples/TestWebPageDataCreator.java
new file mode 100644
index 0000000..fde39ab
--- /dev/null
+++ b/gora-core/src/test/java/org/apache/gora/examples/TestWebPageDataCreator.java
@@ -0,0 +1,19 @@
+package org.apache.gora.examples;
+
+import java.io.IOException;
+
+import org.apache.gora.examples.generated.WebPage;
+import org.apache.gora.memory.store.MemStore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+@RunWith(JUnit4.class)
+public class TestWebPageDataCreator {
+
+  @Test public void testCreatesData() throws IOException{
+    MemStore<String, WebPage> dataStore = new MemStore<String, WebPage>();
+    WebPageDataCreator.createWebPageData(dataStore);
+  }
+  
+}

http://git-wip-us.apache.org/repos/asf/gora/blob/136fc595/gora-core/src/test/java/org/apache/gora/filter/TestMapFieldValueFilter.java
----------------------------------------------------------------------
diff --git a/gora-core/src/test/java/org/apache/gora/filter/TestMapFieldValueFilter.java b/gora-core/src/test/java/org/apache/gora/filter/TestMapFieldValueFilter.java
index 553c788..b53207a 100644
--- a/gora-core/src/test/java/org/apache/gora/filter/TestMapFieldValueFilter.java
+++ b/gora-core/src/test/java/org/apache/gora/filter/TestMapFieldValueFilter.java
@@ -32,6 +32,7 @@ public class TestMapFieldValueFilter {
   @Test
   public void testSerialization() throws IOException {
     MapFieldValueFilter<String, WebPage> filter = new MapFieldValueFilter<String, WebPage>();
+    //set filter field name as metadata
     filter.setFieldName(WebPage.Field.METADATA.toString());
     filter.setMapKey(new Utf8("fetchTime"));
     filter.setFilterOp(FilterOp.EQUALS);
@@ -50,16 +51,17 @@ public class TestMapFieldValueFilter {
   @Test
   public void testFilterBasics() {
     MapFieldValueFilter<String, WebPage> filter = new MapFieldValueFilter<String, WebPage>();
+    //set filter field name as outlinks
     filter.setFieldName(WebPage.Field.OUTLINKS.toString());
     filter.setMapKey(new Utf8("example"));
     filter.setFilterOp(FilterOp.EQUALS);
     filter.setFilterIfMissing(true);
     filter.getOperands().add(new Utf8("http://example.org"));
     
-    WebPage page = new WebPage();
-    page.putToOutlinks(new Utf8("example"), new Utf8("http://example.org"));
+    WebPage page = WebPage.newBuilder().build();
+    page.getOutlinks().put(new Utf8("example"), new Utf8("http://example.org"));
     assertFalse(filter.filter("irrelevant", page));
-    page.putToOutlinks(new Utf8("example"), new Utf8("http://example2.com"));
+    page.getOutlinks().put(new Utf8("example"), new Utf8("http://example2.com"));
     assertTrue(filter.filter("irrelevant", page));
     page = new WebPage();
     assertTrue(filter.filter("irrelevant", page));
@@ -71,17 +73,18 @@ public class TestMapFieldValueFilter {
   @Test
   public void testFilterEntryInMap() {
     MapFieldValueFilter<String, WebPage> filter = new MapFieldValueFilter<String, WebPage>();
+    //set filter field name as outlinks
     filter.setFieldName(WebPage.Field.OUTLINKS.toString());
     filter.setMapKey(new Utf8("foobar.whatever"));
     filter.setFilterOp(FilterOp.EQUALS);
     filter.setFilterIfMissing(true);
     filter.getOperands().add(new Utf8("Click here for foobar!"));
     
-    WebPage page = new WebPage();
+    WebPage page = WebPage.newBuilder().build();
     assertTrue(filter.filter("irrelevant", page));
-    page.putToOutlinks(new Utf8("foobar.whatever"), new Utf8("Mismatch!"));
+    page.getOutlinks().put(new Utf8("foobar.whatever"), new Utf8("Mismatch!"));
     assertTrue(filter.filter("irrelevant", page));
-    page.putToOutlinks(new Utf8("foobar.whatever"), new Utf8("Click here for foobar!"));
+    page.getOutlinks().put(new Utf8("foobar.whatever"), new Utf8("Click here for foobar!"));
     assertFalse(filter.filter("irrelevant", page));
   }
 

http://git-wip-us.apache.org/repos/asf/gora/blob/136fc595/gora-core/src/test/java/org/apache/gora/filter/TestSingleFieldValueFilter.java
----------------------------------------------------------------------
diff --git a/gora-core/src/test/java/org/apache/gora/filter/TestSingleFieldValueFilter.java b/gora-core/src/test/java/org/apache/gora/filter/TestSingleFieldValueFilter.java
index a03ba93..6390b0d 100644
--- a/gora-core/src/test/java/org/apache/gora/filter/TestSingleFieldValueFilter.java
+++ b/gora-core/src/test/java/org/apache/gora/filter/TestSingleFieldValueFilter.java
@@ -33,6 +33,7 @@ public class TestSingleFieldValueFilter {
   @Test
   public void testSerialization() throws IOException {
     SingleFieldValueFilter<String, WebPage> filter = new SingleFieldValueFilter<String, WebPage>();
+    //set filter field to url
     filter.setFieldName(WebPage.Field.URL.toString());
     filter.setFilterOp(FilterOp.EQUALS);
     filter.setFilterIfMissing(true);
@@ -49,12 +50,13 @@ public class TestSingleFieldValueFilter {
   @Test
   public void testFilterBasics() {
     SingleFieldValueFilter<String, WebPage> filter = new SingleFieldValueFilter<String, WebPage>();
+    //set filter field to url
     filter.setFieldName(WebPage.Field.URL.toString());
     filter.setFilterOp(FilterOp.EQUALS);
     filter.setFilterIfMissing(true);
     filter.getOperands().add(new Utf8("example.org"));
     
-    WebPage page = new WebPage();
+    WebPage page = WebPage.newBuilder().build();
     page.setUrl(new Utf8("example.org"));
     assertFalse(filter.filter("irrelevant", page));
     page.setUrl(new Utf8("mismatch.whatever"));
@@ -70,12 +72,13 @@ public class TestSingleFieldValueFilter {
   @Test
   public void testFilterInequals() {
     SingleFieldValueFilter<String, WebPage> filter = new SingleFieldValueFilter<String, WebPage>();
+    //set filter field to url
     filter.setFieldName(WebPage.Field.URL.toString());
     filter.setFilterOp(FilterOp.NOT_EQUALS);
     filter.setFilterIfMissing(true);
     filter.getOperands().add(new Utf8("example.org"));
     
-    WebPage page = new WebPage();
+    WebPage page = WebPage.newBuilder().build();
     page.setUrl(new Utf8("example.org"));
     assertTrue(filter.filter("irrelevant", page));
     

http://git-wip-us.apache.org/repos/asf/gora/blob/136fc595/gora-core/src/test/java/org/apache/gora/mapreduce/DataStoreMapReduceTestBase.java
----------------------------------------------------------------------
diff --git a/gora-core/src/test/java/org/apache/gora/mapreduce/DataStoreMapReduceTestBase.java b/gora-core/src/test/java/org/apache/gora/mapreduce/DataStoreMapReduceTestBase.java
index 28acdcd..c3778af 100644
--- a/gora-core/src/test/java/org/apache/gora/mapreduce/DataStoreMapReduceTestBase.java
+++ b/gora-core/src/test/java/org/apache/gora/mapreduce/DataStoreMapReduceTestBase.java
@@ -37,7 +37,6 @@ import org.slf4j.LoggerFactory;
  * class, which actually only uses {@link MapReduceTestUtils} methods to
  * run the tests.
  */
-@SuppressWarnings("deprecation")
 public abstract class DataStoreMapReduceTestBase extends HadoopTestCase {
   public static final Logger LOG = LoggerFactory.getLogger(DataStoreMapReduceTestBase.class);
 
@@ -71,8 +70,8 @@ public abstract class DataStoreMapReduceTestBase extends HadoopTestCase {
   @Override
   public void tearDown() throws Exception {
     LOG.info("Tearing down Hadoop Test Case...");
-    super.tearDown();
     webPageStore.close();
+    super.tearDown();
   }
 
   protected abstract DataStore<String, WebPage> createWebPageDataStore()

http://git-wip-us.apache.org/repos/asf/gora/blob/136fc595/gora-core/src/test/java/org/apache/gora/mapreduce/MapReduceTestUtils.java
----------------------------------------------------------------------
diff --git a/gora-core/src/test/java/org/apache/gora/mapreduce/MapReduceTestUtils.java b/gora-core/src/test/java/org/apache/gora/mapreduce/MapReduceTestUtils.java
index f837403..12ccea4 100644
--- a/gora-core/src/test/java/org/apache/gora/mapreduce/MapReduceTestUtils.java
+++ b/gora-core/src/test/java/org/apache/gora/mapreduce/MapReduceTestUtils.java
@@ -19,10 +19,10 @@
 package org.apache.gora.mapreduce;
 
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.avro.Schema.Field;
 import org.apache.gora.examples.WebPageDataCreator;
 import org.apache.gora.examples.generated.TokenDatum;
 import org.apache.gora.examples.generated.WebPage;
@@ -32,14 +32,20 @@ import org.apache.gora.query.Query;
 import org.apache.gora.store.DataStore;
 import org.apache.gora.store.impl.DataStoreBase;
 import org.apache.hadoop.conf.Configuration;
+
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class MapReduceTestUtils {
 
   private static final Logger log = LoggerFactory.getLogger(MapReduceTestUtils.class);
   
-  /** Tests by running the {@link QueryCounter} mapreduce job */
+  /** 
+   * Tests by running the {@link org.apache.gora.examples.mapreduce.QueryCounter} 
+   * mapreduce job 
+   */
   public static void testCountQuery(DataStore<String, WebPage> dataStore, Configuration conf)
       throws Exception {
 
@@ -50,7 +56,12 @@ public class MapReduceTestUtils {
 
     QueryCounter<String,WebPage> counter = new QueryCounter<String,WebPage>(conf);
     Query<String,WebPage> query = dataStore.newQuery();
-    query.setFields(WebPage._ALL_FIELDS);
+    List<Field> fields = WebPage.SCHEMA$.getFields();
+    String[] fieldNames = new String[fields.size() - 1];
+    for(int i = 0; i< fieldNames.length; i++){
+      fieldNames[i] = fields.get(i+1).name();
+    }
+    query.setFields(fieldNames);
     
     dataStore.close();
 
@@ -65,7 +76,7 @@ public class MapReduceTestUtils {
  
   public static void testWordCount(Configuration conf, DataStore<String,WebPage> inStore, DataStore<String,
       TokenDatum> outStore) throws Exception {
-	  //Datastore now has to be a Hadoop based datastore
+    //Datastore now has to be a Hadoop based datastore
     ((DataStoreBase<String,WebPage>)inStore).setConf(conf);
     ((DataStoreBase<String,TokenDatum>)outStore).setConf(conf);
     
@@ -97,6 +108,6 @@ public class MapReduceTestUtils {
       String token, int count) throws Exception {
     TokenDatum datum = outStore.get(token, null);
     assertNotNull("token:" + token + " cannot be found in datastore", datum);
-    assertEquals("count for token:" + token + " is wrong", count, datum.getCount());
+    assertEquals("count for token:" + token + " is wrong", count, datum.getCount().intValue());
   }
 }

http://git-wip-us.apache.org/repos/asf/gora/blob/136fc595/gora-core/src/test/java/org/apache/gora/mapreduce/TestGoraInputFormat.java
----------------------------------------------------------------------
diff --git a/gora-core/src/test/java/org/apache/gora/mapreduce/TestGoraInputFormat.java b/gora-core/src/test/java/org/apache/gora/mapreduce/TestGoraInputFormat.java
index eba0400..e9a7b22 100644
--- a/gora-core/src/test/java/org/apache/gora/mapreduce/TestGoraInputFormat.java
+++ b/gora-core/src/test/java/org/apache/gora/mapreduce/TestGoraInputFormat.java
@@ -22,6 +22,7 @@ import java.io.IOException;
 import java.util.Arrays;
 import java.util.List;
 
+import org.apache.avro.Schema.Field;
 import org.apache.gora.examples.generated.Employee;
 import org.apache.gora.mock.persistency.MockPersistent;
 import org.apache.gora.mock.query.MockQuery;
@@ -41,7 +42,8 @@ public class TestGoraInputFormat {
     MockDataStore store = MockDataStore.get();
 
     MockQuery query = store.newQuery();
-    query.setFields(Employee._ALL_FIELDS);
+    
+    query.setFields(getEmployeeFieldNames());
     GoraInputFormat.setInput(job, query, false);
 
     GoraInputFormat<String, MockPersistent> inputFormat
@@ -52,6 +54,15 @@ public class TestGoraInputFormat {
     return inputFormat.getSplits(job);
   }
 
+  /**
+   * First, asserts that the attempt to obtain splits results in 
+   * greater than 0 splits which can be used for computation.
+   * We then check that the partition query (obtained by using the 
+   * splits) has the same fields as we would expect by directly 
+   * accessing the fields of an Employee object.
+   * @throws IOException
+   * @throws InterruptedException
+   */
   @Test
   @SuppressWarnings("rawtypes")
   public void testGetSplits() throws IOException, InterruptedException {
@@ -61,7 +72,16 @@ public class TestGoraInputFormat {
 
     InputSplit split = splits.get(0);
     PartitionQuery query = ((GoraInputSplit)split).getQuery();
-    assertTrue(Arrays.equals(Employee._ALL_FIELDS, query.getFields()));
+    assertTrue(Arrays.equals(getEmployeeFieldNames(), query.getFields()));
+  }
+  
+  private static String[] getEmployeeFieldNames(){
+    List<Field> fields = Employee.SCHEMA$.getFields();
+    String[] fieldNames = new String[fields.size()];
+    for(int i = 0; i< fieldNames.length; i++){
+      fieldNames[i] = fields.get(i).name();
+    }
+    return fieldNames;
   }
 
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/gora/blob/136fc595/gora-core/src/test/java/org/apache/gora/mapreduce/TestPersistentSerialization.java
----------------------------------------------------------------------
diff --git a/gora-core/src/test/java/org/apache/gora/mapreduce/TestPersistentSerialization.java b/gora-core/src/test/java/org/apache/gora/mapreduce/TestPersistentSerialization.java
index 47841a1..7048c0c 100644
--- a/gora-core/src/test/java/org/apache/gora/mapreduce/TestPersistentSerialization.java
+++ b/gora-core/src/test/java/org/apache/gora/mapreduce/TestPersistentSerialization.java
@@ -18,6 +18,9 @@
 
 package org.apache.gora.mapreduce;
 
+import java.util.ArrayList;
+import java.util.HashMap;
+
 import org.apache.avro.util.Utf8;
 import org.apache.gora.examples.WebPageDataCreator;
 import org.apache.gora.examples.generated.Employee;
@@ -31,11 +34,19 @@ import org.apache.hadoop.conf.Configuration;
 import org.junit.Test;
 import static org.junit.Assert.assertEquals;
 
-/** Test class for {@link PersistentSerialization}, {@link PersistentSerializer}
- *  and {@link PersistentDeserializer}
+/**
+ * Test class for {@link PersistentSerialization}, {@link PersistentSerializer}
+ * and {@link PersistentDeserializer}
  */
 public class TestPersistentSerialization {
 
+  /**
+   * Creates an Employee object in-memory setting several fields to dirty. 
+   * Asserts that it can be serialized and 
+   * deserialzed without loosing data. We do this by asserting
+   * what we get 'before' and 'after' (de)serialization processes.
+   * @throws Exception
+   */
   @SuppressWarnings("unchecked")
   @Test
   public void testSerdeEmployee() throws Exception {
@@ -48,23 +59,45 @@ public class TestPersistentSerialization {
     TestIOUtils.testSerializeDeserialize(employee);
   }
 
+  /**
+   * Creates an Employee object but only sets one field as dirty.
+   * We then do (de)serialization and check 'before' and 'after'
+   * states. 
+   * @throws Exception
+   */
   @Test
   public void testSerdeEmployeeOneField() throws Exception {
-    Employee employee = new Employee();
+    Employee employee = Employee.newBuilder().build();
     employee.setSsn(new Utf8("11111"));
 
     TestIOUtils.testSerializeDeserialize(employee);
   }
 
+  /**
+   * Creates an Employee object setting only two fields as dirty.
+   * We then do (de)serialization and check 'before' and 'after'
+   * states. 
+   * @throws Exception
+   */
   @Test
   public void testSerdeEmployeeTwoFields() throws Exception {
-    Employee employee = new Employee();
+    Employee employee = Employee.newBuilder().build();
     employee.setSsn(new Utf8("11111"));
     employee.setSalary(100);
 
     TestIOUtils.testSerializeDeserialize(employee);
   }
 
+  /**
+   * Creates an WebPage object in-memory setting several fields to dirty. 
+   * Run a query over the persistent data.
+   * Asserts that the results can be serialized and 
+   * deserialzed without loosing data. We do this by asserting
+   * what we get 'before' and 'after' (de)serialization processes.
+   * Also simple assertion for equal number of URL's in WebPage 
+   * and results.
+   * @throws Exception
+   */
   @SuppressWarnings("unchecked")
   @Test
   public void testSerdeWebPage() throws Exception {
@@ -75,8 +108,8 @@ public class TestPersistentSerialization {
 
     Result<String, WebPage> result = store.newQuery().execute();
 
-    int i=0;
-    while(result.next()) {
+    int i = 0;
+    while (result.next()) {
       WebPage page = result.get();
       TestIOUtils.testSerializeDeserialize(page);
       i++;
@@ -84,19 +117,26 @@ public class TestPersistentSerialization {
     assertEquals(WebPageDataCreator.URLS.length, i);
   }
 
+  /**
+   * Creates multiple WebPage objects setting several fields to dirty. 
+   * Asserts that the data can be serialized and 
+   * deserialzed without loosing data. We do this by asserting
+   * what we get 'before' and 'after' (de)serialization processes.
+   * @throws Exception
+   */
   @Test
   public void testSerdeMultipleWebPages() throws Exception {
-    WebPage page1 = new WebPage();
-    WebPage page2 = new WebPage();
-    WebPage page3 = new WebPage();
+    WebPage page1 = WebPage.newBuilder().build();
+    WebPage page2 = WebPage.newBuilder().build();
+    WebPage page3 = WebPage.newBuilder().build();
 
     page1.setUrl(new Utf8("foo"));
     page2.setUrl(new Utf8("baz"));
     page3.setUrl(new Utf8("bar"));
-
-    page1.addToParsedContent(new Utf8("coo"));
-
-    page2.putToOutlinks(new Utf8("a"), new Utf8("b"));
+    page1.setParsedContent(new ArrayList<CharSequence>());
+    page1.getParsedContent().add(new Utf8("coo"));
+    page2.setOutlinks(new HashMap<CharSequence, CharSequence>());
+    page2.getOutlinks().put(new Utf8("a"), new Utf8("b"));
 
     TestIOUtils.testSerializeDeserialize(page1, page2, page3);
   }

http://git-wip-us.apache.org/repos/asf/gora/blob/136fc595/gora-core/src/test/java/org/apache/gora/mock/persistency/MockPersistent.java
----------------------------------------------------------------------
diff --git a/gora-core/src/test/java/org/apache/gora/mock/persistency/MockPersistent.java b/gora-core/src/test/java/org/apache/gora/mock/persistency/MockPersistent.java
index 1cf0bfb..b33629b 100644
--- a/gora-core/src/test/java/org/apache/gora/mock/persistency/MockPersistent.java
+++ b/gora-core/src/test/java/org/apache/gora/mock/persistency/MockPersistent.java
@@ -20,7 +20,7 @@ package org.apache.gora.mock.persistency;
 
 import org.apache.avro.Schema;
 import org.apache.gora.persistency.Persistent;
-import org.apache.gora.persistency.StateManager;
+import org.apache.gora.persistency.Tombstone;
 import org.apache.gora.persistency.impl.PersistentBase;
 
 public class MockPersistent extends PersistentBase {
@@ -36,9 +36,6 @@ public class MockPersistent extends PersistentBase {
   public MockPersistent() {
   }
   
-  public MockPersistent(StateManager stateManager) {
-    super(stateManager);
-  }
   
   @Override
   public Object get(int field) {
@@ -79,22 +76,13 @@ public class MockPersistent extends PersistentBase {
   }
 
   @Override
-  public String getField(int index) {
-    return null;
-  }
-
-  @Override
-  public int getFieldIndex(String field) {
-    return 0;
+  public Tombstone getTombstone() {
+    return new Tombstone(){};
   }
 
   @Override
-  public String[] getFields() {
-    return null;
+  public Persistent newInstance() {
+    return new MockPersistent();
   }
 
-  @Override
-  public Persistent newInstance(StateManager stateManager) {
-    return new MockPersistent(stateManager);
-  }
 }

http://git-wip-us.apache.org/repos/asf/gora/blob/136fc595/gora-core/src/test/java/org/apache/gora/persistency/TestListGenericArray.java
----------------------------------------------------------------------
diff --git a/gora-core/src/test/java/org/apache/gora/persistency/TestListGenericArray.java b/gora-core/src/test/java/org/apache/gora/persistency/TestListGenericArray.java
index a049a51..e69de29 100644
--- a/gora-core/src/test/java/org/apache/gora/persistency/TestListGenericArray.java
+++ b/gora-core/src/test/java/org/apache/gora/persistency/TestListGenericArray.java
@@ -1,57 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.gora.persistency;
-
-import org.apache.avro.Schema;
-import org.apache.avro.util.Utf8;
-import static org.junit.Assert.assertFalse;
-import org.junit.Test;
-
-/**
- * Testcase for ListGenericArray class
- */
-public class TestListGenericArray {
-  
-  @Test
-  public void testHashCode() {
-    ListGenericArray array = new ListGenericArray(Schema.create(Schema.Type.STRING)); 
-    boolean stackOverflowError = false;
-    array.add(new Utf8("array test")); 
-    try {
-      int hashCode = array.hashCode();
-    }
-    catch (StackOverflowError e) {
-      stackOverflowError = true;
-    }
-    assertFalse(stackOverflowError);
-  }
-  
-  @Test
-  public void testCompareTo() {
-    ListGenericArray array = new ListGenericArray(Schema.create(Schema.Type.STRING));
-    boolean stackOverflowError = false;
-    array.add(new Utf8("array comparison test"));
-    try {
-      int compareTo = array.compareTo(array);
-    } catch (StackOverflowError e) {
-      stackOverflowError = true;
-    }
-    assertFalse(stackOverflowError);
-  }
-}

http://git-wip-us.apache.org/repos/asf/gora/blob/136fc595/gora-core/src/test/java/org/apache/gora/persistency/impl/TestPersistentBase.java
----------------------------------------------------------------------
diff --git a/gora-core/src/test/java/org/apache/gora/persistency/impl/TestPersistentBase.java b/gora-core/src/test/java/org/apache/gora/persistency/impl/TestPersistentBase.java
index 806743a..ab730be 100644
--- a/gora-core/src/test/java/org/apache/gora/persistency/impl/TestPersistentBase.java
+++ b/gora-core/src/test/java/org/apache/gora/persistency/impl/TestPersistentBase.java
@@ -20,7 +20,10 @@ package org.apache.gora.persistency.impl;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
-
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import org.apache.avro.Schema.Field;
 import org.apache.avro.util.Utf8;
 import org.apache.gora.examples.generated.Employee;
 import org.apache.gora.examples.generated.WebPage;
@@ -28,10 +31,8 @@ import org.apache.gora.memory.store.MemStore;
 import org.apache.gora.store.DataStoreFactory;
 import org.apache.gora.store.DataStoreTestUtil;
 import org.apache.hadoop.conf.Configuration;
-import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
-
 import org.junit.Test;
 
 /**
@@ -39,53 +40,83 @@ import org.junit.Test;
  */
 public class TestPersistentBase {
   
+  /**
+   * Assert that the list of fields from the WebPage Schema
+   * are as we expect. This is done by creating and accessing 
+   * a WebPage object, then comparing the results against 
+   * static fields of the WebPage.SCHEMA$.
+   */
   @Test
   public void testGetFields() {
-    WebPage page = new WebPage();
-    String[] fields = page.getFields();
-    assertArrayEquals(WebPage._ALL_FIELDS, fields);
+    WebPage page = WebPage.newBuilder().build();
+    List<Field> fields = page.getSchema().getFields();
+    assertEquals(WebPage.SCHEMA$.getFields(), fields);
   }
   
+  /**
+   * Assert that individual field values are as we would
+   * expect from directly accessing WebPage.SCHEMA$ values.
+   */
   @Test
   public void testGetField() {
-    WebPage page = new WebPage();
-    for(int i=0; i<WebPage._ALL_FIELDS.length; i++) {
-      String field = page.getField(i);
-      assertEquals(WebPage._ALL_FIELDS[i], field);
+    WebPage page = WebPage.newBuilder().build();
+    for(int i=0; i<WebPage.SCHEMA$.getFields().toArray().length; i++) {
+      Field field = page.getSchema().getFields().get(i);
+      assertEquals(WebPage.SCHEMA$.getFields().get(i), field);
     }
   }
   
+  /**
+   * Assert that field positions as found within the SCHEMA array
+   * are as we would expect by accessing them directly. 
+   */
   @Test
   public void testGetFieldIndex() {
-    WebPage page = new WebPage();
-    for(int i=0; i<WebPage._ALL_FIELDS.length; i++) {
-      int index = page.getFieldIndex(WebPage._ALL_FIELDS[i]);
+    WebPage page = WebPage.newBuilder().build();
+    for(int i=0; i<WebPage.SCHEMA$.getFields().toArray().length; i++) {
+      int index = page.getSchema().getFields().get(i).pos();
       assertEquals(i, index);
     }
   }
   
+  /**
+   * Assert that field positions as found within the SCHEMA array
+   * are as we would expect by accessing them directly. 
+   * This tests for both WebPage and Employee data beans.
+   */
   @Test
   public void testFieldsWithTwoClasses() {
-    WebPage page = new WebPage();
-    for(int i=0; i<WebPage._ALL_FIELDS.length; i++) {
-      int index = page.getFieldIndex(WebPage._ALL_FIELDS[i]);
+    WebPage page = WebPage.newBuilder().build();
+    for(int i=0; i<WebPage.SCHEMA$.getFields().toArray().length; i++) {
+      int index = page.getSchema().getFields().get(i).pos();
       assertEquals(i, index);
     }
-    Employee employee = new Employee();
-    for(int i=0; i<Employee._ALL_FIELDS.length; i++) {
-      int index = employee.getFieldIndex(Employee._ALL_FIELDS[i]);
+    Employee employee = Employee.newBuilder().build();
+    for(int i=0; i<Employee.SCHEMA$.getFields().toArray().length; i++) {
+      int index = employee.getSchema().getFields().get(i).pos();
       assertEquals(i, index);
     }
   }
   
+  /**
+   * First we create a new WebPage object, to which we add some
+   * field values. This makes the fields dirty as we have not 
+   * flushed them to the datastore. We then clear the dirty
+   * fields and assert that the values DO NOT exist for the 
+   * field we previously made dirty.
+   * We then set new values for fields, consequently making them 
+   * dirty, before testing the clearing of an entirely new object
+   * has all fields as null as they should be clean.
+   */
   @Test
   public void testClear() {
     
     //test clear all fields
-    WebPage page = new WebPage();
+    WebPage page = WebPage.newBuilder().build();
+   
     page.setUrl(new Utf8("http://foo.com"));
-    page.addToParsedContent(new Utf8("foo"));
-    page.putToOutlinks(new Utf8("foo"), new Utf8("bar"));
+    page.getParsedContent().add(new Utf8("foo"));
+    page.getOutlinks().put(new Utf8("foo"), new Utf8("bar"));
     page.setContent(ByteBuffer.wrap("foo baz bar".getBytes()));
     
     page.clear();
@@ -97,19 +128,21 @@ public class TestPersistentBase {
     
     //set fields again
     page.setUrl(new Utf8("http://bar.com"));
-    page.addToParsedContent(new Utf8("bar"));
-    page.putToOutlinks(new Utf8("bar"), new Utf8("baz"));
+    page.getParsedContent().add(new Utf8("bar"));
+    page.getOutlinks().put(new Utf8("bar"), new Utf8("baz"));
     page.setContent(ByteBuffer.wrap("foo baz bar barbaz".getBytes()));
     
     //test clear new object
-    page = new WebPage();
+    page = WebPage.newBuilder().build();
     page.clear();
-    
-    //test primitive fields
-    Employee employee = new Employee();
-    employee.clear();
   }
   
+  /**
+   * Tests and asserts that an in-memory representation of the 
+   * Employee object is Equal to a clone of the same object.
+   * @throws IOException
+   * @throws Exception
+   */
   @Test
   public void testClone() throws IOException, Exception {
     //more tests for clone are in TestPersistentDatumReader
@@ -119,6 +152,6 @@ public class TestPersistentBase {
 
     Employee employee = DataStoreTestUtil.createEmployee(store);
     
-    assertEquals(employee, employee.clone());
+    assertEquals(employee, Employee.newBuilder(employee).build());
   }
 }

http://git-wip-us.apache.org/repos/asf/gora/blob/136fc595/gora-core/src/test/java/org/apache/gora/persistency/impl/TestStateManagerImpl.java
----------------------------------------------------------------------
diff --git a/gora-core/src/test/java/org/apache/gora/persistency/impl/TestStateManagerImpl.java b/gora-core/src/test/java/org/apache/gora/persistency/impl/TestStateManagerImpl.java
index e95673a..e69de29 100644
--- a/gora-core/src/test/java/org/apache/gora/persistency/impl/TestStateManagerImpl.java
+++ b/gora-core/src/test/java/org/apache/gora/persistency/impl/TestStateManagerImpl.java
@@ -1,115 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.gora.persistency.impl;
-
-import java.io.IOException;
-
-import org.apache.avro.util.Utf8;
-import org.apache.gora.examples.generated.Employee;
-import org.apache.gora.mock.persistency.MockPersistent;
-import org.apache.gora.persistency.impl.StateManagerImpl;
-import org.junit.Before;
-import org.junit.Test;
-import static junit.framework.Assert.assertFalse;
-import static junit.framework.Assert.assertTrue;
-
-/**
- * Test case for {@link StateManagerImpl}
- */
-public class TestStateManagerImpl {
-
-  private StateManagerImpl stateManager;
-  private MockPersistent persistent;
-  
-  @Before
-  public void setUp() {
-    this.stateManager = new StateManagerImpl();
-    this.persistent = new MockPersistent(stateManager);
-  }
-  
-  @Test
-  public void testDirty() {
-    assertFalse(stateManager.isDirty(persistent));
-    stateManager.setDirty(persistent);
-    assertTrue(stateManager.isDirty(persistent));
-  }
-  
-  @Test
-  public void testDirty2() {
-    assertFalse(stateManager.isDirty(persistent, 0));
-    assertFalse(stateManager.isDirty(persistent, 1));
-    stateManager.setDirty(persistent, 0);
-    assertTrue(stateManager.isDirty(persistent, 0));
-    assertFalse(stateManager.isDirty(persistent, 1));
-  }
-  
-  @Test
-  public void testClearDirty() {
-    assertFalse(stateManager.isDirty(persistent));
-    stateManager.setDirty(persistent, 0);
-    stateManager.clearDirty(persistent);
-    assertFalse(this.stateManager.isDirty(persistent));
-  }
-  
-  @Test
-  public void testReadable() throws IOException {
-    assertFalse(stateManager.isReadable(persistent, 0));
-    assertFalse(stateManager.isReadable(persistent, 1));
-    stateManager.setReadable(persistent, 0);
-    assertTrue(stateManager.isReadable(persistent, 0));
-    assertFalse(stateManager.isReadable(persistent, 1));
-  }
-
-  @Test
-  public void testReadable2() {
-    stateManager = new StateManagerImpl();
-    Employee employee = new Employee(stateManager);
-    assertFalse(stateManager.isReadable(employee, 0));
-    assertFalse(stateManager.isReadable(employee, 1));
-    employee.setName(new Utf8("foo"));
-    assertTrue(stateManager.isReadable(employee, 0));
-    assertFalse(stateManager.isReadable(employee, 1));
-  }
-  
-  @Test
-  public void testClearReadable() {
-    stateManager.setReadable(persistent, 0);
-    stateManager.clearReadable(persistent);
-    assertFalse(stateManager.isReadable(persistent, 0));
-  }
-  
-  @Test
-  public void testIsNew() {
-    //newly created objects should be new
-    assertTrue(persistent.isNew());
-  }
-  
-  @Test
-  public void testNew() {
-    stateManager.setNew(persistent);
-    assertTrue(persistent.isNew());
-  }
-  
-  @Test
-  public void testClearNew() {
-    stateManager.clearNew(persistent);
-    assertFalse(persistent.isNew());
-  }
-  
-}

http://git-wip-us.apache.org/repos/asf/gora/blob/136fc595/gora-core/src/test/java/org/apache/gora/store/DataStoreTestBase.java
----------------------------------------------------------------------
diff --git a/gora-core/src/test/java/org/apache/gora/store/DataStoreTestBase.java b/gora-core/src/test/java/org/apache/gora/store/DataStoreTestBase.java
index 95cd596..e890aa3 100644
--- a/gora-core/src/test/java/org/apache/gora/store/DataStoreTestBase.java
+++ b/gora-core/src/test/java/org/apache/gora/store/DataStoreTestBase.java
@@ -19,22 +19,16 @@
 package org.apache.gora.store;
 
 import java.io.IOException;
-import java.nio.ByteBuffer;
-
-import org.apache.avro.util.Utf8;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.gora.GoraTestDriver;
 import org.apache.gora.examples.generated.Employee;
-import org.apache.gora.examples.generated.Metadata;
 import org.apache.gora.examples.generated.WebPage;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
 
 /**
  * A base class for {@link DataStore} tests. This is just a convenience
@@ -216,7 +210,11 @@ public abstract class DataStoreTestBase {
   public void testUpdate() throws IOException, Exception {
     log.info("test method: testUpdate");
     DataStoreTestUtil.testUpdateEmployee(employeeStore);
-    DataStoreTestUtil.testUpdateWebPage(webPageStore);
+    DataStoreTestUtil.testUpdateWebPagePutToArray(webPageStore);
+    DataStoreTestUtil.testUpdateWebPagePutToNotNullableMap(webPageStore);
+    DataStoreTestUtil.testUpdateWebPagePutToNullableMap(webPageStore);
+    DataStoreTestUtil.testUpdateWebPageRemoveMapEntry(webPageStore);
+    DataStoreTestUtil.testUpdateWebPageRemoveField(webPageStore);
   }
 
   public void testEmptyUpdate() throws IOException, Exception {
@@ -243,7 +241,7 @@ public abstract class DataStoreTestBase {
 
   @Test
   /**
-   * Tests put and get a record with a double  nested recursive record
+   * Tests put and get a record with a double nested recursive record
    * Employee with a boss (nested).
    * @throws IOException
    * @throws Exception
@@ -255,8 +253,9 @@ public abstract class DataStoreTestBase {
 
   @Test
   /**
-   * Tests put and get a record with a nested record (not recursive)
-   * The webpage of an Employee
+   * Tests put and get of an {@link org.apache.gora.examples.generated.Employee} 
+   * record with a nested {@link org.apache.gora.examples.generated.WegPage} record (not recursive)
+   * the webpage of an Employee.
    * @throws IOException
    * @throws Exception
    */

http://git-wip-us.apache.org/repos/asf/gora/blob/136fc595/gora-core/src/test/java/org/apache/gora/store/DataStoreTestUtil.java
----------------------------------------------------------------------
diff --git a/gora-core/src/test/java/org/apache/gora/store/DataStoreTestUtil.java b/gora-core/src/test/java/org/apache/gora/store/DataStoreTestUtil.java
index d093b24..7a39d17 100644
--- a/gora-core/src/test/java/org/apache/gora/store/DataStoreTestUtil.java
+++ b/gora-core/src/test/java/org/apache/gora/store/DataStoreTestUtil.java
@@ -42,7 +42,7 @@ import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.assertNull;
 
-import org.apache.avro.generic.GenericArray;
+import org.apache.avro.Schema.Field;
 import org.apache.avro.util.Utf8;
 import org.apache.gora.examples.WebPageDataCreator;
 import org.apache.gora.examples.generated.Employee;
@@ -53,9 +53,9 @@ import org.apache.gora.persistency.impl.BeanFactoryImpl;
 import org.apache.gora.query.PartitionQuery;
 import org.apache.gora.query.Query;
 import org.apache.gora.query.Result;
+import org.apache.gora.util.AvroUtils;
 import org.apache.gora.util.ByteUtils;
 import org.apache.gora.util.StringUtils;
-import org.junit.Test;
 
 /**
  * Test utilities for DataStores. This utility class provides everything
@@ -89,7 +89,7 @@ public class DataStoreTestUtil {
   public static <K> Employee createEmployee(
       DataStore<K, Employee> dataStore) throws IOException, Exception {
 
-    Employee employee = dataStore.newPersistent();
+    Employee employee = Employee.newBuilder().build();
     employee.setName(new Utf8("Random Joe"));
     employee.setDateOfBirth( System.currentTimeMillis() - 20L *  YEAR_IN_MS );
     employee.setSalary(100000);
@@ -97,12 +97,22 @@ public class DataStoreTestUtil {
     return employee;
   }
 
-  public static <K> Employee createBoss(
-      DataStore<K, Employee> dataStore) throws IOException, Exception {
+  private static <K> WebPage createWebPage(DataStore<K, Employee> dataStore) {
+    WebPage webpage = WebPage.newBuilder().build();
+    webpage.setUrl(new Utf8("url.."));
+    webpage.setContent(ByteBuffer.wrap("test content".getBytes()));
+    webpage.setParsedContent(new ArrayList<CharSequence>());
+    Metadata metadata = Metadata.newBuilder().build();
+    webpage.setMetadata(metadata);
+    return webpage;
+  }
 
-    Employee employee = dataStore.newPersistent();
+  public static <K> Employee createBoss(DataStore<K, Employee> dataStore)
+      throws IOException, Exception {
+
+    Employee employee = Employee.newBuilder().build();
     employee.setName(new Utf8("Random boss"));
-    employee.setDateOfBirth( System.currentTimeMillis() - 22L *  YEAR_IN_MS );
+    employee.setDateOfBirth(System.currentTimeMillis() - 22L * YEAR_IN_MS);
     employee.setSalary(1000000);
     employee.setSsn(new Utf8("202020202020"));
     return employee;
@@ -159,25 +169,23 @@ public class DataStoreTestUtil {
     dataStore.put(ssn, employee);
     dataStore.flush();
 
-    Employee after = dataStore.get(ssn, Employee._ALL_FIELDS);
+    Employee after = dataStore.get(ssn, AvroUtils.getSchemaFieldNames(Employee.SCHEMA$));
 
-    assertEquals(employee, after);
+    assertEqualEmployeeObjects(employee, after);
   }
 
-
   public static void testGetEmployeeRecursive(DataStore<String, Employee> dataStore)
     throws IOException, Exception {
 
     Employee employee = DataStoreTestUtil.createEmployee(dataStore);
     Employee boss = DataStoreTestUtil.createBoss(dataStore);
-    employee.setBoss(boss) ;
+    employee.setBoss(boss);
     
     String ssn = employee.getSsn().toString();
     dataStore.put(ssn, employee);
     dataStore.flush();
-    Employee after = dataStore.get(ssn, Employee._ALL_FIELDS);
-    assertEquals(employee, after);
-    assertEquals(boss, after.getBoss()) ;
+    Employee after = dataStore.get(ssn, AvroUtils.getSchemaFieldNames(Employee.SCHEMA$));
+    assertEqualEmployeeObjects(employee, after);
   }
 
   public static void testGetEmployeeDoubleRecursive(DataStore<String, Employee> dataStore)
@@ -193,10 +201,8 @@ public class DataStoreTestUtil {
       String ssn = employee.getSsn().toString();
       dataStore.put(ssn, employee);
       dataStore.flush();
-      Employee after = dataStore.get(ssn, Employee._ALL_FIELDS);
-      assertEquals(employee, after);
-      assertEquals(boss, after.getBoss()) ;
-      assertEquals(uberBoss, ((Employee)after.getBoss()).getBoss()) ;
+      Employee after = dataStore.get(ssn, AvroUtils.getSchemaFieldNames(Employee.SCHEMA$));
+      assertEqualEmployeeObjects(employee, after);
     }
   
   public static void testGetEmployeeNested(DataStore<String, Employee> dataStore)
@@ -207,7 +213,8 @@ public class DataStoreTestUtil {
     
     webpage.setUrl(new Utf8("url..")) ;
     webpage.setContent(ByteBuffer.wrap("test content".getBytes())) ;
-    Metadata metadata = new BeanFactoryImpl<String,Metadata>(String.class,Metadata.class).newPersistent() ;
+    webpage.setParsedContent(new ArrayList<CharSequence>());
+    Metadata metadata = new BeanFactoryImpl<String,Metadata>(String.class,Metadata.class).newPersistent();
     webpage.setMetadata(metadata) ;
     employee.setWebpage(webpage) ;
     
@@ -215,9 +222,9 @@ public class DataStoreTestUtil {
    
     dataStore.put(ssn, employee);
     dataStore.flush();
-    Employee after = dataStore.get(ssn, Employee._ALL_FIELDS);
-    assertEquals(employee, after);
-    assertEquals(webpage, after.getWebpage()) ;
+    Employee after = dataStore.get(ssn, AvroUtils.getSchemaFieldNames(Employee.SCHEMA$));
+    assertEqualEmployeeObjects(employee, after);
+    assertEqualWebPageObjects(webpage, after.getWebpage());
   }
   
   public static void testGetEmployee3UnionField(DataStore<String, Employee> dataStore)
@@ -225,12 +232,12 @@ public class DataStoreTestUtil {
 
     Employee employee = DataStoreTestUtil.createEmployee(dataStore);
     employee.setBoss(new Utf8("Real boss")) ;
-    
+
     String ssn = employee.getSsn().toString();
     dataStore.put(ssn, employee);
     dataStore.flush();
-    Employee after = dataStore.get(ssn, Employee._ALL_FIELDS);
-    assertEquals(employee, after);
+    Employee after = dataStore.get(ssn, AvroUtils.getSchemaFieldNames(Employee.SCHEMA$));
+    assertEqualEmployeeObjects(employee, after);
     assertEquals("Real boss", ((Utf8)after.getBoss()).toString()) ;
   }
   
@@ -243,32 +250,139 @@ public class DataStoreTestUtil {
   public static void testGetEmployeeWithFields(DataStore<String, Employee> dataStore)
     throws IOException, Exception {
     Employee employee = DataStoreTestUtil.createEmployee(dataStore);
+    WebPage webpage = createWebPage(dataStore);
+    employee.setWebpage(webpage);
+    Employee boss = createBoss(dataStore);
+    employee.setBoss(boss);
     String ssn = employee.getSsn().toString();
     dataStore.put(ssn, employee);
     dataStore.flush();
 
-    // XXX See GORA-216: special case until later reviewed.
-    // Like in K-V stores, if retrieved column does not exists ([webpage] case),
-    // get() must return 'null'.
-    // We prepare an actual weird synthetic test.
-    
-    // String[] fields = employee.getFields();
-    String[] fields = {"name","dateOfBirth","ssn","salary"} ;
-    
+    String[] fields = AvroUtils.getPersistentFieldNames(employee);
     for(Set<String> subset : StringUtils.powerset(fields)) {
       if(subset.isEmpty())
         continue;
       Employee after = dataStore.get(ssn, subset.toArray(new String[subset.size()]));
-      Employee expected = new Employee();
+      Employee expected = Employee.newBuilder().build();
       for(String field:subset) {
-        int index = expected.getFieldIndex(field);
+        int index = expected.getSchema().getField(field).pos();
         expected.put(index, employee.get(index));
       }
 
-      assertEquals(expected, after);        
+      assertEqualEmployeeObjects(expected, after);
+    }
+  }
+  
+  /**
+   * Simple function which iterates through a before (put) and after (get) object
+   * in an attempt to verify if the same field's and values have been obtained.
+   * Within the original employee object we iterate from 1 instead of 0 due to the 
+   * removal of the '__g__' field at position 0 when we put objects into the datastore. 
+   * This field is used to identify whether fields within the object, and 
+   * consequently the object itself, are/is dirty however this field is not 
+   * required when persisting the object.
+   * We explicitly get values from each field as this makes it easier to debug 
+   * if tests go wrong.
+   * @param employee
+   * @param after
+   */
+  private static void assertEqualEmployeeObjects(Employee employee, Employee after) {
+    //for (int i = 1; i < employee.SCHEMA$.getFields().size(); i++) {
+    //  for (int j = 1; j < after.SCHEMA$.getFields().size(); j++) {
+    //    assertEquals(employee.SCHEMA$.getFields().get(i), after.SCHEMA$.getFields().get(j));
+    //  }
+    //}
+    //check name field
+    CharSequence beforeName = employee.getName();
+    CharSequence afterName = after.getName();
+    assertEquals(beforeName, afterName);
+    //check dateOfBirth field
+    Long beforeDOB = employee.getDateOfBirth();
+    Long afterDOB = after.getDateOfBirth();
+    assertEquals(beforeDOB, afterDOB);
+    //check ssn field
+    CharSequence beforeSsn = employee.getSsn();
+    CharSequence afterSsn = after.getSsn();
+    assertEquals(beforeSsn, afterSsn);
+    //check salary field
+    Integer beforeSalary = employee.getSalary();
+    Integer afterSalary = after.getSalary();
+    assertEquals(beforeSalary, afterSalary);
+    //check boss field
+    if (employee.getBoss() != null) {
+      if (employee.getBoss() instanceof Utf8) {
+        String beforeBoss = employee.getBoss().toString();
+        String afterBoss = after.getBoss().toString();
+        assertEquals("Boss String field values in UNION should be the same",
+            beforeBoss, afterBoss);
+      } else {
+        Employee beforeBoss = (Employee) employee.getBoss();
+        Employee afterBoss = (Employee) after.getBoss();
+        assertEqualEmployeeObjects(beforeBoss, afterBoss);
+      }
+    }
+    //check webpage field
+    if (employee.getWebpage() != null) {
+      WebPage beforeWebPage = employee.getWebpage();
+      WebPage afterWebPage = after.getWebpage();
+      assertEqualWebPageObjects(beforeWebPage, afterWebPage);
     }
   }
 
+  /**
+   * Mimics {@link org.apache.gora.store.DataStoreTestUtil#assertEqualEmployeeObjects(Employee, Employee)}
+   * in that we pick our way through fields within before and after 
+   * {@link org.apache.gora.examples.generated.WebPage} objects comparing field values.
+   * @param beforeWebPage
+   * @param afterWebPage
+   */
+  private static void assertEqualWebPageObjects(WebPage beforeWebPage, WebPage afterWebPage) {
+    //check url field
+    CharSequence beforeUrl = beforeWebPage.getUrl();
+    CharSequence afterUrl = afterWebPage.getUrl();
+    assertEquals(beforeUrl, afterUrl);
+    //check content field
+    ByteBuffer beforeContent = beforeWebPage.getContent();
+    ByteBuffer afterContent = afterWebPage.getContent();
+    assertEquals(beforeContent, afterContent);
+    //check parsedContent field
+    List<CharSequence> beforeParsedContent = 
+        (List<CharSequence>) beforeWebPage.getParsedContent();
+    List<CharSequence> afterParsedContent = 
+        (List<CharSequence>) afterWebPage.getParsedContent();
+    assertEquals(beforeParsedContent, afterParsedContent);
+    //check outlinks field
+    Map<CharSequence, CharSequence> beforeOutlinks = 
+        (Map<java.lang.CharSequence,java.lang.CharSequence>) beforeWebPage.getOutlinks();
+    Map<CharSequence, CharSequence> afterOutlinks = 
+        (Map<java.lang.CharSequence,java.lang.CharSequence>) afterWebPage.getOutlinks();
+    assertEquals(beforeOutlinks, afterOutlinks);
+    //check metadata field
+    if (beforeWebPage.get(5) != null) {
+      Metadata beforeMetadata = beforeWebPage.getMetadata();
+      Metadata afterMetadata = afterWebPage.getMetadata();
+      assertEqualMetadataObjects(beforeMetadata, afterMetadata);
+    }
+  }
+
+  /**
+   * Mimics {@link org.apache.gora.store.DataStoreTestUtil#assertEqualEmployeeObjects(Employee, Employee)}
+   * in that we pick our way through fields within before and after 
+   * {@link org.apache.gora.examples.generated.Metadata} objects comparing field values.
+   * @param beforeMetadata
+   * @param afterMetadata
+   */
+  private static void assertEqualMetadataObjects(Metadata beforeMetadata, Metadata afterMetadata) {
+    //check version field
+    int beforeVersion = beforeMetadata.getVersion();
+    int afterVersion = afterMetadata.getVersion();
+    assertEquals(beforeVersion, afterVersion);
+    //check data field
+    Map<CharSequence, CharSequence> beforeData = beforeMetadata.getData();
+    Map<CharSequence, CharSequence> afterData =  afterMetadata.getData();
+    assertEquals(beforeData, afterData);
+  }
+
   public static Employee testPutEmployee(DataStore<String, Employee> dataStore)
   throws IOException, Exception {
     dataStore.createSchema();
@@ -306,6 +420,16 @@ public class DataStoreTestUtil {
     assertNull(employee);
   }
 
+  /**
+   * Here we create 5 {@link org.apache.gora.examples.generated.Employee} objects
+   * before populating fields with data and flushing them to the datastore.
+   * We then update the 1st of the {@link org.apache.gora.examples.generated.Employee}'s
+   * with more data and flush this data. Assertions are then made over the updated
+   * {@link org.apache.gora.examples.generated.Employee} object.
+   * @param dataStore
+   * @throws IOException
+   * @throws Exception
+   */
   public static void testUpdateEmployee(DataStore<String, Employee> dataStore)
   throws IOException, Exception {
     dataStore.createSchema();
@@ -313,7 +437,7 @@ public class DataStoreTestUtil {
     long now = System.currentTimeMillis();
 
     for (int i = 0; i < 5; i++) {
-      Employee employee = dataStore.newPersistent();
+      Employee employee = Employee.newBuilder().build();
       employee.setName(new Utf8("John Doe " + i));
       employee.setDateOfBirth(now - 20L *  YEAR_IN_MS);
       employee.setSalary(100000);
@@ -324,9 +448,9 @@ public class DataStoreTestUtil {
     dataStore.flush();
 
     for (int i = 0; i < 1; i++) {
-      Employee employee = dataStore.newPersistent();
+      Employee employee = Employee.newBuilder().build();
       employee.setName(new Utf8("John Doe " + (i + 5)));
-      employee.setDateOfBirth(now - 18L *  YEAR_IN_MS);
+      employee.setDateOfBirth(now - 18L * YEAR_IN_MS);
       employee.setSalary(120000);
       employee.setSsn(new Utf8(Long.toString(ssn + i)));
       dataStore.put(employee.getSsn().toString(), employee);
@@ -337,33 +461,42 @@ public class DataStoreTestUtil {
     for (int i = 0; i < 1; i++) {
       String key = Long.toString(ssn + i);
       Employee employee = dataStore.get(key);
-      assertEquals(now - 18L * YEAR_IN_MS, employee.getDateOfBirth());
+      assertEquals(now - 18L * YEAR_IN_MS, employee.getDateOfBirth().longValue()); 
       assertEquals("John Doe " + (i + 5), employee.getName().toString());
-      assertEquals(120000, employee.getSalary());
+      assertEquals(120000, employee.getSalary().intValue()); 
     }
   }
 
-  public static void testUpdateWebPage(DataStore<String, WebPage> dataStore)
+  /**
+   * Here we create 7 {@link org.apache.gora.examples.generated.WebPage}
+   * objects and populate field data before flushing the objects to the 
+   * datastore. We then get the objects, adding data to the 'content' and
+   * 'parsedContent' fields before clearing the 'outlinks' field and 
+   * re-populating it. This data is then flushed to the datastore. 
+   * Finally we get the {@link org.apache.gora.examples.generated.WebPage}
+   * objects and make various assertions over verious fields. This tests 
+   * that we can update fields and that data can be written and read correctly.
+   * @param dataStore
+   * @throws IOException
+   * @throws Exception
+   */
+  public static void testUpdateWebPagePutToArray(DataStore<String, WebPage> dataStore)
   throws IOException, Exception {
     dataStore.createSchema();
 
     String[] urls = {"http://a.com/a", "http://b.com/b", "http://c.com/c",
-        "http://d.com/d", "http://e.com/e", "http://f.com/f", "http://g.com/g"};
+        "http://d.com/d", "http://e.com/e", "http://f.com/f", "http://g.com/g" };
     String content = "content";
     String parsedContent = "parsedContent";
-    String anchor = "anchor";
 
     int parsedContentCount = 0;
 
 
     for (int i = 0; i < urls.length; i++) {
-      WebPage webPage = dataStore.newPersistent();
+      WebPage webPage = WebPage.newBuilder().build();
       webPage.setUrl(new Utf8(urls[i]));
       for (parsedContentCount = 0; parsedContentCount < 5; parsedContentCount++) {
-        webPage.addToParsedContent(new Utf8(parsedContent + i + "," + parsedContentCount));
-      }
-      for (int j = 0; j < urls.length; j += 2) {
-        webPage.putToOutlinks(new Utf8(anchor + j), new Utf8(urls[j]));
+        webPage.getParsedContent().add(new Utf8(parsedContent + i + "," + parsedContentCount));
       }
       dataStore.put(webPage.getUrl().toString(), webPage);
     }
@@ -374,15 +507,7 @@ public class DataStoreTestUtil {
       WebPage webPage = dataStore.get(urls[i]);
       webPage.setContent(ByteBuffer.wrap(ByteUtils.toBytes(content + i)));
       for (parsedContentCount = 5; parsedContentCount < 10; parsedContentCount++) {
-        webPage.addToParsedContent(new Utf8(parsedContent + i + "," + parsedContentCount));
-      }
-      webPage.getOutlinks().clear();
-      for (int j = 1; j < urls.length; j += 2) {
-        webPage.putToOutlinks(new Utf8(anchor + j), new Utf8(urls[j]));
-      }
-      //test for double put of same entries
-      for (int j = 1; j < urls.length; j += 2) {
-        webPage.putToOutlinks(new Utf8(anchor + j), new Utf8(urls[j]));
+        webPage.getParsedContent().add(new Utf8(parsedContent + i + "," + parsedContentCount));
       }
       dataStore.put(webPage.getUrl().toString(), webPage);
     }
@@ -394,24 +519,90 @@ public class DataStoreTestUtil {
       assertEquals(content + i, ByteUtils.toString( toByteArray(webPage.getContent()) ));
       assertEquals(10, webPage.getParsedContent().size());
       int j = 0;
-      for (Utf8 pc : webPage.getParsedContent()) {
+      for (CharSequence pc : webPage.getParsedContent()) {
         assertEquals(parsedContent + i + "," + j, pc.toString());
         j++;
       }
+    }
+  }
+
+  public static void testUpdateWebPagePutToNotNullableMap(DataStore<String, WebPage> dataStore)
+  throws IOException, Exception {
+    dataStore.createSchema();
+
+    String[] urls = {"http://a.com/a", "http://b.com/b", "http://c.com/c",
+        "http://d.com/d", "http://e.com/e", "http://f.com/f", "http://g.com/g" };
+    String anchor = "anchor";
+
+    // putting evens
+    for (int i = 0; i < urls.length; i++) {
+      WebPage webPage = WebPage.newBuilder().build();
+      webPage.setUrl(new Utf8(urls[i]));
+      for (int j = 0; j < urls.length; j += 2) {
+        webPage.getOutlinks().put(new Utf8(anchor + j), new Utf8(urls[j]));
+      }
+      dataStore.put(webPage.getUrl().toString(), webPage);
+    }
+    dataStore.flush();
+
+    // putting odds
+    for (int i = 0; i < urls.length; i++) {
+      WebPage webPage = dataStore.get(urls[i]);
+      webPage.getOutlinks().clear();
+      for (int j = 1; j < urls.length; j += 2) {
+        webPage.getOutlinks().put(new Utf8(anchor + j), new Utf8(urls[j]));
+      }
+      // test for double put of same entries
+      for (int j = 1; j < urls.length; j += 2) {
+        webPage.getOutlinks().put(new Utf8(anchor + j), new Utf8(urls[j]));
+      }
+      dataStore.put(webPage.getUrl().toString(), webPage);
+    }
+    dataStore.flush();
+
+    for (int i = 0; i < urls.length; i++) {
+      WebPage webPage = dataStore.get(urls[i]);
       int count = 0;
-      for (j = 1; j < urls.length; j += 2) {
-        Utf8 link = webPage.getOutlinks().get(new Utf8(anchor + j));
+      for (int j = 1; j < urls.length; j += 2) {
+        CharSequence link = webPage.getOutlinks().get(new Utf8(anchor + j));
         assertNotNull(link);
         assertEquals(urls[j], link.toString());
         count++;
       }
       assertEquals(count, webPage.getOutlinks().size());
     }
+  }
+
+  public static void testUpdateWebPagePutToNullableMap(DataStore<String, WebPage> dataStore)
+  throws IOException, Exception {
+    dataStore.createSchema();
+
+    String[] urls = {"http://a.com/a", "http://b.com/b", "http://c.com/c",
+        "http://d.com/d", "http://e.com/e", "http://f.com/f", "http://g.com/g" };
+    String header = "header";
+    String[] headers = { "firstHeader", "secondHeader", "thirdHeader",
+        "fourthHeader", "fifthHeader", "sixthHeader" };
+
+    for (int i = 0; i < urls.length; i++) {
+      WebPage webPage = WebPage.newBuilder().build();
+      webPage.setUrl(new Utf8(urls[i]));
+      //test put for nullable map field 
+      // we put data to the 'headers' field which is a Map with default value of 'null'
+      webPage.setHeaders(new HashMap<CharSequence, CharSequence>());
+      for (int j = 0; j < headers.length; j += 2) {
+        webPage.getHeaders().put(new Utf8(header + j), new Utf8(headers[j]));
+      }
+      dataStore.put(webPage.getUrl().toString(), webPage);
+    }
+
+    dataStore.flush();
 
     for (int i = 0; i < urls.length; i++) {
       WebPage webPage = dataStore.get(urls[i]);
-      for (int j = 0; j < urls.length; j += 2) {
-        webPage.putToOutlinks(new Utf8(anchor + j), new Utf8(urls[j]));
+      //webPage.getHeaders().clear(); //TODO clear method does not work
+      webPage.setHeaders(new HashMap<CharSequence, CharSequence>());
+      for (int j = 1; j < headers.length; j += 2) {
+        webPage.getHeaders().put(new Utf8(header + j), new Utf8(headers[j]));
       }
       dataStore.put(webPage.getUrl().toString(), webPage);
     }
@@ -421,12 +612,93 @@ public class DataStoreTestUtil {
     for (int i = 0; i < urls.length; i++) {
       WebPage webPage = dataStore.get(urls[i]);
       int count = 0;
+      for (int j = 1; j < headers.length; j += 2) {
+        CharSequence headerSample = webPage.getHeaders().get(new Utf8(header + j));
+        assertNotNull(headerSample);
+        assertEquals(headers[j], headerSample.toString());
+        count++;
+      }
+      assertEquals(count, webPage.getHeaders().size());
+    }
+  }
+
+  public static void testUpdateWebPageRemoveMapEntry(DataStore<String, WebPage> dataStore)
+  throws IOException, Exception {
+    dataStore.createSchema();
+
+    String[] urls = {"http://a.com/a", "http://b.com/b", "http://c.com/c",
+        "http://d.com/d", "http://e.com/e", "http://f.com/f", "http://g.com/g" };
+    String anchor = "anchor";
+
+    for (int i = 0; i < urls.length; i++) {
+      WebPage webPage = WebPage.newBuilder().build();
+      webPage.setUrl(new Utf8(urls[i]));
       for (int j = 0; j < urls.length; j++) {
-        Utf8 link = webPage.getOutlinks().get(new Utf8(anchor + j));
-        assertNotNull(link);
-        assertEquals(urls[j], link.toString());
+        webPage.getOutlinks().put(new Utf8(anchor + j), new Utf8(urls[j]));
+      }
+      dataStore.put(webPage.getUrl().toString(), webPage);
+    }
+
+    dataStore.flush();
+
+    // map entry removal test
+    for (int i = 0; i < urls.length; i++) {
+      WebPage webPage = dataStore.get(urls[i]);
+      for (int j = 1; j < urls.length; j += 2) {
+        webPage.getOutlinks().remove(new Utf8(anchor + j));
+      }
+      dataStore.put(webPage.getUrl().toString(), webPage);
+    }
+
+    dataStore.flush();
+
+    for (int i = 0; i < urls.length; i++) {
+      int count = 0;
+      WebPage webPage = dataStore.get(urls[i]);
+      for (int j = 1; j < urls.length; j += 2) {
+        CharSequence link = webPage.getOutlinks().get(new Utf8(anchor + j));
+        assertNull(link);
+        //assertEquals(urls[j], link.toString());
         count++;
       }
+      assertEquals(urls.length - count, webPage.getOutlinks().size());
+    }
+  }
+
+  public static void testUpdateWebPageRemoveField(DataStore<String, WebPage> dataStore)
+  throws IOException, Exception {
+    dataStore.createSchema();
+
+    String[] urls = {"http://a.com/a", "http://b.com/b", "http://c.com/c",
+        "http://d.com/d", "http://e.com/e", "http://f.com/f", "http://g.com/g" };
+    String header = "header";
+    String[] headers = { "firstHeader", "secondHeader", "thirdHeader",
+        "fourthHeader", "fifthHeader", "sixthHeader" };
+
+    for (int i = 0; i < urls.length; i++) {
+      WebPage webPage = WebPage.newBuilder().build();
+      webPage.setUrl(new Utf8(urls[i]));
+      webPage.setHeaders(new HashMap<CharSequence, CharSequence>());
+      for (int j = 0; j < headers.length; j++) {
+        webPage.getHeaders().put(new Utf8(header + j), new Utf8(headers[j]));
+      }
+      dataStore.put(webPage.getUrl().toString(), webPage);
+    }
+
+    dataStore.flush();
+
+    // nullable map field removal test
+    for (int i = 0; i < urls.length; i++) {
+      WebPage webPage = dataStore.get(urls[i]);
+      webPage.setHeaders(null);
+      dataStore.put(webPage.getUrl().toString(), webPage);
+    }
+
+    dataStore.flush();
+
+    for (int i = 0; i < urls.length; i++) {
+      WebPage webPage = dataStore.get(urls[i]);
+      assertNull(webPage.getHeaders());
     }
   }
 
@@ -440,19 +712,20 @@ public class DataStoreTestUtil {
         " actual=" + CONTENTS[i] + " i=" + i
         , Arrays.equals( toByteArray(page.getContent() )
         , CONTENTS[i].getBytes()));
-      GenericArray<Utf8> parsedContent = page.getParsedContent();
+    
+      List<CharSequence> parsedContent = page.getParsedContent();
       assertNotNull(parsedContent);
       assertTrue(parsedContent.size() > 0);
     
       int j=0;
       String[] tokens = CONTENTS[i].split(" ");
-      for(Utf8 token : parsedContent) {
+      for(CharSequence token : parsedContent) {
         assertEquals(tokens[j++], token.toString());
       }
     } else {
       // when page.getContent() is null
       assertTrue(CONTENTS[i] == null) ;
-      GenericArray<Utf8> parsedContent = page.getParsedContent();
+      List<CharSequence> parsedContent = page.getParsedContent();
       assertNotNull(parsedContent);
       assertTrue(parsedContent.size() == 0);
     }
@@ -462,7 +735,7 @@ public class DataStoreTestUtil {
       assertTrue(page.getOutlinks().size() > 0);
       for(int k=0; k<LINKS[i].length; k++) {
         assertEquals(ANCHORS[i][k],
-          page.getFromOutlinks(new Utf8(URLS[LINKS[i][k]])).toString());
+          page.getOutlinks().get(new Utf8(URLS[LINKS[i][k]])).toString());
       }
     } else {
       assertTrue(page.getOutlinks() == null || page.getOutlinks().isEmpty());
@@ -480,7 +753,7 @@ public class DataStoreTestUtil {
   }
 
   public static void testGetWebPage(DataStore<String, WebPage> store) throws IOException, Exception {
-    testGetWebPage(store, WebPage._ALL_FIELDS);
+    testGetWebPage(store, getFields(WebPage.SCHEMA$.getFields()));
   }
 
   public static void testGetWebPageDefaultFields(DataStore<String, WebPage> store)
@@ -507,7 +780,7 @@ public class DataStoreTestUtil {
 
   public static void testQueryWebPageSingleKey(DataStore<String, WebPage> store)
   throws IOException, Exception {
-    testQueryWebPageSingleKey(store, WebPage._ALL_FIELDS);
+    testQueryWebPageSingleKey(store, getFields(WebPage.SCHEMA$.getFields()));
   }
 
   public static void testQueryWebPageSingleKeyDefaultFields(
@@ -714,7 +987,7 @@ public class DataStoreTestUtil {
     WebPageDataCreator.createWebPageData(store);
 
     query = store.newQuery();
-    query.setFields(WebPage._ALL_FIELDS);
+    query.setFields(AvroUtils.getSchemaFieldNames(WebPage.SCHEMA$));
 
     assertNumResults(store.newQuery(), URLS.length);
     store.deleteByQuery(query);
@@ -757,8 +1030,8 @@ public class DataStoreTestUtil {
     WebPageDataCreator.createWebPageData(store);
 
     query = store.newQuery();
-    query.setFields(WebPage.Field.OUTLINKS.getName()
-        , WebPage.Field.PARSED_CONTENT.getName(), WebPage.Field.CONTENT.getName());
+    query.setFields("outlinks"
+        , "parsedContent", "content");
 
     assertNumResults(store.newQuery(), URLS.length);
     store.deleteByQuery(query);
@@ -776,7 +1049,8 @@ public class DataStoreTestUtil {
 
       assertNotNull(page.getUrl());
       assertEquals(page.getUrl().toString(), SORTED_URLS[i]);
-      assertEquals(0, page.getOutlinks().size());
+      assertEquals("Map of Outlinks should have a size of '0' as the deleteByQuery "
+          + "not only removes the data but also the data structure.", 0, page.getOutlinks().size());
       assertEquals(0, page.getParsedContent().size());
       if(page.getContent() != null) {
         System.out.println("url:" + page.getUrl().toString());
@@ -790,7 +1064,7 @@ public class DataStoreTestUtil {
     WebPageDataCreator.createWebPageData(store);
 
     query = store.newQuery();
-    query.setFields(WebPage.Field.URL.getName());
+    query.setFields("url");
     String startKey = SORTED_URLS[NUM_KEYS];
     String endKey = SORTED_URLS[SORTED_URLS.length - NUM_KEYS];
     query.setStartKey(startKey);
@@ -832,10 +1106,10 @@ public class DataStoreTestUtil {
     String url = "http://foo.com/";
 
     store.createSchema();
-    WebPage page = store.newPersistent();
-    Metadata metadata = new Metadata();
+    WebPage page = WebPage.newBuilder().build();
+    Metadata metadata = Metadata.newBuilder().build();
     metadata.setVersion(1);
-    metadata.putToData(new Utf8("foo"), new Utf8("baz"));
+    metadata.getData().put(new Utf8("foo"), new Utf8("baz"));
 
     page.setMetadata(metadata);
     page.setUrl(new Utf8(url));
@@ -846,30 +1120,31 @@ public class DataStoreTestUtil {
     page = store.get(revUrl);
     metadata = page.getMetadata();
     assertNotNull(metadata);
-    assertEquals(1, metadata.getVersion());
+    assertEquals(1, metadata.getVersion().intValue()); 
     assertEquals(new Utf8("baz"), metadata.getData().get(new Utf8("foo")));
   }
 
   public static void testPutArray(DataStore<String, WebPage> store)
           throws IOException, Exception {
     store.createSchema();
-    WebPage page = store.newPersistent();
+    WebPage page = WebPage.newBuilder().build();
 
     String[] tokens = {"example", "content", "in", "example.com"};
-
+    page.setParsedContent(new ArrayList<CharSequence>());
     for(String token: tokens) {
-      page.addToParsedContent(new Utf8(token));
+      page.getParsedContent().add(new Utf8(token));
     }
 
     store.put("com.example/http", page);
     store.close();
+
   }
 
   public static byte[] testPutBytes(DataStore<String, WebPage> store)
           throws IOException, Exception {
 
     store.createSchema();
-    WebPage page = store.newPersistent();
+    WebPage page = WebPage.newBuilder().build();
     page.setUrl(new Utf8("http://example.com"));
     byte[] contentBytes = "example content in example.com".getBytes();
     ByteBuffer buff = ByteBuffer.wrap(contentBytes);
@@ -886,12 +1161,12 @@ public class DataStoreTestUtil {
 
     store.createSchema();
 
-    WebPage page = store.newPersistent();
+    WebPage page = WebPage.newBuilder().build();
 
     page.setUrl(new Utf8("http://example.com"));
-    page.putToOutlinks(new Utf8("http://example2.com"), new Utf8("anchor2"));
-    page.putToOutlinks(new Utf8("http://example3.com"), new Utf8("anchor3"));
-    page.putToOutlinks(new Utf8("http://example3.com"), new Utf8("anchor4"));
+    page.getOutlinks().put(new Utf8("http://example2.com"), new Utf8("anchor2"));
+    page.getOutlinks().put(new Utf8("http://example3.com"), new Utf8("anchor3"));
+    page.getOutlinks().put(new Utf8("http://example3.com"), new Utf8("anchor4"));
     store.put("com.example/http", page);
     store.close();
   }
@@ -905,5 +1180,23 @@ public class DataStoreTestUtil {
     }
     return bytes;
   }
-
+  
+  public static String[] getFields(List<Field> schemaFields) {
+    
+    List<Field> list = new ArrayList<Field>();
+    for (Field field : schemaFields) {
+      if (!Persistent.DIRTY_BYTES_FIELD_NAME.equalsIgnoreCase(field.name())) {
+        list.add(field);
+      }
+    }
+    schemaFields = list;
+    
+    String[] fieldNames = new String[schemaFields.size()];
+    for(int i = 0; i<fieldNames.length; i++ ){
+      fieldNames[i] = schemaFields.get(i).name();
+    }
+    
+    return fieldNames;
+  }
+  
 }

http://git-wip-us.apache.org/repos/asf/gora/blob/136fc595/gora-core/src/test/java/org/apache/gora/store/TestDataStoreFactory.java
----------------------------------------------------------------------
diff --git a/gora-core/src/test/java/org/apache/gora/store/TestDataStoreFactory.java b/gora-core/src/test/java/org/apache/gora/store/TestDataStoreFactory.java
index 5f3436c..a50fd71 100644
--- a/gora-core/src/test/java/org/apache/gora/store/TestDataStoreFactory.java
+++ b/gora-core/src/test/java/org/apache/gora/store/TestDataStoreFactory.java
@@ -20,9 +20,9 @@ package org.apache.gora.store;
 
 import java.util.Properties;
 
-import static junit.framework.Assert.assertNotNull;
-import static junit.framework.Assert.assertEquals;
-import static junit.framework.Assert.assertNotSame;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotSame;
 
 import org.apache.gora.avro.store.DataFileAvroStore;
 import org.apache.gora.mock.persistency.MockPersistent;

http://git-wip-us.apache.org/repos/asf/gora/blob/136fc595/gora-core/src/test/java/org/apache/gora/util/TestIOUtils.java
----------------------------------------------------------------------
diff --git a/gora-core/src/test/java/org/apache/gora/util/TestIOUtils.java b/gora-core/src/test/java/org/apache/gora/util/TestIOUtils.java
index 30efd8c..381d366 100644
--- a/gora-core/src/test/java/org/apache/gora/util/TestIOUtils.java
+++ b/gora-core/src/test/java/org/apache/gora/util/TestIOUtils.java
@@ -27,11 +27,10 @@ import java.io.EOFException;
 import java.io.IOException;
 import java.util.Arrays;
 
-import org.apache.avro.ipc.ByteBufferInputStream;
-import org.apache.avro.ipc.ByteBufferOutputStream;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.avro.util.ByteBufferInputStream;
+import org.apache.avro.util.ByteBufferOutputStream;
 import org.apache.gora.mapreduce.GoraMapReduceUtils;
+import org.apache.gora.util.IOUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.DataInputBuffer;
 import org.apache.hadoop.io.DataOutputBuffer;
@@ -39,6 +38,8 @@ import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.Writable;
 import org.junit.Test;
 import static org.junit.Assert.assertEquals;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Test case for {@link IOUtils} class.

http://git-wip-us.apache.org/repos/asf/gora/blob/136fc595/gora-dynamodb/src/main/java/org/apache/gora/dynamodb/store/DynamoDBStore.java
----------------------------------------------------------------------
diff --git a/gora-dynamodb/src/main/java/org/apache/gora/dynamodb/store/DynamoDBStore.java b/gora-dynamodb/src/main/java/org/apache/gora/dynamodb/store/DynamoDBStore.java
index d84f051..7192c8a 100644
--- a/gora-dynamodb/src/main/java/org/apache/gora/dynamodb/store/DynamoDBStore.java
+++ b/gora-dynamodb/src/main/java/org/apache/gora/dynamodb/store/DynamoDBStore.java
@@ -36,7 +36,6 @@ import org.apache.gora.persistency.Persistent;
 import org.apache.gora.query.PartitionQuery;
 import org.apache.gora.query.Query;
 import org.apache.gora.query.Result;
-import org.apache.gora.store.DataStoreFactory;
 import org.apache.gora.store.ws.impl.WSDataStoreBase;
 import org.apache.gora.util.GoraException;
 import org.slf4j.Logger;
@@ -162,11 +161,11 @@ public class DynamoDBStore<K, T extends Persistent> extends WSDataStoreBase<K, T
       LOG.debug("Initializing DynamoDB store");
       getCredentials();
       setWsProvider(wsProvider);
-      preferredSchema = DataStoreFactory.findProperty(properties, this, PREF_SCH_NAME, null);
-      dynamoDBClient = getClient(DataStoreFactory.findProperty(properties, this, CLI_TYP_PROP, null),(AWSCredentials)getConf());
-      dynamoDBClient.setEndpoint(DataStoreFactory.findProperty(properties, this, ENDPOINT_PROP, null));
+      preferredSchema = properties.getProperty(PREF_SCH_NAME);
+      dynamoDBClient = getClient(properties.getProperty(CLI_TYP_PROP),(AWSCredentials)getConf());
+      dynamoDBClient.setEndpoint(properties.getProperty(ENDPOINT_PROP));
       mapping = readMapping();
-      consistency = DataStoreFactory.findProperty(properties, this, CONSISTENCY_READS, null);
+      consistency = properties.getProperty(CONSISTENCY_READS);
       persistentClass = pPersistentClass;
     }
     catch (Exception e) {


Mime
View raw message