incubator-gora-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e...@apache.org
Subject svn commit: r1021390 [6/9] - in /incubator/gora/trunk: ./ bin/ conf/ docs/ gora-cassandra/ gora-cassandra/ivy/ gora-cassandra/src/examples/java/ gora-cassandra/src/main/java/org/apache/ gora-cassandra/src/main/java/org/apache/gora/ gora-cassandra/src/m...
Date Mon, 11 Oct 2010 15:40:29 GMT
Added: incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/avro/store/TestAvroStore.java
URL: http://svn.apache.org/viewvc/incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/avro/store/TestAvroStore.java?rev=1021390&view=auto
==============================================================================
--- incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/avro/store/TestAvroStore.java (added)
+++ incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/avro/store/TestAvroStore.java Mon Oct 11 15:40:22 2010
@@ -0,0 +1,153 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gora.avro.store;
+
+import static org.apache.gora.examples.WebPageDataCreator.URLS;
+import static org.apache.gora.examples.WebPageDataCreator.URL_INDEXES;
+import static org.apache.gora.examples.WebPageDataCreator.createWebPageData;
+
+import java.io.IOException;
+
+import junit.framework.Assert;
+
+import org.apache.gora.avro.store.AvroStore;
+import org.apache.gora.avro.store.AvroStore.CodecType;
+import org.apache.gora.examples.generated.Employee;
+import org.apache.gora.examples.generated.WebPage;
+import org.apache.gora.query.Query;
+import org.apache.gora.query.Result;
+import org.apache.gora.store.DataStore;
+import org.apache.gora.store.DataStoreFactory;
+import org.apache.gora.store.DataStoreTestUtil;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test case for {@link AvroStore}.
+ */
+public class TestAvroStore {
+
+  public static final String EMPLOYEE_OUTPUT =
+    System.getProperty("test.build.data") + "/testavrostore/employee.data";
+  public static final String WEBPAGE_OUTPUT =
+    System.getProperty("test.build.data") + "/testavrostore/webpage.data";
+
+  protected AvroStore<String,Employee> employeeStore;
+  protected AvroStore<String,WebPage> webPageStore;
+  protected Configuration conf = new Configuration();
+
+  @Before
+  public void setUp() throws Exception {
+    employeeStore = createEmployeeDataStore();
+    employeeStore.initialize(String.class, Employee.class, DataStoreFactory.properties);
+    employeeStore.setOutputPath(EMPLOYEE_OUTPUT);
+    employeeStore.setInputPath(EMPLOYEE_OUTPUT);
+
+    webPageStore = new AvroStore<String, WebPage>();
+    webPageStore.initialize(String.class, WebPage.class, DataStoreFactory.properties);
+    webPageStore.setOutputPath(WEBPAGE_OUTPUT);
+    webPageStore.setInputPath(WEBPAGE_OUTPUT);
+  }
+
+  @SuppressWarnings("unchecked")
+  protected AvroStore<String, Employee> createEmployeeDataStore() {
+    return (AvroStore<String, Employee>) DataStoreFactory.getDataStore(
+        AvroStore.class, String.class, Employee.class);
+  }
+
+  protected AvroStore<String, WebPage> createWebPageDataStore() {
+    return new AvroStore<String, WebPage>();
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    deletePath(employeeStore.getOutputPath());
+    deletePath(webPageStore.getOutputPath());
+
+    employeeStore.close();
+    webPageStore.close();
+  }
+
+  private void deletePath(String output) throws IOException {
+    if(output != null) {
+      Path path = new Path(output);
+      path.getFileSystem(conf).delete(path, true);
+    }
+  }
+
+  @Test
+  public void testNewInstance() throws IOException {
+    DataStoreTestUtil.testNewPersistent(employeeStore);
+  }
+
+  @Test
+  public void testCreateSchema() throws IOException {
+    DataStoreTestUtil.testCreateEmployeeSchema(employeeStore);
+  }
+
+  @Test
+  public void testAutoCreateSchema() throws IOException {
+    DataStoreTestUtil.testAutoCreateSchema(employeeStore);
+  }
+
+  @Test
+  public void testPut() throws IOException {
+    DataStoreTestUtil.testPutEmployee(employeeStore);
+  }
+
+  @Test
+  public void testQuery() throws IOException {
+    createWebPageData(webPageStore);
+    webPageStore.close();
+
+    webPageStore.setInputPath(webPageStore.getOutputPath());
+    testQueryWebPages(webPageStore);
+  }
+
+  @Test
+  public void testQueryBinaryEncoder() throws IOException {
+    webPageStore.setCodecType(CodecType.BINARY);
+    webPageStore.setInputPath(webPageStore.getOutputPath());
+
+    createWebPageData(webPageStore);
+    webPageStore.close();
+    testQueryWebPages(webPageStore);
+  }
+
+  //AvroStore should be closed so that Hadoop file is completely flushed,
+  //so below test is copied and modified to close the store after pushing data
+  public static void testQueryWebPages(DataStore<String, WebPage> store)
+  throws IOException {
+
+    Query<String, WebPage> query = store.newQuery();
+    Result<String, WebPage> result = query.execute();
+
+    int i=0;
+    while(result.next()) {
+      WebPage page = result.get();
+      DataStoreTestUtil.assertWebPage(page, URL_INDEXES.get(page.getUrl().toString()));
+      i++;
+    }
+    Assert.assertEquals(i, URLS.length);
+  }
+
+}

Added: incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/avro/store/TestDataFileAvroStore.java
URL: http://svn.apache.org/viewvc/incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/avro/store/TestDataFileAvroStore.java?rev=1021390&view=auto
==============================================================================
--- incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/avro/store/TestDataFileAvroStore.java (added)
+++ incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/avro/store/TestDataFileAvroStore.java Mon Oct 11 15:40:22 2010
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gora.avro.store;
+
+import org.apache.gora.avro.store.AvroStore;
+import org.apache.gora.avro.store.DataFileAvroStore;
+import org.apache.gora.examples.generated.Employee;
+import org.apache.gora.examples.generated.WebPage;
+
+/**
+ * Test case for {@link DataFileAvroStore}.
+ */
+public class TestDataFileAvroStore extends TestAvroStore {
+
+  @Override
+  protected AvroStore<String, Employee> createEmployeeDataStore() {
+    return new DataFileAvroStore<String, Employee>();
+  }
+  
+  @Override
+  protected AvroStore<String, WebPage> createWebPageDataStore() {
+    return new DataFileAvroStore<String, WebPage>();
+  }
+  
+  //import all tests from super class
+  
+}

Added: incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/DataStoreMapReduceTestBase.java
URL: http://svn.apache.org/viewvc/incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/DataStoreMapReduceTestBase.java?rev=1021390&view=auto
==============================================================================
--- incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/DataStoreMapReduceTestBase.java (added)
+++ incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/DataStoreMapReduceTestBase.java Mon Oct 11 15:40:22 2010
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gora.mapreduce;
+
+import java.io.IOException;
+
+import org.apache.gora.examples.generated.WebPage;
+import org.apache.gora.store.DataStore;
+import org.apache.hadoop.mapred.HadoopTestCase;
+import org.apache.hadoop.mapred.JobConf;
+import org.junit.Before;
+import org.junit.Test;
+
+
+/**
+ * Base class for Mapreduce based tests. This is just a convenience
+ * class, which actually only uses {@link MapReduceTestUtils} methods to
+ * run the tests.
+ */
+@SuppressWarnings("deprecation")
+public abstract class DataStoreMapReduceTestBase extends HadoopTestCase {
+
+  private DataStore<String, WebPage> webPageStore;
+  private JobConf job;
+
+  public DataStoreMapReduceTestBase(int mrMode, int fsMode, int taskTrackers,
+      int dataNodes) throws IOException {
+    super(mrMode, fsMode, taskTrackers, dataNodes);
+  }
+
+  public DataStoreMapReduceTestBase() throws IOException {
+    this(HadoopTestCase.CLUSTER_MR, HadoopTestCase.DFS_FS, 2, 2);
+  }
+
+  @Override
+  @Before
+  public void setUp() throws Exception {
+    super.setUp();
+    webPageStore = createWebPageDataStore();
+    job = createJobConf();
+  }
+
+  @Override
+  public void tearDown() throws Exception {
+    super.tearDown();
+    webPageStore.close();
+  }
+
+  protected abstract DataStore<String, WebPage> createWebPageDataStore()
+    throws IOException;
+
+  @Test
+  public void testCountQuery() throws Exception {
+    MapReduceTestUtils.testCountQuery(webPageStore, job);
+  }
+
+  @Test
+  public void testWordCount() throws Exception {
+    MapReduceTestUtils.testCountQuery(webPageStore, job);
+  }
+}

Added: incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/MapReduceTestUtils.java
URL: http://svn.apache.org/viewvc/incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/MapReduceTestUtils.java?rev=1021390&view=auto
==============================================================================
--- incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/MapReduceTestUtils.java (added)
+++ incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/MapReduceTestUtils.java Mon Oct 11 15:40:22 2010
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gora.mapreduce;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.gora.examples.WebPageDataCreator;
+import org.apache.gora.examples.generated.TokenDatum;
+import org.apache.gora.examples.generated.WebPage;
+import org.apache.gora.examples.mapreduce.QueryCounter;
+import org.apache.gora.examples.mapreduce.WordCount;
+import org.apache.gora.query.Query;
+import org.apache.gora.store.DataStore;
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Assert;
+
+public class MapReduceTestUtils {
+
+  private static final Log log = LogFactory.getLog(MapReduceTestUtils.class);
+  
+  /** Tests by running the {@link QueryCounter} mapreduce job */
+  public static void testCountQuery(DataStore<String, WebPage> dataStore
+      , Configuration conf) 
+  throws Exception {
+    
+    dataStore.setConf(conf);
+    
+    //create input
+    WebPageDataCreator.createWebPageData(dataStore);
+    
+    
+    QueryCounter<String,WebPage> counter = new QueryCounter<String,WebPage>(conf);
+    Query<String,WebPage> query = dataStore.newQuery();
+    query.setFields(WebPage._ALL_FIELDS);
+    
+    dataStore.close();
+    
+    
+    //run the job
+    log.info("running count query job");
+    long result = counter.countQuery(dataStore, query);
+    log.info("finished count query job");
+    
+    //assert results
+    Assert.assertEquals(WebPageDataCreator.URLS.length, result);
+    
+  }
+ 
+  public static void testWordCount(Configuration conf, 
+      DataStore<String,WebPage> inStore, DataStore<String, 
+      TokenDatum> outStore) throws Exception {
+    inStore.setConf(conf);
+    outStore.setConf(conf);
+    
+    //create input
+    WebPageDataCreator.createWebPageData(inStore);
+    
+    //run the job
+    WordCount wordCount = new WordCount(conf);
+    wordCount.wordCount(inStore, outStore);
+    
+    //assert results
+    HashMap<String, Integer> actualCounts = new HashMap<String, Integer>();
+    for(String content : WebPageDataCreator.CONTENTS) {
+      for(String token:content.split(" ")) {
+        Integer count = actualCounts.get(token);
+        if(count == null) 
+          count = 0;
+        actualCounts.put(token, ++count);
+      }
+    }
+    for(Map.Entry<String, Integer> entry:actualCounts.entrySet()) {
+      assertTokenCount(outStore, entry.getKey(), entry.getValue()); 
+    }
+  }
+  
+  private static void assertTokenCount(DataStore<String, TokenDatum> outStore,
+      String token, int count) throws IOException {
+    TokenDatum datum = outStore.get(token, null);
+    Assert.assertNotNull("token:" + token + " cannot be found in datastore", datum);
+    Assert.assertEquals("count for token:" + token + " is wrong", count, datum.getCount());
+  }
+}

Added: incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/TestGoraInputFormat.java
URL: http://svn.apache.org/viewvc/incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/TestGoraInputFormat.java?rev=1021390&view=auto
==============================================================================
--- incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/TestGoraInputFormat.java (added)
+++ incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/TestGoraInputFormat.java Mon Oct 11 15:40:22 2010
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gora.mapreduce;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+import junit.framework.Assert;
+
+import org.apache.gora.examples.generated.Employee;
+import org.apache.gora.mapreduce.GoraInputFormat;
+import org.apache.gora.mapreduce.GoraInputSplit;
+import org.apache.gora.mock.persistency.MockPersistent;
+import org.apache.gora.mock.query.MockQuery;
+import org.apache.gora.mock.store.MockDataStore;
+import org.apache.gora.query.PartitionQuery;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.Job;
+import org.junit.Test;
+
+public class TestGoraInputFormat {
+
+  public List<InputSplit> getInputSplits()
+    throws IOException, InterruptedException {
+
+    Job job = new Job();
+    MockDataStore store = MockDataStore.get();
+
+    MockQuery query = store.newQuery();
+    query.setFields(Employee._ALL_FIELDS);
+    GoraInputFormat.setInput(job, query, false);
+
+    GoraInputFormat<String, MockPersistent> inputFormat
+      = new GoraInputFormat<String, MockPersistent>();
+
+    inputFormat.setConf(job.getConfiguration());
+
+    return inputFormat.getSplits(job);
+  }
+
+  @Test
+  @SuppressWarnings("rawtypes")
+  public void testGetSplits() throws IOException, InterruptedException {
+    List<InputSplit> splits = getInputSplits();
+
+    Assert.assertTrue(splits.size() > 0);
+
+    InputSplit split = splits.get(0);
+    PartitionQuery query = ((GoraInputSplit)split).getQuery();
+    Assert.assertTrue(Arrays.equals(Employee._ALL_FIELDS, query.getFields()));
+  }
+
+}
\ No newline at end of file

Added: incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/TestGoraInputSplit.java
URL: http://svn.apache.org/viewvc/incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/TestGoraInputSplit.java?rev=1021390&view=auto
==============================================================================
--- incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/TestGoraInputSplit.java (added)
+++ incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/TestGoraInputSplit.java Mon Oct 11 15:40:22 2010
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gora.mapreduce;
+
+import java.io.IOException;
+import java.util.List;
+
+import junit.framework.Assert;
+
+import org.apache.gora.mapreduce.GoraInputSplit;
+import org.apache.gora.mock.persistency.MockPersistent;
+import org.apache.gora.mock.query.MockQuery;
+import org.apache.gora.mock.store.MockDataStore;
+import org.apache.gora.query.PartitionQuery;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.TestWritable;
+import org.junit.Test;
+
+/**
+ * Test case for {@link GoraInputSplit}.
+ */
+public class TestGoraInputSplit {
+
+  private Configuration conf = new Configuration();
+  
+  private List<PartitionQuery<String, MockPersistent>> 
+    getPartitions() throws IOException {
+    MockDataStore store = MockDataStore.get();
+    MockQuery query = store.newQuery();
+
+    List<PartitionQuery<String, MockPersistent>> partitions = 
+      store.getPartitions(query);
+    return partitions;
+  }
+  
+  @Test
+  public void testGetLocations() throws IOException {
+    List<PartitionQuery<String, MockPersistent>> partitions = 
+      getPartitions();
+
+    int i=0;;
+    for(PartitionQuery<String, MockPersistent> partition : partitions) {
+      GoraInputSplit split = new GoraInputSplit(conf, partition);
+      Assert.assertEquals(split.getLocations().length, 1);
+      Assert.assertEquals(split.getLocations()[0], MockDataStore.LOCATIONS[i++]);
+    }
+  }
+
+  @Test
+  public void testReadWrite() throws Exception {
+    
+    List<PartitionQuery<String, MockPersistent>> partitions = 
+      getPartitions();
+
+    for(PartitionQuery<String, MockPersistent> partition : partitions) {
+      GoraInputSplit split = new GoraInputSplit(conf, partition);
+      TestWritable.testWritable(split);
+    }
+  }
+  
+}

Added: incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/TestPersistentSerialization.java
URL: http://svn.apache.org/viewvc/incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/TestPersistentSerialization.java?rev=1021390&view=auto
==============================================================================
--- incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/TestPersistentSerialization.java (added)
+++ incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/TestPersistentSerialization.java Mon Oct 11 15:40:22 2010
@@ -0,0 +1,107 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gora.mapreduce;
+
+import junit.framework.Assert;
+
+import org.apache.avro.util.Utf8;
+import org.apache.gora.examples.WebPageDataCreator;
+import org.apache.gora.examples.generated.Employee;
+import org.apache.gora.examples.generated.WebPage;
+import org.apache.gora.mapreduce.PersistentDeserializer;
+import org.apache.gora.mapreduce.PersistentSerialization;
+import org.apache.gora.mapreduce.PersistentSerializer;
+import org.apache.gora.memory.store.MemStore;
+import org.apache.gora.query.Result;
+import org.apache.gora.store.DataStoreFactory;
+import org.apache.gora.store.DataStoreTestUtil;
+import org.apache.gora.util.TestIOUtils;
+import org.junit.Test;
+
+/** Test class for {@link PersistentSerialization}, {@link PersistentSerializer}
+ *  and {@link PersistentDeserializer}
+ */
+public class TestPersistentSerialization {
+
+  @SuppressWarnings("unchecked")
+  @Test
+  public void testSerdeEmployee() throws Exception {
+
+    MemStore<String, Employee> store = DataStoreFactory.getDataStore(
+        MemStore.class, String.class, Employee.class);
+
+    Employee employee = DataStoreTestUtil.createEmployee(store);
+
+    TestIOUtils.testSerializeDeserialize(employee);
+  }
+
+  @Test
+  public void testSerdeEmployeeOneField() throws Exception {
+    Employee employee = new Employee();
+    employee.setSsn(new Utf8("11111"));
+
+    TestIOUtils.testSerializeDeserialize(employee);
+  }
+
+  @Test
+  public void testSerdeEmployeeTwoFields() throws Exception {
+    Employee employee = new Employee();
+    employee.setSsn(new Utf8("11111"));
+    employee.setSalary(100);
+
+    TestIOUtils.testSerializeDeserialize(employee);
+  }
+
+  @SuppressWarnings("unchecked")
+  @Test
+  public void testSerdeWebPage() throws Exception {
+
+    MemStore<String, WebPage> store = DataStoreFactory.getDataStore(
+        MemStore.class, String.class, WebPage.class);
+    WebPageDataCreator.createWebPageData(store);
+
+    Result<String, WebPage> result = store.newQuery().execute();
+
+    int i=0;
+    while(result.next()) {
+      WebPage page = result.get();
+      TestIOUtils.testSerializeDeserialize(page);
+      i++;
+    }
+    Assert.assertEquals(WebPageDataCreator.URLS.length, i);
+  }
+
+  @Test
+  public void testSerdeMultipleWebPages() throws Exception {
+    WebPage page1 = new WebPage();
+    WebPage page2 = new WebPage();
+    WebPage page3 = new WebPage();
+
+    page1.setUrl(new Utf8("foo"));
+    page2.setUrl(new Utf8("baz"));
+    page3.setUrl(new Utf8("bar"));
+
+    page1.addToParsedContent(new Utf8("coo"));
+
+    page2.putToOutlinks(new Utf8("a"), new Utf8("b"));
+
+    TestIOUtils.testSerializeDeserialize(page1, page2, page3);
+  }
+
+}

Added: incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/memory/store/TestMemStore.java
URL: http://svn.apache.org/viewvc/incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/memory/store/TestMemStore.java?rev=1021390&view=auto
==============================================================================
--- incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/memory/store/TestMemStore.java (added)
+++ incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/memory/store/TestMemStore.java Mon Oct 11 15:40:22 2010
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gora.memory.store;
+
+import org.apache.gora.examples.generated.Employee;
+import org.apache.gora.examples.generated.WebPage;
+import org.apache.gora.memory.store.MemStore;
+import org.apache.gora.store.DataStore;
+import org.apache.gora.store.DataStoreFactory;
+import org.apache.gora.store.DataStoreTestBase;
+
+/**
+ * Test case for {@link MemStore}.
+ */
+public class TestMemStore extends DataStoreTestBase {
+
+  @SuppressWarnings("unchecked")
+  @Override
+  protected DataStore<String, Employee> createEmployeeDataStore() {
+    return DataStoreFactory.getDataStore(MemStore.class, String.class, Employee.class);
+  }
+
+  @SuppressWarnings("unchecked")
+  @Override
+  protected DataStore<String, WebPage> createWebPageDataStore() {
+    return DataStoreFactory.getDataStore(MemStore.class, String.class, WebPage.class);
+  }
+}

Added: incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mock/persistency/MockPersistent.java
URL: http://svn.apache.org/viewvc/incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mock/persistency/MockPersistent.java?rev=1021390&view=auto
==============================================================================
--- incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mock/persistency/MockPersistent.java (added)
+++ incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mock/persistency/MockPersistent.java Mon Oct 11 15:40:22 2010
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gora.mock.persistency;
+
+import org.apache.avro.Schema;
+import org.apache.gora.persistency.Persistent;
+import org.apache.gora.persistency.StateManager;
+import org.apache.gora.persistency.impl.PersistentBase;
+
+public class MockPersistent extends PersistentBase {
+
+  public static final String FOO = "foo";
+  public static final String BAZ = "baz";
+  
+  public static final String[] _ALL_FIELDS = {FOO, BAZ};
+  
+  private int foo;
+  private int baz;
+  
+  public MockPersistent() {
+  }
+  
+  public MockPersistent(StateManager stateManager) {
+    super(stateManager);
+  }
+  
+  @Override
+  public Object get(int field) {
+    switch(field) {
+      case 0: return foo;
+      case 1: return baz;
+    }
+    return null;
+  }
+
+  @Override
+  public void put(int field, Object value) {
+    switch(field) {
+      case 0:  foo = (Integer)value;
+      case 1:  baz = (Integer)value;
+    }
+  }
+
+  @Override
+  public Schema getSchema() {
+    return Schema.parse("{\"type\":\"record\",\"name\":\"MockPersistent\",\"namespace\":\"org.apache.gora.mock.persistency\",\"fields\":[{\"name\":\"foo\",\"type\":\"int\"},{\"name\":\"baz\",\"type\":\"int\"}]}");
+  }
+  
+  public void setFoo(int foo) {
+    this.foo = foo;
+  }
+  
+  public void setBaz(int baz) {
+    this.baz = baz;
+  }
+  
+  public int getFoo() {
+    return foo;
+  }
+  
+  public int getBaz() {
+    return baz;
+  }
+
+  @Override
+  public String getField(int index) {
+    return null;
+  }
+
+  @Override
+  public int getFieldIndex(String field) {
+    return 0;
+  }
+
+  @Override
+  public String[] getFields() {
+    return null;
+  }
+
+  @Override
+  public Persistent newInstance(StateManager stateManager) {
+    return new MockPersistent(stateManager);
+  }
+}

Added: incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mock/query/MockQuery.java
URL: http://svn.apache.org/viewvc/incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mock/query/MockQuery.java?rev=1021390&view=auto
==============================================================================
--- incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mock/query/MockQuery.java (added)
+++ incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mock/query/MockQuery.java Mon Oct 11 15:40:22 2010
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gora.mock.query;
+
+import org.apache.gora.mock.persistency.MockPersistent;
+import org.apache.gora.query.impl.QueryBase;
+import org.apache.gora.store.DataStore;
+
+public class MockQuery extends QueryBase<String, MockPersistent> {
+
+  public MockQuery() {
+    super(null);
+  }
+  
+  public MockQuery(DataStore<String, MockPersistent> dataStore) {
+    super(dataStore);
+  }
+
+}

Added: incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mock/store/MockDataStore.java
URL: http://svn.apache.org/viewvc/incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mock/store/MockDataStore.java?rev=1021390&view=auto
==============================================================================
--- incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mock/store/MockDataStore.java (added)
+++ incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mock/store/MockDataStore.java Mon Oct 11 15:40:22 2010
@@ -0,0 +1,139 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gora.mock.store;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.gora.mock.persistency.MockPersistent;
+import org.apache.gora.mock.query.MockQuery;
+import org.apache.gora.query.PartitionQuery;
+import org.apache.gora.query.Query;
+import org.apache.gora.query.Result;
+import org.apache.gora.query.impl.PartitionQueryImpl;
+import org.apache.gora.store.DataStoreFactory;
+import org.apache.gora.store.impl.DataStoreBase;
+
+public class MockDataStore extends DataStoreBase<String, MockPersistent> {
+
+  public static final int NUM_PARTITIONS = 5;
+  public static final String[] LOCATIONS = {"foo1", "foo2", "foo3", "foo4", "foo1"};
+
+  public static MockDataStore get() {
+    MockDataStore dataStore = DataStoreFactory.getDataStore(MockDataStore.class
+        , String.class, MockPersistent.class);
+    return dataStore;
+  }
+
+  public MockDataStore() { }
+
+  @Override
+  public String getSchemaName() {
+    return null;
+  }
+
+  @Override
+  public void close() throws IOException {
+  }
+
+  @Override
+  public void createSchema() throws IOException {
+  }
+
+  @Override
+  public void deleteSchema() throws IOException {
+  }
+
+  @Override
+  public void truncateSchema() throws IOException {
+  }
+
+  @Override
+  public boolean schemaExists() throws IOException {
+    return true;
+  }
+
+  @Override
+  public boolean delete(String key) throws IOException {
+    return false;
+  }
+
+  @Override
+  public long deleteByQuery(Query<String, MockPersistent> query)
+      throws IOException {
+    return 0;
+  }
+
+  @Override
+  public Result<String, MockPersistent> execute(
+      Query<String, MockPersistent> query) throws IOException {
+    return null;
+  }
+
+  @Override
+  public void flush() throws IOException {
+  }
+
+  @Override
+  public MockPersistent get(String key, String[] fields) throws IOException {
+    return null;
+  }
+
+  @Override
+  public Class<String> getKeyClass() {
+    return String.class;
+  }
+
+  @Override
+  public List<PartitionQuery<String, MockPersistent>> getPartitions(
+      Query<String, MockPersistent> query) throws IOException {
+
+    ArrayList<PartitionQuery<String, MockPersistent>> list =
+      new ArrayList<PartitionQuery<String,MockPersistent>>();
+
+    for(int i=0; i<NUM_PARTITIONS; i++) {
+      list.add(new PartitionQueryImpl<String, MockPersistent>(query, LOCATIONS[i]));
+    }
+
+    return list;
+  }
+
+  @Override
+  public Class<MockPersistent> getPersistentClass() {
+    return MockPersistent.class;
+  }
+
+  @Override
+  public MockQuery newQuery() {
+    return new MockQuery(this);
+  }
+
+  @Override
+  public void put(String key, MockPersistent obj) throws IOException {
+  }
+
+  @Override
+  public void setKeyClass(Class<String> keyClass) {
+  }
+
+  @Override
+  public void setPersistentClass(Class<MockPersistent> persistentClass) {
+  }
+}

Added: incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/persistency/impl/TestPersistentBase.java
URL: http://svn.apache.org/viewvc/incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/persistency/impl/TestPersistentBase.java?rev=1021390&view=auto
==============================================================================
--- incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/persistency/impl/TestPersistentBase.java (added)
+++ incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/persistency/impl/TestPersistentBase.java Mon Oct 11 15:40:22 2010
@@ -0,0 +1,120 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gora.persistency.impl;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.avro.util.Utf8;
+import org.apache.gora.examples.generated.Employee;
+import org.apache.gora.examples.generated.WebPage;
+import org.apache.gora.memory.store.MemStore;
+import org.apache.gora.store.DataStoreFactory;
+import org.apache.gora.store.DataStoreTestUtil;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Testcase for PersistentBase class
+ */
+public class TestPersistentBase {
+  
+  @Test
+  public void testGetFields() {
+    WebPage page = new WebPage();
+    String[] fields = page.getFields();
+    Assert.assertArrayEquals(WebPage._ALL_FIELDS, fields);
+  }
+  
+  @Test
+  public void testGetField() {
+    WebPage page = new WebPage();
+    for(int i=0; i<WebPage._ALL_FIELDS.length; i++) {
+      String field = page.getField(i);
+      Assert.assertEquals(WebPage._ALL_FIELDS[i], field);
+    }
+  }
+  
+  @Test
+  public void testGetFieldIndex() {
+    WebPage page = new WebPage();
+    for(int i=0; i<WebPage._ALL_FIELDS.length; i++) {
+      int index = page.getFieldIndex(WebPage._ALL_FIELDS[i]);
+      Assert.assertEquals(i, index);
+    }
+  }
+  
+  @Test
+  public void testFieldsWithTwoClasses() {
+    WebPage page = new WebPage();
+    for(int i=0; i<WebPage._ALL_FIELDS.length; i++) {
+      int index = page.getFieldIndex(WebPage._ALL_FIELDS[i]);
+      Assert.assertEquals(i, index);
+    }
+    Employee employee = new Employee();
+    for(int i=0; i<Employee._ALL_FIELDS.length; i++) {
+      int index = employee.getFieldIndex(Employee._ALL_FIELDS[i]);
+      Assert.assertEquals(i, index);
+    }
+  }
+  
+  @Test
+  public void testClear() {
+    
+    //test clear all fields
+    WebPage page = new WebPage();
+    page.setUrl(new Utf8("http://foo.com"));
+    page.addToParsedContent(new Utf8("foo"));
+    page.putToOutlinks(new Utf8("foo"), new Utf8("bar"));
+    page.setContent(ByteBuffer.wrap("foo baz bar".getBytes()));
+    
+    page.clear();
+    
+    Assert.assertNull(page.getUrl());
+    Assert.assertEquals(0, page.getParsedContent().size());
+    Assert.assertEquals(0, page.getOutlinks().size());
+    Assert.assertNull(page.getContent());
+    
+    //set fields again
+    page.setUrl(new Utf8("http://bar.com"));
+    page.addToParsedContent(new Utf8("bar"));
+    page.putToOutlinks(new Utf8("bar"), new Utf8("baz"));
+    page.setContent(ByteBuffer.wrap("foo baz bar barbaz".getBytes()));
+    
+    //test clear new object
+    page = new WebPage();
+    page.clear();
+    
+    //test primitive fields
+    Employee employee = new Employee();
+    employee.clear();
+  }
+  
+  @Test
+  public void testClone() throws IOException {
+    //more tests for clone are in TestPersistentDatumReader
+    @SuppressWarnings("unchecked")
+    MemStore<String, Employee> store = DataStoreFactory.getDataStore(
+        MemStore.class, String.class, Employee.class);
+
+    Employee employee = DataStoreTestUtil.createEmployee(store);
+    
+    Assert.assertEquals(employee, employee.clone());
+  }
+}

Added: incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/persistency/impl/TestStateManagerImpl.java
URL: http://svn.apache.org/viewvc/incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/persistency/impl/TestStateManagerImpl.java?rev=1021390&view=auto
==============================================================================
--- incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/persistency/impl/TestStateManagerImpl.java (added)
+++ incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/persistency/impl/TestStateManagerImpl.java Mon Oct 11 15:40:22 2010
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gora.persistency.impl;
+
+import java.io.IOException;
+
+import junit.framework.Assert;
+
+import org.apache.avro.util.Utf8;
+import org.apache.gora.examples.generated.Employee;
+import org.apache.gora.mock.persistency.MockPersistent;
+import org.apache.gora.persistency.impl.StateManagerImpl;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test case for {@link StateManagerImpl}
+ */
+public class TestStateManagerImpl {
+
+  private StateManagerImpl stateManager;
+  private MockPersistent persistent;
+  
+  @Before
+  public void setUp() {
+    this.stateManager = new StateManagerImpl();
+    this.persistent = new MockPersistent(stateManager);
+  }
+  
+  @Test
+  public void testDirty() {
+    Assert.assertFalse(stateManager.isDirty(persistent));
+    stateManager.setDirty(persistent);
+    Assert.assertTrue(stateManager.isDirty(persistent));
+  }
+  
+  @Test
+  public void testDirty2() {
+    Assert.assertFalse(stateManager.isDirty(persistent, 0));
+    Assert.assertFalse(stateManager.isDirty(persistent, 1));
+    stateManager.setDirty(persistent, 0);
+    Assert.assertTrue(stateManager.isDirty(persistent, 0));
+    Assert.assertFalse(stateManager.isDirty(persistent, 1));
+  }
+  
+  @Test
+  public void testClearDirty() {
+    Assert.assertFalse(stateManager.isDirty(persistent));
+    stateManager.setDirty(persistent, 0);
+    stateManager.clearDirty(persistent);
+    Assert.assertFalse(this.stateManager.isDirty(persistent));
+  }
+  
+  @Test
+  public void testReadable() throws IOException {
+    Assert.assertFalse(stateManager.isReadable(persistent, 0));
+    Assert.assertFalse(stateManager.isReadable(persistent, 1));
+    stateManager.setReadable(persistent, 0);
+    Assert.assertTrue(stateManager.isReadable(persistent, 0));
+    Assert.assertFalse(stateManager.isReadable(persistent, 1));
+  }
+
+  @Test
+  public void testReadable2() {
+    stateManager = new StateManagerImpl();
+    Employee employee = new Employee(stateManager);
+    Assert.assertFalse(stateManager.isReadable(employee, 0));
+    Assert.assertFalse(stateManager.isReadable(employee, 1));
+    employee.setName(new Utf8("foo"));
+    Assert.assertTrue(stateManager.isReadable(employee, 0));
+    Assert.assertFalse(stateManager.isReadable(employee, 1));
+  }
+  
+  @Test
+  public void testClearReadable() {
+    stateManager.setReadable(persistent, 0);
+    stateManager.clearReadable(persistent);
+    Assert.assertFalse(stateManager.isReadable(persistent, 0));
+  }
+  
+  @Test
+  public void testIsNew() {
+    //newly created objects should be new
+    Assert.assertTrue(persistent.isNew());
+  }
+  
+  @Test
+  public void testNew() {
+    stateManager.setNew(persistent);
+    Assert.assertTrue(persistent.isNew());
+  }
+  
+  @Test
+  public void testClearNew() {
+    stateManager.clearNew(persistent);
+    Assert.assertFalse(persistent.isNew());
+  }
+  
+}

Added: incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/query/impl/TestPartitionQueryImpl.java
URL: http://svn.apache.org/viewvc/incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/query/impl/TestPartitionQueryImpl.java?rev=1021390&view=auto
==============================================================================
--- incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/query/impl/TestPartitionQueryImpl.java (added)
+++ incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/query/impl/TestPartitionQueryImpl.java Mon Oct 11 15:40:22 2010
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gora.query.impl;
+
+import org.apache.gora.mock.persistency.MockPersistent;
+import org.apache.gora.mock.query.MockQuery;
+import org.apache.gora.mock.store.MockDataStore;
+import org.apache.gora.query.impl.PartitionQueryImpl;
+import org.apache.hadoop.io.TestWritable;
+import org.junit.Test;
+
+/**
+ * Test case for {@link PartitionQueryImpl}
+ */
+public class TestPartitionQueryImpl {
+
+  private MockDataStore dataStore = MockDataStore.get();
+  
+  @Test
+  public void testReadWrite() throws Exception {
+    
+    MockQuery baseQuery = dataStore.newQuery();
+    baseQuery.setStartKey("start");
+    baseQuery.setLimit(42);
+    
+    PartitionQueryImpl<String, MockPersistent> 
+      query = new PartitionQueryImpl<String, MockPersistent>(baseQuery);
+    
+    TestWritable.testWritable(query);
+  }
+  
+}

Added: incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/query/impl/TestQueryBase.java
URL: http://svn.apache.org/viewvc/incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/query/impl/TestQueryBase.java?rev=1021390&view=auto
==============================================================================
--- incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/query/impl/TestQueryBase.java (added)
+++ incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/query/impl/TestQueryBase.java Mon Oct 11 15:40:22 2010
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.gora.query.impl;
+
+import junit.framework.Assert;
+
+import org.apache.gora.mock.query.MockQuery;
+import org.apache.gora.mock.store.MockDataStore;
+import org.apache.gora.query.impl.QueryBase;
+import org.apache.gora.util.TestIOUtils;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test case for {@link QueryBase}.
+ */
+public class TestQueryBase {
+
+  private MockDataStore dataStore = MockDataStore.get();
+  private MockQuery query;
+  
+  private static final String[] FIELDS = {"foo", "baz", "bar"};
+  private static final String START_KEY = "1_start";
+  private static final String END_KEY = "2_end";
+  
+  @Before
+  public void setUp() {
+    query = dataStore.newQuery(); //MockQuery extends QueryBase
+  }
+  
+  @Test
+  public void testReadWrite() throws Exception {
+    query.setFields(FIELDS);
+    query.setKeyRange(START_KEY, END_KEY);
+    TestIOUtils.testSerializeDeserialize(query);
+    
+    Assert.assertNotNull(query.getDataStore());
+  }
+  
+  @Test
+  public void testReadWrite2() throws Exception {
+    query.setLimit(1000);
+    query.setTimeRange(0, System.currentTimeMillis());
+    TestIOUtils.testSerializeDeserialize(query);
+  }
+
+}

Added: incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/store/DataStoreTestBase.java
URL: http://svn.apache.org/viewvc/incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/store/DataStoreTestBase.java?rev=1021390&view=auto
==============================================================================
--- incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/store/DataStoreTestBase.java (added)
+++ incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/store/DataStoreTestBase.java Mon Oct 11 15:40:22 2010
@@ -0,0 +1,368 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gora.store;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import junit.framework.Assert;
+
+import org.apache.avro.util.Utf8;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.gora.GoraTestDriver;
+import org.apache.gora.examples.generated.Employee;
+import org.apache.gora.examples.generated.Metadata;
+import org.apache.gora.examples.generated.WebPage;
+import org.apache.gora.store.DataStore;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * A base class for {@link DataStore} tests. This is just a convenience
+ * class, which actually only uses {@link DataStoreTestUtil} methods to
+ * run the tests. Not all test cases can extend this class (like TestHBaseStore),
+ * so all test logic shuold reside in DataStoreTestUtil class.
+ */
+public abstract class DataStoreTestBase {
+
+  public static final Log log = LogFactory.getLog(DataStoreTestBase.class);
+
+  protected static GoraTestDriver testDriver;
+
+  protected DataStore<String,Employee> employeeStore;
+  protected DataStore<String,WebPage> webPageStore;
+
+  @Deprecated
+  protected abstract DataStore<String,Employee> createEmployeeDataStore() throws IOException ;
+
+  @Deprecated
+  protected abstract DataStore<String,WebPage> createWebPageDataStore() throws IOException;
+
+  /** junit annoyingly forces BeforeClass to be static, so this method
+   * should be called from a static block
+   */
+  protected static void setTestDriver(GoraTestDriver driver) {
+    testDriver = driver;
+  }
+
+  private static boolean setUpClassCalled = false;
+  
+  @BeforeClass
+  public static void setUpClass() throws Exception {
+    if(testDriver != null && !setUpClassCalled) {
+      log.info("setting up class");
+      testDriver.setUpClass();
+      setUpClassCalled = true;
+    }
+  }
+
+  @AfterClass
+  public static void tearDownClass() throws Exception {
+    if(testDriver != null) {
+      log.info("tearing down class");
+      testDriver.tearDownClass();
+    }
+  }
+
+  @Before
+  public void setUp() throws Exception {
+    //There is an issue in JUnit 4 tests in Eclipse where TestSqlStore static
+    //methods are not called BEFORE setUpClass. I think this is a bug in 
+    //JUnitRunner in Eclipse. Below is a workaround for that problem.
+    if(!setUpClassCalled) setUpClass();  
+    
+    log.info("setting up test");
+    if(testDriver != null) {
+      employeeStore = testDriver.createDataStore(String.class, Employee.class);
+      webPageStore = testDriver.createDataStore(String.class, WebPage.class);
+      testDriver.setUp();
+    } else {
+      employeeStore =  createEmployeeDataStore();
+      webPageStore = createWebPageDataStore();
+
+      employeeStore.truncateSchema();
+      webPageStore.truncateSchema();
+    }
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    log.info("tearing down test");
+    if(testDriver != null) {
+      testDriver.tearDown();
+    }
+    //employeeStore.close();
+    //webPageStore.close();
+  }
+
+  @Test
+  public void testNewInstance() throws IOException {
+    log.info("test method: testNewInstance");
+    DataStoreTestUtil.testNewPersistent(employeeStore);
+  }
+
+  @Test
+  public void testCreateSchema() throws Exception {
+    log.info("test method: testCreateSchema");
+    DataStoreTestUtil.testCreateEmployeeSchema(employeeStore);
+    assertSchemaExists("Employee");
+  }
+
+  // Override this to assert that schema is created correctly
+  public void assertSchemaExists(String schemaName) throws Exception {
+  }
+
+  @Test
+  public void testAutoCreateSchema() throws Exception {
+    log.info("test method: testAutoCreateSchema");
+    DataStoreTestUtil.testAutoCreateSchema(employeeStore);
+    assertAutoCreateSchema();
+  }
+
+  public void assertAutoCreateSchema() throws Exception {
+    assertSchemaExists("Employee");
+  }
+
+  @Test
+  public  void testTruncateSchema() throws Exception {
+    log.info("test method: testTruncateSchema");
+    DataStoreTestUtil.testTruncateSchema(webPageStore);
+    assertSchemaExists("WebPage");
+  }
+
+  @Test
+  public void testDeleteSchema() throws IOException {
+    log.info("test method: testDeleteSchema");
+    DataStoreTestUtil.testDeleteSchema(webPageStore);
+  }
+
+  @Test
+  public void testSchemaExists() throws Exception {
+    log.info("test method: testSchemaExists");
+    DataStoreTestUtil.testSchemaExists(webPageStore);
+    assertSchemaExists("WebPage");
+  }
+
+  @Test
+  public void testPut() throws IOException {
+    log.info("test method: testPut");
+    Employee employee = DataStoreTestUtil.testPutEmployee(employeeStore);
+    assertPut(employee);
+  }
+
+  public void assertPut(Employee employee) throws IOException {
+  }
+
+  @Test
+  public void testPutNested() throws IOException {
+    log.info("test method: testPutNested");
+
+    String revUrl = "foo.com:http/";
+    String url = "http://foo.com/";
+
+    webPageStore.createSchema();
+    WebPage page = webPageStore.newPersistent();
+    Metadata metadata = new Metadata();  
+    metadata.setVersion(1);
+    metadata.putToData(new Utf8("foo"), new Utf8("baz"));
+
+    page.setMetadata(metadata);
+    page.setUrl(new Utf8(url));
+
+    webPageStore.put(revUrl, page);
+    webPageStore.flush();
+
+    page = webPageStore.get(revUrl);
+    metadata = page.getMetadata();
+    Assert.assertNotNull(metadata);
+    Assert.assertEquals(1, metadata.getVersion());
+    Assert.assertEquals(new Utf8("baz"), metadata.getData().get(new Utf8("foo")));
+  }
+
+  @Test
+  public void testPutArray() throws IOException {
+    log.info("test method: testPutArray");
+    webPageStore.createSchema();
+    WebPage page = webPageStore.newPersistent();
+
+    String[] tokens = {"example", "content", "in", "example.com"};
+
+    for(String token: tokens) {
+      page.addToParsedContent(new Utf8(token));
+    }
+
+    webPageStore.put("com.example/http", page);
+    webPageStore.close();
+
+    assertPutArray();
+  }
+
+  public void assertPutArray() throws IOException {
+  }
+
+  @Test
+  public void testPutBytes() throws IOException {
+    log.info("test method: testPutBytes");
+    webPageStore.createSchema();
+    WebPage page = webPageStore.newPersistent();
+    page.setUrl(new Utf8("http://example.com"));
+    byte[] contentBytes = "example content in example.com".getBytes();
+    ByteBuffer buff = ByteBuffer.wrap(contentBytes);
+    page.setContent(buff);
+
+    webPageStore.put("com.example/http", page);
+    webPageStore.close();
+
+    assertPutBytes(contentBytes);
+  }
+
+  public void assertPutBytes(byte[] contentBytes) throws IOException {
+  }
+
+  @Test
+  public void testPutMap() throws IOException {
+    log.info("test method: testPutMap");
+    webPageStore.createSchema();
+
+    WebPage page = webPageStore.newPersistent();
+
+    page.setUrl(new Utf8("http://example.com"));
+    page.putToOutlinks(new Utf8("http://example2.com"), new Utf8("anchor2"));
+    page.putToOutlinks(new Utf8("http://example3.com"), new Utf8("anchor3"));
+    page.putToOutlinks(new Utf8("http://example3.com"), new Utf8("anchor4"));
+    webPageStore.put("com.example/http", page);
+    webPageStore.close();
+
+    assertPutMap();
+  }
+
+  public void assertPutMap() throws IOException {
+  }
+
+  @Test
+  public void testUpdate() throws IOException {
+    log.info("test method: testUpdate");
+    DataStoreTestUtil.testUpdateEmployee(employeeStore);
+    DataStoreTestUtil.testUpdateWebPage(webPageStore);
+  }
+
+  public void testEmptyUpdate() throws IOException {
+    DataStoreTestUtil.testEmptyUpdateEmployee(employeeStore);
+  }
+
+  @Test
+  public void testGet() throws IOException {
+    log.info("test method: testGet");
+    DataStoreTestUtil.testGetEmployee(employeeStore);
+  }
+
+  @Test
+  public void testGetWithFields() throws IOException {
+    log.info("test method: testGetWithFields");
+    DataStoreTestUtil.testGetEmployeeWithFields(employeeStore);
+  }
+
+  @Test
+  public void testGetWebPage() throws IOException {
+    log.info("test method: testGetWebPage");
+    DataStoreTestUtil.testGetWebPage(webPageStore);
+  }
+
+  @Test
+  public void testGetWebPageDefaultFields() throws IOException {
+    log.info("test method: testGetWebPageDefaultFields");
+    DataStoreTestUtil.testGetWebPageDefaultFields(webPageStore);
+  }
+
+  @Test
+  public void testGetNonExisting() throws Exception {
+    log.info("test method: testGetNonExisting");
+    DataStoreTestUtil.testGetEmployeeNonExisting(employeeStore);
+  }
+
+ @Test
+  public void testQuery() throws IOException {
+    log.info("test method: testQuery");
+    DataStoreTestUtil.testQueryWebPages(webPageStore);
+  }
+
+  @Test
+  public void testQueryStartKey() throws IOException {
+    log.info("test method: testQueryStartKey");
+    DataStoreTestUtil.testQueryWebPageStartKey(webPageStore);
+  }
+
+  @Test
+  public void testQueryEndKey() throws IOException {
+    log.info("test method: testQueryEndKey");
+    DataStoreTestUtil.testQueryWebPageEndKey(webPageStore);
+  }
+
+  @Test
+  public void testQueryKeyRange() throws IOException {
+    log.info("test method: testQueryKetRange");
+    DataStoreTestUtil.testQueryWebPageKeyRange(webPageStore);
+  }
+
+ @Test
+  public void testQueryWebPageSingleKey() throws IOException {
+   log.info("test method: testQueryWebPageSingleKey");
+    DataStoreTestUtil.testQueryWebPageSingleKey(webPageStore);
+  }
+
+  @Test
+  public void testQueryWebPageSingleKeyDefaultFields() throws IOException {
+    log.info("test method: testQuerySingleKeyDefaultFields");
+    DataStoreTestUtil.testQueryWebPageSingleKeyDefaultFields(webPageStore);
+  }
+
+  @Test
+  public void testQueryWebPageQueryEmptyResults() throws IOException {
+    log.info("test method: testQueryEmptyResults");
+    DataStoreTestUtil.testQueryWebPageEmptyResults(webPageStore);
+  }
+
+  @Test
+  public void testDelete() throws IOException {
+    log.info("test method: testDelete");
+    DataStoreTestUtil.testDelete(webPageStore);
+  }
+
+  @Test
+  public void testDeleteByQuery() throws IOException {
+    log.info("test method: testDeleteByQuery");
+    DataStoreTestUtil.testDeleteByQuery(webPageStore);
+  }
+
+  @Test
+  public void testDeleteByQueryFields() throws IOException {
+    log.info("test method: testQueryByQueryFields");
+    DataStoreTestUtil.testDeleteByQueryFields(webPageStore);
+  }
+
+  @Test
+  public void testGetPartitions() throws IOException {
+    log.info("test method: testGetPartitions");
+    DataStoreTestUtil.testGetPartitions(webPageStore);
+  }
+}

Added: incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/store/DataStoreTestUtil.java
URL: http://svn.apache.org/viewvc/incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/store/DataStoreTestUtil.java?rev=1021390&view=auto
==============================================================================
--- incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/store/DataStoreTestUtil.java (added)
+++ incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/store/DataStoreTestUtil.java Mon Oct 11 15:40:22 2010
@@ -0,0 +1,704 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gora.store;
+
+import static org.apache.gora.examples.WebPageDataCreator.ANCHORS;
+import static org.apache.gora.examples.WebPageDataCreator.CONTENTS;
+import static org.apache.gora.examples.WebPageDataCreator.LINKS;
+import static org.apache.gora.examples.WebPageDataCreator.SORTED_URLS;
+import static org.apache.gora.examples.WebPageDataCreator.URLS;
+import static org.apache.gora.examples.WebPageDataCreator.URL_INDEXES;
+import static org.apache.gora.examples.WebPageDataCreator.createWebPageData;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import junit.framework.Assert;
+
+import org.apache.avro.generic.GenericArray;
+import org.apache.avro.util.Utf8;
+import org.apache.gora.examples.WebPageDataCreator;
+import org.apache.gora.examples.generated.Employee;
+import org.apache.gora.examples.generated.WebPage;
+import org.apache.gora.persistency.Persistent;
+import org.apache.gora.query.PartitionQuery;
+import org.apache.gora.query.Query;
+import org.apache.gora.query.Result;
+import org.apache.gora.store.DataStore;
+import org.apache.gora.util.ByteUtils;
+import org.apache.gora.util.StringUtils;
+
+/**
+ * Test utilities for DataStores
+ */
+public class DataStoreTestUtil {
+
+  public static final long YEAR_IN_MS = 365L * 24L * 60L * 60L * 1000L;
+  private static final int NUM_KEYS = 4;
+
+  public static <K, T extends Persistent> void testNewPersistent(
+      DataStore<K,T> dataStore) throws IOException {
+
+    T obj1 = dataStore.newPersistent();
+    T obj2 = dataStore.newPersistent();
+
+    Assert.assertEquals(dataStore.getPersistentClass(),
+        obj1.getClass());
+    Assert.assertNotNull(obj1);
+    Assert.assertNotNull(obj2);
+    Assert.assertFalse( obj1 == obj2 );
+  }
+
+  public static <K> Employee createEmployee(
+      DataStore<K, Employee> dataStore) throws IOException {
+
+    Employee employee = dataStore.newPersistent();
+    employee.setName(new Utf8("Random Joe"));
+    employee.setDateOfBirth( System.currentTimeMillis() - 20L *  YEAR_IN_MS );
+    employee.setSalary(100000);
+    employee.setSsn(new Utf8("101010101010"));
+    return employee;
+  }
+
+  public static void testAutoCreateSchema(DataStore<String,Employee> dataStore)
+  throws IOException {
+    //should not throw exception
+    dataStore.put("foo", createEmployee(dataStore));
+  }
+
+  public static void testCreateEmployeeSchema(DataStore<String, Employee> dataStore)
+  throws IOException {
+    dataStore.createSchema();
+
+    //should not throw exception
+    dataStore.createSchema();
+  }
+
+  public static void testTruncateSchema(DataStore<String, WebPage> dataStore)
+  throws IOException {
+    dataStore.createSchema();
+    WebPageDataCreator.createWebPageData(dataStore);
+    dataStore.truncateSchema();
+
+    assertEmptyResults(dataStore.newQuery());
+  }
+
+  public static void testDeleteSchema(DataStore<String, WebPage> dataStore)
+  throws IOException {
+    dataStore.createSchema();
+    WebPageDataCreator.createWebPageData(dataStore);
+    dataStore.deleteSchema();
+    dataStore.createSchema();
+
+    assertEmptyResults(dataStore.newQuery());
+  }
+
+  public static<K, T extends Persistent> void testSchemaExists(
+      DataStore<K, T> dataStore) throws IOException {
+    dataStore.createSchema();
+
+    Assert.assertTrue(dataStore.schemaExists());
+
+    dataStore.deleteSchema();
+    Assert.assertFalse(dataStore.schemaExists());
+  }
+
+  public static void testGetEmployee(DataStore<String, Employee> dataStore)
+    throws IOException {
+    dataStore.createSchema();
+    Employee employee = DataStoreTestUtil.createEmployee(dataStore);
+    String ssn = employee.getSsn().toString();
+    dataStore.put(ssn, employee);
+    dataStore.flush();
+
+    Employee after = dataStore.get(ssn, Employee._ALL_FIELDS);
+
+    Assert.assertEquals(employee, after);
+  }
+
+  public static void testGetEmployeeNonExisting(DataStore<String, Employee> dataStore)
+    throws IOException {
+    Employee employee = dataStore.get("_NON_EXISTING_SSN_FOR_EMPLOYEE_");
+    Assert.assertNull(employee);
+  }
+
+  public static void testGetEmployeeWithFields(DataStore<String, Employee> dataStore)
+    throws IOException {
+    Employee employee = DataStoreTestUtil.createEmployee(dataStore);
+    String ssn = employee.getSsn().toString();
+    dataStore.put(ssn, employee);
+    dataStore.flush();
+
+    String[] fields = employee.getFields();
+    for(Set<String> subset : StringUtils.powerset(fields)) {
+      if(subset.isEmpty())
+        continue;
+      Employee after = dataStore.get(ssn, subset.toArray(new String[subset.size()]));
+      Employee expected = new Employee();
+      for(String field:subset) {
+        int index = expected.getFieldIndex(field);
+        expected.put(index, employee.get(index));
+      }
+
+      Assert.assertEquals(expected, after);
+    }
+  }
+
+  public static Employee testPutEmployee(DataStore<String, Employee> dataStore)
+  throws IOException {
+    dataStore.createSchema();
+    Employee employee = DataStoreTestUtil.createEmployee(dataStore);
+    return employee;
+  }
+
+  public static void testEmptyUpdateEmployee(DataStore<String, Employee> dataStore)
+  throws IOException {
+    dataStore.createSchema();
+    long ssn = 1234567890L;
+    String ssnStr = Long.toString(ssn);
+    long now = System.currentTimeMillis();
+
+    Employee employee = dataStore.newPersistent();
+    employee.setName(new Utf8("John Doe"));
+    employee.setDateOfBirth(now - 20L *  YEAR_IN_MS);
+    employee.setSalary(100000);
+    employee.setSsn(new Utf8(ssnStr));
+    dataStore.put(employee.getSsn().toString(), employee);
+
+    dataStore.flush();
+
+    employee = dataStore.get(ssnStr);
+    dataStore.put(ssnStr, employee);
+
+    dataStore.flush();
+
+    employee = dataStore.newPersistent();
+    dataStore.put(Long.toString(ssn + 1), employee);
+
+    dataStore.flush();
+
+    employee = dataStore.get(Long.toString(ssn + 1));
+    Assert.assertNull(employee);
+  }
+
+  public static void testUpdateEmployee(DataStore<String, Employee> dataStore)
+  throws IOException {
+    dataStore.createSchema();
+    long ssn = 1234567890L;
+    long now = System.currentTimeMillis();
+
+    for (int i = 0; i < 5; i++) {
+      Employee employee = dataStore.newPersistent();
+      employee.setName(new Utf8("John Doe " + i));
+      employee.setDateOfBirth(now - 20L *  YEAR_IN_MS);
+      employee.setSalary(100000);
+      employee.setSsn(new Utf8(Long.toString(ssn + i)));
+      dataStore.put(employee.getSsn().toString(), employee);
+    }
+
+    dataStore.flush();
+
+    for (int i = 0; i < 1; i++) {
+      Employee employee = dataStore.newPersistent();
+      employee.setName(new Utf8("John Doe " + (i + 5)));
+      employee.setDateOfBirth(now - 18L *  YEAR_IN_MS);
+      employee.setSalary(120000);
+      employee.setSsn(new Utf8(Long.toString(ssn + i)));
+      dataStore.put(employee.getSsn().toString(), employee);
+    }
+
+    dataStore.flush();
+
+    for (int i = 0; i < 1; i++) {
+      String key = Long.toString(ssn + i);
+      Employee employee = dataStore.get(key);
+      Assert.assertEquals(now - 18L * YEAR_IN_MS, employee.getDateOfBirth());
+      Assert.assertEquals("John Doe " + (i + 5), employee.getName().toString());
+      Assert.assertEquals(120000, employee.getSalary());
+    }
+  }
+
+  public static void testUpdateWebPage(DataStore<String, WebPage> dataStore)
+  throws IOException {
+    dataStore.createSchema();
+
+    String[] urls = {"http://a.com/a", "http://b.com/b", "http://c.com/c",
+        "http://d.com/d", "http://e.com/e", "http://f.com/f", "http://g.com/g"};
+    String content = "content";
+    String parsedContent = "parsedContent";
+    String anchor = "anchor";
+
+    int parsedContentCount = 0;
+
+
+    for (int i = 0; i < urls.length; i++) {
+      WebPage webPage = dataStore.newPersistent();
+      webPage.setUrl(new Utf8(urls[i]));
+      for (parsedContentCount = 0; parsedContentCount < 5; parsedContentCount++) {
+        webPage.addToParsedContent(new Utf8(parsedContent + i + "," + parsedContentCount));
+      }
+      for (int j = 0; j < urls.length; j += 2) {
+        webPage.putToOutlinks(new Utf8(anchor + j), new Utf8(urls[j]));
+      }
+      dataStore.put(webPage.getUrl().toString(), webPage);
+    }
+
+    dataStore.flush();
+
+    for (int i = 0; i < urls.length; i++) {
+      WebPage webPage = dataStore.get(urls[i]);
+      webPage.setContent(ByteBuffer.wrap(ByteUtils.toBytes(content + i)));
+      for (parsedContentCount = 5; parsedContentCount < 10; parsedContentCount++) {
+        webPage.addToParsedContent(new Utf8(parsedContent + i + "," + parsedContentCount));
+      }
+      webPage.getOutlinks().clear();
+      for (int j = 1; j < urls.length; j += 2) {
+        webPage.putToOutlinks(new Utf8(anchor + j), new Utf8(urls[j]));
+      }
+      dataStore.put(webPage.getUrl().toString(), webPage);
+    }
+
+    dataStore.flush();
+
+    for (int i = 0; i < urls.length; i++) {
+      WebPage webPage = dataStore.get(urls[i]);
+      Assert.assertEquals(content + i, ByteUtils.toString(webPage.getContent().array()));
+      Assert.assertEquals(10, webPage.getParsedContent().size());
+      int j = 0;
+      for (Utf8 pc : webPage.getParsedContent()) {
+        Assert.assertEquals(parsedContent + i + "," + j, pc.toString());
+        j++;
+      }
+      int count = 0;
+      for (j = 1; j < urls.length; j += 2) {
+        Utf8 link = webPage.getOutlinks().get(new Utf8(anchor + j));
+        Assert.assertNotNull(link);
+        Assert.assertEquals(urls[j], link.toString());
+        count++;
+      }
+      Assert.assertEquals(count, webPage.getOutlinks().size());
+    }
+
+    for (int i = 0; i < urls.length; i++) {
+      WebPage webPage = dataStore.get(urls[i]);
+      for (int j = 0; j < urls.length; j += 2) {
+        webPage.putToOutlinks(new Utf8(anchor + j), new Utf8(urls[j]));
+      }
+      dataStore.put(webPage.getUrl().toString(), webPage);
+    }
+
+    dataStore.flush();
+
+    for (int i = 0; i < urls.length; i++) {
+      WebPage webPage = dataStore.get(urls[i]);
+      int count = 0;
+      for (int j = 0; j < urls.length; j++) {
+        Utf8 link = webPage.getOutlinks().get(new Utf8(anchor + j));
+        Assert.assertNotNull(link);
+        Assert.assertEquals(urls[j], link.toString());
+        count++;
+      }
+    }
+  }
+
+  public static void assertWebPage(WebPage page, int i) {
+    Assert.assertNotNull(page);
+
+    Assert.assertEquals(URLS[i], page.getUrl().toString());
+    Assert.assertTrue("content error:" + new String(page.getContent().array()) +
+        " actual=" + CONTENTS[i] + " i=" + i
+    , Arrays.equals(page.getContent().array()
+        , CONTENTS[i].getBytes()));
+
+    GenericArray<Utf8> parsedContent = page.getParsedContent();
+    Assert.assertNotNull(parsedContent);
+    Assert.assertTrue(parsedContent.size() > 0);
+
+    int j=0;
+    String[] tokens = CONTENTS[i].split(" ");
+    for(Utf8 token : parsedContent) {
+      Assert.assertEquals(tokens[j++], token.toString());
+    }
+
+    if(LINKS[i].length > 0) {
+      Assert.assertNotNull(page.getOutlinks());
+      Assert.assertTrue(page.getOutlinks().size() > 0);
+      for(j=0; j<LINKS[i].length; j++) {
+        Assert.assertEquals(ANCHORS[i][j],
+            page.getFromOutlinks(new Utf8(URLS[LINKS[i][j]])).toString());
+      }
+    } else {
+      Assert.assertTrue(page.getOutlinks() == null || page.getOutlinks().isEmpty());
+    }
+  }
+
+  private static void testGetWebPage(DataStore<String, WebPage> store, String[] fields)
+    throws IOException {
+    createWebPageData(store);
+
+    for(int i=0; i<URLS.length; i++) {
+      WebPage page = store.get(URLS[i], fields);
+      assertWebPage(page, i);
+    }
+  }
+
+  public static void testGetWebPage(DataStore<String, WebPage> store) throws IOException {
+    testGetWebPage(store, WebPage._ALL_FIELDS);
+  }
+
+  public static void testGetWebPageDefaultFields(DataStore<String, WebPage> store)
+  throws IOException {
+    testGetWebPage(store, null);
+  }
+
+  private static void testQueryWebPageSingleKey(DataStore<String, WebPage> store
+      , String[] fields) throws IOException {
+
+    createWebPageData(store);
+
+    for(int i=0; i<URLS.length; i++) {
+      Query<String, WebPage> query = store.newQuery();
+      query.setFields(fields);
+      query.setKey(URLS[i]);
+      Result<String, WebPage> result = query.execute();
+      Assert.assertTrue(result.next());
+      WebPage page = result.get();
+      assertWebPage(page, i);
+      Assert.assertFalse(result.next());
+    }
+  }
+
+  public static void testQueryWebPageSingleKey(DataStore<String, WebPage> store)
+  throws IOException {
+    testQueryWebPageSingleKey(store, WebPage._ALL_FIELDS);
+  }
+
+  public static void testQueryWebPageSingleKeyDefaultFields(
+      DataStore<String, WebPage> store) throws IOException {
+    testQueryWebPageSingleKey(store, null);
+  }
+
+  public static void testQueryWebPageKeyRange(DataStore<String, WebPage> store,
+      boolean setStartKeys, boolean setEndKeys)
+  throws IOException {
+    createWebPageData(store);
+
+    //create sorted set of urls
+    List<String> sortedUrls = new ArrayList<String>();
+    for(String url: URLS) {
+      sortedUrls.add(url);
+    }
+    Collections.sort(sortedUrls);
+
+    //try all ranges
+    for(int i=0; i<sortedUrls.size(); i++) {
+      for(int j=i; j<sortedUrls.size(); j++) {
+        Query<String, WebPage> query = store.newQuery();
+        if(setStartKeys)
+          query.setStartKey(sortedUrls.get(i));
+        if(setEndKeys)
+          query.setEndKey(sortedUrls.get(j));
+        Result<String, WebPage> result = query.execute();
+
+        int r=0;
+        while(result.next()) {
+          WebPage page = result.get();
+          assertWebPage(page, URL_INDEXES.get(page.getUrl().toString()));
+          r++;
+        }
+
+        int expectedLength = (setEndKeys ? j+1: sortedUrls.size()) -
+                             (setStartKeys ? i: 0);
+        Assert.assertEquals(expectedLength, r);
+        if(!setEndKeys)
+          break;
+      }
+      if(!setStartKeys)
+        break;
+    }
+  }
+
+  public static void testQueryWebPages(DataStore<String, WebPage> store)
+  throws IOException {
+    testQueryWebPageKeyRange(store, false, false);
+  }
+
+  public static void testQueryWebPageStartKey(DataStore<String, WebPage> store)
+  throws IOException {
+    testQueryWebPageKeyRange(store, true, false);
+  }
+
+  public static void testQueryWebPageEndKey(DataStore<String, WebPage> store)
+  throws IOException {
+    testQueryWebPageKeyRange(store, false, true);
+  }
+
+  public static void testQueryWebPageKeyRange(DataStore<String, WebPage> store)
+  throws IOException {
+    testQueryWebPageKeyRange(store, true, true);
+  }
+
+  public static void testQueryWebPageEmptyResults(DataStore<String, WebPage> store)
+    throws IOException {
+    createWebPageData(store);
+
+    //query empty results
+    Query<String, WebPage> query = store.newQuery();
+    query.setStartKey("aa");
+    query.setEndKey("ab");
+    assertEmptyResults(query);
+
+    //query empty results for one key
+    query = store.newQuery();
+    query.setKey("aa");
+    assertEmptyResults(query);
+  }
+
+  public static<K,T extends Persistent> void assertEmptyResults(Query<K, T> query)
+    throws IOException {
+    assertNumResults(query, 0);
+  }
+
+  public static<K,T extends Persistent> void assertNumResults(Query<K, T>query
+      , long numResults) throws IOException {
+    Result<K, T> result = query.execute();
+    int actualNumResults = 0;
+    while(result.next()) {
+      actualNumResults++;
+    }
+    result.close();
+    Assert.assertEquals(numResults, actualNumResults);
+  }
+
+  public static void testGetPartitions(DataStore<String, WebPage> store)
+  throws IOException {
+    createWebPageData(store);
+    testGetPartitions(store, store.newQuery());
+  }
+
+  public static void testGetPartitions(DataStore<String, WebPage> store
+      , Query<String, WebPage> query) throws IOException {
+    List<PartitionQuery<String, WebPage>> partitions = store.getPartitions(query);
+
+    Assert.assertNotNull(partitions);
+    Assert.assertTrue(partitions.size() > 0);
+
+    for(PartitionQuery<String, WebPage> partition:partitions) {
+      Assert.assertNotNull(partition);
+    }
+
+    assertPartitions(store, query, partitions);
+  }
+
+  public static void assertPartitions(DataStore<String, WebPage> store,
+      Query<String, WebPage> query, List<PartitionQuery<String,WebPage>> partitions)
+  throws IOException {
+
+    int count = 0, partitionsCount = 0;
+    Map<String, Integer> results = new HashMap<String, Integer>();
+    Map<String, Integer> partitionResults = new HashMap<String, Integer>();
+
+    //execute query and count results
+    Result<String, WebPage> result = store.execute(query);
+    Assert.assertNotNull(result);
+
+    while(result.next()) {
+      Assert.assertNotNull(result.getKey());
+      Assert.assertNotNull(result.get());
+      results.put(result.getKey(), result.get().hashCode()); //keys are not reused, so this is safe
+      count++;
+    }
+    result.close();
+
+    Assert.assertTrue(count > 0); //assert that results is not empty
+    Assert.assertEquals(count, results.size()); //assert that keys are unique
+
+    for(PartitionQuery<String, WebPage> partition:partitions) {
+      Assert.assertNotNull(partition);
+
+      result = store.execute(partition);
+      Assert.assertNotNull(result);
+
+      while(result.next()) {
+        Assert.assertNotNull(result.getKey());
+        Assert.assertNotNull(result.get());
+        partitionResults.put(result.getKey(), result.get().hashCode());
+        partitionsCount++;
+      }
+      result.close();
+
+      Assert.assertEquals(partitionsCount, partitionResults.size()); //assert that keys are unique
+    }
+
+    Assert.assertTrue(partitionsCount > 0);
+    Assert.assertEquals(count, partitionsCount);
+
+    for(Map.Entry<String, Integer> r : results.entrySet()) {
+      Integer p = partitionResults.get(r.getKey());
+      Assert.assertNotNull(p);
+      Assert.assertEquals(r.getValue(), p);
+    }
+  }
+
+  public static void testDelete(DataStore<String, WebPage> store) throws IOException {
+    WebPageDataCreator.createWebPageData(store);
+    //delete one by one
+
+    int deletedSoFar = 0;
+    for(String url : URLS) {
+      Assert.assertTrue(store.delete(url));
+      store.flush();
+
+      //assert that it is actually deleted
+      Assert.assertNull(store.get(url));
+
+      //assert that other records are not deleted
+      assertNumResults(store.newQuery(), URLS.length - ++deletedSoFar);
+    }
+  }
+
+  public static void testDeleteByQuery(DataStore<String, WebPage> store)
+    throws IOException {
+
+    Query<String, WebPage> query;
+
+    //test 1 - delete all
+    WebPageDataCreator.createWebPageData(store);
+
+    query = store.newQuery();
+
+    assertNumResults(store.newQuery(), URLS.length);
+    store.deleteByQuery(query);
+    assertEmptyResults(store.newQuery());
+
+
+    //test 2 - delete all
+    WebPageDataCreator.createWebPageData(store);
+
+    query = store.newQuery();
+    query.setFields(WebPage._ALL_FIELDS);
+
+    assertNumResults(store.newQuery(), URLS.length);
+    store.deleteByQuery(query);
+    assertEmptyResults(store.newQuery());
+
+
+    //test 3 - delete all
+    WebPageDataCreator.createWebPageData(store);
+
+    query = store.newQuery();
+    query.setKeyRange("a", "z"); //all start with "http://"
+
+    assertNumResults(store.newQuery(), URLS.length);
+    store.deleteByQuery(query);
+    assertEmptyResults(store.newQuery());
+
+
+    //test 4 - delete some
+    WebPageDataCreator.createWebPageData(store);
+    query = store.newQuery();
+    query.setEndKey(SORTED_URLS[NUM_KEYS]);
+
+    assertNumResults(store.newQuery(), URLS.length);
+    store.deleteByQuery(query);
+    assertNumResults(store.newQuery(), URLS.length - (NUM_KEYS+1));
+
+    store.truncateSchema();
+
+  }
+
+  public static void testDeleteByQueryFields(DataStore<String, WebPage> store)
+  throws IOException {
+
+    Query<String, WebPage> query;
+
+    //test 5 - delete all with some fields
+    WebPageDataCreator.createWebPageData(store);
+
+    query = store.newQuery();
+    query.setFields(WebPage.Field.OUTLINKS.getName()
+        , WebPage.Field.PARSED_CONTENT.getName(), WebPage.Field.CONTENT.getName());
+
+    assertNumResults(store.newQuery(), URLS.length);
+    store.deleteByQuery(query);
+    store.deleteByQuery(query);
+    store.deleteByQuery(query);//don't you love that HBase sometimes does not delete arbitrarily
+    assertNumResults(store.newQuery(), URLS.length);
+
+    //assert that data is deleted
+    for (int i = 0; i < SORTED_URLS.length; i++) {
+      WebPage page = store.get(SORTED_URLS[i]);
+      Assert.assertNotNull(page);
+
+      Assert.assertNotNull(page.getUrl());
+      Assert.assertEquals(page.getUrl().toString(), SORTED_URLS[i]);
+      Assert.assertEquals(0, page.getOutlinks().size());
+      Assert.assertEquals(0, page.getParsedContent().size());
+      if(page.getContent() != null) {
+        System.out.println("url:" + page.getUrl().toString());
+        System.out.println( "limit:" + page.getContent().limit());
+      } else {
+        Assert.assertNull(page.getContent());
+      }
+    }
+
+    //test 6 - delete some with some fields
+    WebPageDataCreator.createWebPageData(store);
+
+    query = store.newQuery();
+    query.setFields(WebPage.Field.URL.getName());
+    String startKey = SORTED_URLS[NUM_KEYS];
+    String endKey = SORTED_URLS[SORTED_URLS.length - NUM_KEYS];
+    query.setStartKey(startKey);
+    query.setEndKey(endKey);
+
+    assertNumResults(store.newQuery(), URLS.length);
+    store.deleteByQuery(query);
+    store.deleteByQuery(query);
+    store.deleteByQuery(query);//don't you love that HBase sometimes does not delete arbitrarily
+
+    assertNumResults(store.newQuery(), URLS.length);
+
+    //assert that data is deleted
+    for (int i = 0; i < URLS.length; i++) {
+      WebPage page = store.get(URLS[i]);
+      Assert.assertNotNull(page);
+      if( URLS[i].compareTo(startKey) < 0 || URLS[i].compareTo(endKey) >= 0) {
+        //not deleted
+        assertWebPage(page, i);
+      } else {
+        //deleted
+        Assert.assertNull(page.getUrl());
+        Assert.assertNotNull(page.getOutlinks());
+        Assert.assertNotNull(page.getParsedContent());
+        Assert.assertNotNull(page.getContent());
+        Assert.assertTrue(page.getOutlinks().size() > 0);
+        Assert.assertTrue(page.getParsedContent().size() > 0);
+      }
+    }
+
+  }
+}



Mime
View raw message