hama-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tjungb...@apache.org
Subject svn commit: r1387891 - in /hama/trunk: core/src/main/java/org/apache/hama/bsp/ graph/src/main/java/org/apache/hama/graph/ jdbm/src/test/java/org/apache/hama/jdbm/
Date Thu, 20 Sep 2012 07:18:47 GMT
Author: tjungblut
Date: Thu Sep 20 07:18:47 2012
New Revision: 1387891

URL: http://svn.apache.org/viewvc?rev=1387891&view=rev
Log:
fix build for java 6

Added:
    hama/trunk/core/src/main/java/org/apache/hama/bsp/WritableComparator.java
    hama/trunk/core/src/main/java/org/apache/hama/bsp/WritableSerialization.java
Removed:
    hama/trunk/graph/src/main/java/org/apache/hama/graph/WritableComparator.java
    hama/trunk/graph/src/main/java/org/apache/hama/graph/WritableSerialization.java
Modified:
    hama/trunk/graph/src/main/java/org/apache/hama/graph/GraphJobRunner.java
    hama/trunk/graph/src/main/java/org/apache/hama/graph/VertexWritableSerialization.java
    hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/DBCacheTest.java
    hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/DBTest.java
    hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/DefragTest.java
    hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/SerializationTest.java

Added: hama/trunk/core/src/main/java/org/apache/hama/bsp/WritableComparator.java
URL: http://svn.apache.org/viewvc/hama/trunk/core/src/main/java/org/apache/hama/bsp/WritableComparator.java?rev=1387891&view=auto
==============================================================================
--- hama/trunk/core/src/main/java/org/apache/hama/bsp/WritableComparator.java (added)
+++ hama/trunk/core/src/main/java/org/apache/hama/bsp/WritableComparator.java Thu Sep 20 07:18:47
2012
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.bsp;
+
+import java.io.Serializable;
+import java.util.Comparator;
+
+/**
+ * Comparator that uses Writable Comparable instance to compare.
+ */
+public final class WritableComparator<T extends Comparable<T>> implements
+    Comparator<T>, Serializable {
+
+  private static final long serialVersionUID = 1L;
+
+  @Override
+  public int compare(T o1, T o2) {
+    return o1.compareTo(o2);
+  }
+
+}

Added: hama/trunk/core/src/main/java/org/apache/hama/bsp/WritableSerialization.java
URL: http://svn.apache.org/viewvc/hama/trunk/core/src/main/java/org/apache/hama/bsp/WritableSerialization.java?rev=1387891&view=auto
==============================================================================
--- hama/trunk/core/src/main/java/org/apache/hama/bsp/WritableSerialization.java (added)
+++ hama/trunk/core/src/main/java/org/apache/hama/bsp/WritableSerialization.java Thu Sep 20
07:18:47 2012
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.bsp;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hama.jdbm.Serializer;
+
+/**
+ * Writable serialization for Hadoop objects with a class cache to reduce the
+ * amount of writing the classname instead of a single integer (with V
+ * compression, so most of the time it just takes a single byte).
+ */
+public class WritableSerialization<K extends Writable> implements
+    Serializer<K>, Serializable {
+
+  private static final long serialVersionUID = 1L;
+
+  // clazzname as string -> index in the lookuplist
+  protected transient final HashMap<String, Integer> CLAZZ_CACHE = new HashMap<String,
Integer>();
+  protected transient final ArrayList<Class<?>> LOOKUP_LIST = new ArrayList<Class<?>>();
+  private transient int lastAssigned = 0;
+
+  protected transient Writable instance;
+  protected transient int writableClassIndex;
+
+  protected transient Configuration conf;
+
+  public WritableSerialization() {
+  }
+
+  public WritableSerialization(Class<?> writableClazz) {
+    Integer integer = CLAZZ_CACHE.get(writableClazz);
+    if (integer == null) {
+      integer = lastAssigned++;
+      CLAZZ_CACHE.put(writableClazz.getName(), integer);
+      LOOKUP_LIST.add(writableClazz);
+    }
+    this.writableClassIndex = integer;
+  }
+
+  public WritableSerialization(Class<?> writableClazz, Configuration conf) {
+    this(writableClazz);
+    this.conf = conf;
+  }
+
+  @Override
+  public void serialize(DataOutput out, K obj) throws IOException {
+    WritableUtils.writeVInt(out, writableClassIndex);
+    obj.write(out);
+  }
+
+  @SuppressWarnings("unchecked")
+  @Override
+  public K deserialize(DataInput in) throws IOException, ClassNotFoundException {
+    writableClassIndex = WritableUtils.readVInt(in);
+    instance = newInstance();
+    instance.readFields(in);
+    return (K) instance;
+  }
+
+  public Writable newInstance() {
+    return (Writable) ReflectionUtils.newInstance(
+        LOOKUP_LIST.get(writableClassIndex), conf);
+  }
+
+  @Override
+  public int hashCode() {
+    final int prime = 31;
+    int result = 1;
+    result = prime * result + ((instance == null) ? 0 : instance.hashCode());
+    return result;
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj)
+      return true;
+    if (obj == null)
+      return false;
+    if (getClass() != obj.getClass())
+      return false;
+    @SuppressWarnings("rawtypes")
+    WritableSerialization other = (WritableSerialization) obj;
+    if (instance == null) {
+      if (other.instance != null)
+        return false;
+    } else if (!instance.equals(other.instance))
+      return false;
+    return true;
+  }
+
+}

Modified: hama/trunk/graph/src/main/java/org/apache/hama/graph/GraphJobRunner.java
URL: http://svn.apache.org/viewvc/hama/trunk/graph/src/main/java/org/apache/hama/graph/GraphJobRunner.java?rev=1387891&r1=1387890&r2=1387891&view=diff
==============================================================================
--- hama/trunk/graph/src/main/java/org/apache/hama/graph/GraphJobRunner.java (original)
+++ hama/trunk/graph/src/main/java/org/apache/hama/graph/GraphJobRunner.java Thu Sep 20 07:18:47
2012
@@ -46,6 +46,8 @@ import org.apache.hama.bsp.BSPPeer;
 import org.apache.hama.bsp.Combiner;
 import org.apache.hama.bsp.HashPartitioner;
 import org.apache.hama.bsp.Partitioner;
+import org.apache.hama.bsp.WritableComparator;
+import org.apache.hama.bsp.WritableSerialization;
 import org.apache.hama.bsp.sync.SyncException;
 import org.apache.hama.jdbm.DB;
 import org.apache.hama.jdbm.DBMaker;
@@ -312,7 +314,7 @@ public final class GraphJobRunner<V exte
 
       Comparator<V> writableComparator = new WritableComparator<V>();
       vertices = db.createTreeMap("graph-db", writableComparator,
-          new WritableSerialization<V>(vertexIdClass),
+          new WritableSerialization<V>(vertexIdClass, peer.getConfiguration()),
           new VertexWritableSerialization<Vertex<V, E, M>>(vertexClass, this));
     } else {
       vertices = new HashMap<V, Vertex<V, E, M>>();

Modified: hama/trunk/graph/src/main/java/org/apache/hama/graph/VertexWritableSerialization.java
URL: http://svn.apache.org/viewvc/hama/trunk/graph/src/main/java/org/apache/hama/graph/VertexWritableSerialization.java?rev=1387891&r1=1387890&r2=1387891&view=diff
==============================================================================
--- hama/trunk/graph/src/main/java/org/apache/hama/graph/VertexWritableSerialization.java
(original)
+++ hama/trunk/graph/src/main/java/org/apache/hama/graph/VertexWritableSerialization.java
Thu Sep 20 07:18:47 2012
@@ -19,6 +19,7 @@ package org.apache.hama.graph;
 
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hama.bsp.WritableSerialization;
 
 import com.google.common.base.Preconditions;
 
@@ -54,7 +55,7 @@ public final class VertexWritableSeriali
   @Override
   public Writable newInstance() {
     Writable newInstance = (Writable) ReflectionUtils.newInstance(
-        LOOKUP_LIST.get(writableClassIndex), null);
+        LOOKUP_LIST.get(writableClassIndex), conf);
     ((Vertex) newInstance).runner = this.runner;
     return newInstance;
   }

Modified: hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/DBCacheTest.java
URL: http://svn.apache.org/viewvc/hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/DBCacheTest.java?rev=1387891&r1=1387890&r2=1387891&view=diff
==============================================================================
--- hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/DBCacheTest.java (original)
+++ hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/DBCacheTest.java Thu Sep 20 07:18:47
2012
@@ -17,7 +17,7 @@
  */
 package org.apache.hama.jdbm;
 
-import java.util.Map;
+import java.util.NavigableMap;
 import java.util.Set;
 
 public class DBCacheTest extends TestCaseWithTestFile {
@@ -50,7 +50,7 @@ public class DBCacheTest extends TestCas
   public void test_issue_xyz() {
     org.apache.hama.jdbm.DB db = DBMaker.openFile(newTestFile()).enableSoftCache()
         .make();
-    Map m = db.createTreeMap("test");
+    NavigableMap<String,String> m = db.createTreeMap("test");
 
     for (int i = 0; i < 1e5; i++) {
       m.put("test" + i, "test" + i);

Modified: hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/DBTest.java
URL: http://svn.apache.org/viewvc/hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/DBTest.java?rev=1387891&r1=1387890&r2=1387891&view=diff
==============================================================================
--- hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/DBTest.java (original)
+++ hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/DBTest.java Thu Sep 20 07:18:47 2012
@@ -27,6 +27,7 @@ import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.NavigableMap;
 import java.util.Set;
 import java.util.SortedMap;
 import java.util.SortedSet;
@@ -244,7 +245,7 @@ public class DBTest extends TestCaseWith
 
   public void testGetCollections() throws IOException {
     DB db = newDBCache();
-    db.createTreeMap("treemap");
+    NavigableMap<String, Object> createTreeMap = db.createTreeMap("treemap");
     db.createHashMap("hashmap");
     db.createTreeSet("treeset");
     db.createHashSet("hashset");
@@ -323,7 +324,7 @@ public class DBTest extends TestCaseWith
                                                // map are initilaized
     d.commit();
     long recCount = d.countRecords();
-    Map l = d.createTreeMap("test");
+    NavigableMap<String,String> l = d.createTreeMap("test");
     l.put("1", "b");
     l.put("2", "b");
     d.commit();
@@ -369,7 +370,7 @@ public class DBTest extends TestCaseWith
                                                // map are initilaized
     d.commit();
     long recCount = d.countRecords();
-    Map l = d.createTreeMap("test");
+    NavigableMap<String,String> l = d.createTreeMap("test");
     d.commit();
     assertFalse(recCount == d.countRecords());
     d.deleteCollection("test");
@@ -417,7 +418,7 @@ public class DBTest extends TestCaseWith
   public void testCollectionSize() throws IOException {
     DB d = newDBNoCache();
 
-    Map tm = d.createTreeMap("t1");
+    Map<Integer,Integer> tm = d.createTreeMap("t1");
     tm.put(1, 1);
     tm.put(2, 2);
     assertEquals(d.collectionSize(tm), 2);

Modified: hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/DefragTest.java
URL: http://svn.apache.org/viewvc/hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/DefragTest.java?rev=1387891&r1=1387890&r2=1387891&view=diff
==============================================================================
--- hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/DefragTest.java (original)
+++ hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/DefragTest.java Thu Sep 20 07:18:47
2012
@@ -57,7 +57,7 @@ public class DefragTest extends TestCase
   public void testDefragBtree() throws IOException {
     String file = newTestFile();
     DBStore m = new DBStore(file, false, false, false);
-    Map t = m.createTreeMap("aa");
+    Map<Integer,String> t = m.createTreeMap("aa");
     TreeMap t2 = new TreeMap();
     for (int i = 0; i < 10000; i++) {
       t.put(i, "" + i);

Modified: hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/SerializationTest.java
URL: http://svn.apache.org/viewvc/hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/SerializationTest.java?rev=1387891&r1=1387890&r2=1387891&view=diff
==============================================================================
--- hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/SerializationTest.java (original)
+++ hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/SerializationTest.java Thu Sep 20 07:18:47
2012
@@ -42,7 +42,6 @@ import java.util.Map;
 import java.util.Properties;
 import java.util.TreeMap;
 import java.util.TreeSet;
-import java.util.UUID;
 import java.util.Vector;
 
 import junit.framework.TestCase;



Mime
View raw message