incubator-hama-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From edwardy...@apache.org
Subject svn commit: r956746 - in /incubator/hama/trunk: ./ src/examples/org/apache/hama/examples/ src/examples/org/apache/hama/examples/mapreduce/ src/java/org/apache/hama/io/ src/test/org/apache/hama/examples/ src/test/org/apache/hama/io/
Date Tue, 22 Jun 2010 00:18:11 GMT
Author: edwardyoon
Date: Tue Jun 22 00:18:09 2010
New Revision: 956746

URL: http://svn.apache.org/viewvc?rev=956746&view=rev
Log:
Moving BlockID, Pair classes to examples package

Added:
    incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/BlockID.java
    incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/Pair.java
    incubator/hama/trunk/src/test/org/apache/hama/examples/TestBlockID.java
Removed:
    incubator/hama/trunk/src/java/org/apache/hama/io/BlockID.java
    incubator/hama/trunk/src/java/org/apache/hama/io/Pair.java
    incubator/hama/trunk/src/test/org/apache/hama/io/TestBlockID.java
Modified:
    incubator/hama/trunk/CHANGES.txt
    incubator/hama/trunk/src/examples/org/apache/hama/examples/JacobiEigen.java
    incubator/hama/trunk/src/examples/org/apache/hama/examples/MatrixMultiplication.java
    incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/BlockMultMap.java
    incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/BlockMultReduce.java
    incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/CollectBlocksMapper.java
    incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/CollectBlocksReducer.java
    incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/PivotInputFormat.java
    incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/PivotMap.java
    incubator/hama/trunk/src/test/org/apache/hama/examples/TestFileMatrixBlockMult.java

Modified: incubator/hama/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/CHANGES.txt?rev=956746&r1=956745&r2=956746&view=diff
==============================================================================
--- incubator/hama/trunk/CHANGES.txt (original)
+++ incubator/hama/trunk/CHANGES.txt Tue Jun 22 00:18:09 2010
@@ -43,6 +43,7 @@ Trunk (unreleased changes)
 
   IMPROVEMENTS
 
+    HAMA-264: Moving BlockID, Pair classes to examples package (edwardyoon)
     HAMA-269: Add getter for serverName to BSPPeerInterface (edwardyoon)
     HAMA-257: Design BSP program code interface (edwardyoon)
     HAMA-261: Remove meaningless unit tests (edwardyoon)

Modified: incubator/hama/trunk/src/examples/org/apache/hama/examples/JacobiEigen.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/examples/org/apache/hama/examples/JacobiEigen.java?rev=956746&r1=956745&r2=956746&view=diff
==============================================================================
--- incubator/hama/trunk/src/examples/org/apache/hama/examples/JacobiEigen.java (original)
+++ incubator/hama/trunk/src/examples/org/apache/hama/examples/JacobiEigen.java Tue Jun 22
00:18:09 2010
@@ -24,10 +24,10 @@ import org.apache.hama.HamaAdmin;
 import org.apache.hama.HamaAdminImpl;
 import org.apache.hama.examples.mapreduce.DummyMapper;
 import org.apache.hama.examples.mapreduce.JacobiInitMap;
+import org.apache.hama.examples.mapreduce.Pair;
 import org.apache.hama.examples.mapreduce.PivotInputFormat;
 import org.apache.hama.examples.mapreduce.PivotMap;
 import org.apache.hama.examples.mapreduce.RotationInputFormat;
-import org.apache.hama.io.Pair;
 import org.apache.hama.matrix.DenseMatrix;
 import org.apache.hama.matrix.Matrix;
 import org.apache.hama.util.BytesUtil;

Modified: incubator/hama/trunk/src/examples/org/apache/hama/examples/MatrixMultiplication.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/examples/org/apache/hama/examples/MatrixMultiplication.java?rev=956746&r1=956745&r2=956746&view=diff
==============================================================================
--- incubator/hama/trunk/src/examples/org/apache/hama/examples/MatrixMultiplication.java (original)
+++ incubator/hama/trunk/src/examples/org/apache/hama/examples/MatrixMultiplication.java Tue
Jun 22 00:18:09 2010
@@ -37,12 +37,12 @@ import org.apache.hadoop.mapreduce.Job;
 import org.apache.hama.Constants;
 import org.apache.hama.HamaAdmin;
 import org.apache.hama.HamaAdminImpl;
+import org.apache.hama.examples.mapreduce.BlockID;
 import org.apache.hama.examples.mapreduce.BlockMultMap;
 import org.apache.hama.examples.mapreduce.BlockMultReduce;
 import org.apache.hama.examples.mapreduce.CollectBlocksMapper;
 import org.apache.hama.examples.mapreduce.DenseMatrixVectorMultMap;
 import org.apache.hama.examples.mapreduce.DenseMatrixVectorMultReduce;
-import org.apache.hama.io.BlockID;
 import org.apache.hama.matrix.DenseMatrix;
 import org.apache.hama.matrix.Matrix;
 import org.apache.hama.util.RandomVariable;

Added: incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/BlockID.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/BlockID.java?rev=956746&view=auto
==============================================================================
--- incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/BlockID.java (added)
+++ incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/BlockID.java Tue
Jun 22 00:18:09 2010
@@ -0,0 +1,150 @@
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.examples.mapreduce;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.log4j.Logger;
+
+/** A WritableComparable for BlockIDs. */
+@SuppressWarnings("unchecked")
+public class BlockID implements WritableComparable {
+  static final Logger LOG = Logger.getLogger(BlockID.class);
+  public static final int PAD_SIZE = 15;
+  private int row;
+  private int column;
+  private int seq = -1;
+  
+  public BlockID() {
+  }
+
+  public BlockID(int row, int column) {
+    set(row, column);
+  }
+
+  public BlockID(byte[] bytes) throws IOException {
+    String rKey = Bytes.toString(bytes);
+    String keys[] = null;
+    if (rKey.substring(0, 8).equals("00000000")) {
+      int i = 8;
+      while (rKey.charAt(i) == '0') {
+        i++;
+      }
+      keys = rKey.substring(i, rKey.length()).split("[,]");
+    } else {
+      int i = 0;
+      while (rKey.charAt(i) == '0') {
+        i++;
+      }
+      keys = rKey.substring(i, rKey.length()).split("[,]");
+    }
+
+    try {
+      this.row = Integer.parseInt(keys[1]);
+      String[] columns = keys[2].split("[-]");
+      this.column = Integer.parseInt(columns[0]);
+    } catch (ArrayIndexOutOfBoundsException e) {
+      throw new ArrayIndexOutOfBoundsException(rKey + "\n" + e);
+    }
+  }
+
+  public BlockID(int row, int column, int seq) {
+    set(row, column);
+    this.seq = seq;
+  }
+
+  public void set(int row, int column) {
+    this.row = row;
+    this.column = column;
+  }
+
+  public int getRow() {
+    return row;
+  }
+
+  public int getColumn() {
+    return column;
+  }
+
+  public void readFields(DataInput in) throws IOException {
+    BlockID value = new BlockID(Bytes.readByteArray(in));
+    this.row = value.getRow();
+    this.column = value.getColumn();
+  }
+
+  public void write(DataOutput out) throws IOException {
+    Bytes.writeByteArray(out, Bytes.toBytes(this.toString()));
+  }
+
+  /**
+   * Make BlockID's string representation be same format.
+   */
+  public String toString() {
+    int zeros = PAD_SIZE - String.valueOf(row).length()
+        - String.valueOf(column).length();
+    StringBuffer buf = new StringBuffer();
+    for (int i = 0; i < zeros; ++i) {
+      buf.append("0");
+    }
+
+    if(seq > -1) {
+      return buf.toString() + "," + row + "," + column + "-" + seq;
+    } else {
+      return buf.toString() + "," + row + "," + column;
+    }
+  }
+
+  @Override
+  public int hashCode() {
+    // simply use a prime number
+    // may be need a more balance hash function
+    return row * 37 + column;
+  }
+
+  public int compareTo(Object o) {
+    int thisRow = this.row;
+    int thatRow = ((BlockID) o).row;
+    int thisColumn = this.column;
+    int thatColumn = ((BlockID) o).column;
+
+    if (thisRow != thatRow) {
+      return (thisRow < thatRow ? -1 : 1);
+    } else {
+      return (thisColumn < thatColumn ? -1 : (thisColumn == thatColumn ? 0 : 1));
+    }
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (o == null)
+      return false;
+    if (!(o instanceof BlockID))
+      return false;
+    return compareTo(o) == 0;
+  }
+
+  public byte[] getBytes() {
+    return Bytes.toBytes(this.toString());
+  }
+}

Modified: incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/BlockMultMap.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/BlockMultMap.java?rev=956746&r1=956745&r2=956746&view=diff
==============================================================================
--- incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/BlockMultMap.java
(original)
+++ incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/BlockMultMap.java
Tue Jun 22 00:18:09 2010
@@ -8,7 +8,6 @@ import org.apache.hadoop.hbase.mapreduce
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.io.BytesWritable;
 import org.apache.hama.Constants;
-import org.apache.hama.io.BlockID;
 import org.apache.hama.matrix.SubMatrix;
 
 public class BlockMultMap extends TableMapper<BlockID, BytesWritable> {

Modified: incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/BlockMultReduce.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/BlockMultReduce.java?rev=956746&r1=956745&r2=956746&view=diff
==============================================================================
--- incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/BlockMultReduce.java
(original)
+++ incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/BlockMultReduce.java
Tue Jun 22 00:18:09 2010
@@ -9,7 +9,6 @@ import org.apache.hadoop.hbase.util.Byte
 import org.apache.hadoop.io.BytesWritable;
 import org.apache.hadoop.io.Writable;
 import org.apache.hama.Constants;
-import org.apache.hama.io.BlockID;
 import org.apache.hama.matrix.SubMatrix;
 import org.apache.hama.util.BytesUtil;
 

Modified: incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/CollectBlocksMapper.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/CollectBlocksMapper.java?rev=956746&r1=956745&r2=956746&view=diff
==============================================================================
--- incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/CollectBlocksMapper.java
(original)
+++ incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/CollectBlocksMapper.java
Tue Jun 22 00:18:09 2010
@@ -8,7 +8,6 @@ import org.apache.hadoop.hbase.client.Re
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.mapreduce.TableMapper;
 import org.apache.hadoop.io.MapWritable;
-import org.apache.hama.io.BlockID;
 import org.apache.hama.matrix.DenseVector;
 import org.apache.hama.util.BytesUtil;
 

Modified: incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/CollectBlocksReducer.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/CollectBlocksReducer.java?rev=956746&r1=956745&r2=956746&view=diff
==============================================================================
--- incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/CollectBlocksReducer.java
(original)
+++ incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/CollectBlocksReducer.java
Tue Jun 22 00:18:09 2010
@@ -11,7 +11,6 @@ import org.apache.hadoop.hbase.util.Byte
 import org.apache.hadoop.io.MapWritable;
 import org.apache.hadoop.io.Writable;
 import org.apache.hama.Constants;
-import org.apache.hama.io.BlockID;
 import org.apache.hama.matrix.DenseVector;
 import org.apache.hama.matrix.SubMatrix;
 

Added: incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/Pair.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/Pair.java?rev=956746&view=auto
==============================================================================
--- incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/Pair.java (added)
+++ incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/Pair.java Tue Jun
22 00:18:09 2010
@@ -0,0 +1,65 @@
+package org.apache.hama.examples.mapreduce;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.io.WritableComparable;
+
+/** A Pair stands for (row, column) pair **/
+public class Pair implements WritableComparable<Pair> {
+
+  int row, col;
+  
+  public Pair() {}
+  
+  public Pair(int row_, int col_) {
+    set(row_, col_);
+  }
+  
+  public int getRow() { return row; }
+  public int getColumn() { return col; }
+  
+  public void setRow(int row_) { row = row_; }
+  public void setColumn(int col_) { col = col_; }
+  public void set(int row_, int col_) {
+    row = row_;
+    col = col_;
+  }
+  
+  @Override
+  public void readFields(DataInput in) throws IOException {
+    row = in.readInt();
+    col = in.readInt();
+  }
+
+  @Override
+  public void write(DataOutput out) throws IOException {
+    out.writeInt(row);
+    out.writeInt(col);
+  }
+
+  @Override
+  public int compareTo(Pair p) {
+    return row == p.row ? col - p.col : row - p.row;
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    Pair pair = (Pair)obj;
+    return compareTo(pair) == 0;
+  }
+
+  @Override
+  public int hashCode() {
+    return row;
+  }
+
+  @Override
+  public String toString() {
+    StringBuilder sb = new StringBuilder();
+    sb.append('(').append(row).append(',').append(col).append(')');
+    return sb.toString();
+  }
+
+}

Modified: incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/PivotInputFormat.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/PivotInputFormat.java?rev=956746&r1=956745&r2=956746&view=diff
==============================================================================
--- incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/PivotInputFormat.java
(original)
+++ incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/PivotInputFormat.java
Tue Jun 22 00:18:09 2010
@@ -30,7 +30,6 @@ import org.apache.hadoop.mapreduce.Recor
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hama.examples.JacobiEigen;
-import org.apache.hama.io.Pair;
 import org.apache.hama.util.BytesUtil;
 
 public class PivotInputFormat extends InputFormat<Pair, DoubleWritable>

Modified: incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/PivotMap.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/PivotMap.java?rev=956746&r1=956745&r2=956746&view=diff
==============================================================================
--- incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/PivotMap.java (original)
+++ incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/PivotMap.java Tue
Jun 22 00:18:09 2010
@@ -4,7 +4,6 @@ import java.io.IOException;
 
 import org.apache.hadoop.io.DoubleWritable;
 import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hama.io.Pair;
 
 public class PivotMap extends
     Mapper<Pair, DoubleWritable, Pair, DoubleWritable> {

Added: incubator/hama/trunk/src/test/org/apache/hama/examples/TestBlockID.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/test/org/apache/hama/examples/TestBlockID.java?rev=956746&view=auto
==============================================================================
--- incubator/hama/trunk/src/test/org/apache/hama/examples/TestBlockID.java (added)
+++ incubator/hama/trunk/src/test/org/apache/hama/examples/TestBlockID.java Tue Jun 22 00:18:09
2010
@@ -0,0 +1,69 @@
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.examples;
+
+import java.io.IOException;
+
+import junit.framework.TestCase;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.io.DataInputBuffer;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hama.examples.mapreduce.BlockID;
+
+public class TestBlockID extends TestCase {
+  final static Log LOG = LogFactory.getLog(TestBlockID.class.getName());
+
+  /**
+   * BlockID object compare
+   */
+  public void testCompare() {
+    BlockID a = new BlockID(1, 3);
+    BlockID b = new BlockID(1, 1);
+    assertEquals(a.compareTo(b), 1);
+
+    BlockID c = new BlockID(3, 1);
+    BlockID d = new BlockID(1, 1);
+    assertEquals(a.compareTo(c), -1);
+
+    assertEquals(b.compareTo(d), 0);
+  }
+  
+  /**
+   * BlockID object IO
+   * @throws IOException 
+   */
+  public void testIO() throws IOException {
+    DataOutputBuffer outBuf = new DataOutputBuffer();
+    DataInputBuffer inBuf = new DataInputBuffer();
+    
+    BlockID a = new BlockID(1, 3);
+    outBuf.reset();
+    a.write(outBuf);
+    
+    inBuf.reset(outBuf.getData(), outBuf.getLength());
+    BlockID b = new BlockID();
+    b.readFields(inBuf);
+    
+    assertEquals(0, a.compareTo(b));
+  }
+
+}

Modified: incubator/hama/trunk/src/test/org/apache/hama/examples/TestFileMatrixBlockMult.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/test/org/apache/hama/examples/TestFileMatrixBlockMult.java?rev=956746&r1=956745&r2=956746&view=diff
==============================================================================
--- incubator/hama/trunk/src/test/org/apache/hama/examples/TestFileMatrixBlockMult.java (original)
+++ incubator/hama/trunk/src/test/org/apache/hama/examples/TestFileMatrixBlockMult.java Tue
Jun 22 00:18:09 2010
@@ -28,9 +28,9 @@ import org.apache.hadoop.mapreduce.lib.i
 import org.apache.hama.Constants;
 import org.apache.hama.HamaCluster;
 import org.apache.hama.HamaConfiguration;
+import org.apache.hama.examples.mapreduce.BlockID;
 import org.apache.hama.examples.mapreduce.BlockMultMap;
 import org.apache.hama.examples.mapreduce.BlockMultReduce;
-import org.apache.hama.io.BlockID;
 import org.apache.hama.matrix.DenseMatrix;
 import org.apache.hama.matrix.DenseVector;
 import org.apache.hama.matrix.Matrix;



Mime
View raw message