incubator-hama-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From edwardy...@apache.org
Subject svn commit: r727296 - in /incubator/hama/trunk: ./ src/java/org/apache/hama/ src/java/org/apache/hama/io/ src/java/org/apache/hama/util/ src/test/org/apache/hama/mapred/
Date Wed, 17 Dec 2008 06:20:40 GMT
Author: edwardyoon
Date: Tue Dec 16 22:20:39 2008
New Revision: 727296

URL: http://svn.apache.org/viewvc?rev=727296&view=rev
Log:
left pad with zeroes to integer key

Modified:
    incubator/hama/trunk/CHANGES.txt
    incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java
    incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java
    incubator/hama/trunk/src/java/org/apache/hama/io/VectorUpdate.java
    incubator/hama/trunk/src/java/org/apache/hama/util/BytesUtil.java
    incubator/hama/trunk/src/test/org/apache/hama/mapred/TestBlockMatrixMapReduce.java

Modified: incubator/hama/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/CHANGES.txt?rev=727296&r1=727295&r2=727296&view=diff
==============================================================================
--- incubator/hama/trunk/CHANGES.txt (original)
+++ incubator/hama/trunk/CHANGES.txt Tue Dec 16 22:20:39 2008
@@ -33,6 +33,7 @@
     
   IMPROVEMENTS
     
+    HAMA-138: To order, left pad with zeroes to integer key (edwardyoon)
     HAMA-135, HAMA-137: Refactor mapred, I/O package (edwardyoon)
     HAMA-134: We don't need to fill C with zeros (edwardyoon)
     HAMA-131: Add argument for the number of blocks (edwardyoon)

Modified: incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java?rev=727296&r1=727295&r2=727296&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java Tue Dec 16 22:20:39
2008
@@ -74,7 +74,7 @@
       this.tableDesc.addFamily(new HColumnDescriptor(Constants.ATTRIBUTE));
       this.tableDesc.addFamily(new HColumnDescriptor(Constants.ALIASEFAMILY));
       this.tableDesc.addFamily(new HColumnDescriptor(Constants.BLOCK));
-      
+
       LOG.info("Initializing the matrix storage.");
       this.admin.createTable(this.tableDesc);
       LOG.info("Create Matrix " + matrixPath);
@@ -131,8 +131,8 @@
 
   public String getRowLabel(int row) throws IOException {
     Cell rows = null;
-    rows = table.get(BytesUtil.intToBytes(row), Bytes.toBytes(Constants.ATTRIBUTE
-        + "string"));
+    rows = table.get(BytesUtil.getRowIndex(row), Bytes
+        .toBytes(Constants.ATTRIBUTE + "string"));
 
     return (rows != null) ? Bytes.toString(rows.getValue()) : null;
   }

Modified: incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java?rev=727296&r1=727295&r2=727296&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java Tue Dec 16 22:20:39 2008
@@ -20,6 +20,7 @@
 package org.apache.hama;
 
 import java.io.IOException;
+import java.util.Iterator;
 
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -337,7 +338,7 @@
    * @throws IOException
    */
   public double get(int i, int j) throws IOException {
-    Cell c = table.get(BytesUtil.intToBytes(i), BytesUtil.getColumnIndex(j));
+    Cell c = table.get(BytesUtil.getRowIndex(i), BytesUtil.getColumnIndex(j));
     return (c != null) ? BytesUtil.bytesToDouble(c.getValue()) : 0;
   }
 
@@ -366,7 +367,7 @@
   }
 
   public DenseVector getRow(int row) throws IOException {
-    return new DenseVector(table.getRow(BytesUtil.intToBytes(row)));
+    return new DenseVector(table.getRow(BytesUtil.getRowIndex(row)));
   }
 
   public DenseVector getColumn(int column) throws IOException {
@@ -449,7 +450,6 @@
     return this.getClass().getSimpleName();
   }
 
-  // TODO: Scanner should be used. -- Edward J. Yoon
   public SubMatrix subMatrix(int i0, int i1, int j0, int j1) throws IOException {
     int columnSize = (j1 - j0) + 1;
     SubMatrix result = new SubMatrix((i1 - i0) + 1, columnSize);
@@ -459,12 +459,17 @@
       cols[jj] = BytesUtil.getColumnIndex(j);
     }
 
+    Scanner scan = table.getScanner(cols, BytesUtil.getRowIndex(i0), 
+        BytesUtil.getRowIndex(i1 + 1));
+    Iterator<RowResult> it = scan.iterator();
+    int i = 0;
     RowResult rs = null;
-    for (int i = i0, ii = 0; i <= i1; i++, ii++) {
-      rs = table.getRow(BytesUtil.intToBytes(i), cols);
+    while (it.hasNext()) {
+      rs = it.next();
       for (int j = j0, jj = 0; j <= j1; j++, jj++) {
-        result.set(ii, jj, rs.get(BytesUtil.getColumnIndex(j)).getValue());
+        result.set(i, jj, rs.get(BytesUtil.getColumnIndex(j)).getValue());
       }
+      i++;
     }
 
     return result;
@@ -476,7 +481,7 @@
   }
 
   public SubMatrix getBlock(int i, int j) throws IOException {
-    return new SubMatrix(table.get(new BlockID(i, j).getBytes(), 
+    return new SubMatrix(table.get(new BlockID(i, j).getBytes(),
         Bytes.toBytes(Constants.BLOCK)).getValue());
   }
 
@@ -521,8 +526,8 @@
           endColumn = this.getColumns() - 1;
 
         BatchUpdate update = new BatchUpdate(new BlockID(i, j).getBytes());
-        update.put(Constants.BLOCK_POSITION, 
-            new BlockPosition(startRow, endRow, startColumn, endColumn).getBytes());
+        update.put(Constants.BLOCK_POSITION, new BlockPosition(startRow,
+            endRow, startColumn, endColumn).getBytes());
         table.commit(update);
 
         j++;
@@ -533,7 +538,7 @@
   }
 
   protected BlockPosition getBlockPosition(int i, int j) throws IOException {
-    byte[] rs = table.get(new BlockID(i, j).getBytes(), 
+    byte[] rs = table.get(new BlockID(i, j).getBytes(),
         Bytes.toBytes(Constants.BLOCK_POSITION)).getValue();
     return new BlockPosition(rs);
   }
@@ -567,17 +572,17 @@
    */
   public void blocking(int blockNum) throws IOException {
     this.checkBlockNum(blockNum);
-    
+
     String[] columns = new String[] { Constants.BLOCK_POSITION };
     Scanner scan = table.getScanner(columns);
 
     for (RowResult row : scan) {
       BlockID bID = new BlockID(row.getRow());
-      BlockPosition pos = 
-        new BlockPosition(row.get(Constants.BLOCK_POSITION).getValue());
-      
-      setBlock(bID.getRow(), bID.getColumn(), subMatrix(pos.getStartRow(), 
-          pos.getEndRow(), pos.getStartColumn(), pos.getEndColumn()));
+      BlockPosition pos = new BlockPosition(row.get(Constants.BLOCK_POSITION)
+          .getValue());
+
+      setBlock(bID.getRow(), bID.getColumn(), subMatrix(pos.getStartRow(), pos
+          .getEndRow(), pos.getStartColumn(), pos.getEndColumn()));
     }
   }
 

Modified: incubator/hama/trunk/src/java/org/apache/hama/io/VectorUpdate.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/io/VectorUpdate.java?rev=727296&r1=727295&r2=727296&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/io/VectorUpdate.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/io/VectorUpdate.java Tue Dec 16 22:20:39
2008
@@ -32,7 +32,7 @@
   private BatchUpdate batchUpdate;
 
   public VectorUpdate(int i) {
-    this.batchUpdate = new BatchUpdate(BytesUtil.intToBytes(i));
+    this.batchUpdate = new BatchUpdate(BytesUtil.getRowIndex(i));
   }
 
   public VectorUpdate(String row) {

Modified: incubator/hama/trunk/src/java/org/apache/hama/util/BytesUtil.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/util/BytesUtil.java?rev=727296&r1=727295&r2=727296&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/util/BytesUtil.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/util/BytesUtil.java Tue Dec 16 22:20:39
2008
@@ -29,6 +29,7 @@
  */
 public class BytesUtil {
   public static final int SIZEOF_DOUBLE = Double.SIZE/Byte.SIZE;
+  public static final int PAD_SIZE = 15; 
   
   /**
    * Bytes to integer conversion
@@ -72,6 +73,47 @@
   }
 
   /**
+   * Gets the row index
+   * 
+   * @param bytes
+   * @return the converted value
+   */
+  public static int getRowIndex(byte[] bytes) {
+    String rKey = new String(bytes);
+    
+    if(rKey.substring(0, 8).equals("00000000")){
+      int i = 8;
+      while (rKey.charAt(i) == '0') {
+        i++;
+      }
+      return Integer.parseInt(rKey.substring(i, rKey.length()));
+    } else {
+      int i = 0;
+      while (rKey.charAt(i) == '0') {
+        i++;
+      }
+      return Integer.parseInt(rKey.substring(i, rKey.length()));
+    }
+  }
+
+  /**
+   * Gets the row index
+   * 
+   * @param integer
+   * @return the converted value
+   */
+  public static byte[] getRowIndex(int integer) {
+    String index = String.valueOf(integer);
+    int zeros = PAD_SIZE - index.length();
+    StringBuffer buf = new StringBuffer();
+    for (int i = 0; i < zeros; ++i) {
+      buf.append("0");
+    }
+    
+    return Bytes.toBytes(buf.toString() + index);
+  }
+  
+  /**
    * Gets the column index
    * 
    * @param bytes

Modified: incubator/hama/trunk/src/test/org/apache/hama/mapred/TestBlockMatrixMapReduce.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/test/org/apache/hama/mapred/TestBlockMatrixMapReduce.java?rev=727296&r1=727295&r2=727296&view=diff
==============================================================================
--- incubator/hama/trunk/src/test/org/apache/hama/mapred/TestBlockMatrixMapReduce.java (original)
+++ incubator/hama/trunk/src/test/org/apache/hama/mapred/TestBlockMatrixMapReduce.java Tue
Dec 16 22:20:39 2008
@@ -35,7 +35,7 @@
 public class TestBlockMatrixMapReduce extends HCluster {
   static final Logger LOG = Logger.getLogger(TestBlockMatrixMapReduce.class);
   static Matrix c;
-  static final int SIZE = 20;
+  static final int SIZE = 32;
 
   /** constructor */
   public TestBlockMatrixMapReduce() {
@@ -46,8 +46,9 @@
       ClassNotFoundException {
     Matrix m1 = DenseMatrix.random(conf, SIZE, SIZE);
     Matrix m2 = DenseMatrix.random(conf, SIZE, SIZE);
-    ((DenseMatrix) m1).blocking_mapred(4);
-    ((DenseMatrix) m2).blocking_mapred(4);
+    // Partitioning 8 * 8 submatrix. It also the test submatrix() and scanner.
+    ((DenseMatrix) m1).blocking_mapred(16);
+    ((DenseMatrix) m2).blocking_mapred(16);
 
     miniMRJob(m1.getPath(), m2.getPath());
 



Mime
View raw message