hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Apache Wiki <wikidi...@apache.org>
Subject [Hadoop Wiki] Update of "DataProcessingBenchmarks" by udanax
Date Thu, 24 Jan 2008 10:40:35 GMT
Dear Wiki user,

You have subscribed to a wiki page or wiki category on "Hadoop Wiki" for change notification.

The following page has been changed by udanax:
http://wiki.apache.org/hadoop/DataProcessingBenchmarks

------------------------------------------------------------------------------
  [[TableOfContents(4)]]
  ----
- == Data Processing Benchmarks ==
+ == Hadoop Map/Reduce Data Processing Benchmarks ==
  
   * Reporter : [:udanax:Edward Yoon]
    * TODO-LIST :
@@ -43, +43 @@

   * '''m''': number of data points; '''n''': number of features; '''P''': number of mappers;
   * ''~+O+~(mn^2^/P + n^3^/P' + n^2^log(p))''
  
+ == Hbase Matrix Computing Benchmarks ==
+ 
+ === Matrix Multiplication ===
+ {{{ }}}For i:=0 step 1 until N-1 (Find the i-th row of C)[[BR]]
+ {{{       }}}begin[[BR]]
+ {{{       }}}C[i,j] := 0; (o<=j<=n-1)[[BR]]
+ {{{           }}}For k:=0 step 1 until N-1 (Find and add the k-th)[[BR]]
+ {{{               }}}C[i,j]:=C[i,j] + A[i,k]*B[k,j]; (0<=j<=n-1)[[BR]]
+ {{{       }}}end;[[BR]]
+ 
+ ==== MapReduce Flow ====
+ {{{
+ Map:
+ 
+   public void map(HStoreKey key, MapWritable value,
+       OutputCollector<Text, MapWritable> output, Reporter reporter)
+       throws IOException {
+     Text tKey = key.getRow();
+     MapWritable col = new MapWritable();
+ 
+     for (Map.Entry<Writable, Writable> e : value.entrySet()) {
+       Text row = new Text(e.getKey().toString().replace(Matrix.C, ""));
+       double c = Matrix.toDouble(((ImmutableBytesWritable) e.getValue()).get());
+       
+       for (Map.Entry<Text, byte[]> f : table.getRow(row).entrySet()) {
+         byte[] multiplied = Matrix.toBytes(Matrix.toDouble(f.getValue()) * c);
+         Writable val = new ImmutableBytesWritable(multiplied);
+         col.put(f.getKey(), val);
+       }
+ 
+       output.collect(tKey, col);
+     }
+   }
+ 
+ Reduce:
+ 
+   public void reduce(Text key, Iterator<MapWritable> values,
+       OutputCollector<Text, MapWritable> output, Reporter reporter)
+       throws IOException {
+     Map<String, Double> merge = new HashMap<String, Double>();
+ 
+     while (values.hasNext()) {
+       for (Map.Entry<Writable, Writable> e : values.next().entrySet()) {
+         if (merge.containsKey(e.getKey().toString())) {
+           double a = merge.get(e.getKey().toString());
+           merge.put(e.getKey().toString(), a + getDouble(e.getValue()));
+         } else {
+           merge.put(e.getKey().toString(), getDouble(e.getValue()));
+         }
+       }
+     }
+ 
+     MapWritable r = new MapWritable();
+     for (Map.Entry<String, Double> f : merge.entrySet()) {
+       Writable val = new ImmutableBytesWritable(Matrix.toBytes(f.getValue()));
+       r.put(new Text(f.getKey()), val);
+     }
+     output.collect(key, r);
+   }
+ 
+ }}}
+ 

Mime
View raw message