hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tomwh...@apache.org
Subject svn commit: r519461 [3/3] - in /lucene/hadoop/branches/branch-0.12: ./ bin/ docs/ lib/ src/c++/librecordio/ src/c++/librecordio/test/ src/contrib/streaming/src/java/org/apache/hadoop/streaming/ src/docs/src/documentation/content/xdocs/ src/java/org/apa...
Date Sat, 17 Mar 2007 22:15:24 GMT
Modified: lucene/hadoop/branches/branch-0.12/src/docs/src/documentation/content/xdocs/index.xml
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/docs/src/documentation/content/xdocs/index.xml?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/docs/src/documentation/content/xdocs/index.xml (original)
+++ lucene/hadoop/branches/branch-0.12/src/docs/src/documentation/content/xdocs/index.xml Sat Mar 17 15:15:19 2007
@@ -15,6 +15,15 @@
       <title>News</title>
 
       <section>
+      <title>17 March, 2007: release 0.12.1 available</title>
+      <p>For details see the <a
+      href="http://tinyurl.com/2kynuc">release notes</a>. The release can
+      be obtained from <a
+      href="http://www.apache.org/dyn/closer.cgi/lucene/hadoop/"> a
+      nearby mirror</a>.
+      </p> </section>
+
+      <section>
       <title>2 March, 2007: release 0.12.0 available</title>
       <p>For details see the <a
       href="http://tinyurl.com/2vp248">release notes</a>. The release can

Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/dfs/FSDataset.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/dfs/FSDataset.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/dfs/FSDataset.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/dfs/FSDataset.java Sat Mar 17 15:15:19 2007
@@ -42,17 +42,13 @@
     class FSDir {
         File dir;
         int numBlocks = 0;
-        int myIdx = 0;
         FSDir children[];
-        FSDir siblings[];
-
+        int lastChildIdx = 0;
         /**
          */
-        public FSDir(File dir, int myIdx, FSDir[] siblings) 
+        public FSDir(File dir) 
             throws IOException {
             this.dir = dir;
-            this.myIdx = myIdx;
-            this.siblings = siblings;
             this.children = null;
             if (! dir.exists()) {
               if (! dir.mkdirs()) {
@@ -74,36 +70,61 @@
                 int curdir = 0;
                 for (int idx = 0; idx < files.length; idx++) {
                   if (files[idx].isDirectory()) {
-                    children[curdir] = new FSDir(files[idx], curdir, children);
+                    children[curdir] = new FSDir(files[idx]);
                     curdir++;
                   }
                 }
               }
             }
         }
+        
+        public File addBlock( Block b, File src ) throws IOException {
+          //First try without creating subdirectories
+          File file = addBlock( b, src, false, false );          
+          return ( file != null ) ? file : addBlock( b, src, true, true );
+        }
 
-        /**
-         */
-        public File addBlock(Block b, File src) throws IOException {
+        private File addBlock( Block b, File src, boolean createOk, 
+                               boolean resetIdx ) throws IOException {
             if (numBlocks < maxBlocksPerDir) {
               File dest = new File(dir, b.getBlockName());
               src.renameTo(dest);
               numBlocks += 1;
               return dest;
-            } else {
-              if (siblings != null && myIdx != (siblings.length-1)) {
-                File dest = siblings[myIdx+1].addBlock(b, src);
-                if (dest != null) { return dest; }
-              }
-              if (children == null) {
-                children = new FSDir[maxBlocksPerDir];
-                for (int idx = 0; idx < maxBlocksPerDir; idx++) {
-                  children[idx] = new FSDir(
-                      new File(dir, "subdir"+idx), idx, children);
+            }
+            
+            if ( lastChildIdx < 0 && resetIdx ) {
+              //reset so that all children will be checked
+              lastChildIdx = random.nextInt( children.length );              
+            }
+            
+            if ( lastChildIdx >= 0 && children != null ) {
+              //Check if any child-tree has room for a block.
+              for (int i=0; i < children.length; i++) {
+                int idx = ( lastChildIdx + i )%children.length;
+                File file = children[idx].addBlock( b, src, false, resetIdx );
+                if ( file != null ) {
+                  lastChildIdx = idx;
+                  return file; 
                 }
               }
-              return children[0].addBlock(b, src);
+              lastChildIdx = -1;
+            }
+            
+            if ( !createOk ) {
+              return null;
+            }
+            
+            if ( children == null || children.length == 0 ) {
+              children = new FSDir[maxBlocksPerDir];
+              for (int idx = 0; idx < maxBlocksPerDir; idx++) {
+                children[idx] = new FSDir( new File(dir, "subdir"+idx) );
+              }
             }
+            
+            //now pick a child randomly for creating a new set of subdirs.
+            lastChildIdx = random.nextInt( children.length );
+            return children[ lastChildIdx ].addBlock( b, src, true, false ); 
         }
 
         /**
@@ -171,13 +192,57 @@
         }
         
         void clearPath(File f) {
-          if (dir.compareTo(f) == 0) numBlocks--;
-          else {
-            if ((siblings != null) && (myIdx != (siblings.length - 1)))
-              siblings[myIdx + 1].clearPath(f);
-            else if (children != null)
-              children[0].clearPath(f);
+          String root = dir.getAbsolutePath();
+          String dir = f.getAbsolutePath();
+          if ( dir.startsWith( root ) ) {
+            String[] dirNames = dir.substring( root.length() ).
+                         split( File.separator + "subdir" );
+            if ( clearPath( f, dirNames, 1 ) )
+              return;
+          }
+          clearPath( f, null, -1 );
+        }
+        
+        /*
+         * dirNames is an array of string integers derived from
+         * usual directory structure data/subdirN/subdirXY/subdirM ...
+         * If dirName array is non-null, we only check the child at 
+         * the children[dirNames[idx]]. This avoids iterating over
+         * children in common case. If directory structure changes 
+         * in later versions, we need to revisit this.
+         */
+        private boolean clearPath( File f, String[] dirNames, int idx ) {
+          if ( ( dirNames == null || idx == dirNames.length ) &&
+               dir.compareTo(f) == 0) {
+            numBlocks--;
+            return true;
+          }
+          
+          if ( dirNames != null ) {
+            //guess the child index from the directory name
+            if ( idx > ( dirNames.length - 1 ) || children == null ) {
+              return false;
+            }
+            int childIdx; 
+            try {
+              childIdx = Integer.parseInt( dirNames[idx] );
+            } catch ( NumberFormatException ignored ) {
+              // layout changed? we could print a warning.
+              return false;
+            }
+            return ( childIdx >= 0 && childIdx < children.length ) ?
+              children[childIdx].clearPath( f, dirNames, idx+1 ) : false;
+          }
+
+          //guesses failed. back to blind iteration.
+          if ( children != null ) {
+            for(int i=0; i < children.length; i++) {
+              if ( children[i].clearPath( f, null, -1 ) ){
+                return true;
+              }
+            }
           }
+          return false;
         }
         
         public String toString() {
@@ -203,7 +268,7 @@
         this.usableDiskPct = conf.getFloat("dfs.datanode.du.pct",
             (float) USABLE_DISK_PCT_DEFAULT);
         this.dir = dir;
-        this.dataDir = new FSDir(new File(dir, "data"), 0, null);
+        this.dataDir = new FSDir(new File(dir, "data"));
         this.tmpDir = new File(dir, "tmp");
         if (tmpDir.exists()) {
           FileUtil.fullyDelete(tmpDir);
@@ -361,6 +426,7 @@
     private int maxBlocksPerDir = 0;
     private HashMap<Block,FSVolume> volumeMap = null;
     private HashMap<Block,File> blockMap = null;
+    static  Random random = new Random();
 
     /**
      * An FSDataset has a directory where it loads its data files.

Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/dfs/FSNamesystem.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/dfs/FSNamesystem.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/dfs/FSNamesystem.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/dfs/FSNamesystem.java Sat Mar 17 15:15:19 2007
@@ -155,7 +155,7 @@
     // We also store pending replication-orders.
     // Set of: Block
     //
-    private UnderReplicationBlocks neededReplications = new UnderReplicationBlocks();
+    private UnderReplicatedBlocks neededReplications = new UnderReplicatedBlocks();
     private PendingReplicationBlocks pendingReplications;
 
     //
@@ -334,12 +334,12 @@
      * Blocks have replication priority, with priority 0 indicating the highest
      * Blocks have only one replicas has the highest
      */
-    private class UnderReplicationBlocks {
+    private class UnderReplicatedBlocks {
         private static final int LEVEL = 3;
         TreeSet<Block>[] priorityQueues = new TreeSet[LEVEL];
         
         /* constructor */
-        UnderReplicationBlocks() {
+        UnderReplicatedBlocks() {
             for(int i=0; i<LEVEL; i++) {
                 priorityQueues[i] = new TreeSet<Block>();
             }
@@ -369,7 +369,7 @@
         */
         private int getPriority(Block block, 
                 int curReplicas, int expectedReplicas) {
-            if (curReplicas>=expectedReplicas) {
+            if (curReplicas<=0 || curReplicas>=expectedReplicas) {
                 return LEVEL; // no need to replicate
             } else if(curReplicas==1) {
                 return 0; // highest priority
@@ -414,16 +414,14 @@
         /* remove a block from a under replication queue */
         synchronized boolean remove(Block block, 
                 int oldReplicas, int oldExpectedReplicas) {
-            if(oldExpectedReplicas <= oldReplicas) {
-                return false;
-            }
             int priLevel = getPriority(block, oldReplicas, oldExpectedReplicas);
             return remove(block, priLevel);
         }
         
         /* remove a block from a under replication queue given a priority*/
         private boolean remove(Block block, int priLevel ) {
-            if( priorityQueues[priLevel].remove(block) ) {
+            if( priLevel >= 0 && priLevel < LEVEL 
+                    && priorityQueues[priLevel].remove(block) ) {
                 NameNode.stateChangeLog.debug(
                      "BLOCK* NameSystem.UnderReplicationBlock.remove: "
                    + "Removing block " + block.getBlockName()
@@ -1562,6 +1560,7 @@
       DatanodeDescriptor nodeDescr 
               = new DatanodeDescriptor( nodeReg, networkLocation, hostName );
       unprotectedAddDatanode( nodeDescr );
+      clusterMap.add(nodeDescr);
       getEditLog().logAddDatanode( nodeDescr );
       
       // also treat the registration message as a heartbeat
@@ -1890,7 +1889,6 @@
     
     void unprotectedAddDatanode( DatanodeDescriptor nodeDescr ) {
       datanodeMap.put( nodeDescr.getStorageID(), nodeDescr );
-      clusterMap.add(nodeDescr);
       NameNode.stateChangeLog.debug(
           "BLOCK* NameSystem.unprotectedAddDatanode: "
           + "node " + nodeDescr.getName() + " is added to datanodeMap." );
@@ -2115,16 +2113,19 @@
             return block;
         
         // filter out containingNodes that are marked for decommission.
-        int numCurrentReplica = countContainingNodes(containingNodes);
+        int numCurrentReplica = countContainingNodes(containingNodes)
+                              + pendingReplications.getNumReplicas(block);
         
         // check whether safe replication is reached for the block
         // only if it is a part of a files
         incrementSafeBlockCount( numCurrentReplica );
-        
+ 
         // handle underReplication/overReplication
         short fileReplication = fileINode.getReplication();
-        if(neededReplications.contains(block)) {
-            neededReplications.update(block, curReplicaDelta, 0);
+        if (numCurrentReplica >= fileReplication) {
+          neededReplications.remove(block);
+        } else {
+          neededReplications.update(block, curReplicaDelta, 0);
         }
         proccessOverReplicatedBlock( block, fileReplication );
         return block;
@@ -2501,6 +2502,7 @@
      * that are marked for decommission.
      */
     private int countContainingNodes(Collection<DatanodeDescriptor> nodelist) {
+      if( nodelist == null ) return 0;
       int count = 0;
       for (Iterator<DatanodeDescriptor> it = nodelist.iterator(); 
            it.hasNext(); ) {
@@ -2638,17 +2640,21 @@
                   filterDecommissionedNodes(containingNodes);
               int numCurrentReplica = nodes.size() +
                                       pendingReplications.getNumReplicas(block);
-              DatanodeDescriptor targets[] = replicator.chooseTarget(
+              if (numCurrentReplica >= fileINode.getReplication()) {
+                it.remove();
+              } else {
+                DatanodeDescriptor targets[] = replicator.chooseTarget(
                   Math.min( fileINode.getReplication() - numCurrentReplica,
                             needed),
                   datanodeMap.get(srcNode.getStorageID()),
                   nodes, null, blockSize);
-              if (targets.length > 0) {
-                // Build items to return
-                replicateBlocks.add(block);
-                numCurrentReplicas.add(new Integer(numCurrentReplica));
-                replicateTargetSets.add(targets);
-                needed -= targets.length;
+                if (targets.length > 0) {
+                  // Build items to return
+                  replicateBlocks.add(block);
+                  numCurrentReplicas.add(new Integer(numCurrentReplica));
+                  replicateTargetSets.add(targets);
+                  needed -= targets.length;
+                }
               }
             }
           }
@@ -3528,6 +3534,8 @@
         NameNode.stateChangeLog.info("STATE* Network topology has "
                 +clusterMap.getNumOfRacks()+" racks and "
                 +clusterMap.getNumOfLeaves()+ " datanodes");
+        NameNode.stateChangeLog.info("STATE* UnderReplicatedBlocks has "
+                +neededReplications.size()+" blocks" );
       }
       
       /** 

Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/fs/ChecksumFileSystem.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/fs/ChecksumFileSystem.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/fs/ChecksumFileSystem.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/fs/ChecksumFileSystem.java Sat Mar 17 15:15:19 2007
@@ -135,11 +135,15 @@
       long checksumBoundary = desired/bytesPerSum*bytesPerSum;
       if(checksumBoundary != getPos()) {
         datas.seek(checksumBoundary);
-        sums.seek(HEADER_LENGTH + 4*(checksumBoundary/bytesPerSum));
+        if(sums != null) {
+          sums.seek(HEADER_LENGTH + 4*(checksumBoundary/bytesPerSum));
+        }
       }
       
-      sum.reset();
-      inSum = 0;
+      if(sums != null) {
+        sum.reset();
+        inSum = 0;
+      }
       
       // scan to desired position
       int delta = (int)(desired - checksumBoundary);
@@ -217,14 +221,14 @@
             datas.seek(oldPos);
             
             if (seekToNewSource(oldPos)) {
-              // Neither the data stream nor the checksum stream are being read
-              // from different sources, meaning we'll still get a checksum error 
-              // if we try to do the read again.  We throw an exception instead.
-              throw ce;
+                // Since at least one of the sources is different, 
+                // the read might succeed, so we'll retry.
+                retry = true;
             } else {
-              // Since at least one of the sources is different, 
-              // the read might succeed, so we'll retry.
-              retry = true;
+                // Neither the data stream nor the checksum stream are being read
+                // from different sources, meaning we'll still get a checksum error 
+                // if we try to do the read again.  We throw an exception instead.
+                throw ce;
             }
           }
         }
@@ -307,7 +311,7 @@
 
     @Override
       public boolean seekToNewSource(long targetPos) throws IOException {
-      return datas.seekToNewSource(targetPos) &&
+      return datas.seekToNewSource(targetPos) ||
         sums.seekToNewSource(targetPos/bytesPerSum);
     }
 
@@ -397,8 +401,10 @@
     
     public void close() throws IOException {
       writeSum();
-      sums.close();
-      super.close();
+      if(sums != null) {
+        sums.close();
+      }
+      out.close();
     }
     
     public static long getChecksumLength(long size, int bytesPerSum) {
@@ -599,7 +605,7 @@
     } else {
       Path[] srcs = listPaths(src);
       for (Path srcFile : srcs) {
-        copyToLocalFile(srcFile, dst, copyCrc);
+        copyToLocalFile(srcFile, new Path(dst, srcFile.getName()), copyCrc);
       }
     }
   }

Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/fs/FSDataOutputStream.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/fs/FSDataOutputStream.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/fs/FSDataOutputStream.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/fs/FSDataOutputStream.java Sat Mar 17 15:15:19 2007
@@ -43,6 +43,10 @@
       return position;                            // return cached position
     }
     
+    public void close() throws IOException {
+      flush();
+      out.close();
+    }
   }
 
   private static class Buffer extends BufferedOutputStream {
@@ -62,6 +66,11 @@
         buf[count++] = (byte)b;
       }
     }
+
+    public void close() throws IOException {
+      flush();
+      out.close();
+    }
   }
 
   public FSDataOutputStream(OutputStream out, int bufferSize)
@@ -76,5 +85,10 @@
 
   public long getPos() throws IOException {
     return ((Buffer)out).getPos();
+  }
+
+  public void close() throws IOException {
+    flush();
+    out.close();
   }
 }

Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/io/WritableComparator.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/io/WritableComparator.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/io/WritableComparator.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/io/WritableComparator.java Sat Mar 17 15:15:19 2007
@@ -187,7 +187,7 @@
           i = i << 8;
           i = i | (bytes[start+1+idx] & 0xFF);
       }
-      return (isNegative ? (i | 0x8000000000000000L) : i);
+      return (isNegative ? (i ^ -1L) : i);
   }
   
   /**

Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/io/WritableUtils.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/io/WritableUtils.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/io/WritableUtils.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/io/WritableUtils.java Sat Mar 17 15:15:19 2007
@@ -279,7 +279,7 @@
       
       int len = -112;
       if (i < 0) {
-          i &= 0x7FFFFFFFFFFFFFFFL; // reset the sign bit
+          i ^= -1L; // take one's complement'
           len = -120;
       }
       
@@ -320,7 +320,7 @@
           i = i << 8;
           i = i | (b & 0xFF);
       }
-      return (isNegative ? (i | 0x8000000000000000L) : i);
+      return (isNegative ? (i ^ -1L) : i);
   }
 
   /**
@@ -345,7 +345,7 @@
       
       int len = -112;
       if (i < 0) {
-          i &= 0x7FFFFFFFFFFFFFFFL; // reset the sign bit
+          i ^= -1L; // take one's complement'
           len = -120;
       }
       

Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/mapred/Counters.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/mapred/Counters.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/mapred/Counters.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/mapred/Counters.java Sat Mar 17 15:15:19 2007
@@ -21,13 +21,13 @@
 import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.MissingResourceException;
 import java.util.ResourceBundle;
-import java.util.SortedMap;
 import java.util.TreeMap;
 
 import org.apache.commons.logging.*;
@@ -174,7 +174,7 @@
    * @return Set of counter names.
    */
   public synchronized Collection<String> getGroupNames() {
-    return counters.keySet();
+    return new ArrayList<String>(counters.keySet());
   }
   
   /**

Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/mapred/JobClient.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/mapred/JobClient.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/mapred/JobClient.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/mapred/JobClient.java Sat Mar 17 15:15:19 2007
@@ -617,11 +617,14 @@
 
     private static void displayTaskLogs(String taskId, String baseUrl)
     throws IOException {
-      // Copy tasks's stdout of the JobClient
-      getTaskLogs(taskId, new URL(baseUrl+"&filter=stdout"), System.out);
-      
-      // Copy task's stderr to stderr of the JobClient 
-      getTaskLogs(taskId, new URL(baseUrl+"&filter=stderr"), System.err);
+      // The tasktracker for a 'failed/killed' job might not be around...
+      if (baseUrl != null) {
+        // Copy tasks's stdout of the JobClient
+        getTaskLogs(taskId, new URL(baseUrl+"&filter=stdout"), System.out);
+        
+        // Copy task's stderr to stderr of the JobClient 
+        getTaskLogs(taskId, new URL(baseUrl+"&filter=stderr"), System.err);
+      }
     }
     
     private static void getTaskLogs(String taskId, URL taskLogUrl, 

Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/mapred/JobInProgress.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/mapred/JobInProgress.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/mapred/JobInProgress.java Sat Mar 17 15:15:19 2007
@@ -315,26 +315,29 @@
               ttStatus.getHttpPort() + "/tasklog.jsp?plaintext=true&taskid=" +
               status.getTaskId() + "&all=true";
           }
-          
+
+          TaskCompletionEvent taskEvent = null;
           if (state == TaskStatus.State.SUCCEEDED) {
-            this.taskCompletionEvents.add( new TaskCompletionEvent(
-                taskCompletionEventTracker, 
-                status.getTaskId(),
-                tip.idWithinJob(),
-                status.getIsMap(),
-                TaskCompletionEvent.Status.SUCCEEDED,
-                httpTaskLogLocation ));
+            taskEvent = new TaskCompletionEvent(
+                          taskCompletionEventTracker, 
+                          status.getTaskId(),
+                          tip.idWithinJob(),
+                          status.getIsMap(),
+                          TaskCompletionEvent.Status.SUCCEEDED,
+                          httpTaskLogLocation 
+                          );
             tip.setSuccessEventNumber(taskCompletionEventTracker);
             completedTask(tip, status, metrics);
           } else if (state == TaskStatus.State.FAILED ||
                      state == TaskStatus.State.KILLED) {
-            this.taskCompletionEvents.add( new TaskCompletionEvent(
-                taskCompletionEventTracker, 
-                status.getTaskId(),
-                tip.idWithinJob(),
-                status.getIsMap(),
-                TaskCompletionEvent.Status.FAILED, 
-                httpTaskLogLocation ));
+            taskEvent = new TaskCompletionEvent(
+                          taskCompletionEventTracker, 
+                          status.getTaskId(),
+                          tip.idWithinJob(),
+                          status.getIsMap(),
+                          TaskCompletionEvent.Status.FAILED, 
+                          httpTaskLogLocation
+                          );
             // Get the event number for the (possibly) previously successful
             // task. If there exists one, then set that status to OBSOLETE 
             int eventNumber;
@@ -348,9 +351,13 @@
             failedTask(tip, status.getTaskId(), status, status.getTaskTracker(),
                        wasRunning, wasComplete);
           }          
-        }
 
-        taskCompletionEventTracker++;
+          // Add the 'complete' task i.e. successful/failed
+          if (taskEvent != null) {
+            this.taskCompletionEvents.add(taskEvent);
+            taskCompletionEventTracker++;
+          }
+        }
         
         //
         // Update JobInProgress status
@@ -936,8 +943,8 @@
        return null;
     }
     
-    public TaskCompletionEvent[] getTaskCompletionEvents(int fromEventId, 
-        int maxEvents) {
+    synchronized public TaskCompletionEvent[] getTaskCompletionEvents(
+            int fromEventId, int maxEvents) {
       TaskCompletionEvent[] events = TaskCompletionEvent.EMPTY_ARRAY;
       if( taskCompletionEvents.size() > fromEventId) {
         int actualMax = Math.min(maxEvents, 

Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/mapred/LineRecordReader.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/mapred/LineRecordReader.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/mapred/LineRecordReader.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/mapred/LineRecordReader.java Sat Mar 17 15:15:19 2007
@@ -113,7 +113,7 @@
     if (start == end) {
       return 0.0f;
     } else {
-      return (pos - start) / (end - start);
+      return Math.min(1.0f, (pos - start) / (float)(end - start));
     }
   }
   

Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/mapred/ReduceTaskRunner.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/mapred/ReduceTaskRunner.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/mapred/ReduceTaskRunner.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/mapred/ReduceTaskRunner.java Sat Mar 17 15:15:19 2007
@@ -144,6 +144,12 @@
    */
   private Map<Integer, MapOutputLocation> retryFetches = new HashMap();
   
+  /** 
+   * a TreeSet for needed map outputs
+   */
+  private Set <Integer> neededOutputs = 
+    Collections.synchronizedSet(new TreeSet<Integer>());
+  
   /** Represents the result of an attempt to copy a map output */
   private class CopyResult {
     
@@ -152,7 +158,10 @@
     
     // the size of the file copied, -1 if the transfer failed
     private final long size;
-        
+    
+    //a flag signifying whether a copy result is obsolete
+    private static final int OBSOLETE = -2;
+    
     CopyResult(MapOutputLocation loc, long size) {
       this.loc = loc;
       this.size = size;
@@ -160,6 +169,9 @@
     
     public int getMapId() { return loc.getMapId(); }
     public boolean getSuccess() { return size >= 0; }
+    public boolean isObsolete() { 
+      return size == OBSOLETE;
+    }
     public long getSize() { return size; }
     public String getHost() { return loc.getHost(); }
     public MapOutputLocation getLocation() { return loc; }
@@ -284,7 +296,9 @@
      */
     private long copyOutput(MapOutputLocation loc
                             ) throws IOException, InterruptedException {
-
+      if (!neededOutputs.contains(loc.getMapId())) {
+        return CopyResult.OBSOLETE;
+      }
       String reduceId = reduceTask.getTaskId();
       LOG.info(reduceId + " Copying " + loc.getMapTaskId() +
                " output from " + loc.getHost() + ".");
@@ -297,16 +311,28 @@
       tmpFilename = loc.getFile(inMemFileSys, localFileSys, shuffleMetrics,
                                tmpFilename, reduceTask.getPartition(),
                                STALLED_COPY_TIMEOUT);
+      if (!neededOutputs.contains(loc.getMapId())) {
+        if (tmpFilename != null) {
+          FileSystem fs = tmpFilename.getFileSystem(conf);
+          fs.delete(tmpFilename);
+        }
+        return CopyResult.OBSOLETE;
+      }
       if (tmpFilename == null)
         throw new IOException("File " + finalFilename + "-" + id + 
                               " not created");
       long bytes = -1;
       // lock the ReduceTaskRunner while we do the rename
       synchronized (ReduceTaskRunner.this) {
-        // if we can't rename the file, something is broken (and IOException
-        // will be thrown). This file could have been created in the inmemory
+        // This file could have been created in the inmemory
         // fs or the localfs. So need to get the filesystem owning the path. 
         FileSystem fs = tmpFilename.getFileSystem(conf);
+        if (!neededOutputs.contains(loc.getMapId())) {
+          fs.delete(tmpFilename);
+          return CopyResult.OBSOLETE;
+        }
+        // if we can't rename the file, something is broken (and IOException
+        // will be thrown). 
         if (!fs.rename(tmpFilename, finalFilename)) {
           fs.delete(tmpFilename);
           throw new IOException("failure to rename map output " + tmpFilename);
@@ -332,6 +358,7 @@
           mergeInProgress = true;
           m.start();
         }
+        neededOutputs.remove(loc.getMapId());
       }
       return bytes;
     }
@@ -424,7 +451,6 @@
     this.mapOutputFile.removeAll(reduceTask.getTaskId());
     
     final int      numOutputs = reduceTask.getNumMaps();
-    List           neededOutputs = new ArrayList(numOutputs);
     Map<Integer, MapOutputLocation> knownOutputs = 
                                     new HashMap<Integer, MapOutputLocation>();
     int            numInFlight = 0, numCopied = 0;
@@ -484,23 +510,12 @@
           List <MapOutputLocation> locs = queryJobTracker(fromEventId, 
                                                           jobClient);
           
-          // remove discovered outputs from needed list
-          // and put them on the known list
-          int gotLocs = (locs == null ? 0 : locs.size());
+          // put discovered them on the known list
           for (int i=0; i < locs.size(); i++) {
-            // check whether we actually need an output. It could happen
-            // that a map task that successfully ran earlier got lost, but
-            // if we already have copied the output of that unfortunate task
-            // we need not copy it again from the new TT (we will ignore 
-            // the event for the new rescheduled execution)
-            if(neededOutputs.remove(new Integer(locs.get(i).getMapId()))) {
-              knownOutputs.put(new Integer(locs.get(i).getMapId()), locs.get(i));
-            }
-            else gotLocs--; //we don't need this output
-            
+            knownOutputs.put(new Integer(locs.get(i).getMapId()), locs.get(i));
           }
           LOG.info(reduceTask.getTaskId() +
-                " Got " + gotLocs + 
+                " Got " + locs.size() + 
                 " new map outputs from jobtracker and " + retryFetches.size() +
                 " map outputs from previous failures");
           // clear the "failed" fetches hashmap
@@ -575,9 +590,13 @@
             copyPhase.setStatus("copy (" + numCopied + " of " + numOutputs + 
                                 " at " +
                                 mbpsFormat.format(transferRate) +  " MB/s)");          
+          } else if (cr.isObsolete()) {
+            //ignore
+            LOG.info(reduceTask.getTaskId() + 
+                " Ignoring obsolete copy result for Map Task: " + 
+                cr.getLocation().getMapTaskId() + " from host: " + 
+                cr.getHost());
           } else {
-            // this copy failed, put it back onto neededOutputs
-            neededOutputs.add(new Integer(cr.getMapId()));
             retryFetches.put(new Integer(cr.getMapId()), cr.getLocation());
           
             // wait a random amount of time for next contact
@@ -600,7 +619,6 @@
               if (cr.getHost().equals(loc.getHost())) {
                 retryFetches.put(new Integer(loc.getMapId()), loc);
                 locIt.remove();
-                neededOutputs.add(new Integer(loc.getMapId()));
               }
             }
           }

Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/mapred/SequenceFileRecordReader.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/mapred/SequenceFileRecordReader.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/mapred/SequenceFileRecordReader.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/mapred/SequenceFileRecordReader.java Sat Mar 17 15:15:19 2007
@@ -44,7 +44,8 @@
     if (split.getStart() > in.getPosition())
       in.sync(split.getStart());                  // sync to start
 
-    more = in.getPosition() < end;
+    this.start = in.getPosition();
+    more = start < end;
   }
 
 
@@ -104,7 +105,7 @@
     if (end == start) {
       return 0.0f;
     } else {
-      return (in.getPosition() - start) / (end - start);
+      return Math.min(1.0f, (in.getPosition() - start) / (float)(end - start));
     }
   }
   

Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/metrics/spi/AbstractMetricsContext.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/metrics/spi/AbstractMetricsContext.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/metrics/spi/AbstractMetricsContext.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/metrics/spi/AbstractMetricsContext.java Sat Mar 17 15:15:19 2007
@@ -323,7 +323,7 @@
         //Map<String,MetricValue> metricUpdates = record.getMetricTable();
         Map metricUpdates = record.getMetricTable();
         
-        RecordMap recordMap = (RecordMap) bufferedData.get(recordName);
+        RecordMap recordMap = getRecordMap(recordName);
         synchronized (recordMap) {
             MetricMap metricMap = (MetricMap) recordMap.get(tagTable);
             if (metricMap == null) {
@@ -349,6 +349,10 @@
         }
     }
     
+    private synchronized RecordMap getRecordMap(String recordName) {
+        return (RecordMap) bufferedData.get(recordName);
+    }
+    
     /**
      * Adds two numbers, coercing the second to the type of the first.
      *
@@ -382,7 +386,7 @@
         String recordName = record.getRecordName();
         TagMap tagTable = record.getTagTable();
         
-        RecordMap recordMap = (RecordMap) bufferedData.get(recordName);
+        RecordMap recordMap = getRecordMap(recordName);
         synchronized (recordMap) {
             recordMap.remove(tagTable);
         }

Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/metrics/spi/MetricsRecordImpl.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/metrics/spi/MetricsRecordImpl.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/metrics/spi/MetricsRecordImpl.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/metrics/spi/MetricsRecordImpl.java Sat Mar 17 15:15:19 2007
@@ -64,6 +64,9 @@
      * @throws MetricsException if the tagName conflicts with the configuration
      */
     public void setTag(String tagName, String tagValue) {
+        if (tagValue == null) {
+            tagValue = "";
+        }
         tagTable.put(tagName, tagValue);
     }
     

Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/net/NetworkTopology.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/net/NetworkTopology.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/net/NetworkTopology.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/net/NetworkTopology.java Sat Mar 17 15:15:19 2007
@@ -197,9 +197,7 @@
                     }
                 }
                 if(parentNode==null) {
-                    throw new IllegalArgumentException( n.getName()
-                            + ", which is located at "
-                            + parent+", is not a decendent of " + currentPath);
+                    return false;
                 }
                 // remove n from the parent node
                 boolean isRemoved = parentNode.remove( n );

Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/Record.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/Record.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/Record.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/Record.java Sat Mar 17 15:15:19 2007
@@ -18,16 +18,62 @@
 
 package org.apache.hadoop.record;
 
+import java.io.DataInput;
+import java.io.DataOutput;
 import java.io.IOException;
+import org.apache.hadoop.io.WritableComparable;
 
 /**
- * Interface that is implemented by generated classes.
+ * Abstract class that is extended by generated classes.
  * 
  * @author Milind Bhandarkar
  */
-public interface Record extends Cloneable {
-    void serialize(OutputArchive archive, String tag)
-        throws IOException;
-    void deserialize(InputArchive archive, String tag)
-        throws IOException;
+public abstract class Record implements WritableComparable, Cloneable {
+  
+  /**
+   * Serialize a record with tag (ususally field name)
+   * @param rout Record output destination
+   * @param tag record tag (Used only in tagged serialization e.g. XML)
+   */
+  public abstract void serialize(RecordOutput rout, String tag)
+  throws IOException;
+  
+  /**
+   * Deserialize a record with a tag (usually field name)
+   * @param rin Record input source
+   * @param tag Record tag (Used only in tagged serialization e.g. XML)
+   */
+  public abstract void deserialize(RecordInput rin, String tag)
+  throws IOException;
+  
+  // inheric javadoc
+  public abstract int compareTo (final Object peer) throws ClassCastException;
+  
+  /**
+   * Serialize a record without a tag
+   * @param rout Record output destination
+   */
+  public void serialize(RecordOutput rout) throws IOException {
+    this.serialize(rout, "");
+  }
+  
+  /**
+   * Deserialize a record without a tag
+   * @param rin Record input source
+   */
+  public void deserialize(RecordInput rin) throws IOException {
+    this.deserialize(rin, "");
+  }
+  
+  // inherit javadoc
+  public void write(final DataOutput out) throws java.io.IOException {
+    BinaryRecordOutput bout = new BinaryRecordOutput(out);
+    this.serialize(bout);
+  }
+  
+  // inherit javadoc
+  public void readFields(final DataInput din) throws java.io.IOException {
+    BinaryRecordInput rin = new BinaryRecordInput(din);
+    this.deserialize(rin);
+  }
 }

Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/Utils.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/Utils.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/Utils.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/Utils.java Sat Mar 17 15:15:19 2007
@@ -51,11 +51,15 @@
       } else if (ch == '&') {
         sb.append("&amp;");
       } else if (ch == '%') {
-        sb.append("%25");
-      } else if (ch < 0x20) {
+        sb.append("%0025");
+      } else if (ch < 0x20 ||
+          (ch > 0xD7FF && ch < 0xE000) ||
+          (ch > 0xFFFD)) {
         sb.append("%");
-        sb.append(hexchars[ch/16]);
-        sb.append(hexchars[ch%16]);
+        sb.append(hexchars[(ch & 0xF000) >> 12]);
+        sb.append(hexchars[(ch & 0x0F00) >> 8]);
+        sb.append(hexchars[(ch & 0x00F0) >> 4]);
+        sb.append(hexchars[(ch & 0x000F)]);
       } else {
         sb.append(ch);
       }
@@ -67,9 +71,9 @@
     if (ch >= '0' && ch <= '9') {
       return ch - '0';
     } else if (ch >= 'A' && ch <= 'F') {
-      return ch - 'A';
+      return ch - 'A' + 10;
     } else if (ch >= 'a' && ch <= 'f') {
-      return ch - 'a';
+      return ch - 'a' + 10;
     }
     return 0;
   }
@@ -84,15 +88,16 @@
     for (int idx = 0; idx < s.length();) {
       char ch = s.charAt(idx++);
       if (ch == '%') {
-        char ch1 = s.charAt(idx++);
-        char ch2 = s.charAt(idx++);
-        char res = (char)(h2c(ch1)*16 + h2c(ch2));
+        int ch1 = h2c(s.charAt(idx++)) << 12;
+        int ch2 = h2c(s.charAt(idx++)) << 8;
+        int ch3 = h2c(s.charAt(idx++)) << 4;
+        int ch4 = h2c(s.charAt(idx++));
+        char res = (char)(ch1 | ch2 | ch3 | ch4);
         sb.append(res);
       } else {
         sb.append(ch);
       }
     }
-    
     return sb.toString();
   }
   
@@ -196,7 +201,7 @@
     for (int idx = 0; idx < blen; idx++) {
       char c1 = s.charAt(2*idx);
       char c2 = s.charAt(2*idx+1);
-      barr[idx] = Byte.parseByte(""+c1+c2, 16);
+      barr[idx] = (byte)Integer.parseInt(""+c1+c2, 16);
     }
     return new Buffer(barr);
   }
@@ -230,9 +235,162 @@
     for (int idx = 0; idx < blen; idx++) {
       char c1 = s.charAt(2*idx+1);
       char c2 = s.charAt(2*idx+2);
-      barr[idx] = Byte.parseByte(""+c1+c2, 16);
+      barr[idx] = (byte)Integer.parseInt(""+c1+c2, 16);
     }
     return new Buffer(barr);
+  }
+  
+  private static int utf8LenForCodePoint(final int cpt) throws IOException {
+    if (cpt >=0 && cpt <= 0x7F) {
+      return 1;
+    }
+    if (cpt >= 0x80 && cpt <= 0x07FF) {
+      return 2;
+    }
+    if ((cpt >= 0x0800 && cpt < 0xD800) ||
+        (cpt > 0xDFFF && cpt <= 0xFFFD)) {
+      return 3;
+    }
+    if (cpt >= 0x10000 && cpt <= 0x10FFFF) {
+      return 4;
+    }
+    throw new IOException("Illegal Unicode Codepoint "+
+        Integer.toHexString(cpt)+" in string.");
+  }
+  
+  private static final int B10 =    Integer.parseInt("10000000", 2);
+  private static final int B110 =   Integer.parseInt("11000000", 2);
+  private static final int B1110 =  Integer.parseInt("11100000", 2);
+  private static final int B11110 = Integer.parseInt("11110000", 2);
+  private static final int B11 =    Integer.parseInt("11000000", 2);
+  private static final int B111 =   Integer.parseInt("11100000", 2);
+  private static final int B1111 =  Integer.parseInt("11110000", 2);
+  private static final int B11111 = Integer.parseInt("11111000", 2);
+  
+  private static int writeUtf8(int cpt, final byte[] bytes, final int offset)
+  throws IOException {
+    if (cpt >=0 && cpt <= 0x7F) {
+      bytes[offset] = (byte) cpt;
+      return 1;
+    }
+    if (cpt >= 0x80 && cpt <= 0x07FF) {
+      bytes[offset+1] = (byte) (B10 | (cpt & 0x3F));
+      cpt = cpt >> 6;
+      bytes[offset] = (byte) (B110 | (cpt & 0x1F));
+      return 2;
+    }
+    if ((cpt >= 0x0800 && cpt < 0xD800) ||
+        (cpt > 0xDFFF && cpt <= 0xFFFD)) {
+      bytes[offset+2] = (byte) (B10 | (cpt & 0x3F));
+      cpt = cpt >> 6;
+      bytes[offset+1] = (byte) (B10 | (cpt & 0x3F));
+      cpt = cpt >> 6;
+      bytes[offset] = (byte) (B1110 | (cpt & 0x0F));
+      return 3;
+    }
+    if (cpt >= 0x10000 && cpt <= 0x10FFFF) {
+      bytes[offset+3] = (byte) (B10 | (cpt & 0x3F));
+      cpt = cpt >> 6;
+      bytes[offset+2] = (byte) (B10 | (cpt & 0x3F));
+      cpt = cpt >> 6;
+      bytes[offset+1] = (byte) (B10 | (cpt & 0x3F));
+      cpt = cpt >> 6;
+      bytes[offset] = (byte) (B11110 | (cpt & 0x07));
+      return 4;
+    }
+    throw new IOException("Illegal Unicode Codepoint "+
+        Integer.toHexString(cpt)+" in string.");
+  }
+  
+  static void toBinaryString(final DataOutput out, final String str)
+  throws IOException {
+    final int strlen = str.length();
+    byte[] bytes = new byte[strlen*4]; // Codepoints expand to 4 bytes max
+    int utf8Len = 0;
+    int idx = 0;
+    while(idx < strlen) {
+      final int cpt = str.codePointAt(idx);
+      idx += Character.isSupplementaryCodePoint(cpt) ? 2 : 1;
+      utf8Len += writeUtf8(cpt, bytes, utf8Len);
+    }
+    writeVInt(out, utf8Len);
+    out.write(bytes, 0, utf8Len);
+  }
+  
+  static boolean isValidCodePoint(int cpt) {
+    return !((cpt > 0x10FFFF) ||
+        (cpt >= 0xD800 && cpt <= 0xDFFF) ||
+        (cpt >= 0xFFFE && cpt <=0xFFFF));
+  }
+  
+  private static int utf8ToCodePoint(int b1, int b2, int b3, int b4) {
+    int cpt = 0;
+    cpt = (((b1 & ~B11111) << 18) |
+        ((b2 & ~B11) << 12) |
+        ((b3 & ~B11) << 6) |
+        (b4 & ~B11));
+    return cpt;
+  }
+  
+  private static int utf8ToCodePoint(int b1, int b2, int b3) {
+    int cpt = 0;
+    cpt = (((b1 & ~B1111) << 12) | ((b2 & ~B11) << 6) | (b3 & ~B11));
+    return cpt;
+  }
+  
+  private static int utf8ToCodePoint(int b1, int b2) {
+    int cpt = 0;
+    cpt = (((b1 & ~B111) << 6) | (b2 & ~B11));
+    return cpt;
+  }
+  
+  private static void checkB10(int b) throws IOException {
+    if ((b & B11) != B10) {
+      throw new IOException("Invalid UTF-8 representation.");
+    }
+  }
+  
+  static String fromBinaryString(final DataInput din) throws IOException {
+    final int utf8Len = readVInt(din);
+    final byte[] bytes = new byte[utf8Len];
+    din.readFully(bytes);
+    int len = 0;
+    // For the most commmon case, i.e. ascii, numChars = utf8Len
+    StringBuilder sb = new StringBuilder(utf8Len);
+    while(len < utf8Len) {
+      int cpt = 0;
+      final int b1 = bytes[len++] & 0xFF;
+      if (b1 <= 0x7F) {
+        cpt = b1;
+      } else if ((b1 & B11111) == B11110) {
+        int b2 = bytes[len++] & 0xFF;
+        checkB10(b2);
+        int b3 = bytes[len++] & 0xFF;
+        checkB10(b3);
+        int b4 = bytes[len++] & 0xFF;
+        checkB10(b4);
+        cpt = utf8ToCodePoint(b1, b2, b3, b4);
+      } else if ((b1 & B1111) == B1110) {
+        int b2 = bytes[len++] & 0xFF;
+        checkB10(b2);
+        int b3 = bytes[len++] & 0xFF;
+        checkB10(b3);
+        cpt = utf8ToCodePoint(b1, b2, b3);
+      } else if ((b1 & B111) == B110) {
+        int b2 = bytes[len++] & 0xFF;
+        checkB10(b2);
+        cpt = utf8ToCodePoint(b1, b2);
+      } else {
+        throw new IOException("Invalid UTF-8 byte "+Integer.toHexString(b1)+
+            " at offset "+(len-1)+" in length of "+utf8Len);
+      }
+      if (!isValidCodePoint(cpt)) {
+        throw new IOException("Illegal Unicode Codepoint "+
+          Integer.toHexString(cpt)+" in stream.");
+      }
+      sb.appendCodePoint(cpt);
+    }
+    return sb.toString();
   }
   
   /** Parse a float from a byte array. */

Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/CppGenerator.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/CppGenerator.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/CppGenerator.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/CppGenerator.java Sat Mar 17 15:15:19 2007
@@ -46,20 +46,21 @@
     FileWriter cc = new FileWriter(name+".cc");
     FileWriter hh = new FileWriter(name+".hh");
     
-    hh.write("#ifndef __"+name.toUpperCase().replace('.','_')+"__\n");
-    hh.write("#define __"+name.toUpperCase().replace('.','_')+"__\n");
+    String fileName = (new File(name)).getName();
+    hh.write("#ifndef __"+fileName.toUpperCase().replace('.','_')+"__\n");
+    hh.write("#define __"+fileName.toUpperCase().replace('.','_')+"__\n");
     hh.write("#include \"recordio.hh\"\n");
     for (Iterator<JFile> iter = ilist.iterator(); iter.hasNext();) {
       hh.write("#include \""+iter.next().getName()+".hh\"\n");
     }
     
-    cc.write("#include \""+name+".hh\"\n");
+    cc.write("#include \""+fileName+".hh\"\n");
     
     for (Iterator<JRecord> iter = rlist.iterator(); iter.hasNext();) {
       iter.next().genCppCode(hh, cc, options);
     }
     
-    hh.write("#endif //"+name.toUpperCase().replace('.','_')+"__\n");
+    hh.write("#endif //"+fileName.toUpperCase().replace('.','_')+"__\n");
     
     hh.close();
     cc.close();

Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/JBuffer.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/JBuffer.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/JBuffer.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/JBuffer.java Sat Mar 17 15:15:19 2007
@@ -33,10 +33,6 @@
       super("org.apache.hadoop.record.Buffer", "Buffer", "org.apache.hadoop.record.Buffer");
     }
     
-    void genWriteMethod(CodeBuffer cb, String fname, String tag) {
-      cb.append("a_.writeBuffer("+fname+",\""+tag+"\");\n");
-    }
-    
     void genCompareTo(CodeBuffer cb, String fname, String other) {
       cb.append("ret = "+fname+".compareTo("+other+");\n");
     }
@@ -65,7 +61,7 @@
       cb.append("int z1 = org.apache.hadoop.record.Utils.getVIntSize(i1);\n");
       cb.append("int z2 = org.apache.hadoop.record.Utils.getVIntSize(i2);\n");
       cb.append("s1+=z1; s2+=z2; l1-=z1; l2-=z2;\n");
-      cb.append("int r1 = org.apache.hadoop.record.Utils.compareBytes(b1,s1,l1,b2,s2,l2);\n");
+      cb.append("int r1 = org.apache.hadoop.record.Utils.compareBytes(b1,s1,i1,b2,s2,i2);\n");
       cb.append("if (r1 != 0) { return (r1<0)?-1:0; }\n");
       cb.append("s1+=i1; s2+=i2; l1-=i1; l1-=i2;\n");
       cb.append("}\n");

Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/JMap.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/JMap.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/JMap.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/JMap.java Sat Mar 17 15:15:19 2007
@@ -79,14 +79,14 @@
       }
       cb.append("{\n");
       incrLevel();
-      cb.append("org.apache.hadoop.record.Index "+getId("midx")+" = a_.startMap(\""+tag+"\");\n");
+      cb.append("org.apache.hadoop.record.Index "+getId("midx")+" = a.startMap(\""+tag+"\");\n");
       cb.append(fname+"=new "+getType()+"();\n");
       cb.append("for (; !"+getId("midx")+".done(); "+getId("midx")+".incr()) {\n");
       key.genReadMethod(cb, getId("k"),getId("k"),true);
       value.genReadMethod(cb, getId("v"),getId("v"),true);
       cb.append(fname+".put("+getId("k")+","+getId("v")+");\n");
       cb.append("}\n");
-      cb.append("a_.endMap(\""+tag+"\");\n");
+      cb.append("a.endMap(\""+tag+"\");\n");
       decrLevel();
       cb.append("}\n");
     }
@@ -100,7 +100,7 @@
           key.getWrapperType()+","+value.getWrapperType()+">> ";
       cb.append("{\n");
       incrLevel();
-      cb.append("a_.startMap("+fname+",\""+tag+"\");\n");
+      cb.append("a.startMap("+fname+",\""+tag+"\");\n");
       cb.append(setType+getId("es")+" = "+fname+".entrySet();\n");
       cb.append("for("+iterType+getId("midx")+" = "+getId("es")+".iterator(); "+getId("midx")+".hasNext(); ) {\n");
       cb.append(entryType+getId("me")+" = "+getId("midx")+".next();\n");
@@ -109,7 +109,7 @@
       key.genWriteMethod(cb, getId("k"),getId("k"));
       value.genWriteMethod(cb, getId("v"),getId("v"));
       cb.append("}\n");
-      cb.append("a_.endMap("+fname+",\""+tag+"\");\n");
+      cb.append("a.endMap("+fname+",\""+tag+"\");\n");
       cb.append("}\n");
       decrLevel();
     }
@@ -161,7 +161,7 @@
   /** Creates a new instance of JMap */
   public JMap(JType t1, JType t2) {
     setJavaType(new JavaMap(t1.getJavaType(), t2.getJavaType()));
-    setCppType(new CppType(" ::std::map<"+t1.getCppType().getType()+","+
+    setCppType(new CppCompType(" ::std::map<"+t1.getCppType().getType()+","+
         t2.getCppType().getType()+">"));
     setCType(new CType());
     keyType = t1;

Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/JRecord.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/JRecord.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/JRecord.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/JRecord.java Sat Mar 17 15:15:19 2007
@@ -55,11 +55,11 @@
         cb.append(fullName+" "+fname+";\n");
       }
       cb.append(fname+"= new "+fullName+"();\n");
-      cb.append("a_.readRecord("+fname+",\""+tag+"\");\n");
+      cb.append(fname+".deserialize(a,\""+tag+"\");\n");
     }
     
     void genWriteMethod(CodeBuffer cb, String fname, String tag) {
-      cb.append("a_.writeRecord("+fname+",\""+tag+"\");\n");
+      cb.append(fname+".serialize(a,\""+tag+"\");\n");
     }
     
     void genSlurpBytes(CodeBuffer cb, String b, String s, String l) {
@@ -100,9 +100,7 @@
       cb.append("// File generated by hadoop record compiler. Do not edit.\n");
       cb.append("package "+module+";\n\n");
       cb.append("public class "+name+
-          " implements org.apache.hadoop.record.Record");
-      cb.append(", org.apache.hadoop.io.WritableComparable");
-      cb.append(" {\n");
+          " extends org.apache.hadoop.record.Record {\n");
       
       for (Iterator<JField<JavaType>> i = fields.iterator(); i.hasNext();) {
         JField<JavaType> jf = i.next();
@@ -138,57 +136,44 @@
         type.genGetSet(cb, name);
       }
       cb.append("public void serialize("+
-          "final org.apache.hadoop.record.OutputArchive a_, final String tag)\n"+
+          "final org.apache.hadoop.record.RecordOutput a, final String tag)\n"+
           "throws java.io.IOException {\n");
-      cb.append("a_.startRecord(this,tag);\n");
+      cb.append("a.startRecord(this,tag);\n");
       for (Iterator<JField<JavaType>> i = fields.iterator(); i.hasNext();) {
         JField<JavaType> jf = i.next();
         String name = jf.getName();
         JavaType type = jf.getType();
         type.genWriteMethod(cb, name, name);
       }
-      cb.append("a_.endRecord(this,tag);\n");
+      cb.append("a.endRecord(this,tag);\n");
       cb.append("}\n");
       
       cb.append("public void deserialize("+
-          "final org.apache.hadoop.record.InputArchive a_, final String tag)\n"+
+          "final org.apache.hadoop.record.RecordInput a, final String tag)\n"+
           "throws java.io.IOException {\n");
-      cb.append("a_.startRecord(tag);\n");
+      cb.append("a.startRecord(tag);\n");
       for (Iterator<JField<JavaType>> i = fields.iterator(); i.hasNext();) {
         JField<JavaType> jf = i.next();
         String name = jf.getName();
         JavaType type = jf.getType();
         type.genReadMethod(cb, name, name, false);
       }
-      cb.append("a_.endRecord(tag);\n");
+      cb.append("a.endRecord(tag);\n");
       cb.append("}\n");
       
       cb.append("public String toString() {\n");
       cb.append("try {\n");
       cb.append("java.io.ByteArrayOutputStream s =\n");
       cb.append("  new java.io.ByteArrayOutputStream();\n");
-      cb.append("org.apache.hadoop.record.CsvOutputArchive a_ = \n");
-      cb.append("  new org.apache.hadoop.record.CsvOutputArchive(s);\n");
-      cb.append("this.serialize(a_,\"\");\n");
+      cb.append("org.apache.hadoop.record.CsvRecordOutput a = \n");
+      cb.append("  new org.apache.hadoop.record.CsvRecordOutput(s);\n");
+      cb.append("this.serialize(a);\n");
       cb.append("return new String(s.toByteArray(), \"UTF-8\");\n");
       cb.append("} catch (Throwable ex) {\n");
       cb.append("throw new RuntimeException(ex);\n");
       cb.append("}\n");
       cb.append("}\n");
       
-      cb.append("public void write(final java.io.DataOutput out)\n"+
-          "throws java.io.IOException {\n");
-      cb.append("org.apache.hadoop.record.BinaryOutputArchive archive =\n"+
-          "new org.apache.hadoop.record.BinaryOutputArchive(out);\n");
-      cb.append("this.serialize(archive, \"\");\n");
-      cb.append("}\n");
-      
-      cb.append("public void readFields(final java.io.DataInput in)\n"+
-          "throws java.io.IOException {\n");
-      cb.append("org.apache.hadoop.record.BinaryInputArchive archive =\n"+
-          "new org.apache.hadoop.record.BinaryInputArchive(in);\n");
-      cb.append("this.deserialize(archive, \"\");\n");
-      cb.append("}\n");
       cb.append("public int compareTo (final Object peer_) throws ClassCastException {\n");
       cb.append("if (!(peer_ instanceof "+name+")) {\n");
       cb.append("throw new ClassCastException(\"Comparing different types of records.\");\n");
@@ -253,7 +238,7 @@
       cb.append("}\n");
       
       cb.append("public static class Comparator extends"+
-          " org.apache.hadoop.io.WritableComparator {\n");
+          " org.apache.hadoop.record.RecordComparator {\n");
       cb.append("public Comparator() {\n");
       cb.append("super("+name+".class);\n");
       cb.append("}\n");
@@ -295,7 +280,7 @@
       cb.append("}\n");
       cb.append("}\n\n");
       cb.append("static {\n");
-      cb.append("org.apache.hadoop.io.WritableComparator.define("
+      cb.append("org.apache.hadoop.record.RecordComparator.define("
           +name+".class, new Comparator());\n");
       cb.append("}\n");
       cb.append("}\n");

Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/JString.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/JString.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/JString.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/JString.java Sat Mar 17 15:15:19 2007
@@ -48,7 +48,7 @@
       cb.append("int z1 = org.apache.hadoop.record.Utils.getVIntSize(i1);\n");
       cb.append("int z2 = org.apache.hadoop.record.Utils.getVIntSize(i2);\n");
       cb.append("s1+=z1; s2+=z2; l1-=z1; l2-=z2;\n");
-      cb.append("int r1 = org.apache.hadoop.record.Utils.compareBytes(b1,s1,l1,b2,s2,l2);\n");
+      cb.append("int r1 = org.apache.hadoop.record.Utils.compareBytes(b1,s1,i1,b2,s2,i2);\n");
       cb.append("if (r1 != 0) { return (r1<0)?-1:0; }\n");
       cb.append("s1+=i1; s2+=i2; l1-=i1; l1-=i2;\n");
       cb.append("}\n");

Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/JType.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/JType.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/JType.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/JType.java Sat Mar 17 15:15:19 2007
@@ -80,14 +80,14 @@
     }
     
     void genWriteMethod(CodeBuffer cb, String fname, String tag) {
-      cb.append("a_.write"+methodSuffix+"("+fname+",\""+tag+"\");\n");
+      cb.append("a.write"+methodSuffix+"("+fname+",\""+tag+"\");\n");
     }
     
     void genReadMethod(CodeBuffer cb, String fname, String tag, boolean decl) {
       if (decl) {
         cb.append(name+" "+fname+";\n");
       }
-      cb.append(fname+"=a_.read"+methodSuffix+"(\""+tag+"\");\n");
+      cb.append(fname+"=a.read"+methodSuffix+"(\""+tag+"\");\n");
     }
     
     void genCompareTo(CodeBuffer cb, String fname, String other) {

Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/JVector.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/JVector.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/JVector.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/JVector.java Sat Mar 17 15:15:19 2007
@@ -75,13 +75,13 @@
       }
       cb.append("{\n");
       incrLevel();
-      cb.append("org.apache.hadoop.record.Index "+getId("vidx")+" = a_.startVector(\""+tag+"\");\n");
+      cb.append("org.apache.hadoop.record.Index "+getId("vidx")+" = a.startVector(\""+tag+"\");\n");
       cb.append(fname+"=new "+getType()+"();\n");
       cb.append("for (; !"+getId("vidx")+".done(); "+getId("vidx")+".incr()) {\n");
       element.genReadMethod(cb, getId("e"), getId("e"), true);
       cb.append(fname+".add("+getId("e")+");\n");
       cb.append("}\n");
-      cb.append("a_.endVector(\""+tag+"\");\n");
+      cb.append("a.endVector(\""+tag+"\");\n");
       decrLevel();
       cb.append("}\n");
     }
@@ -89,13 +89,13 @@
     void genWriteMethod(CodeBuffer cb, String fname, String tag) {
       cb.append("{\n");
       incrLevel();
-      cb.append("a_.startVector("+fname+",\""+tag+"\");\n");
+      cb.append("a.startVector("+fname+",\""+tag+"\");\n");
       cb.append("int "+getId("len")+" = "+fname+".size();\n");
       cb.append("for(int "+getId("vidx")+" = 0; "+getId("vidx")+"<"+getId("len")+"; "+getId("vidx")+"++) {\n");
       cb.append(element.getType()+" "+getId("e")+" = "+fname+".get("+getId("vidx")+");\n");
       element.genWriteMethod(cb, getId("e"), getId("e"));
       cb.append("}\n");
-      cb.append("a_.endVector("+fname+",\""+tag+"\");\n");
+      cb.append("a.endVector("+fname+",\""+tag+"\");\n");
       cb.append("}\n");
       decrLevel();
     }

Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/generated/Rcc.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/generated/Rcc.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/generated/Rcc.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/generated/Rcc.java Sat Mar 17 15:15:19 2007
@@ -43,6 +43,10 @@
         System.exit(driver(args));
     }
 
+    public static void usage() {
+        System.err.println("Usage: rcc --language [java|c++] ddl-files");
+    }
+
     public static int driver(String[] args) {
         for (int i=0; i<args.length; i++) {
             if ("-l".equalsIgnoreCase(args[i]) ||
@@ -64,7 +68,7 @@
             }
         }
         if (recFiles.size() == 0) {
-            System.err.println("No record files specified. Exiting.");
+            usage();
             return 1;
         }
         for (int i=0; i<recFiles.size(); i++) {

Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/generated/rcc.jj
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/generated/rcc.jj?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/generated/rcc.jj (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/compiler/generated/rcc.jj Sat Mar 17 15:15:19 2007
@@ -47,6 +47,10 @@
         System.exit(driver(args));
     }
  
+    public static void usage() {
+        System.err.println("Usage: rcc --language [java|c++] ddl-files");
+    }
+
     public static int driver(String[] args) {
         for (int i=0; i<args.length; i++) {
             if ("-l".equalsIgnoreCase(args[i]) ||
@@ -68,7 +72,7 @@
             }
         }
         if (recFiles.size() == 0) {
-            System.err.println("No record files specified. Exiting.");
+            usage();
             return 1;
         }
         for (int i=0; i<recFiles.size(); i++) {

Modified: lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/package.html
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/package.html?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/package.html (original)
+++ lucene/hadoop/branches/branch-0.12/src/java/org/apache/hadoop/record/package.html Sat Mar 17 15:15:19 2007
@@ -479,9 +479,9 @@
 
   private:
 
-    int32_t      mI32;
-    double       mD;
-    std::string  mS;
+    int32_t      I32;
+    double       D;
+    std::string  S;
 
   public:
 
@@ -491,14 +491,14 @@
     virtual bool operator==(const RI& peer) const;
     virtual bool operator<(const RI& peer) const;
 
-    virtual int32_t getI32(void) const { return mI32; }
-    virtual void setI32(int32_t v) { mI32 = v; }
+    virtual int32_t getI32(void) const { return I32; }
+    virtual void setI32(int32_t v) { I32 = v; }
 
-    virtual double getD(void) const { return mD; }
-    virtual void setD(double v) { mD = v; }
+    virtual double getD(void) const { return D; }
+    virtual void setD(double v) { D = v; }
 
-    virtual std::string& getS(void) const { return mS; }
-    virtual const std::string& getS(void) const { return mS; }
+    virtual std::string& getS(void) const { return S; }
+    virtual const std::string& getS(void) const { return S; }
 
     virtual std::string type(void) const;
     virtual std::string signature(void) const;
@@ -507,8 +507,6 @@
 
     virtual void serialize(hadoop::OArchive& a) const;
     virtual void deserialize(hadoop::IArchive& a);
-
-    virtual bool validate(void);
   };
 } // end namespace inclrec
 
@@ -531,9 +529,9 @@
 
   private:
 
-    std::vector<float> mVF;
-    inclrec::RI        mRec;
-    std::string        mBuf;
+    std::vector<float> VF;
+    inclrec::RI        Rec;
+    std::string        Buf;
 
   public:
 
@@ -595,11 +593,11 @@
 long            int64_t             long
 float           float               float
 double          double              double
-ustring         std::string         Text
-buffer          std::string         java.io.ByteArrayOutputStream
+ustring         std::string         java.lang.String
+buffer          std::string         org.apache.hadoop.record.Buffer
 class type      class type          class type
-vector<type>    std::vector<type>   java.util.ArrayList
-map<type,type>  std::map<type,type> java.util.TreeMap
+vector<type>    std::vector<type>   java.util.ArrayList<type>
+map<type,type>  std::map<type,type> java.util.TreeMap<type,type>
 </code></pre>
 
 <h2>Data encodings</h2>

Modified: lucene/hadoop/branches/branch-0.12/src/test/org/apache/hadoop/record/FromCpp.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/test/org/apache/hadoop/record/FromCpp.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/test/org/apache/hadoop/record/FromCpp.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/test/org/apache/hadoop/record/FromCpp.java Sat Mar 17 15:15:19 2007
@@ -18,7 +18,6 @@
 
 package org.apache.hadoop.record;
 
-import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
@@ -58,9 +57,9 @@
             r1.setVectorVal(new ArrayList<String>());
             r1.setMapVal(new TreeMap<String,String>());
             FileInputStream istream = new FileInputStream(tmpfile);
-            RecordReader in = new RecordReader(istream, "binary");
+            BinaryRecordInput in = new BinaryRecordInput(istream);
             RecRecord1 r2 = new RecRecord1();
-            in.read(r2);
+            r2.deserialize(in, "");
             istream.close();
             assertTrue(r1.equals(r2));
         } catch (IOException ex) {
@@ -84,9 +83,9 @@
             r1.setVectorVal(new ArrayList<String>());
             r1.setMapVal(new TreeMap<String,String>());
             FileInputStream istream = new FileInputStream(tmpfile);
-            RecordReader in = new RecordReader(istream, "csv");
+            CsvRecordInput in = new CsvRecordInput(istream);
             RecRecord1 r2 = new RecRecord1();
-            in.read(r2);
+            r2.deserialize(in, "");
             istream.close();
             assertTrue(r1.equals(r2));
         } catch (IOException ex) {
@@ -110,9 +109,9 @@
             r1.setVectorVal(new ArrayList<String>());
             r1.setMapVal(new TreeMap<String,String>());
             FileInputStream istream = new FileInputStream(tmpfile);
-            RecordReader in = new RecordReader(istream, "xml");
+            XmlRecordInput in = new XmlRecordInput(istream);
             RecRecord1 r2 = new RecRecord1();
-            in.read(r2);
+            r2.deserialize(in, "");
             istream.close();
             assertTrue(r1.equals(r2));
         } catch (IOException ex) {

Modified: lucene/hadoop/branches/branch-0.12/src/test/org/apache/hadoop/record/TestRecordIO.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/test/org/apache/hadoop/record/TestRecordIO.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/test/org/apache/hadoop/record/TestRecordIO.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/test/org/apache/hadoop/record/TestRecordIO.java Sat Mar 17 15:15:19 2007
@@ -47,7 +47,7 @@
         try {
             tmpfile = File.createTempFile("hadooprec", ".dat");
             FileOutputStream ostream = new FileOutputStream(tmpfile);
-            RecordWriter out = new RecordWriter(ostream, "binary");
+            BinaryRecordOutput out = new BinaryRecordOutput(ostream);
             RecRecord1 r1 = new RecRecord1();
             r1.setBoolVal(true);
             r1.setByteVal((byte)0x66);
@@ -62,12 +62,12 @@
             RecRecord0 r0 = new RecRecord0();
             r0.setStringVal("other random text");
             r1.setRecordVal(r0);
-            out.write(r1);
+            r1.serialize(out, "");
             ostream.close();
             FileInputStream istream = new FileInputStream(tmpfile);
-            RecordReader in = new RecordReader(istream, "binary");
+            BinaryRecordInput in = new BinaryRecordInput(istream);
             RecRecord1 r2 = new RecRecord1();
-            in.read(r2);
+            r2.deserialize(in, "");
             istream.close();
             tmpfile.delete();
             assertTrue("Serialized and deserialized records do not match.", r1.equals(r2));
@@ -81,7 +81,7 @@
         try {
             tmpfile = File.createTempFile("hadooprec", ".txt");
             FileOutputStream ostream = new FileOutputStream(tmpfile);
-            RecordWriter out = new RecordWriter(ostream, "csv");
+            CsvRecordOutput out = new CsvRecordOutput(ostream);
             RecRecord1 r1 = new RecRecord1();
             r1.setBoolVal(true);
             r1.setByteVal((byte)0x66);
@@ -96,12 +96,12 @@
             RecRecord0 r0 = new RecRecord0();
             r0.setStringVal("other random text");
             r1.setRecordVal(r0);
-            out.write(r1);
+            r1.serialize(out, "");
             ostream.close();
             FileInputStream istream = new FileInputStream(tmpfile);
-            RecordReader in = new RecordReader(istream, "csv");
+            CsvRecordInput in = new CsvRecordInput(istream);
             RecRecord1 r2 = new RecRecord1();
-            in.read(r2);
+            r2.deserialize(in, "");
             istream.close();
             tmpfile.delete();
             assertTrue("Serialized and deserialized records do not match.", r1.equals(r2));
@@ -140,7 +140,7 @@
         try {
             tmpfile = File.createTempFile("hadooprec", ".xml");
             FileOutputStream ostream = new FileOutputStream(tmpfile);
-            RecordWriter out = new RecordWriter(ostream, "xml");
+            XmlRecordOutput out = new XmlRecordOutput(ostream);
             RecRecord1 r1 = new RecRecord1();
             r1.setBoolVal(true);
             r1.setByteVal((byte)0x66);
@@ -148,19 +148,19 @@
             r1.setDoubleVal(1.5234);
             r1.setIntVal(4567);
             r1.setLongVal(0x5a5a5a5a5a5aL);
-            r1.setStringVal("ran\002dom &lt; %text<&more");
+            r1.setStringVal("ran\002dom &lt; %text<&more\uffff");
             r1.setBufferVal(new Buffer());
             r1.setVectorVal(new ArrayList<String>());
             r1.setMapVal(new TreeMap<String,String>());
             RecRecord0 r0 = new RecRecord0();
             r0.setStringVal("other %rando\007m &amp; >&more text");
             r1.setRecordVal(r0);
-            out.write(r1);
+            r1.serialize(out, "");
             ostream.close();
             FileInputStream istream = new FileInputStream(tmpfile);
-            RecordReader in = new RecordReader(istream, "xml");
+            XmlRecordInput in = new XmlRecordInput(istream);
             RecRecord1 r2 = new RecRecord1();
-            in.read(r2);
+            r2.deserialize(in, "");
             istream.close();
             tmpfile.delete();
             assertTrue("Serialized and deserialized records do not match.", r1.equals(r2));

Modified: lucene/hadoop/branches/branch-0.12/src/test/org/apache/hadoop/record/ToCpp.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/test/org/apache/hadoop/record/ToCpp.java?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/test/org/apache/hadoop/record/ToCpp.java (original)
+++ lucene/hadoop/branches/branch-0.12/src/test/org/apache/hadoop/record/ToCpp.java Sat Mar 17 15:15:19 2007
@@ -20,8 +20,6 @@
 
 import java.io.IOException;
 import junit.framework.*;
-import org.apache.hadoop.record.RecordWriter;
-import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.util.ArrayList;
@@ -46,9 +44,9 @@
     public void testBinary() {
         File tmpfile;
         try {
-            tmpfile = new File("/temp/hadooptemp.dat");
+            tmpfile = new File("/tmp/hadooptemp.dat");
             FileOutputStream ostream = new FileOutputStream(tmpfile);
-            RecordWriter out = new RecordWriter(ostream, "binary");
+            BinaryRecordOutput out = new BinaryRecordOutput(ostream);
             RecRecord1 r1 = new RecRecord1();
             r1.setBoolVal(true);
             r1.setByteVal((byte)0x66);
@@ -60,7 +58,7 @@
             r1.setBufferVal(new Buffer());
             r1.setVectorVal(new ArrayList<String>());
             r1.setMapVal(new TreeMap<String,String>());
-            out.write(r1);
+            r1.serialize(out, "");
             ostream.close();
         } catch (IOException ex) {
             ex.printStackTrace();
@@ -70,9 +68,9 @@
     public void testCsv() {
         File tmpfile;
         try {
-            tmpfile = new File("/temp/hadooptemp.txt");
+            tmpfile = new File("/tmp/hadooptemp.txt");
             FileOutputStream ostream = new FileOutputStream(tmpfile);
-            RecordWriter out = new RecordWriter(ostream, "csv");
+            CsvRecordOutput out = new CsvRecordOutput(ostream);
             RecRecord1 r1 = new RecRecord1();
             r1.setBoolVal(true);
             r1.setByteVal((byte)0x66);
@@ -84,7 +82,7 @@
             r1.setBufferVal(new Buffer());
             r1.setVectorVal(new ArrayList<String>());
             r1.setMapVal(new TreeMap<String,String>());
-            out.write(r1);
+            r1.serialize(out, "");
             ostream.close();
         } catch (IOException ex) {
             ex.printStackTrace();
@@ -94,9 +92,9 @@
     public void testXml() {
         File tmpfile;
         try {
-            tmpfile = new File("/temp/hadooptemp.xml");
+            tmpfile = new File("/tmp/hadooptemp.xml");
             FileOutputStream ostream = new FileOutputStream(tmpfile);
-            RecordWriter out = new RecordWriter(ostream, "xml");
+            XmlRecordOutput out = new XmlRecordOutput(ostream);
             RecRecord1 r1 = new RecRecord1();
             r1.setBoolVal(true);
             r1.setByteVal((byte)0x66);
@@ -108,7 +106,7 @@
             r1.setBufferVal(new Buffer());
             r1.setVectorVal(new ArrayList<String>());
             r1.setMapVal(new TreeMap<String,String>());
-            out.write(r1);
+            r1.serialize(out, "");
             ostream.close();
         } catch (IOException ex) {
             ex.printStackTrace();

Modified: lucene/hadoop/branches/branch-0.12/src/webapps/job/jobfailures.jsp
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.12/src/webapps/job/jobfailures.jsp?view=diff&rev=519461&r1=519460&r2=519461
==============================================================================
--- lucene/hadoop/branches/branch-0.12/src/webapps/job/jobfailures.jsp (original)
+++ lucene/hadoop/branches/branch-0.12/src/webapps/job/jobfailures.jsp Sat Mar 17 15:15:19 2007
@@ -24,7 +24,7 @@
         TaskTrackerStatus taskTracker = tracker.getTaskTracker(taskTrackerName);
         out.print("<tr><td>" + statuses[i].getTaskId() +
                   "</td><td><a href=\"/taskdetails.jsp?jobid="+ jobId + 
-                  "&taskid=" + tipId + "\">" + tipId +
+                  "&tipid=" + tipId + "\">" + tipId +
                   "</a></td>");
         if (taskTracker == null) {
           out.print("<td>" + taskTrackerName + "</td>");



Mime
View raw message