accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bil...@apache.org
Subject svn commit: r1382004 - in /accumulo/trunk: core/src/main/java/org/apache/accumulo/core/client/ core/src/main/java/org/apache/accumulo/core/client/mapreduce/ core/src/test/java/org/apache/accumulo/core/client/mapreduce/ core/src/test/java/org/apache/acc...
Date Fri, 07 Sep 2012 12:56:20 GMT
Author: billie
Date: Fri Sep  7 12:56:20 2012
New Revision: 1382004

URL: http://svn.apache.org/viewvc?rev=1382004&view=rev
Log:
ACCUMULO-700 ACCUMULO-507 changed InputFormat internal configuration serialization for iterators
(now uses IteratorSetting) and ranges (now written to a file)

Modified:
    accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java
    accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
    accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
    accumulo/trunk/core/src/test/java/org/apache/accumulo/core/security/AuthorizationsTest.java
    accumulo/trunk/test/system/continuous/   (props changed)

Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java?rev=1382004&r1=1382003&r2=1382004&view=diff
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java
(original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/IteratorSetting.java
Fri Sep  7 12:56:20 2012
@@ -16,6 +16,9 @@
  */
 package org.apache.accumulo.core.client;
 
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
@@ -28,6 +31,8 @@ import org.apache.accumulo.core.iterator
 import org.apache.accumulo.core.util.ArgumentChecker;
 import org.apache.accumulo.core.util.Pair;
 import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableUtils;
 
 /**
  * Configure an iterator for minc, majc, and/or scan. By default, IteratorSetting will be
configured for scan.
@@ -42,7 +47,7 @@ import org.apache.hadoop.io.Text;
  * scanner.addScanIterator(cfg);
  * </pre>
  */
-public class IteratorSetting {
+public class IteratorSetting implements Writable {
   private int priority;
   private String name;
   private String iteratorClass;
@@ -185,6 +190,11 @@ public class IteratorSetting {
     this(priority, name, iteratorClass.getName());
   }
   
+  public IteratorSetting(DataInput din) throws IOException {
+    this.properties = new HashMap<String,String>();
+    this.readFields(din);
+  }
+
   /**
    * Add another option to the iterator.
    * 
@@ -297,4 +307,29 @@ public class IteratorSetting {
     }
     
   }
+  
+  @Override
+  public void readFields(DataInput din) throws IOException {
+    priority = WritableUtils.readVInt(din);
+    name = WritableUtils.readString(din);
+    iteratorClass = WritableUtils.readString(din);
+    properties.clear();
+    int size = WritableUtils.readVInt(din);
+    while (size > 0) {
+      properties.put(WritableUtils.readString(din), WritableUtils.readString(din));
+      size--;
+    }
+  }
+  
+  @Override
+  public void write(DataOutput dout) throws IOException {
+    WritableUtils.writeVInt(dout, priority);
+    WritableUtils.writeString(dout, name);
+    WritableUtils.writeString(dout, iteratorClass);
+    WritableUtils.writeVInt(dout, properties.size());
+    for (Entry<String,String> e : properties.entrySet()) {
+      WritableUtils.writeString(dout, e.getKey());
+      WritableUtils.writeString(dout, e.getValue());
+    }
+  }
 }

Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java?rev=1382004&r1=1382003&r2=1382004&view=diff
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
(original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
Fri Sep  7 12:56:20 2012
@@ -23,11 +23,8 @@ import java.io.DataInputStream;
 import java.io.DataOutput;
 import java.io.DataOutputStream;
 import java.io.IOException;
-import java.io.UnsupportedEncodingException;
 import java.math.BigInteger;
 import java.net.InetAddress;
-import java.net.URLDecoder;
-import java.net.URLEncoder;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -141,7 +138,6 @@ public abstract class InputFormatBase<K,
   
   // Used for specifying the iterators to be applied
   private static final String ITERATORS = PREFIX + ".iterators";
-  private static final String ITERATORS_OPTIONS = PREFIX + ".iterators.options";
   private static final String ITERATORS_DELIM = ",";
   
   private static final String READ_OFFLINE = PREFIX + ".read.offline";
@@ -212,7 +208,7 @@ public abstract class InputFormatBase<K,
     } catch (IOException ioe) {
       throw new RuntimeException(ioe);
     }
-
+    
   }
   
   /**
@@ -259,17 +255,24 @@ public abstract class InputFormatBase<K,
    */
   public static void setRanges(Configuration conf, Collection<Range> ranges) {
     ArgumentChecker.notNull(ranges);
-    ArrayList<String> rangeStrings = new ArrayList<String>(ranges.size());
     try {
+      FileSystem fs = FileSystem.get(conf);
+      Path file = new Path(fs.getWorkingDirectory(), conf.get("mapred.job.name") + System.currentTimeMillis()
+ ".ranges");
+      conf.set(RANGES, file.toString());
+      FSDataOutputStream fos = fs.create(file, false);
+      fs.setPermission(file, new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE));
+      fs.deleteOnExit(file);
+      
+      fos.writeInt(ranges.size());
       for (Range r : ranges) {
-        ByteArrayOutputStream baos = new ByteArrayOutputStream();
-        r.write(new DataOutputStream(baos));
-        rangeStrings.add(new String(Base64.encodeBase64(baos.toByteArray())));
+        r.write(fos);
       }
-    } catch (IOException ex) {
-      throw new IllegalArgumentException("Unable to encode ranges to Base64", ex);
+      fos.close();
+      
+      DistributedCache.addCacheFile(file.toUri(), conf);
+    } catch (IOException e) {
+      throw new RuntimeException("Unable to write ranges to file", e);
     }
-    conf.setStrings(RANGES, rangeStrings.toArray(new String[0]));
   }
   
   /**
@@ -381,32 +384,25 @@ public abstract class InputFormatBase<K,
     // First check to see if anything has been set already
     String iterators = conf.get(ITERATORS);
     
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    String newIter;
+    try {
+      cfg.write(new DataOutputStream(baos));
+      newIter = new String(Base64.encodeBase64(baos.toByteArray()));
+      baos.close();
+    } catch (IOException e) {
+      throw new IllegalArgumentException("unable to serialize IteratorSetting");
+    }
+    
     // No iterators specified yet, create a new string
     if (iterators == null || iterators.isEmpty()) {
-      iterators = new AccumuloIterator(cfg.getPriority(), cfg.getIteratorClass(), cfg.getName()).toString();
+      iterators = newIter;
     } else {
       // append the next iterator & reset
-      iterators = iterators.concat(ITERATORS_DELIM + new AccumuloIterator(cfg.getPriority(),
cfg.getIteratorClass(), cfg.getName()).toString());
+      iterators = iterators.concat(ITERATORS_DELIM + newIter);
     }
     // Store the iterators w/ the job
     conf.set(ITERATORS, iterators);
-    for (Entry<String,String> entry : cfg.getOptions().entrySet()) {
-      if (entry.getValue() == null)
-        continue;
-      
-      String iteratorOptions = conf.get(ITERATORS_OPTIONS);
-      
-      // No options specified yet, create a new string
-      if (iteratorOptions == null || iteratorOptions.isEmpty()) {
-        iteratorOptions = new AccumuloIteratorOption(cfg.getName(), entry.getKey(), entry.getValue()).toString();
-      } else {
-        // append the next option & reset
-        iteratorOptions = iteratorOptions.concat(ITERATORS_DELIM + new AccumuloIteratorOption(cfg.getName(),
entry.getKey(), entry.getValue()));
-      }
-      
-      // Store the options w/ the job
-      conf.set(ITERATORS_OPTIONS, iteratorOptions);
-    }
   }
   
   /**
@@ -541,12 +537,21 @@ public abstract class InputFormatBase<K,
    */
   protected static List<Range> getRanges(Configuration conf) throws IOException {
     ArrayList<Range> ranges = new ArrayList<Range>();
-    for (String rangeString : conf.getStringCollection(RANGES)) {
-      ByteArrayInputStream bais = new ByteArrayInputStream(Base64.decodeBase64(rangeString.getBytes()));
-      Range range = new Range();
-      range.readFields(new DataInputStream(bais));
-      ranges.add(range);
+    FileSystem fs = FileSystem.get(conf);
+    String rangePath = conf.get(RANGES);
+    if (rangePath == null)
+      return ranges;
+    Path file = new Path(rangePath);
+    
+    FSDataInputStream fdis = fs.open(file);
+    int numRanges = fdis.readInt();
+    while (numRanges > 0) {
+      Range r = new Range();
+      r.readFields(fdis);
+      ranges.add(r);
+      numRanges--;
     }
+    fdis.close();
     return ranges;
   }
   
@@ -617,7 +622,7 @@ public abstract class InputFormatBase<K,
       
       if (!usesLocalIterators(conf)) {
         // validate that any scan-time iterators can be loaded by the the tablet servers
-        for (AccumuloIterator iter : getIterators(conf)) {
+        for (IteratorSetting iter : getIterators(conf)) {
           if (!c.instanceOperations().testClassLoad(iter.getIteratorClass(), SortedKeyValueIterator.class.getName()))
             throw new AccumuloException("Servers are unable to load " + iter.getIteratorClass()
+ " as a " + SortedKeyValueIterator.class.getName());
         }
@@ -656,45 +661,26 @@ public abstract class InputFormatBase<K,
    * @return a list of iterators
    * @see #addIterator(Configuration, IteratorSetting)
    */
-  protected static List<AccumuloIterator> getIterators(Configuration conf) {
+  protected static List<IteratorSetting> getIterators(Configuration conf) {
     
     String iterators = conf.get(ITERATORS);
     
     // If no iterators are present, return an empty list
     if (iterators == null || iterators.isEmpty())
-      return new ArrayList<AccumuloIterator>();
+      return new ArrayList<IteratorSetting>();
     
     // Compose the set of iterators encoded in the job configuration
     StringTokenizer tokens = new StringTokenizer(conf.get(ITERATORS), ITERATORS_DELIM);
-    List<AccumuloIterator> list = new ArrayList<AccumuloIterator>();
-    while (tokens.hasMoreTokens()) {
-      String itstring = tokens.nextToken();
-      list.add(new AccumuloIterator(itstring));
-    }
-    return list;
-  }
-  
-  /**
-   * Gets a list of the iterator options specified on this configuration.
-   * 
-   * @param conf
-   *          the Hadoop configuration object
-   * @return a list of iterator options
-   * @see #addIterator(Configuration, IteratorSetting)
-   */
-  protected static List<AccumuloIteratorOption> getIteratorOptions(Configuration conf)
{
-    String iteratorOptions = conf.get(ITERATORS_OPTIONS);
-    
-    // If no options are present, return an empty list
-    if (iteratorOptions == null || iteratorOptions.isEmpty())
-      return new ArrayList<AccumuloIteratorOption>();
-    
-    // Compose the set of options encoded in the job configuration
-    StringTokenizer tokens = new StringTokenizer(conf.get(ITERATORS_OPTIONS), ITERATORS_DELIM);
-    List<AccumuloIteratorOption> list = new ArrayList<AccumuloIteratorOption>();
-    while (tokens.hasMoreTokens()) {
-      String optionString = tokens.nextToken();
-      list.add(new AccumuloIteratorOption(optionString));
+    List<IteratorSetting> list = new ArrayList<IteratorSetting>();
+    try {
+      while (tokens.hasMoreTokens()) {
+        String itstring = tokens.nextToken();
+        ByteArrayInputStream bais = new ByteArrayInputStream(Base64.decodeBase64(itstring.getBytes()));
+        list.add(new IteratorSetting(new DataInputStream(bais)));
+        bais.close();
+      }
+    } catch (IOException e) {
+      throw new IllegalArgumentException("couldn't decode iterator settings");
     }
     return list;
   }
@@ -714,18 +700,9 @@ public abstract class InputFormatBase<K,
      * @throws AccumuloException
      */
     protected void setupIterators(Configuration conf, Scanner scanner) throws AccumuloException
{
-      List<AccumuloIterator> iterators = getIterators(conf);
-      List<AccumuloIteratorOption> options = getIteratorOptions(conf);
-      
-      Map<String,IteratorSetting> scanIterators = new HashMap<String,IteratorSetting>();
-      for (AccumuloIterator iterator : iterators) {
-        scanIterators.put(iterator.getIteratorName(), new IteratorSetting(iterator.getPriority(),
iterator.getIteratorName(), iterator.getIteratorClass()));
-      }
-      for (AccumuloIteratorOption option : options) {
-        scanIterators.get(option.iteratorName).addOption(option.getKey(), option.getValue());
-      }
-      for (AccumuloIterator iterator : iterators) {
-        scanner.addScanIterator(scanIterators.get(iterator.getIteratorName()));
+      List<IteratorSetting> iterators = getIterators(conf);
+      for (IteratorSetting iterator : iterators) {
+        scanner.addScanIterator(iterator);
       }
     }
     
@@ -1142,101 +1119,4 @@ public abstract class InputFormatBase<K,
         out.writeUTF(locations[i]);
     }
   }
-  
-  /**
-   * The Class IteratorSetting. Encapsulates specifics for an Accumulo iterator's name &
priority.
-   */
-  static class AccumuloIterator {
-    
-    private static final String FIELD_SEP = ":";
-    
-    private int priority;
-    private String iteratorClass;
-    private String iteratorName;
-    
-    public AccumuloIterator(int priority, String iteratorClass, String iteratorName) {
-      this.priority = priority;
-      this.iteratorClass = iteratorClass;
-      this.iteratorName = iteratorName;
-    }
-    
-    // Parses out a setting given an string supplied from an earlier toString() call
-    public AccumuloIterator(String iteratorSetting) {
-      // Parse the string to expand the iterator
-      StringTokenizer tokenizer = new StringTokenizer(iteratorSetting, FIELD_SEP);
-      priority = Integer.parseInt(tokenizer.nextToken());
-      iteratorClass = tokenizer.nextToken();
-      iteratorName = tokenizer.nextToken();
-    }
-    
-    public int getPriority() {
-      return priority;
-    }
-    
-    public String getIteratorClass() {
-      return iteratorClass;
-    }
-    
-    public String getIteratorName() {
-      return iteratorName;
-    }
-    
-    @Override
-    public String toString() {
-      return new String(priority + FIELD_SEP + iteratorClass + FIELD_SEP + iteratorName);
-    }
-    
-  }
-  
-  /**
-   * The Class AccumuloIteratorOption. Encapsulates specifics for an Accumulo iterator's
optional configuration details - associated via the iteratorName.
-   */
-  static class AccumuloIteratorOption {
-    private static final String FIELD_SEP = ":";
-    
-    private String iteratorName;
-    private String key;
-    private String value;
-    
-    public AccumuloIteratorOption(String iteratorName, String key, String value) {
-      this.iteratorName = iteratorName;
-      this.key = key;
-      this.value = value;
-    }
-    
-    // Parses out an option given a string supplied from an earlier toString() call
-    public AccumuloIteratorOption(String iteratorOption) {
-      StringTokenizer tokenizer = new StringTokenizer(iteratorOption, FIELD_SEP);
-      this.iteratorName = tokenizer.nextToken();
-      try {
-        this.key = URLDecoder.decode(tokenizer.nextToken(), "UTF-8");
-        this.value = URLDecoder.decode(tokenizer.nextToken(), "UTF-8");
-      } catch (UnsupportedEncodingException e) {
-        throw new RuntimeException(e);
-      }
-    }
-    
-    public String getIteratorName() {
-      return iteratorName;
-    }
-    
-    public String getKey() {
-      return key;
-    }
-    
-    public String getValue() {
-      return value;
-    }
-    
-    @Override
-    public String toString() {
-      try {
-        return new String(iteratorName + FIELD_SEP + URLEncoder.encode(key, "UTF-8") + FIELD_SEP
+ URLEncoder.encode(value, "UTF-8"));
-      } catch (UnsupportedEncodingException e) {
-        throw new RuntimeException(e);
-      }
-    }
-    
-  }
-  
 }

Modified: accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java?rev=1382004&r1=1382003&r2=1382004&view=diff
==============================================================================
--- accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
(original)
+++ accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
Fri Sep  7 12:56:20 2012
@@ -19,23 +19,26 @@ package org.apache.accumulo.core.client.
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
 import java.io.IOException;
+import java.util.HashSet;
 import java.util.List;
 
 import org.apache.accumulo.core.client.BatchWriter;
 import org.apache.accumulo.core.client.Connector;
 import org.apache.accumulo.core.client.IteratorSetting;
-import org.apache.accumulo.core.client.mapreduce.InputFormatBase.AccumuloIterator;
-import org.apache.accumulo.core.client.mapreduce.InputFormatBase.AccumuloIteratorOption;
 import org.apache.accumulo.core.client.mapreduce.InputFormatBase.RangeInputSplit;
 import org.apache.accumulo.core.client.mock.MockInstance;
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
 import org.apache.accumulo.core.data.Value;
 import org.apache.accumulo.core.iterators.user.RegExFilter;
 import org.apache.accumulo.core.iterators.user.WholeRowIterator;
 import org.apache.accumulo.core.security.Authorizations;
 import org.apache.accumulo.core.util.ContextFactory;
+import org.apache.commons.codec.binary.Base64;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapreduce.InputSplit;
@@ -89,15 +92,19 @@ public class AccumuloInputFormatTest {
   
   /**
    * Check that the iterator configuration is getting stored in the Job conf correctly.
+   * @throws IOException 
    */
   @Test
-  public void testSetIterator() {
+  public void testSetIterator() throws IOException {
     JobContext job = ContextFactory.createJobContext();
     
-    AccumuloInputFormat.addIterator(job.getConfiguration(), new IteratorSetting(1, "WholeRow",
"org.apache.accumulo.core.iterators.WholeRowIterator"));
+    IteratorSetting is = new IteratorSetting(1, "WholeRow", "org.apache.accumulo.core.iterators.WholeRowIterator");
+    AccumuloInputFormat.addIterator(job.getConfiguration(), is);
     Configuration conf = job.getConfiguration();
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    is.write(new DataOutputStream(baos));
     String iterators = conf.get("AccumuloInputFormat.iterators");
-    assertEquals("1:org.apache.accumulo.core.iterators.WholeRowIterator:WholeRow", iterators);
+    assertEquals(new String(Base64.encodeBase64(baos.toByteArray())), iterators);
   }
   
   @Test
@@ -111,35 +118,31 @@ public class AccumuloInputFormatTest {
     iter.addOption("junk", "\0omg:!\\xyzzy");
     AccumuloInputFormat.addIterator(job.getConfiguration(), iter);
     
-    List<AccumuloIterator> list = AccumuloInputFormat.getIterators(job.getConfiguration());
+    List<IteratorSetting> list = AccumuloInputFormat.getIterators(job.getConfiguration());
     
     // Check the list size
     assertTrue(list.size() == 3);
     
     // Walk the list and make sure our settings are correct
-    AccumuloIterator setting = list.get(0);
+    IteratorSetting setting = list.get(0);
     assertEquals(1, setting.getPriority());
     assertEquals("org.apache.accumulo.core.iterators.user.WholeRowIterator", setting.getIteratorClass());
-    assertEquals("WholeRow", setting.getIteratorName());
+    assertEquals("WholeRow", setting.getName());
+    assertEquals(0, setting.getOptions().size());
     
     setting = list.get(1);
     assertEquals(2, setting.getPriority());
     assertEquals("org.apache.accumulo.core.iterators.VersioningIterator", setting.getIteratorClass());
-    assertEquals("Versions", setting.getIteratorName());
+    assertEquals("Versions", setting.getName());
+    assertEquals(0, setting.getOptions().size());
     
     setting = list.get(2);
     assertEquals(3, setting.getPriority());
     assertEquals("org.apache.accumulo.core.iterators.CountingIterator", setting.getIteratorClass());
-    assertEquals("Count", setting.getIteratorName());
-    
-    List<AccumuloIteratorOption> iteratorOptions = AccumuloInputFormat.getIteratorOptions(job.getConfiguration());
-    assertEquals(2, iteratorOptions.size());
-    assertEquals("Count", iteratorOptions.get(0).getIteratorName());
-    assertEquals("Count", iteratorOptions.get(1).getIteratorName());
-    assertEquals("v1", iteratorOptions.get(0).getKey());
-    assertEquals("1", iteratorOptions.get(0).getValue());
-    assertEquals("junk", iteratorOptions.get(1).getKey());
-    assertEquals("\0omg:!\\xyzzy", iteratorOptions.get(1).getValue());
+    assertEquals("Count", setting.getName());
+    assertEquals(2, setting.getOptions().size());
+    assertEquals("1", setting.getOptions().get("v1"));
+    assertEquals("\0omg:!\\xyzzy", setting.getOptions().get("junk"));
   }
   
   /**
@@ -157,25 +160,22 @@ public class AccumuloInputFormatTest {
     Job job = new Job();
     AccumuloInputFormat.addIterator(job.getConfiguration(), someSetting);
     
-    final String rawConfigOpt = new AccumuloIteratorOption("iterator", key, value).toString();
-    
-    assertEquals(rawConfigOpt, job.getConfiguration().get("AccumuloInputFormat.iterators.options"));
-    
-    List<AccumuloIteratorOption> opts = AccumuloInputFormat.getIteratorOptions(job.getConfiguration());
-    assertEquals(1, opts.size());
-    assertEquals(opts.get(0).getKey(), key);
-    assertEquals(opts.get(0).getValue(), value);
+    List<IteratorSetting> list = AccumuloInputFormat.getIterators(job.getConfiguration());
+    assertEquals(1, list.size());
+    assertEquals(1, list.get(0).getOptions().size());
+    assertEquals(list.get(0).getOptions().get(key), value);
     
     someSetting.addOption(key + "2", value);
     someSetting.setPriority(2);
     someSetting.setName("it2");
     AccumuloInputFormat.addIterator(job.getConfiguration(), someSetting);
-    opts = AccumuloInputFormat.getIteratorOptions(job.getConfiguration());
-    assertEquals(3, opts.size());
-    for (AccumuloIteratorOption opt : opts) {
-      assertEquals(opt.getKey().substring(0, key.length()), key);
-      assertEquals(opt.getValue(), value);
-    }
+    list = AccumuloInputFormat.getIterators(job.getConfiguration());
+    assertEquals(2, list.size());
+    assertEquals(1, list.get(0).getOptions().size());
+    assertEquals(list.get(0).getOptions().get(key), value);
+    assertEquals(2, list.get(1).getOptions().size());
+    assertEquals(list.get(1).getOptions().get(key), value);
+    assertEquals(list.get(1).getOptions().get(key + "2"), value);
   }
   
   /**
@@ -189,26 +189,26 @@ public class AccumuloInputFormatTest {
     AccumuloInputFormat.addIterator(job.getConfiguration(), new IteratorSetting(2, "Versions",
"org.apache.accumulo.core.iterators.VersioningIterator"));
     AccumuloInputFormat.addIterator(job.getConfiguration(), new IteratorSetting(3, "Count",
"org.apache.accumulo.core.iterators.CountingIterator"));
     
-    List<AccumuloIterator> list = AccumuloInputFormat.getIterators(job.getConfiguration());
+    List<IteratorSetting> list = AccumuloInputFormat.getIterators(job.getConfiguration());
     
     // Check the list size
     assertTrue(list.size() == 3);
     
     // Walk the list and make sure our settings are correct
-    AccumuloIterator setting = list.get(0);
+    IteratorSetting setting = list.get(0);
     assertEquals(1, setting.getPriority());
     assertEquals("org.apache.accumulo.core.iterators.WholeRowIterator", setting.getIteratorClass());
-    assertEquals("WholeRow", setting.getIteratorName());
+    assertEquals("WholeRow", setting.getName());
     
     setting = list.get(1);
     assertEquals(2, setting.getPriority());
     assertEquals("org.apache.accumulo.core.iterators.VersioningIterator", setting.getIteratorClass());
-    assertEquals("Versions", setting.getIteratorName());
+    assertEquals("Versions", setting.getName());
     
     setting = list.get(2);
     assertEquals(3, setting.getPriority());
     assertEquals("org.apache.accumulo.core.iterators.CountingIterator", setting.getIteratorClass());
-    assertEquals("Count", setting.getIteratorName());
+    assertEquals("Count", setting.getName());
     
   }
   
@@ -222,19 +222,22 @@ public class AccumuloInputFormatTest {
     RegExFilter.setRegexs(is, regex, null, null, null, false);
     AccumuloInputFormat.addIterator(job.getConfiguration(), is);
     
-    assertTrue(regex.equals(AccumuloInputFormat.getIterators(job.getConfiguration()).get(0).getIteratorName()));
+    assertTrue(regex.equals(AccumuloInputFormat.getIterators(job.getConfiguration()).get(0).getName()));
   }
   
   static class TestMapper extends Mapper<Key,Value,Key,Value> {
     Key key = null;
+    int first = 0;
     int count = 0;
     
     @Override
     protected void map(Key k, Value v, Context context) throws IOException, InterruptedException
{
       if (key != null)
         assertEquals(key.getRow().toString(), new String(v.get()));
-      assertEquals(k.getRow(), new Text(String.format("%09x", count + 1)));
-      assertEquals(new String(v.get()), String.format("%09x", count));
+      else
+        first = Integer.parseInt(k.getRow().toString(), 16) - 1;
+      assertEquals(k.getRow(), new Text(String.format("%09x", first + count + 1)));
+      assertEquals(new String(v.get()), String.format("%09x", first + count));
       key = new Key(k);
       count++;
     }
@@ -259,10 +262,14 @@ public class AccumuloInputFormatTest {
     job.setNumReduceTasks(0);
     AccumuloInputFormat.setInputInfo(job.getConfiguration(), "root", "".getBytes(), "testtable",
new Authorizations());
     AccumuloInputFormat.setMockInstance(job.getConfiguration(), "testmapinstance");
+    HashSet<Range> ranges = new HashSet<Range>();
+    ranges.add(new Range("000000000", "000000010"));
+    ranges.add(new Range("000000100", "000000110"));
+    AccumuloInputFormat.setRanges(job.getConfiguration(), ranges);
     
     AccumuloInputFormat input = new AccumuloInputFormat();
     List<InputSplit> splits = input.getSplits(job);
-    assertEquals(splits.size(), 1);
+    assertEquals(splits.size(), 2);
     
     TestMapper mapper = (TestMapper) job.getMapperClass().newInstance();
     for (InputSplit split : splits) {
@@ -271,6 +278,7 @@ public class AccumuloInputFormatTest {
       Mapper<Key,Value,Key,Value>.Context context = ContextFactory.createMapContext(mapper,
tac, reader, null, split);
       reader.initialize(split, context);
       mapper.run(context);
+      assertEquals(mapper.count, 16);
     }
   }
   
@@ -302,5 +310,6 @@ public class AccumuloInputFormatTest {
     while (rr.nextKeyValue()) {
       mapper.map(rr.getCurrentKey(), rr.getCurrentValue(), (TestMapper.Context) context);
     }
+    assertEquals(mapper.count, 100);
   }
 }

Modified: accumulo/trunk/core/src/test/java/org/apache/accumulo/core/security/AuthorizationsTest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/test/java/org/apache/accumulo/core/security/AuthorizationsTest.java?rev=1382004&r1=1382003&r2=1382004&view=diff
==============================================================================
--- accumulo/trunk/core/src/test/java/org/apache/accumulo/core/security/AuthorizationsTest.java
(original)
+++ accumulo/trunk/core/src/test/java/org/apache/accumulo/core/security/AuthorizationsTest.java
Fri Sep  7 12:56:20 2012
@@ -35,8 +35,6 @@ public class AuthorizationsTest {
     byte[] array = a.getAuthorizationsArray();
     Authorizations b = new Authorizations(array);
     assertEquals(a, b);
-
-    System.out.println(b);
   }
   
 }

Propchange: accumulo/trunk/test/system/continuous/
------------------------------------------------------------------------------
--- svn:ignore (original)
+++ svn:ignore Fri Sep  7 12:56:20 2012
@@ -1,5 +1,5 @@
 continuous-env.sh
-
 walkers.txt
-
 ingesters.txt
+batch_walkers.txt
+scanners.txt



Mime
View raw message