accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ctubb...@apache.org
Subject svn commit: r1441692 - in /accumulo/trunk/core/src: main/java/org/apache/accumulo/core/client/mapreduce/ test/java/org/apache/accumulo/core/client/mapred/ test/java/org/apache/accumulo/core/client/mapreduce/
Date Sat, 02 Feb 2013 02:56:41 GMT
Author: ctubbsii
Date: Sat Feb  2 02:56:40 2013
New Revision: 1441692

URL: http://svn.apache.org/viewvc?rev=1441692&view=rev
Log:
ACCUMULO-993 Added unit tests for mapred package. Reverted deprecated tests for maxVersions and found a bug with deprecated method in mapreduce package.
Prevented M/R tests from stepping on each other with different table names for mock (based on the test class name).

Added:
    accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapred/
    accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormatTest.java
    accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java
    accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloOutputFormatTest.java
    accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloRowInputFormatTest.java
Modified:
    accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
    accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloFileOutputFormatTest.java
    accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
    accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormatTest.java
    accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloRowInputFormatTest.java

Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java?rev=1441692&r1=1441691&r2=1441692&view=diff
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java (original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java Sat Feb  2 02:56:40 2013
@@ -506,7 +506,7 @@ public abstract class InputFormatBase<K,
    * @return an Accumulo tablet locator
    * @throws TableNotFoundException
    *           if the table name set on the configuration doesn't exist
-   * @throws AccumuloSecurityException 
+   * @throws AccumuloSecurityException
    * @since 1.5.0
    */
   protected static TabletLocator getTabletLocator(JobContext context) throws TableNotFoundException, AccumuloSecurityException {
@@ -1024,8 +1024,12 @@ public abstract class InputFormatBase<K,
    */
   @Deprecated
   public static void setMaxVersions(Configuration conf, int maxVersions) throws IOException {
-    IteratorSetting vers = new IteratorSetting(0, "vers", VersioningIterator.class);
-    VersioningIterator.setMaxVersions(vers, maxVersions);
+    IteratorSetting vers = new IteratorSetting(1, "vers", VersioningIterator.class);
+    try {
+      VersioningIterator.setMaxVersions(vers, maxVersions);
+    } catch (IllegalArgumentException e) {
+      throw new IOException(e);
+    }
     InputConfigurator.addIterator(CLASS, conf, vers);
   }
   
@@ -1192,7 +1196,7 @@ public abstract class InputFormatBase<K,
     // deconstruct to get at this option in the first place, but to preserve correct behavior, this appears necessary.
     List<IteratorSetting> iteratorSettings = InputConfigurator.getIterators(CLASS, conf);
     for (IteratorSetting setting : iteratorSettings) {
-      if ("vers".equals(setting.getName()) && 0 == setting.getPriority() && VersioningIterator.class.getName().equals(setting.getIteratorClass())) {
+      if ("vers".equals(setting.getName()) && 1 == setting.getPriority() && VersioningIterator.class.getName().equals(setting.getIteratorClass())) {
         if (setting.getOptions().containsKey("maxVersions"))
           return Integer.parseInt(setting.getOptions().get("maxVersions"));
         else

Added: accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormatTest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormatTest.java?rev=1441692&view=auto
==============================================================================
--- accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormatTest.java (added)
+++ accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormatTest.java Sat Feb  2 02:56:40 2013
@@ -0,0 +1,250 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapred;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.File;
+import java.io.FileFilter;
+import java.io.IOException;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.tokens.UserPassToken;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.lib.IdentityMapper;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+public class AccumuloFileOutputFormatTest {
+  private static final String PREFIX = AccumuloFileOutputFormatTest.class.getSimpleName();
+  private static final String INSTANCE_NAME = PREFIX + "_mapred_instance";
+  private static final String BAD_TABLE = PREFIX + "_mapred_bad_table";
+  private static final String TEST_TABLE = PREFIX + "_mapred_test_table";
+  private static final String EMPTY_TABLE = PREFIX + "_mapred_empty_table";
+  
+  public static TemporaryFolder folder = new TemporaryFolder();
+  private static AssertionError e1 = null;
+  private static AssertionError e2 = null;
+  
+  @BeforeClass
+  public static void setup() throws Exception {
+    folder.create();
+    
+    MockInstance mockInstance = new MockInstance(INSTANCE_NAME);
+    Connector c = mockInstance.getConnector(new UserPassToken("root", new byte[0]));
+    c.tableOperations().create(EMPTY_TABLE);
+    c.tableOperations().create(TEST_TABLE);
+    c.tableOperations().create(BAD_TABLE);
+    BatchWriter bw = c.createBatchWriter(TEST_TABLE, new BatchWriterConfig());
+    Mutation m = new Mutation("Key");
+    m.put("", "", "");
+    bw.addMutation(m);
+    bw.close();
+    bw = c.createBatchWriter(BAD_TABLE, new BatchWriterConfig());
+    m = new Mutation("r1");
+    m.put("cf1", "cq1", "A&B");
+    m.put("cf1", "cq1", "A&B");
+    m.put("cf1", "cq2", "A&");
+    bw.addMutation(m);
+    bw.close();
+  }
+  
+  @AfterClass
+  public static void teardown() throws IOException {
+    folder.delete();
+  }
+  
+  @Test
+  public void testEmptyWrite() throws Exception {
+    handleWriteTests(false);
+  }
+  
+  @Test
+  public void testRealWrite() throws Exception {
+    handleWriteTests(true);
+  }
+  
+  private static class MRTester extends Configured implements Tool {
+    private static class BadKeyMapper implements Mapper<Key,Value,Key,Value> {
+      
+      int index = 0;
+      
+      @Override
+      public void map(Key key, Value value, OutputCollector<Key,Value> output, Reporter reporter) throws IOException {
+        try {
+          try {
+            output.collect(key, value);
+            if (index == 2)
+              fail();
+          } catch (Exception e) {
+            assertEquals(2, index);
+          }
+        } catch (AssertionError e) {
+          e1 = e;
+        }
+        index++;
+      }
+      
+      @Override
+      public void configure(JobConf job) {}
+      
+      @Override
+      public void close() throws IOException {
+        try {
+          assertEquals(2, index);
+        } catch (AssertionError e) {
+          e2 = e;
+        }
+      }
+      
+    }
+    
+    @Override
+    public int run(String[] args) throws Exception {
+      
+      if (args.length != 4) {
+        throw new IllegalArgumentException("Usage : " + MRTester.class.getName() + " <user> <pass> <table> <outputfile>");
+      }
+      
+      String user = args[0];
+      String pass = args[1];
+      String table = args[2];
+      
+      JobConf job = new JobConf(getConf());
+      job.setJarByClass(this.getClass());
+      
+      job.setInputFormat(AccumuloInputFormat.class);
+      
+      AccumuloInputFormat.setConnectorInfo(job, new UserPassToken(user, pass.getBytes(Constants.UTF8)));
+      AccumuloInputFormat.setInputTableName(job, table);
+      AccumuloInputFormat.setMockInstance(job, INSTANCE_NAME);
+      AccumuloFileOutputFormat.setOutputPath(job, new Path(args[3]));
+      
+      job.setMapperClass(BAD_TABLE.equals(table) ? BadKeyMapper.class : IdentityMapper.class);
+      job.setMapOutputKeyClass(Key.class);
+      job.setMapOutputValueClass(Value.class);
+      job.setOutputFormat(AccumuloFileOutputFormat.class);
+      
+      job.setNumReduceTasks(0);
+      
+      return JobClient.runJob(job).isSuccessful() ? 0 : 1;
+    }
+    
+    public static void main(String[] args) throws Exception {
+      assertEquals(0, ToolRunner.run(CachedConfiguration.getInstance(), new MRTester(), args));
+    }
+  }
+  
+  public void handleWriteTests(boolean content) throws Exception {
+    File f = folder.newFile();
+    f.delete();
+    MRTester.main(new String[] {"root", "", content ? TEST_TABLE : EMPTY_TABLE, f.getAbsolutePath()});
+    
+    assertTrue(f.exists());
+    File[] files = f.listFiles(new FileFilter() {
+      @Override
+      public boolean accept(File file) {
+        return file.getName().startsWith("part-m-");
+      }
+    });
+    if (content) {
+      assertEquals(1, files.length);
+      assertTrue(files[0].exists());
+    } else {
+      assertEquals(0, files.length);
+    }
+  }
+  
+  @Test
+  public void writeBadVisibility() throws Exception {
+    File f = folder.newFile();
+    f.delete();
+    MRTester.main(new String[] {"root", "", BAD_TABLE, f.getAbsolutePath()});
+    assertNull(e1);
+    assertNull(e2);
+  }
+  
+  @Test
+  public void validateConfiguration() throws IOException, InterruptedException {
+    
+    int a = 7;
+    long b = 300l;
+    long c = 50l;
+    long d = 10l;
+    String e = "snappy";
+    
+    JobConf job = new JobConf();
+    AccumuloFileOutputFormat.setReplication(job, a);
+    AccumuloFileOutputFormat.setFileBlockSize(job, b);
+    AccumuloFileOutputFormat.setDataBlockSize(job, c);
+    AccumuloFileOutputFormat.setIndexBlockSize(job, d);
+    AccumuloFileOutputFormat.setCompressionType(job, e);
+    
+    AccumuloConfiguration acuconf = AccumuloFileOutputFormat.getAccumuloConfiguration(job);
+    
+    assertEquals(7, acuconf.getCount(Property.TABLE_FILE_REPLICATION));
+    assertEquals(300l, acuconf.getMemoryInBytes(Property.TABLE_FILE_BLOCK_SIZE));
+    assertEquals(50l, acuconf.getMemoryInBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE));
+    assertEquals(10l, acuconf.getMemoryInBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX));
+    assertEquals("snappy", acuconf.get(Property.TABLE_FILE_COMPRESSION_TYPE));
+    
+    a = 17;
+    b = 1300l;
+    c = 150l;
+    d = 110l;
+    e = "lzo";
+    
+    job = new JobConf();
+    AccumuloFileOutputFormat.setReplication(job, a);
+    AccumuloFileOutputFormat.setFileBlockSize(job, b);
+    AccumuloFileOutputFormat.setDataBlockSize(job, c);
+    AccumuloFileOutputFormat.setIndexBlockSize(job, d);
+    AccumuloFileOutputFormat.setCompressionType(job, e);
+    
+    acuconf = AccumuloFileOutputFormat.getAccumuloConfiguration(job);
+    
+    assertEquals(17, acuconf.getCount(Property.TABLE_FILE_REPLICATION));
+    assertEquals(1300l, acuconf.getMemoryInBytes(Property.TABLE_FILE_BLOCK_SIZE));
+    assertEquals(150l, acuconf.getMemoryInBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE));
+    assertEquals(110l, acuconf.getMemoryInBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX));
+    assertEquals("lzo", acuconf.get(Property.TABLE_FILE_COMPRESSION_TYPE));
+    
+  }
+}

Added: accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java?rev=1441692&view=auto
==============================================================================
--- accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java (added)
+++ accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java Sat Feb  2 02:56:40 2013
@@ -0,0 +1,284 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapred;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.user.RegExFilter;
+import org.apache.accumulo.core.iterators.user.WholeRowIterator;
+import org.apache.accumulo.core.security.tokens.UserPassToken;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.lib.NullOutputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.junit.Test;
+
+public class AccumuloInputFormatTest {
+  
+  private static final String PREFIX = AccumuloInputFormatTest.class.getSimpleName();
+  private static final String INSTANCE_NAME = PREFIX + "_mapred_instance";
+  private static final String TEST_TABLE_1 = PREFIX + "_mapred_table_1";
+  
+  /**
+   * Check that the iterator configuration is getting stored in the Job conf correctly.
+   * 
+   * @throws IOException
+   */
+  @Test
+  public void testSetIterator() throws IOException {
+    JobConf job = new JobConf();
+    
+    IteratorSetting is = new IteratorSetting(1, "WholeRow", "org.apache.accumulo.core.iterators.WholeRowIterator");
+    AccumuloInputFormat.addIterator(job, is);
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    is.write(new DataOutputStream(baos));
+    String iterators = job.get("AccumuloInputFormat.ScanOpts.Iterators");
+    assertEquals(new String(Base64.encodeBase64(baos.toByteArray())), iterators);
+  }
+  
+  @Test
+  public void testAddIterator() throws IOException {
+    JobConf job = new JobConf();
+    
+    AccumuloInputFormat.addIterator(job, new IteratorSetting(1, "WholeRow", WholeRowIterator.class));
+    AccumuloInputFormat.addIterator(job, new IteratorSetting(2, "Versions", "org.apache.accumulo.core.iterators.VersioningIterator"));
+    IteratorSetting iter = new IteratorSetting(3, "Count", "org.apache.accumulo.core.iterators.CountingIterator");
+    iter.addOption("v1", "1");
+    iter.addOption("junk", "\0omg:!\\xyzzy");
+    AccumuloInputFormat.addIterator(job, iter);
+    
+    List<IteratorSetting> list = AccumuloInputFormat.getIterators(job);
+    
+    // Check the list size
+    assertTrue(list.size() == 3);
+    
+    // Walk the list and make sure our settings are correct
+    IteratorSetting setting = list.get(0);
+    assertEquals(1, setting.getPriority());
+    assertEquals("org.apache.accumulo.core.iterators.user.WholeRowIterator", setting.getIteratorClass());
+    assertEquals("WholeRow", setting.getName());
+    assertEquals(0, setting.getOptions().size());
+    
+    setting = list.get(1);
+    assertEquals(2, setting.getPriority());
+    assertEquals("org.apache.accumulo.core.iterators.VersioningIterator", setting.getIteratorClass());
+    assertEquals("Versions", setting.getName());
+    assertEquals(0, setting.getOptions().size());
+    
+    setting = list.get(2);
+    assertEquals(3, setting.getPriority());
+    assertEquals("org.apache.accumulo.core.iterators.CountingIterator", setting.getIteratorClass());
+    assertEquals("Count", setting.getName());
+    assertEquals(2, setting.getOptions().size());
+    assertEquals("1", setting.getOptions().get("v1"));
+    assertEquals("\0omg:!\\xyzzy", setting.getOptions().get("junk"));
+  }
+  
+  /**
+   * Test adding iterator options where the keys and values contain both the FIELD_SEPARATOR character (':') and ITERATOR_SEPARATOR (',') characters. There
+   * should be no exceptions thrown when trying to parse these types of option entries.
+   * 
+   * This test makes sure that the expected raw values, as appears in the Job, are equal to what's expected.
+   */
+  @Test
+  public void testIteratorOptionEncoding() throws Throwable {
+    String key = "colon:delimited:key";
+    String value = "comma,delimited,value";
+    IteratorSetting someSetting = new IteratorSetting(1, "iterator", "Iterator.class");
+    someSetting.addOption(key, value);
+    JobConf job = new JobConf();
+    AccumuloInputFormat.addIterator(job, someSetting);
+    
+    List<IteratorSetting> list = AccumuloInputFormat.getIterators(job);
+    assertEquals(1, list.size());
+    assertEquals(1, list.get(0).getOptions().size());
+    assertEquals(list.get(0).getOptions().get(key), value);
+    
+    someSetting.addOption(key + "2", value);
+    someSetting.setPriority(2);
+    someSetting.setName("it2");
+    AccumuloInputFormat.addIterator(job, someSetting);
+    list = AccumuloInputFormat.getIterators(job);
+    assertEquals(2, list.size());
+    assertEquals(1, list.get(0).getOptions().size());
+    assertEquals(list.get(0).getOptions().get(key), value);
+    assertEquals(2, list.get(1).getOptions().size());
+    assertEquals(list.get(1).getOptions().get(key), value);
+    assertEquals(list.get(1).getOptions().get(key + "2"), value);
+  }
+  
+  /**
+   * Test getting iterator settings for multiple iterators set
+   * 
+   * @throws IOException
+   */
+  @Test
+  public void testGetIteratorSettings() throws IOException {
+    JobConf job = new JobConf();
+    
+    AccumuloInputFormat.addIterator(job, new IteratorSetting(1, "WholeRow", "org.apache.accumulo.core.iterators.WholeRowIterator"));
+    AccumuloInputFormat.addIterator(job, new IteratorSetting(2, "Versions", "org.apache.accumulo.core.iterators.VersioningIterator"));
+    AccumuloInputFormat.addIterator(job, new IteratorSetting(3, "Count", "org.apache.accumulo.core.iterators.CountingIterator"));
+    
+    List<IteratorSetting> list = AccumuloInputFormat.getIterators(job);
+    
+    // Check the list size
+    assertTrue(list.size() == 3);
+    
+    // Walk the list and make sure our settings are correct
+    IteratorSetting setting = list.get(0);
+    assertEquals(1, setting.getPriority());
+    assertEquals("org.apache.accumulo.core.iterators.WholeRowIterator", setting.getIteratorClass());
+    assertEquals("WholeRow", setting.getName());
+    
+    setting = list.get(1);
+    assertEquals(2, setting.getPriority());
+    assertEquals("org.apache.accumulo.core.iterators.VersioningIterator", setting.getIteratorClass());
+    assertEquals("Versions", setting.getName());
+    
+    setting = list.get(2);
+    assertEquals(3, setting.getPriority());
+    assertEquals("org.apache.accumulo.core.iterators.CountingIterator", setting.getIteratorClass());
+    assertEquals("Count", setting.getName());
+    
+  }
+  
+  @Test
+  public void testSetRegex() throws IOException {
+    JobConf job = new JobConf();
+    
+    String regex = ">\"*%<>\'\\";
+    
+    IteratorSetting is = new IteratorSetting(50, regex, RegExFilter.class);
+    RegExFilter.setRegexs(is, regex, null, null, null, false);
+    AccumuloInputFormat.addIterator(job, is);
+    
+    assertTrue(regex.equals(AccumuloInputFormat.getIterators(job).get(0).getName()));
+  }
+  
+  private static AssertionError e1 = null;
+  private static AssertionError e2 = null;
+  
+  private static class MRTester extends Configured implements Tool {
+    private static class TestMapper implements Mapper<Key,Value,Key,Value> {
+      Key key = null;
+      int count = 0;
+      
+      @Override
+      public void map(Key k, Value v, OutputCollector<Key,Value> output, Reporter reporter) throws IOException {
+        try {
+          if (key != null)
+            assertEquals(key.getRow().toString(), new String(v.get()));
+          assertEquals(k.getRow(), new Text(String.format("%09x", count + 1)));
+          assertEquals(new String(v.get()), String.format("%09x", count));
+        } catch (AssertionError e) {
+          e1 = e;
+        }
+        key = new Key(k);
+        count++;
+      }
+      
+      @Override
+      public void configure(JobConf job) {}
+      
+      @Override
+      public void close() throws IOException {
+        try {
+          assertEquals(100, count);
+        } catch (AssertionError e) {
+          e2 = e;
+        }
+      }
+      
+    }
+    
+    @Override
+    public int run(String[] args) throws Exception {
+      
+      if (args.length != 3) {
+        throw new IllegalArgumentException("Usage : " + MRTester.class.getName() + " <user> <pass> <table>");
+      }
+      
+      String user = args[0];
+      String pass = args[1];
+      String table = args[2];
+      
+      JobConf job = new JobConf(getConf());
+      job.setJarByClass(this.getClass());
+      
+      job.setInputFormat(AccumuloInputFormat.class);
+      
+      AccumuloInputFormat.setConnectorInfo(job, new UserPassToken(user, pass));
+      AccumuloInputFormat.setInputTableName(job, table);
+      AccumuloInputFormat.setMockInstance(job, INSTANCE_NAME);
+      
+      job.setMapperClass(TestMapper.class);
+      job.setMapOutputKeyClass(Key.class);
+      job.setMapOutputValueClass(Value.class);
+      job.setOutputFormat(NullOutputFormat.class);
+      
+      job.setNumReduceTasks(0);
+      
+      return JobClient.runJob(job).isSuccessful() ? 0 : 1;
+    }
+    
+    public static void main(String[] args) throws Exception {
+      assertEquals(0, ToolRunner.run(CachedConfiguration.getInstance(), new MRTester(), args));
+    }
+  }
+  
+  @Test
+  public void testMap() throws Exception {
+    MockInstance mockInstance = new MockInstance(INSTANCE_NAME);
+    Connector c = mockInstance.getConnector(new UserPassToken("root", ""));
+    c.tableOperations().create(TEST_TABLE_1);
+    BatchWriter bw = c.createBatchWriter(TEST_TABLE_1, new BatchWriterConfig());
+    for (int i = 0; i < 100; i++) {
+      Mutation m = new Mutation(new Text(String.format("%09x", i + 1)));
+      m.put(new Text(), new Text(), new Value(String.format("%09x", i).getBytes()));
+      bw.addMutation(m);
+    }
+    bw.close();
+    
+    MRTester.main(new String[] {"root", "", TEST_TABLE_1});
+    assertNull(e1);
+    assertNull(e2);
+  }
+}

Added: accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloOutputFormatTest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloOutputFormatTest.java?rev=1441692&view=auto
==============================================================================
--- accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloOutputFormatTest.java (added)
+++ accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloOutputFormatTest.java Sat Feb  2 02:56:40 2013
@@ -0,0 +1,203 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapred;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Map.Entry;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.tokens.UserPassToken;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.junit.Test;
+
+/**
+ * 
+ */
+public class AccumuloOutputFormatTest {
+  private static AssertionError e1 = null;
+  private static final String PREFIX = AccumuloOutputFormatTest.class.getSimpleName();
+  private static final String INSTANCE_NAME = PREFIX + "_mapred_instance";
+  private static final String TEST_TABLE_1 = PREFIX + "_mapred_table_1";
+  private static final String TEST_TABLE_2 = PREFIX + "_mapred_table_2";
+  
+  private static class MRTester extends Configured implements Tool {
+    private static class TestMapper implements Mapper<Key,Value,Text,Mutation> {
+      Key key = null;
+      int count = 0;
+      OutputCollector<Text,Mutation> finalOutput;
+      
+      @Override
+      public void map(Key k, Value v, OutputCollector<Text,Mutation> output, Reporter reporter) throws IOException {
+        finalOutput = output;
+        try {
+          if (key != null)
+            assertEquals(key.getRow().toString(), new String(v.get()));
+          assertEquals(k.getRow(), new Text(String.format("%09x", count + 1)));
+          assertEquals(new String(v.get()), String.format("%09x", count));
+        } catch (AssertionError e) {
+          e1 = e;
+        }
+        key = new Key(k);
+        count++;
+      }
+      
+      @Override
+      public void configure(JobConf job) {}
+      
+      @Override
+      public void close() throws IOException {
+        Mutation m = new Mutation("total");
+        m.put("", "", Integer.toString(count));
+        finalOutput.collect(new Text(), m);
+      }
+      
+    }
+    
+    @Override
+    public int run(String[] args) throws Exception {
+      
+      if (args.length != 4) {
+        throw new IllegalArgumentException("Usage : " + MRTester.class.getName() + " <user> <pass> <inputtable> <outputtable>");
+      }
+      
+      String user = args[0];
+      String pass = args[1];
+      String table1 = args[2];
+      String table2 = args[3];
+      
+      JobConf job = new JobConf(getConf());
+      job.setJarByClass(this.getClass());
+      
+      job.setInputFormat(AccumuloInputFormat.class);
+      
+      AccumuloInputFormat.setConnectorInfo(job, new UserPassToken(user, pass));
+      AccumuloInputFormat.setInputTableName(job, table1);
+      AccumuloInputFormat.setMockInstance(job, INSTANCE_NAME);
+      
+      job.setMapperClass(TestMapper.class);
+      job.setMapOutputKeyClass(Key.class);
+      job.setMapOutputValueClass(Value.class);
+      job.setOutputFormat(AccumuloOutputFormat.class);
+      job.setOutputKeyClass(Text.class);
+      job.setOutputValueClass(Mutation.class);
+      
+      AccumuloOutputFormat.setConnectorInfo(job, new UserPassToken(user, pass));
+      AccumuloOutputFormat.setCreateTables(job, false);
+      AccumuloOutputFormat.setDefaultTableName(job, table2);
+      AccumuloOutputFormat.setMockInstance(job, INSTANCE_NAME);
+      
+      job.setNumReduceTasks(0);
+      
+      return JobClient.runJob(job).isSuccessful() ? 0 : 1;
+    }
+    
+    public static void main(String[] args) throws Exception {
+      assertEquals(0, ToolRunner.run(CachedConfiguration.getInstance(), new MRTester(), args));
+    }
+  }
+  
+  @Test
+  public void testBWSettings() throws IOException {
+    JobConf job = new JobConf();
+    
+    // make sure we aren't testing defaults
+    final BatchWriterConfig bwDefaults = new BatchWriterConfig();
+    assertNotEquals(7654321l, bwDefaults.getMaxLatency(TimeUnit.MILLISECONDS));
+    assertNotEquals(9898989l, bwDefaults.getTimeout(TimeUnit.MILLISECONDS));
+    assertNotEquals(42, bwDefaults.getMaxWriteThreads());
+    assertNotEquals(1123581321l, bwDefaults.getMaxMemory());
+    
+    final BatchWriterConfig bwConfig = new BatchWriterConfig();
+    bwConfig.setMaxLatency(7654321l, TimeUnit.MILLISECONDS);
+    bwConfig.setTimeout(9898989l, TimeUnit.MILLISECONDS);
+    bwConfig.setMaxWriteThreads(42);
+    bwConfig.setMaxMemory(1123581321l);
+    AccumuloOutputFormat.setBatchWriterOptions(job, bwConfig);
+    
+    AccumuloOutputFormat myAOF = new AccumuloOutputFormat() {
+      @Override
+      public void checkOutputSpecs(FileSystem ignored, JobConf job) throws IOException {
+        BatchWriterConfig bwOpts = getBatchWriterOptions(job);
+        
+        // passive check
+        assertEquals(bwConfig.getMaxLatency(TimeUnit.MILLISECONDS), bwOpts.getMaxLatency(TimeUnit.MILLISECONDS));
+        assertEquals(bwConfig.getTimeout(TimeUnit.MILLISECONDS), bwOpts.getTimeout(TimeUnit.MILLISECONDS));
+        assertEquals(bwConfig.getMaxWriteThreads(), bwOpts.getMaxWriteThreads());
+        assertEquals(bwConfig.getMaxMemory(), bwOpts.getMaxMemory());
+        
+        // explicit check
+        assertEquals(7654321l, bwOpts.getMaxLatency(TimeUnit.MILLISECONDS));
+        assertEquals(9898989l, bwOpts.getTimeout(TimeUnit.MILLISECONDS));
+        assertEquals(42, bwOpts.getMaxWriteThreads());
+        assertEquals(1123581321l, bwOpts.getMaxMemory());
+        
+      }
+    };
+    myAOF.checkOutputSpecs(null, job);
+  }
+  
+  @Test
+  public void testMR() throws Exception {
+    MockInstance mockInstance = new MockInstance(INSTANCE_NAME);
+    Connector c = mockInstance.getConnector(new UserPassToken("root", ""));
+    c.tableOperations().create(TEST_TABLE_1);
+    c.tableOperations().create(TEST_TABLE_2);
+    BatchWriter bw = c.createBatchWriter(TEST_TABLE_1, new BatchWriterConfig());
+    for (int i = 0; i < 100; i++) {
+      Mutation m = new Mutation(new Text(String.format("%09x", i + 1)));
+      m.put(new Text(), new Text(), new Value(String.format("%09x", i).getBytes()));
+      bw.addMutation(m);
+    }
+    bw.close();
+    
+    MRTester.main(new String[] {"root", "", TEST_TABLE_1, TEST_TABLE_2});
+    assertNull(e1);
+    
+    Scanner scanner = c.createScanner(TEST_TABLE_2, new Authorizations());
+    Iterator<Entry<Key,Value>> iter = scanner.iterator();
+    assertTrue(iter.hasNext());
+    Entry<Key,Value> entry = iter.next();
+    assertEquals(Integer.parseInt(new String(entry.getValue().get())), 100);
+    assertFalse(iter.hasNext());
+  }
+}

Added: accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloRowInputFormatTest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloRowInputFormatTest.java?rev=1441692&view=auto
==============================================================================
--- accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloRowInputFormatTest.java (added)
+++ accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloRowInputFormatTest.java Sat Feb  2 02:56:40 2013
@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapred;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map.Entry;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyValue;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.accumulo.core.security.tokens.UserPassToken;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.util.PeekingIterator;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.lib.NullOutputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.junit.Test;
+
+public class AccumuloRowInputFormatTest {
+  private static final String PREFIX = AccumuloRowInputFormatTest.class.getSimpleName();
+  private static final String INSTANCE_NAME = PREFIX + "_mapred_instance";
+  private static final String TEST_TABLE_1 = PREFIX + "_mapred_table_1";
+  
+  private static final String ROW1 = "row1";
+  private static final String ROW2 = "row2";
+  private static final String ROW3 = "row3";
+  private static final String COLF1 = "colf1";
+  private static List<Entry<Key,Value>> row1;
+  private static List<Entry<Key,Value>> row2;
+  private static List<Entry<Key,Value>> row3;
+  private static AssertionError e1 = null;
+  private static AssertionError e2 = null;
+  
+  public AccumuloRowInputFormatTest() {
+    row1 = new ArrayList<Entry<Key,Value>>();
+    row1.add(new KeyValue(new Key(ROW1, COLF1, "colq1"), "v1".getBytes()));
+    row1.add(new KeyValue(new Key(ROW1, COLF1, "colq2"), "v2".getBytes()));
+    row1.add(new KeyValue(new Key(ROW1, "colf2", "colq3"), "v3".getBytes()));
+    row2 = new ArrayList<Entry<Key,Value>>();
+    row2.add(new KeyValue(new Key(ROW2, COLF1, "colq4"), "v4".getBytes()));
+    row3 = new ArrayList<Entry<Key,Value>>();
+    row3.add(new KeyValue(new Key(ROW3, COLF1, "colq5"), "v5".getBytes()));
+  }
+  
+  public static void checkLists(final List<Entry<Key,Value>> first, final List<Entry<Key,Value>> second) {
+    assertEquals("Sizes should be the same.", first.size(), second.size());
+    for (int i = 0; i < first.size(); i++) {
+      assertEquals("Keys should be equal.", first.get(i).getKey(), second.get(i).getKey());
+      assertEquals("Values should be equal.", first.get(i).getValue(), second.get(i).getValue());
+    }
+  }
+  
+  public static void checkLists(final List<Entry<Key,Value>> first, final Iterator<Entry<Key,Value>> second) {
+    int entryIndex = 0;
+    while (second.hasNext()) {
+      final Entry<Key,Value> entry = second.next();
+      assertEquals("Keys should be equal", first.get(entryIndex).getKey(), entry.getKey());
+      assertEquals("Values should be equal", first.get(entryIndex).getValue(), entry.getValue());
+      entryIndex++;
+    }
+  }
+  
+  public static void insertList(final BatchWriter writer, final List<Entry<Key,Value>> list) throws MutationsRejectedException {
+    for (Entry<Key,Value> e : list) {
+      final Key key = e.getKey();
+      final Mutation mutation = new Mutation(key.getRow());
+      ColumnVisibility colVisibility = new ColumnVisibility(key.getColumnVisibility());
+      mutation.put(key.getColumnFamily(), key.getColumnQualifier(), colVisibility, key.getTimestamp(), e.getValue());
+      writer.addMutation(mutation);
+    }
+  }
+  
+  private static class MRTester extends Configured implements Tool {
+    public static class TestMapper implements Mapper<Text,PeekingIterator<Entry<Key,Value>>,Key,Value> {
+      int count = 0;
+      
+      @Override
+      public void map(Text k, PeekingIterator<Entry<Key,Value>> v, OutputCollector<Key,Value> output, Reporter reporter) throws IOException {
+        try {
+          switch (count) {
+            case 0:
+              assertEquals("Current key should be " + ROW1, new Text(ROW1), k);
+              checkLists(row1, v);
+              break;
+            case 1:
+              assertEquals("Current key should be " + ROW2, new Text(ROW2), k);
+              checkLists(row2, v);
+              break;
+            case 2:
+              assertEquals("Current key should be " + ROW3, new Text(ROW3), k);
+              checkLists(row3, v);
+              break;
+            default:
+              assertTrue(false);
+          }
+        } catch (AssertionError e) {
+          e1 = e;
+        }
+        count++;
+      }
+      
+      @Override
+      public void configure(JobConf job) {}
+      
+      @Override
+      public void close() throws IOException {
+        try {
+          assertEquals(3, count);
+        } catch (AssertionError e) {
+          e2 = e;
+        }
+      }
+      
+    }
+    
+    @Override
+    public int run(String[] args) throws Exception {
+      
+      if (args.length != 3) {
+        throw new IllegalArgumentException("Usage : " + MRTester.class.getName() + " <user> <pass> <table>");
+      }
+      
+      String user = args[0];
+      String pass = args[1];
+      String table = args[2];
+      
+      JobConf job = new JobConf(getConf());
+      job.setJarByClass(this.getClass());
+      
+      job.setInputFormat(AccumuloRowInputFormat.class);
+      
+      AccumuloInputFormat.setConnectorInfo(job, new UserPassToken(user, pass));
+      AccumuloInputFormat.setInputTableName(job, table);
+      AccumuloRowInputFormat.setMockInstance(job, INSTANCE_NAME);
+      
+      job.setMapperClass(TestMapper.class);
+      job.setMapOutputKeyClass(Key.class);
+      job.setMapOutputValueClass(Value.class);
+      job.setOutputFormat(NullOutputFormat.class);
+      
+      job.setNumReduceTasks(0);
+      
+      return JobClient.runJob(job).isSuccessful() ? 0 : 1;
+    }
+    
+    public static void main(String[] args) throws Exception {
+      assertEquals(0, ToolRunner.run(CachedConfiguration.getInstance(), new MRTester(), args));
+    }
+  }
+  
+  @Test
+  public void test() throws Exception {
+    final MockInstance instance = new MockInstance(INSTANCE_NAME);
+    final Connector conn = instance.getConnector(new UserPassToken("root", ""));
+    conn.tableOperations().create(TEST_TABLE_1);
+    BatchWriter writer = null;
+    try {
+      writer = conn.createBatchWriter(TEST_TABLE_1, new BatchWriterConfig());
+      insertList(writer, row1);
+      insertList(writer, row2);
+      insertList(writer, row3);
+    } finally {
+      if (writer != null) {
+        writer.close();
+      }
+    }
+    MRTester.main(new String[] {"root", "", TEST_TABLE_1});
+    assertNull(e1);
+    assertNull(e2);
+  }
+}

Modified: accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloFileOutputFormatTest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloFileOutputFormatTest.java?rev=1441692&r1=1441691&r2=1441692&view=diff
==============================================================================
--- accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloFileOutputFormatTest.java (original)
+++ accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloFileOutputFormatTest.java Sat Feb  2 02:56:40 2013
@@ -48,6 +48,12 @@ import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
 
 public class AccumuloFileOutputFormatTest {
+  private static final String PREFIX = AccumuloFileOutputFormatTest.class.getSimpleName();
+  private static final String INSTANCE_NAME = PREFIX + "_mapreduce_instance";
+  private static final String BAD_TABLE = PREFIX + "_mapreduce_bad_table";
+  private static final String TEST_TABLE = PREFIX + "_mapreduce_test_table";
+  private static final String EMPTY_TABLE = PREFIX + "_mapreduce_empty_table";
+  
   public static TemporaryFolder folder = new TemporaryFolder();
   private static AssertionError e1 = null;
   private static AssertionError e2 = null;
@@ -56,17 +62,17 @@ public class AccumuloFileOutputFormatTes
   public static void setup() throws Exception {
     folder.create();
     
-    MockInstance mockInstance = new MockInstance("testinstance");
+    MockInstance mockInstance = new MockInstance(INSTANCE_NAME);
     Connector c = mockInstance.getConnector(new UserPassToken("root", new byte[0]));
-    c.tableOperations().create("emptytable");
-    c.tableOperations().create("testtable");
-    c.tableOperations().create("badtable");
-    BatchWriter bw = c.createBatchWriter("testtable", new BatchWriterConfig());
+    c.tableOperations().create(EMPTY_TABLE);
+    c.tableOperations().create(TEST_TABLE);
+    c.tableOperations().create(BAD_TABLE);
+    BatchWriter bw = c.createBatchWriter(TEST_TABLE, new BatchWriterConfig());
     Mutation m = new Mutation("Key");
     m.put("", "", "");
     bw.addMutation(m);
     bw.close();
-    bw = c.createBatchWriter("badtable", new BatchWriterConfig());
+    bw = c.createBatchWriter(BAD_TABLE, new BatchWriterConfig());
     m = new Mutation("r1");
     m.put("cf1", "cq1", "A&B");
     m.put("cf1", "cq1", "A&B");
@@ -138,10 +144,10 @@ public class AccumuloFileOutputFormatTes
       
       AccumuloInputFormat.setConnectorInfo(job, new UserPassToken(user, pass.getBytes(Charset.forName("UTF-8"))));
       AccumuloInputFormat.setInputTableName(job, table);
-      AccumuloInputFormat.setMockInstance(job, "testinstance");
+      AccumuloInputFormat.setMockInstance(job, INSTANCE_NAME);
       AccumuloFileOutputFormat.setOutputPath(job, new Path(args[3]));
       
-      job.setMapperClass("badtable".equals(table) ? BadKeyMapper.class : Mapper.class);
+      job.setMapperClass(BAD_TABLE.equals(table) ? BadKeyMapper.class : Mapper.class);
       job.setMapOutputKeyClass(Key.class);
       job.setMapOutputValueClass(Value.class);
       job.setOutputFormatClass(AccumuloFileOutputFormat.class);
@@ -161,7 +167,7 @@ public class AccumuloFileOutputFormatTes
   public void handleWriteTests(boolean content) throws Exception {
     File f = folder.newFile();
     f.delete();
-    MRTester.main(new String[] {"root", "", content ? "testtable" : "emptytable", f.getAbsolutePath()});
+    MRTester.main(new String[] {"root", "", content ? TEST_TABLE : EMPTY_TABLE, f.getAbsolutePath()});
     
     assertTrue(f.exists());
     File[] files = f.listFiles(new FileFilter() {
@@ -182,7 +188,7 @@ public class AccumuloFileOutputFormatTes
   public void writeBadVisibility() throws Exception {
     File f = folder.newFile();
     f.delete();
-    MRTester.main(new String[] {"root", "", "badtable", f.getAbsolutePath()});
+    MRTester.main(new String[] {"root", "", BAD_TABLE, f.getAbsolutePath()});
     assertNull(e1);
     assertNull(e2);
   }

Modified: accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java?rev=1441692&r1=1441691&r2=1441692&view=diff
==============================================================================
--- accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java (original)
+++ accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java Sat Feb  2 02:56:40 2013
@@ -50,6 +50,50 @@ import org.junit.Test;
 
 public class AccumuloInputFormatTest {
   
+  private static final String PREFIX = AccumuloInputFormatTest.class.getSimpleName();
+  private static final String INSTANCE_NAME = PREFIX + "_mapreduce_instance";
+  private static final String TEST_TABLE_1 = PREFIX + "_mapreduce_table_1";
+  
+  /**
+   * Test basic setting & getting of max versions.
+   * 
+   * @throws IOException
+   *           Signals that an I/O exception has occurred.
+   */
+  @Deprecated
+  @Test
+  public void testMaxVersions() throws IOException {
+    Job job = new Job();
+    AccumuloInputFormat.setMaxVersions(job.getConfiguration(), 1);
+    int version = AccumuloInputFormat.getMaxVersions(job.getConfiguration());
+    assertEquals(1, version);
+  }
+  
+  /**
+   * Test max versions with an invalid value.
+   * 
+   * @throws IOException
+   *           Signals that an I/O exception has occurred.
+   */
+  @Deprecated
+  @Test(expected = IOException.class)
+  public void testMaxVersionsLessThan1() throws IOException {
+    Job job = new Job();
+    AccumuloInputFormat.setMaxVersions(job.getConfiguration(), 0);
+  }
+  
+  /**
+   * Test no max version configured.
+   * 
+   * @throws IOException
+   */
+  @Deprecated
+  @Test
+  public void testNoMaxVersion() throws IOException {
+    Job job = new Job();
+    assertEquals(-1, AccumuloInputFormat.getMaxVersions(job.getConfiguration()));
+  }
+  
   /**
    * Check that the iterator configuration is getting stored in the Job conf correctly.
    * 
@@ -238,7 +282,7 @@ public class AccumuloInputFormatTest {
       
       AccumuloInputFormat.setConnectorInfo(job, new UserPassToken(user, pass));
       AccumuloInputFormat.setInputTableName(job, table);
-      AccumuloInputFormat.setMockInstance(job, "testmapinstance");
+      AccumuloInputFormat.setMockInstance(job, INSTANCE_NAME);
       
       job.setMapperClass(TestMapper.class);
       job.setMapOutputKeyClass(Key.class);
@@ -259,10 +303,10 @@ public class AccumuloInputFormatTest {
   
   @Test
   public void testMap() throws Exception {
-    MockInstance mockInstance = new MockInstance("testmapinstance");
+    MockInstance mockInstance = new MockInstance(INSTANCE_NAME);
     Connector c = mockInstance.getConnector(new UserPassToken("root", ""));
-    c.tableOperations().create("testtable");
-    BatchWriter bw = c.createBatchWriter("testtable", new BatchWriterConfig());
+    c.tableOperations().create(TEST_TABLE_1);
+    BatchWriter bw = c.createBatchWriter(TEST_TABLE_1, new BatchWriterConfig());
     for (int i = 0; i < 100; i++) {
       Mutation m = new Mutation(new Text(String.format("%09x", i + 1)));
       m.put(new Text(), new Text(), new Value(String.format("%09x", i).getBytes()));
@@ -270,7 +314,7 @@ public class AccumuloInputFormatTest {
     }
     bw.close();
     
-    MRTester.main(new String[] {"root", "", "testtable"});
+    MRTester.main(new String[] {"root", "", TEST_TABLE_1});
     assertNull(e1);
     assertNull(e2);
   }

Modified: accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormatTest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormatTest.java?rev=1441692&r1=1441691&r2=1441692&view=diff
==============================================================================
--- accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormatTest.java (original)
+++ accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormatTest.java Sat Feb  2 02:56:40 2013
@@ -52,6 +52,10 @@ import org.junit.Test;
  */
 public class AccumuloOutputFormatTest {
   private static AssertionError e1 = null;
+  private static final String PREFIX = AccumuloOutputFormatTest.class.getSimpleName();
+  private static final String INSTANCE_NAME = PREFIX + "_mapreduce_instance";
+  private static final String TEST_TABLE_1 = PREFIX + "_mapreduce_table_1";
+  private static final String TEST_TABLE_2 = PREFIX + "_mapreduce_table_2";
   
   private static class MRTester extends Configured implements Tool {
     private static class TestMapper extends Mapper<Key,Value,Text,Mutation> {
@@ -99,7 +103,7 @@ public class AccumuloOutputFormatTest {
       
       AccumuloInputFormat.setConnectorInfo(job, new UserPassToken(user, pass));
       AccumuloInputFormat.setInputTableName(job, table1);
-      AccumuloInputFormat.setMockInstance(job, "testmrinstance");
+      AccumuloInputFormat.setMockInstance(job, INSTANCE_NAME);
       
       job.setMapperClass(TestMapper.class);
       job.setMapOutputKeyClass(Key.class);
@@ -111,7 +115,7 @@ public class AccumuloOutputFormatTest {
       AccumuloOutputFormat.setConnectorInfo(job, new UserPassToken(user, pass));
       AccumuloOutputFormat.setCreateTables(job, false);
       AccumuloOutputFormat.setDefaultTableName(job, table2);
-      AccumuloOutputFormat.setMockInstance(job, "testmrinstance");
+      AccumuloOutputFormat.setMockInstance(job, INSTANCE_NAME);
       
       job.setNumReduceTasks(0);
       
@@ -167,11 +171,11 @@ public class AccumuloOutputFormatTest {
   
   @Test
   public void testMR() throws Exception {
-    MockInstance mockInstance = new MockInstance("testmrinstance");
+    MockInstance mockInstance = new MockInstance(INSTANCE_NAME);
     Connector c = mockInstance.getConnector(new UserPassToken("root", ""));
-    c.tableOperations().create("testtable1");
-    c.tableOperations().create("testtable2");
-    BatchWriter bw = c.createBatchWriter("testtable1", new BatchWriterConfig());
+    c.tableOperations().create(TEST_TABLE_1);
+    c.tableOperations().create(TEST_TABLE_2);
+    BatchWriter bw = c.createBatchWriter(TEST_TABLE_1, new BatchWriterConfig());
     for (int i = 0; i < 100; i++) {
       Mutation m = new Mutation(new Text(String.format("%09x", i + 1)));
       m.put(new Text(), new Text(), new Value(String.format("%09x", i).getBytes()));
@@ -179,10 +183,10 @@ public class AccumuloOutputFormatTest {
     }
     bw.close();
     
-    MRTester.main(new String[] {"root", "", "testtable1", "testtable2"});
+    MRTester.main(new String[] {"root", "", TEST_TABLE_1, TEST_TABLE_2});
     assertNull(e1);
     
-    Scanner scanner = c.createScanner("testtable2", new Authorizations());
+    Scanner scanner = c.createScanner(TEST_TABLE_2, new Authorizations());
     Iterator<Entry<Key,Value>> iter = scanner.iterator();
     assertTrue(iter.hasNext());
     Entry<Key,Value> entry = iter.next();

Modified: accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloRowInputFormatTest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloRowInputFormatTest.java?rev=1441692&r1=1441691&r2=1441692&view=diff
==============================================================================
--- accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloRowInputFormatTest.java (original)
+++ accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloRowInputFormatTest.java Sat Feb  2 02:56:40 2013
@@ -49,6 +49,10 @@ import org.apache.hadoop.util.ToolRunner
 import org.junit.Test;
 
 public class AccumuloRowInputFormatTest {
+  private static final String PREFIX = AccumuloRowInputFormatTest.class.getSimpleName();
+  private static final String INSTANCE_NAME = PREFIX + "_mapreduce_instance";
+  private static final String TEST_TABLE_1 = PREFIX + "_mapreduce_table_1";
+  
   private static final String ROW1 = "row1";
   private static final String ROW2 = "row2";
   private static final String ROW3 = "row3";
@@ -155,7 +159,7 @@ public class AccumuloRowInputFormatTest 
       
       AccumuloInputFormat.setConnectorInfo(job, new UserPassToken(user, pass));
       AccumuloInputFormat.setInputTableName(job, table);
-      AccumuloRowInputFormat.setMockInstance(job, "instance1");
+      AccumuloRowInputFormat.setMockInstance(job, INSTANCE_NAME);
       
       job.setMapperClass(TestMapper.class);
       job.setMapOutputKeyClass(Key.class);
@@ -176,12 +180,12 @@ public class AccumuloRowInputFormatTest 
   
   @Test
   public void test() throws Exception {
-    final MockInstance instance = new MockInstance("instance1");
+    final MockInstance instance = new MockInstance(INSTANCE_NAME);
     final Connector conn = instance.getConnector(new UserPassToken("root", ""));
-    conn.tableOperations().create("test");
+    conn.tableOperations().create(TEST_TABLE_1);
     BatchWriter writer = null;
     try {
-      writer = conn.createBatchWriter("test", new BatchWriterConfig());
+      writer = conn.createBatchWriter(TEST_TABLE_1, new BatchWriterConfig());
       insertList(writer, row1);
       insertList(writer, row2);
       insertList(writer, row3);
@@ -190,7 +194,7 @@ public class AccumuloRowInputFormatTest 
         writer.close();
       }
     }
-    MRTester.main(new String[] {"root", "", "test"});
+    MRTester.main(new String[] {"root", "", TEST_TABLE_1});
     assertNull(e1);
     assertNull(e2);
   }



Mime
View raw message