accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vi...@apache.org
Subject svn commit: r1437726 [6/10] - in /accumulo/branches/ACCUMULO-259: ./ assemble/ conf/examples/1GB/native-standalone/ conf/examples/1GB/standalone/ conf/examples/2GB/native-standalone/ conf/examples/2GB/standalone/ conf/examples/3GB/native-standalone/ co...
Date Wed, 23 Jan 2013 20:52:04 GMT
Modified: accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java?rev=1437726&r1=1437725&r2=1437726&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java (original)
+++ accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/util/shell/Shell.java Wed Jan 23 20:51:59 2013
@@ -62,6 +62,7 @@ import org.apache.accumulo.core.util.for
 import org.apache.accumulo.core.util.format.Formatter;
 import org.apache.accumulo.core.util.format.FormatterFactory;
 import org.apache.accumulo.core.util.shell.commands.AboutCommand;
+import org.apache.accumulo.core.util.shell.commands.AddAuthsCommand;
 import org.apache.accumulo.core.util.shell.commands.AddSplitsCommand;
 import org.apache.accumulo.core.util.shell.commands.AuthenticateCommand;
 import org.apache.accumulo.core.util.shell.commands.ByeCommand;
@@ -104,6 +105,7 @@ import org.apache.accumulo.core.util.she
 import org.apache.accumulo.core.util.shell.commands.InfoCommand;
 import org.apache.accumulo.core.util.shell.commands.InsertCommand;
 import org.apache.accumulo.core.util.shell.commands.InterpreterCommand;
+import org.apache.accumulo.core.util.shell.commands.ListCompactionsCommand;
 import org.apache.accumulo.core.util.shell.commands.ListIterCommand;
 import org.apache.accumulo.core.util.shell.commands.ListScansCommand;
 import org.apache.accumulo.core.util.shell.commands.MaxRowCommand;
@@ -112,6 +114,7 @@ import org.apache.accumulo.core.util.she
 import org.apache.accumulo.core.util.shell.commands.OfflineCommand;
 import org.apache.accumulo.core.util.shell.commands.OnlineCommand;
 import org.apache.accumulo.core.util.shell.commands.PasswdCommand;
+import org.apache.accumulo.core.util.shell.commands.PingCommand;
 import org.apache.accumulo.core.util.shell.commands.QuestionCommand;
 import org.apache.accumulo.core.util.shell.commands.QuitCommand;
 import org.apache.accumulo.core.util.shell.commands.QuotedStringTokenizer;
@@ -293,7 +296,8 @@ public class Shell extends ShellOptions 
     
     Command[] dataCommands = {new DeleteCommand(), new DeleteManyCommand(), new DeleteRowsCommand(), new EGrepCommand(), new FormatterCommand(),
         new InterpreterCommand(), new GrepCommand(), new ImportDirectoryCommand(), new InsertCommand(), new MaxRowCommand(), new ScanCommand()};
-    Command[] debuggingCommands = {new ClasspathCommand(), new DebugCommand(), new ListScansCommand(), new TraceCommand()};
+    Command[] debuggingCommands = {new ClasspathCommand(), new DebugCommand(), new ListScansCommand(), new ListCompactionsCommand(), new TraceCommand(),
+        new PingCommand()};
     Command[] execCommands = {new ExecfileCommand(), new HistoryCommand()};
     Command[] exitCommands = {new ByeCommand(), new ExitCommand(), new QuitCommand()};
     Command[] helpCommands = {new AboutCommand(), new HelpCommand(), new InfoCommand(), new QuestionCommand()};
@@ -308,7 +312,7 @@ public class Shell extends ShellOptions 
         new TablesCommand()};
     Command[] tableControlCommands = {new AddSplitsCommand(), new CompactCommand(), new ConstraintCommand(), new FlushCommand(), new GetGroupsCommand(),
         new GetSplitsCommand(), new MergeCommand(), new SetGroupsCommand()};
-    Command[] userCommands = {new CreateUserCommand(), new DeleteUserCommand(), new DropUserCommand(), new GetAuthsCommand(), new PasswdCommand(),
+    Command[] userCommands = {new AddAuthsCommand(), new CreateUserCommand(), new DeleteUserCommand(), new DropUserCommand(), new GetAuthsCommand(), new PasswdCommand(),
         new SetAuthsCommand(), new UsersCommand()};
     commandGrouping.put("-- Writing, Reading, and Removing Data --", dataCommands);
     commandGrouping.put("-- Debugging Commands -------------------", debuggingCommands);

Modified: accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/util/shell/commands/DeleteCommand.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/util/shell/commands/DeleteCommand.java?rev=1437726&r1=1437725&r2=1437726&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/util/shell/commands/DeleteCommand.java (original)
+++ accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/util/shell/commands/DeleteCommand.java Wed Jan 23 20:51:59 2013
@@ -47,8 +47,9 @@ public class DeleteCommand extends Comma
     return Long.MAX_VALUE;
   }
   
-  public int execute(final String fullCommand, final CommandLine cl, final Shell shellState) throws AccumuloException, AccumuloSecurityException, TableNotFoundException,
-      IOException, ConstraintViolationException {
+  @Override
+  public int execute(final String fullCommand, final CommandLine cl, final Shell shellState) throws AccumuloException, AccumuloSecurityException,
+      TableNotFoundException, IOException, ConstraintViolationException {
     shellState.checkTableState();
     
     final Mutation m = new Mutation(new Text(cl.getArgs()[0].getBytes(Shell.CHARSET)));
@@ -68,7 +69,7 @@ public class DeleteCommand extends Comma
       m.putDelete(colf, colq);
     }
     final BatchWriter bw = shellState.getConnector().createBatchWriter(shellState.getTableName(),
-        new BatchWriterConfig().setMaxMemory(m.estimatedMemoryUsed()).setMaxWriteThreads(1).setTimeout(getTimeout(cl), TimeUnit.MILLISECONDS));
+        new BatchWriterConfig().setMaxMemory(Math.max(m.estimatedMemoryUsed(), 1024)).setMaxWriteThreads(1).setTimeout(getTimeout(cl), TimeUnit.MILLISECONDS));
     bw.addMutation(m);
     bw.close();
     return 0;
@@ -100,7 +101,7 @@ public class DeleteCommand extends Comma
         "time before insert should fail if no data is written. If no unit is given assumes seconds.  Units d,h,m,s,and ms are supported.  e.g. 30s or 100ms");
     timeoutOption.setArgName("timeout");
     o.addOption(timeoutOption);
-
+    
     return o;
   }
   

Modified: accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/util/shell/commands/DeleteManyCommand.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/util/shell/commands/DeleteManyCommand.java?rev=1437726&r1=1437725&r2=1437726&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/util/shell/commands/DeleteManyCommand.java (original)
+++ accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/util/shell/commands/DeleteManyCommand.java Wed Jan 23 20:51:59 2013
@@ -46,6 +46,9 @@ public class DeleteManyCommand extends S
     
     scanner.addScanIterator(new IteratorSetting(Integer.MAX_VALUE, "NOVALUE", SortedKeyIterator.class));
     
+    // handle session-specific scan iterators
+    addScanIterators(shellState, scanner, tableName);
+    
     // handle remaining optional arguments
     scanner.setRange(getRange(cl, interpeter));
     

Modified: accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/util/shell/commands/InsertCommand.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/util/shell/commands/InsertCommand.java?rev=1437726&r1=1437725&r2=1437726&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/util/shell/commands/InsertCommand.java (original)
+++ accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/util/shell/commands/InsertCommand.java Wed Jan 23 20:51:59 2013
@@ -54,9 +54,10 @@ public class InsertCommand extends Comma
     
     return Long.MAX_VALUE;
   }
-
-  public int execute(final String fullCommand, final CommandLine cl, final Shell shellState) throws AccumuloException, AccumuloSecurityException, TableNotFoundException,
-      IOException, ConstraintViolationException {
+  
+  @Override
+  public int execute(final String fullCommand, final CommandLine cl, final Shell shellState) throws AccumuloException, AccumuloSecurityException,
+      TableNotFoundException, IOException, ConstraintViolationException {
     shellState.checkTableState();
     
     final Mutation m = new Mutation(new Text(cl.getArgs()[0].getBytes(Shell.CHARSET)));
@@ -78,7 +79,7 @@ public class InsertCommand extends Comma
       m.put(colf, colq, val);
     
     final BatchWriter bw = shellState.getConnector().createBatchWriter(shellState.getTableName(),
-        new BatchWriterConfig().setMaxMemory(m.estimatedMemoryUsed()).setMaxWriteThreads(1).setTimeout(getTimeout(cl), TimeUnit.MILLISECONDS));
+        new BatchWriterConfig().setMaxMemory(Math.max(m.estimatedMemoryUsed(), 1024)).setMaxWriteThreads(1).setTimeout(getTimeout(cl), TimeUnit.MILLISECONDS));
     bw.addMutation(m);
     try {
       bw.close();
@@ -103,7 +104,7 @@ public class InsertCommand extends Comma
         if (e.getCause() != null)
           lines.add("   Caused by : " + e.getCause().getClass().getName() + " : " + e.getCause().getMessage());
       }
-
+      
       shellState.printLines(lines.iterator(), false);
     }
     return 0;
@@ -134,7 +135,7 @@ public class InsertCommand extends Comma
         "time before insert should fail if no data is written. If no unit is given assumes seconds.  Units d,h,m,s,and ms are supported.  e.g. 30s or 100ms");
     timeoutOption.setArgName("timeout");
     o.addOption(timeoutOption);
-
+    
     return o;
   }
   

Modified: accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/util/shell/commands/InterpreterCommand.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/util/shell/commands/InterpreterCommand.java?rev=1437726&r1=1437725&r2=1437726&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/util/shell/commands/InterpreterCommand.java (original)
+++ accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/util/shell/commands/InterpreterCommand.java Wed Jan 23 20:51:59 2013
@@ -25,15 +25,10 @@ import org.apache.accumulo.core.util.she
  */
 public class InterpreterCommand extends ShellPluginConfigurationCommand {
   
-  /**
-   * @param typeName
-   * @param tableProp
-   * @param classOpt
-   */
   public InterpreterCommand() {
     super("interpreter", Property.TABLE_INTERPRETER_CLASS, "i");
   }
-
+  
   @Override
   public String description() {
     return "specifies a scan interpreter to interpret scan range and column arguments";

Modified: accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/util/shell/commands/MergeCommand.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/util/shell/commands/MergeCommand.java?rev=1437726&r1=1437725&r2=1437726&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/util/shell/commands/MergeCommand.java (original)
+++ accumulo/branches/ACCUMULO-259/core/src/main/java/org/apache/accumulo/core/util/shell/commands/MergeCommand.java Wed Jan 23 20:51:59 2013
@@ -92,7 +92,9 @@ public class MergeCommand extends Comman
     verboseOpt = new Option("v", "verbose", false, "verbose output during merge");
     sizeOpt = new Option("s", "size", true, "merge tablets to the given size over the entire table");
     forceOpt = new Option("f", "force", false, "merge small tablets to large tablets, even if it goes over the given size");
-    o.addOption(OptUtil.startRowOpt());
+    Option startRowOpt = OptUtil.startRowOpt();
+    startRowOpt.setDescription("begin row (NOT inclusive)");
+    o.addOption(startRowOpt);
     o.addOption(OptUtil.endRowOpt());
     o.addOption(OptUtil.tableOpt("table to be merged"));
     o.addOption(verboseOpt);

Modified: accumulo/branches/ACCUMULO-259/core/src/main/thrift/tabletserver.thrift
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/core/src/main/thrift/tabletserver.thrift?rev=1437726&r1=1437725&r2=1437726&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/core/src/main/thrift/tabletserver.thrift (original)
+++ accumulo/branches/ACCUMULO-259/core/src/main/thrift/tabletserver.thrift Wed Jan 23 20:51:59 2013
@@ -87,6 +87,35 @@ struct ActiveScan {
     13:list<binary> authorizations
 }
 
+enum CompactionType {
+   MINOR,
+   MERGE,
+   MAJOR,
+   FULL
+}
+
+enum CompactionReason {
+   USER,
+   SYSTEM,
+   CHOP,
+   IDLE,
+   CLOSE
+}
+
+struct ActiveCompaction {
+    1:data.TKeyExtent extent
+    2:i64 age
+    3:list<string> inputFiles
+    4:string outputFile
+    5:CompactionType type
+    6:CompactionReason reason
+    7:string localityGroup
+    8:i64 entriesRead
+    9:i64 entriesWritten
+    10:list<data.IterInfo> ssiList
+    11:map<string, map<string, string>> ssio 
+}
+
 struct TIteratorSetting {
     1:i32 priority;
     2:string name;
@@ -157,6 +186,7 @@ service TabletClientService extends clie
   oneway void fastHalt(3:cloudtrace.TInfo tinfo, 1:security.ThriftInstanceTokenWrapper credentials, 2:string lock);
   
   list<ActiveScan> getActiveScans(2:cloudtrace.TInfo tinfo, 1:security.ThriftInstanceTokenWrapper credentials) throws (1:security.ThriftSecurityException sec)
+  list<ActiveCompaction> getActiveCompactions(2:cloudtrace.TInfo tinfo, 1:security.ThriftInstanceTokenWrapper credentials) throws (1:security.ThriftSecurityException sec)
   oneway void removeLogs(1:cloudtrace.TInfo tinfo, 2:security.ThriftInstanceTokenWrapper credentials, 3:list<string> filenames)
 }
 

Modified: accumulo/branches/ACCUMULO-259/core/src/test/java/org/apache/accumulo/core/cli/TestClientOpts.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/core/src/test/java/org/apache/accumulo/core/cli/TestClientOpts.java?rev=1437726&r1=1437725&r2=1437726&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/core/src/test/java/org/apache/accumulo/core/cli/TestClientOpts.java (original)
+++ accumulo/branches/ACCUMULO-259/core/src/test/java/org/apache/accumulo/core/cli/TestClientOpts.java Wed Jan 23 20:51:59 2013
@@ -27,6 +27,7 @@ import java.util.concurrent.TimeUnit;
 import org.apache.accumulo.core.Constants;
 import org.apache.accumulo.core.client.BatchWriterConfig;
 import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.tokens.UserPassToken;
 import org.junit.Test;
 
 import com.beust.jcommander.JCommander;
@@ -43,7 +44,7 @@ public class TestClientOpts {
     BatchScannerOpts bsOpts = new BatchScannerOpts();
     assertEquals(System.getProperty("user.name"), args.user);
     assertNull(args.securePassword);
-    assertArrayEquals("secret".getBytes(), args.getPassword());
+    assertArrayEquals("secret".getBytes(), ((UserPassToken) args.getAccumuloToken()).getPassword());
     assertEquals(new Long(cfg.getMaxLatency(TimeUnit.MILLISECONDS)), bwOpts.batchLatency);
     assertEquals(new Long(cfg.getTimeout(TimeUnit.MILLISECONDS)), bwOpts.batchTimeout);
     assertEquals(new Long(cfg.getMaxMemory()), bwOpts.batchMemory);
@@ -74,7 +75,7 @@ public class TestClientOpts {
         "--help");
     assertEquals("bar", args.user);
     assertNull(args.securePassword);
-    assertArrayEquals("foo".getBytes(), args.getPassword());
+    assertArrayEquals("foo".getBytes(), ((UserPassToken) args.getAccumuloToken()).getPassword());
     assertEquals(new Long(3000), bwOpts.batchLatency);
     assertEquals(new Long(2000), bwOpts.batchTimeout);
     assertEquals(new Long(1024*1024), bwOpts.batchMemory);

Modified: accumulo/branches/ACCUMULO-259/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloFileOutputFormatTest.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloFileOutputFormatTest.java?rev=1437726&r1=1437725&r2=1437726&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloFileOutputFormatTest.java (original)
+++ accumulo/branches/ACCUMULO-259/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloFileOutputFormatTest.java Wed Jan 23 20:51:59 2013
@@ -17,92 +17,176 @@
 package org.apache.accumulo.core.client.mapreduce;
 
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
+import java.io.File;
+import java.io.FileFilter;
 import java.io.IOException;
+import java.nio.charset.Charset;
 
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.mock.MockInstance;
 import org.apache.accumulo.core.conf.AccumuloConfiguration;
 import org.apache.accumulo.core.conf.Property;
 import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
 import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.util.ContextFactory;
+import org.apache.accumulo.core.security.tokens.UserPassToken;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.RecordWriter;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.junit.After;
-import org.junit.Before;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
 import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
 
 public class AccumuloFileOutputFormatTest {
-  static Job job;
-  static TaskAttemptContext tac;
-  static Path f = null;
-  
-  @Before
-  public void setup() throws IOException {
-    job = new Job();
-    
-    Path file = new Path("target/");
-    f = new Path(file, "_temporary");
-    job.getConfiguration().set("mapred.output.dir", file.toString());
-    
-    tac = ContextFactory.createTaskAttemptContext(job);
-  }
-  
-  @After
-  public void teardown() throws IOException {
-    if (f != null && f.getFileSystem(job.getConfiguration()).exists(f)) {
-      f.getFileSystem(job.getConfiguration()).delete(f, true);
-    }
+  public static TemporaryFolder folder = new TemporaryFolder();
+  private static AssertionError e1 = null;
+  private static AssertionError e2 = null;
+  
+  @BeforeClass
+  public static void setup() throws Exception {
+    folder.create();
+    
+    MockInstance mockInstance = new MockInstance("testinstance");
+    Connector c = mockInstance.getConnector(new UserPassToken("root", new byte[0]));
+    c.tableOperations().create("emptytable");
+    c.tableOperations().create("testtable");
+    c.tableOperations().create("badtable");
+    BatchWriter bw = c.createBatchWriter("testtable", new BatchWriterConfig());
+    Mutation m = new Mutation("Key");
+    m.put("", "", "");
+    bw.addMutation(m);
+    bw.close();
+    bw = c.createBatchWriter("badtable", new BatchWriterConfig());
+    m = new Mutation("r1");
+    m.put("cf1", "cq1", "A&B");
+    m.put("cf1", "cq1", "A&B");
+    m.put("cf1", "cq2", "A&");
+    bw.addMutation(m);
+    bw.close();
+  }
+  
+  @AfterClass
+  public static void teardown() throws IOException {
+    folder.delete();
   }
   
   @Test
-  public void testEmptyWrite() throws IOException, InterruptedException {
+  public void testEmptyWrite() throws Exception {
     handleWriteTests(false);
   }
   
   @Test
-  public void testRealWrite() throws IOException, InterruptedException {
+  public void testRealWrite() throws Exception {
     handleWriteTests(true);
   }
   
-  public void handleWriteTests(boolean content) throws IOException, InterruptedException {
-    AccumuloFileOutputFormat afof = new AccumuloFileOutputFormat();
-    RecordWriter<Key,Value> rw = afof.getRecordWriter(tac);
-    
-    if (content)
-      rw.write(new Key("Key"), new Value("".getBytes()));
-    
-    Path file = afof.getDefaultWorkFile(tac, ".rf");
-    System.out.println(file);
-    rw.close(tac);
-    
-    if (content)
-      assertTrue(file.getFileSystem(job.getConfiguration()).exists(file));
-    else
-      assertFalse(file.getFileSystem(job.getConfiguration()).exists(file));
-    file.getFileSystem(tac.getConfiguration()).delete(file.getParent(), true);
+  private static class MRTester extends Configured implements Tool {
+    private static class BadKeyMapper extends Mapper<Key,Value,Key,Value> {
+      int index = 0;
+      
+      @Override
+      protected void map(Key key, Value value, Context context) throws IOException, InterruptedException {
+        try {
+          try {
+            context.write(key, value);
+            if (index == 2)
+              assertTrue(false);
+          } catch (Exception e) {
+            assertEquals(2, index);
+          }
+        } catch (AssertionError e) {
+          e1 = e;
+        }
+        index++;
+      }
+      
+      @Override
+      protected void cleanup(Context context) throws IOException, InterruptedException {
+        try {
+          assertEquals(2, index);
+        } catch (AssertionError e) {
+          e2 = e;
+        }
+      }
+    }
+    
+    @Override
+    public int run(String[] args) throws Exception {
+      
+      if (args.length != 4) {
+        throw new IllegalArgumentException("Usage : " + MRTester.class.getName() + " <user> <pass> <table> <outputfile>");
+      }
+      
+      String user = args[0];
+      String pass = args[1];
+      String table = args[2];
+      
+      Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
+      job.setJarByClass(this.getClass());
+      
+      job.setInputFormatClass(AccumuloInputFormat.class);
+      
+      AccumuloInputFormat.setConnectorInfo(job, new UserPassToken(user, pass.getBytes(Charset.forName("UTF-8"))));
+      AccumuloInputFormat.setInputTableName(job, table);
+      AccumuloInputFormat.setMockInstance(job, "testinstance");
+      AccumuloFileOutputFormat.setOutputPath(job, new Path(args[3]));
+      
+      job.setMapperClass("badtable".equals(table) ? BadKeyMapper.class : Mapper.class);
+      job.setMapOutputKeyClass(Key.class);
+      job.setMapOutputValueClass(Value.class);
+      job.setOutputFormatClass(AccumuloFileOutputFormat.class);
+      
+      job.setNumReduceTasks(0);
+      
+      job.waitForCompletion(true);
+      
+      return job.isSuccessful() ? 0 : 1;
+    }
+    
+    public static void main(String[] args) throws Exception {
+      assertEquals(0, ToolRunner.run(CachedConfiguration.getInstance(), new MRTester(), args));
+    }
+  }
+  
+  public void handleWriteTests(boolean content) throws Exception {
+    File f = folder.newFile();
+    f.delete();
+    MRTester.main(new String[] {"root", "", content ? "testtable" : "emptytable", f.getAbsolutePath()});
+    
+    assertTrue(f.exists());
+    File[] files = f.listFiles(new FileFilter() {
+      @Override
+      public boolean accept(File file) {
+        return file.getName().startsWith("part-m-");
+      }
+    });
+    if (content) {
+      assertEquals(1, files.length);
+      assertTrue(files[0].exists());
+    } else {
+      assertEquals(0, files.length);
+    }
   }
   
   @Test
-  public void writeBadVisibility() throws IOException, InterruptedException {
-    AccumuloFileOutputFormat afof = new AccumuloFileOutputFormat();
-    RecordWriter<Key,Value> rw = afof.getRecordWriter(tac);
-    
-    Path file = afof.getDefaultWorkFile(tac, ".rf");
-
-    rw.write(new Key("r1", "cf1", "cq1", "A&B"), new Value("".getBytes()));
-    rw.write(new Key("r1", "cf1", "cq2", "A&B"), new Value("".getBytes()));
-    try {
-      rw.write(new Key("r1", "cf1", "cq2", "A&"), new Value("".getBytes()));
-      assertFalse(true);
-    } catch (Exception e) {}
-    
-    file.getFileSystem(tac.getConfiguration()).delete(file.getParent(), true);
+  public void writeBadVisibility() throws Exception {
+    File f = folder.newFile();
+    f.delete();
+    MRTester.main(new String[] {"root", "", "badtable", f.getAbsolutePath()});
+    assertNull(e1);
+    assertNull(e2);
   }
-
+  
   @Test
   public void validateConfiguration() throws IOException, InterruptedException {
     
@@ -110,8 +194,9 @@ public class AccumuloFileOutputFormatTes
     long b = 300l;
     long c = 50l;
     long d = 10l;
-    String e = "type";
+    String e = "snappy";
     
+    Job job = new Job();
     AccumuloFileOutputFormat.setReplication(job, a);
     AccumuloFileOutputFormat.setFileBlockSize(job, b);
     AccumuloFileOutputFormat.setDataBlockSize(job, c);
@@ -120,10 +205,32 @@ public class AccumuloFileOutputFormatTes
     
     AccumuloConfiguration acuconf = AccumuloFileOutputFormat.getAccumuloConfiguration(job);
     
-    assertEquals(a, acuconf.getCount(Property.TABLE_FILE_REPLICATION));
-    assertEquals(b, acuconf.getMemoryInBytes(Property.TABLE_FILE_BLOCK_SIZE));
-    assertEquals(c, acuconf.getMemoryInBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE));
-    assertEquals(d, acuconf.getMemoryInBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX));
-    assertEquals(e, acuconf.get(Property.TABLE_FILE_COMPRESSION_TYPE));
+    assertEquals(7, acuconf.getCount(Property.TABLE_FILE_REPLICATION));
+    assertEquals(300l, acuconf.getMemoryInBytes(Property.TABLE_FILE_BLOCK_SIZE));
+    assertEquals(50l, acuconf.getMemoryInBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE));
+    assertEquals(10l, acuconf.getMemoryInBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX));
+    assertEquals("snappy", acuconf.get(Property.TABLE_FILE_COMPRESSION_TYPE));
+    
+    a = 17;
+    b = 1300l;
+    c = 150l;
+    d = 110l;
+    e = "lzo";
+    
+    job = new Job();
+    AccumuloFileOutputFormat.setReplication(job, a);
+    AccumuloFileOutputFormat.setFileBlockSize(job, b);
+    AccumuloFileOutputFormat.setDataBlockSize(job, c);
+    AccumuloFileOutputFormat.setIndexBlockSize(job, d);
+    AccumuloFileOutputFormat.setCompressionType(job, e);
+    
+    acuconf = AccumuloFileOutputFormat.getAccumuloConfiguration(job);
+    
+    assertEquals(17, acuconf.getCount(Property.TABLE_FILE_REPLICATION));
+    assertEquals(1300l, acuconf.getMemoryInBytes(Property.TABLE_FILE_BLOCK_SIZE));
+    assertEquals(150l, acuconf.getMemoryInBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE));
+    assertEquals(110l, acuconf.getMemoryInBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX));
+    assertEquals("lzo", acuconf.get(Property.TABLE_FILE_COMPRESSION_TYPE));
+    
   }
 }

Modified: accumulo/branches/ACCUMULO-259/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java?rev=1437726&r1=1437725&r2=1437726&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java (original)
+++ accumulo/branches/ACCUMULO-259/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java Wed Jan 23 20:51:59 2013
@@ -17,6 +17,7 @@
 package org.apache.accumulo.core.client.mapreduce;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
 import java.io.ByteArrayOutputStream;
@@ -28,97 +29,57 @@ import org.apache.accumulo.core.client.B
 import org.apache.accumulo.core.client.BatchWriterConfig;
 import org.apache.accumulo.core.client.Connector;
 import org.apache.accumulo.core.client.IteratorSetting;
-import org.apache.accumulo.core.client.mapreduce.InputFormatBase.RangeInputSplit;
 import org.apache.accumulo.core.client.mock.MockInstance;
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.Mutation;
 import org.apache.accumulo.core.data.Value;
 import org.apache.accumulo.core.iterators.user.RegExFilter;
 import org.apache.accumulo.core.iterators.user.WholeRowIterator;
-import org.apache.accumulo.core.security.Authorizations;
 import org.apache.accumulo.core.security.tokens.UserPassToken;
-import org.apache.accumulo.core.util.ContextFactory;
+import org.apache.accumulo.core.util.CachedConfiguration;
 import org.apache.commons.codec.binary.Base64;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.InputSplit;
 import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.JobContext;
 import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.RecordReader;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.junit.After;
+import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
 import org.junit.Test;
 
 public class AccumuloInputFormatTest {
   
-  @After
-  public void tearDown() throws Exception {}
-  
   /**
-   * Test basic setting & getting of max versions.
-   * 
-   * @throws IOException
-   *           Signals that an I/O exception has occurred.
-   */
-  @Test
-  public void testMaxVersions() throws IOException {
-    JobContext job = ContextFactory.createJobContext();
-    AccumuloInputFormat.setMaxVersions(job.getConfiguration(), 1);
-    int version = AccumuloInputFormat.getMaxVersions(job.getConfiguration());
-    assertEquals(1, version);
-  }
-  
-  /**
-   * Test max versions with an invalid value.
+   * Check that the iterator configuration is getting stored in the Job conf correctly.
    * 
    * @throws IOException
-   *           Signals that an I/O exception has occurred.
-   */
-  @Test(expected = IOException.class)
-  public void testMaxVersionsLessThan1() throws IOException {
-    JobContext job = ContextFactory.createJobContext();
-    AccumuloInputFormat.setMaxVersions(job.getConfiguration(), 0);
-  }
-  
-  /**
-   * Test no max version configured.
-   */
-  @Test
-  public void testNoMaxVersion() {
-    JobContext job = ContextFactory.createJobContext();
-    assertEquals(-1, AccumuloInputFormat.getMaxVersions(job.getConfiguration()));
-  }
-  
-  /**
-   * Check that the iterator configuration is getting stored in the Job conf correctly.
-   * @throws IOException 
    */
   @Test
   public void testSetIterator() throws IOException {
-    JobContext job = ContextFactory.createJobContext();
+    Job job = new Job();
     
     IteratorSetting is = new IteratorSetting(1, "WholeRow", "org.apache.accumulo.core.iterators.WholeRowIterator");
-    AccumuloInputFormat.addIterator(job.getConfiguration(), is);
+    AccumuloInputFormat.addIterator(job, is);
     Configuration conf = job.getConfiguration();
     ByteArrayOutputStream baos = new ByteArrayOutputStream();
     is.write(new DataOutputStream(baos));
-    String iterators = conf.get("AccumuloInputFormat.iterators");
+    String iterators = conf.get("AccumuloInputFormat.ScanOpts.Iterators");
     assertEquals(new String(Base64.encodeBase64(baos.toByteArray())), iterators);
   }
   
   @Test
-  public void testAddIterator() {
-    JobContext job = ContextFactory.createJobContext();
+  public void testAddIterator() throws IOException {
+    Job job = new Job();
     
-    AccumuloInputFormat.addIterator(job.getConfiguration(), new IteratorSetting(1, "WholeRow", WholeRowIterator.class));
-    AccumuloInputFormat.addIterator(job.getConfiguration(), new IteratorSetting(2, "Versions", "org.apache.accumulo.core.iterators.VersioningIterator"));
+    AccumuloInputFormat.addIterator(job, new IteratorSetting(1, "WholeRow", WholeRowIterator.class));
+    AccumuloInputFormat.addIterator(job, new IteratorSetting(2, "Versions", "org.apache.accumulo.core.iterators.VersioningIterator"));
     IteratorSetting iter = new IteratorSetting(3, "Count", "org.apache.accumulo.core.iterators.CountingIterator");
     iter.addOption("v1", "1");
     iter.addOption("junk", "\0omg:!\\xyzzy");
-    AccumuloInputFormat.addIterator(job.getConfiguration(), iter);
+    AccumuloInputFormat.addIterator(job, iter);
     
-    List<IteratorSetting> list = AccumuloInputFormat.getIterators(job.getConfiguration());
+    List<IteratorSetting> list = AccumuloInputFormat.getIterators(job);
     
     // Check the list size
     assertTrue(list.size() == 3);
@@ -158,9 +119,9 @@ public class AccumuloInputFormatTest {
     IteratorSetting someSetting = new IteratorSetting(1, "iterator", "Iterator.class");
     someSetting.addOption(key, value);
     Job job = new Job();
-    AccumuloInputFormat.addIterator(job.getConfiguration(), someSetting);
+    AccumuloInputFormat.addIterator(job, someSetting);
     
-    List<IteratorSetting> list = AccumuloInputFormat.getIterators(job.getConfiguration());
+    List<IteratorSetting> list = AccumuloInputFormat.getIterators(job);
     assertEquals(1, list.size());
     assertEquals(1, list.get(0).getOptions().size());
     assertEquals(list.get(0).getOptions().get(key), value);
@@ -168,8 +129,8 @@ public class AccumuloInputFormatTest {
     someSetting.addOption(key + "2", value);
     someSetting.setPriority(2);
     someSetting.setName("it2");
-    AccumuloInputFormat.addIterator(job.getConfiguration(), someSetting);
-    list = AccumuloInputFormat.getIterators(job.getConfiguration());
+    AccumuloInputFormat.addIterator(job, someSetting);
+    list = AccumuloInputFormat.getIterators(job);
     assertEquals(2, list.size());
     assertEquals(1, list.get(0).getOptions().size());
     assertEquals(list.get(0).getOptions().get(key), value);
@@ -180,16 +141,18 @@ public class AccumuloInputFormatTest {
   
   /**
    * Test getting iterator settings for multiple iterators set
+   * 
+   * @throws IOException
    */
   @Test
-  public void testGetIteratorSettings() {
-    JobContext job = ContextFactory.createJobContext();
+  public void testGetIteratorSettings() throws IOException {
+    Job job = new Job();
     
-    AccumuloInputFormat.addIterator(job.getConfiguration(), new IteratorSetting(1, "WholeRow", "org.apache.accumulo.core.iterators.WholeRowIterator"));
-    AccumuloInputFormat.addIterator(job.getConfiguration(), new IteratorSetting(2, "Versions", "org.apache.accumulo.core.iterators.VersioningIterator"));
-    AccumuloInputFormat.addIterator(job.getConfiguration(), new IteratorSetting(3, "Count", "org.apache.accumulo.core.iterators.CountingIterator"));
+    AccumuloInputFormat.addIterator(job, new IteratorSetting(1, "WholeRow", "org.apache.accumulo.core.iterators.WholeRowIterator"));
+    AccumuloInputFormat.addIterator(job, new IteratorSetting(2, "Versions", "org.apache.accumulo.core.iterators.VersioningIterator"));
+    AccumuloInputFormat.addIterator(job, new IteratorSetting(3, "Count", "org.apache.accumulo.core.iterators.CountingIterator"));
     
-    List<IteratorSetting> list = AccumuloInputFormat.getIterators(job.getConfiguration());
+    List<IteratorSetting> list = AccumuloInputFormat.getIterators(job);
     
     // Check the list size
     assertTrue(list.size() == 3);
@@ -213,30 +176,84 @@ public class AccumuloInputFormatTest {
   }
   
   @Test
-  public void testSetRegex() {
-    JobContext job = ContextFactory.createJobContext();
+  public void testSetRegex() throws IOException {
+    Job job = new Job();
     
     String regex = ">\"*%<>\'\\";
     
     IteratorSetting is = new IteratorSetting(50, regex, RegExFilter.class);
     RegExFilter.setRegexs(is, regex, null, null, null, false);
-    AccumuloInputFormat.addIterator(job.getConfiguration(), is);
+    AccumuloInputFormat.addIterator(job, is);
     
-    assertTrue(regex.equals(AccumuloInputFormat.getIterators(job.getConfiguration()).get(0).getName()));
+    assertTrue(regex.equals(AccumuloInputFormat.getIterators(job).get(0).getName()));
   }
   
-  static class TestMapper extends Mapper<Key,Value,Key,Value> {
-    Key key = null;
-    int count = 0;
+  private static AssertionError e1 = null;
+  private static AssertionError e2 = null;
+  
+  private static class MRTester extends Configured implements Tool {
+    private static class TestMapper extends Mapper<Key,Value,Key,Value> {
+      Key key = null;
+      int count = 0;
+      
+      @Override
+      protected void map(Key k, Value v, Context context) throws IOException, InterruptedException {
+        try {
+          if (key != null)
+            assertEquals(key.getRow().toString(), new String(v.get()));
+          assertEquals(k.getRow(), new Text(String.format("%09x", count + 1)));
+          assertEquals(new String(v.get()), String.format("%09x", count));
+        } catch (AssertionError e) {
+          e1 = e;
+        }
+        key = new Key(k);
+        count++;
+      }
+      
+      @Override
+      protected void cleanup(Context context) throws IOException, InterruptedException {
+        try {
+          assertEquals(100, count);
+        } catch (AssertionError e) {
+          e2 = e;
+        }
+      }
+    }
     
     @Override
-    protected void map(Key k, Value v, Context context) throws IOException, InterruptedException {
-      if (key != null)
-        assertEquals(key.getRow().toString(), new String(v.get()));
-      assertEquals(k.getRow(), new Text(String.format("%09x", count + 1)));
-      assertEquals(new String(v.get()), String.format("%09x", count));
-      key = new Key(k);
-      count++;
+    public int run(String[] args) throws Exception {
+      
+      if (args.length != 3) {
+        throw new IllegalArgumentException("Usage : " + MRTester.class.getName() + " <user> <pass> <table>");
+      }
+      
+      String user = args[0];
+      String pass = args[1];
+      String table = args[2];
+      
+      Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
+      job.setJarByClass(this.getClass());
+      
+      job.setInputFormatClass(AccumuloInputFormat.class);
+      
+      AccumuloInputFormat.setConnectorInfo(job, new UserPassToken(user, pass));
+      AccumuloInputFormat.setInputTableName(job, table);
+      AccumuloInputFormat.setMockInstance(job, "testmapinstance");
+      
+      job.setMapperClass(TestMapper.class);
+      job.setMapOutputKeyClass(Key.class);
+      job.setMapOutputValueClass(Value.class);
+      job.setOutputFormatClass(NullOutputFormat.class);
+      
+      job.setNumReduceTasks(0);
+      
+      job.waitForCompletion(true);
+      
+      return job.isSuccessful() ? 0 : 1;
+    }
+    
+    public static void main(String[] args) throws Exception {
+      assertEquals(0, ToolRunner.run(CachedConfiguration.getInstance(), new MRTester(), args));
     }
   }
   
@@ -253,54 +270,8 @@ public class AccumuloInputFormatTest {
     }
     bw.close();
     
-    Job job = new Job(new Configuration());
-    job.setInputFormatClass(AccumuloInputFormat.class);
-    job.setMapperClass(TestMapper.class);
-    job.setNumReduceTasks(0);
-    AccumuloInputFormat.setInputInfo(job.getConfiguration(), "root", "".getBytes(), "testtable", new Authorizations());
-    AccumuloInputFormat.setMockInstance(job.getConfiguration(), "testmapinstance");
-    
-    AccumuloInputFormat input = new AccumuloInputFormat();
-    List<InputSplit> splits = input.getSplits(job);
-    assertEquals(splits.size(), 1);
-    
-    TestMapper mapper = (TestMapper) job.getMapperClass().newInstance();
-    for (InputSplit split : splits) {
-      TaskAttemptContext tac = ContextFactory.createTaskAttemptContext(job);
-      RecordReader<Key,Value> reader = input.createRecordReader(split, tac);
-      Mapper<Key,Value,Key,Value>.Context context = ContextFactory.createMapContext(mapper, tac, reader, null, split);
-      reader.initialize(split, context);
-      mapper.run(context);
-    }
-  }
-  
-  @Test
-  public void testSimple() throws Exception {
-    MockInstance mockInstance = new MockInstance("testmapinstance");
-    Connector c = mockInstance.getConnector(new UserPassToken("root", ""));
-    c.tableOperations().create("testtable2");
-    BatchWriter bw = c.createBatchWriter("testtable2", new BatchWriterConfig());
-    for (int i = 0; i < 100; i++) {
-      Mutation m = new Mutation(new Text(String.format("%09x", i + 1)));
-      m.put(new Text(), new Text(), new Value(String.format("%09x", i).getBytes()));
-      bw.addMutation(m);
-    }
-    bw.close();
-    
-    JobContext job = ContextFactory.createJobContext();
-    AccumuloInputFormat.setInputInfo(job.getConfiguration(), "root", "".getBytes(), "testtable2", new Authorizations());
-    AccumuloInputFormat.setMockInstance(job.getConfiguration(), "testmapinstance");
-    AccumuloInputFormat input = new AccumuloInputFormat();
-    RangeInputSplit ris = new RangeInputSplit();
-    TaskAttemptContext tac = ContextFactory.createTaskAttemptContext(job);
-    RecordReader<Key,Value> rr = input.createRecordReader(ris, tac);
-    rr.initialize(ris, tac);
-    
-    TestMapper mapper = new TestMapper();
-    Mapper<Key,Value,Key,Value>.Context context = ContextFactory.createMapContext(mapper, tac, rr, null, ris);
-    rr.initialize(ris, tac);
-    while (rr.nextKeyValue()) {
-      mapper.map(rr.getCurrentKey(), rr.getCurrentValue(), (TestMapper.Context) context);
-    }
+    MRTester.main(new String[] {"root", "", "testtable"});
+    assertNull(e1);
+    assertNull(e2);
   }
 }

Modified: accumulo/branches/ACCUMULO-259/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormatTest.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormatTest.java?rev=1437726&r1=1437725&r2=1437726&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormatTest.java (original)
+++ accumulo/branches/ACCUMULO-259/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormatTest.java Wed Jan 23 20:51:59 2013
@@ -18,12 +18,14 @@ package org.apache.accumulo.core.client.
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
 import java.util.Iterator;
-import java.util.List;
 import java.util.Map.Entry;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.accumulo.core.client.BatchWriter;
 import org.apache.accumulo.core.client.BatchWriterConfig;
@@ -35,46 +37,135 @@ import org.apache.accumulo.core.data.Mut
 import org.apache.accumulo.core.data.Value;
 import org.apache.accumulo.core.security.Authorizations;
 import org.apache.accumulo.core.security.tokens.UserPassToken;
-import org.apache.accumulo.core.util.ContextFactory;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.InputSplit;
 import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
 import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.RecordReader;
-import org.apache.hadoop.mapreduce.RecordWriter;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
 import org.junit.Test;
 
 /**
  * 
  */
 public class AccumuloOutputFormatTest {
-  static class TestMapper extends Mapper<Key,Value,Text,Mutation> {
-    Key key = null;
-    int count = 0;
+  private static AssertionError e1 = null;
+  
+  private static class MRTester extends Configured implements Tool {
+    private static class TestMapper extends Mapper<Key,Value,Text,Mutation> {
+      Key key = null;
+      int count = 0;
+      
+      @Override
+      protected void map(Key k, Value v, Context context) throws IOException, InterruptedException {
+        try {
+          if (key != null)
+            assertEquals(key.getRow().toString(), new String(v.get()));
+          assertEquals(k.getRow(), new Text(String.format("%09x", count + 1)));
+          assertEquals(new String(v.get()), String.format("%09x", count));
+        } catch (AssertionError e) {
+          e1 = e;
+        }
+        key = new Key(k);
+        count++;
+      }
+      
+      @Override
+      protected void cleanup(Context context) throws IOException, InterruptedException {
+        Mutation m = new Mutation("total");
+        m.put("", "", Integer.toString(count));
+        context.write(new Text(), m);
+      }
+    }
     
     @Override
-    protected void map(Key k, Value v, Context context) throws IOException, InterruptedException {
-      if (key != null)
-        assertEquals(key.getRow().toString(), new String(v.get()));
-      assertEquals(k.getRow(), new Text(String.format("%09x", count + 1)));
-      assertEquals(new String(v.get()), String.format("%09x", count));
-      key = new Key(k);
-      count++;
+    public int run(String[] args) throws Exception {
+      
+      if (args.length != 4) {
+        throw new IllegalArgumentException("Usage : " + MRTester.class.getName() + " <user> <pass> <inputtable> <outputtable>");
+      }
+      
+      String user = args[0];
+      String pass = args[1];
+      String table1 = args[2];
+      String table2 = args[3];
+      
+      Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
+      job.setJarByClass(this.getClass());
+      
+      job.setInputFormatClass(AccumuloInputFormat.class);
+      
+      AccumuloInputFormat.setConnectorInfo(job, new UserPassToken(user, pass));
+      AccumuloInputFormat.setInputTableName(job, table1);
+      AccumuloInputFormat.setMockInstance(job, "testmrinstance");
+      
+      job.setMapperClass(TestMapper.class);
+      job.setMapOutputKeyClass(Key.class);
+      job.setMapOutputValueClass(Value.class);
+      job.setOutputFormatClass(AccumuloOutputFormat.class);
+      job.setOutputKeyClass(Text.class);
+      job.setOutputValueClass(Mutation.class);
+      
+      AccumuloOutputFormat.setConnectorInfo(job, new UserPassToken(user, pass));
+      AccumuloOutputFormat.setCreateTables(job, false);
+      AccumuloOutputFormat.setDefaultTableName(job, table2);
+      AccumuloOutputFormat.setMockInstance(job, "testmrinstance");
+      
+      job.setNumReduceTasks(0);
+      
+      job.waitForCompletion(true);
+      
+      return job.isSuccessful() ? 0 : 1;
     }
     
-    @Override
-    protected void cleanup(Context context) throws IOException, InterruptedException {
-      super.cleanup(context);
-      Mutation m = new Mutation("total");
-      m.put("", "", Integer.toString(count));
-      try {
-        context.write(new Text(), m);
-      } catch (NullPointerException e) {}
+    public static void main(String[] args) throws Exception {
+      assertEquals(0, ToolRunner.run(CachedConfiguration.getInstance(), new MRTester(), args));
     }
   }
   
   @Test
+  public void testBWSettings() throws IOException {
+    Job job = new Job();
+    
+    // make sure we aren't testing defaults
+    final BatchWriterConfig bwDefaults = new BatchWriterConfig();
+    assertNotEquals(7654321l, bwDefaults.getMaxLatency(TimeUnit.MILLISECONDS));
+    assertNotEquals(9898989l, bwDefaults.getTimeout(TimeUnit.MILLISECONDS));
+    assertNotEquals(42, bwDefaults.getMaxWriteThreads());
+    assertNotEquals(1123581321l, bwDefaults.getMaxMemory());
+    
+    final BatchWriterConfig bwConfig = new BatchWriterConfig();
+    bwConfig.setMaxLatency(7654321l, TimeUnit.MILLISECONDS);
+    bwConfig.setTimeout(9898989l, TimeUnit.MILLISECONDS);
+    bwConfig.setMaxWriteThreads(42);
+    bwConfig.setMaxMemory(1123581321l);
+    AccumuloOutputFormat.setBatchWriterOptions(job, bwConfig);
+    
+    AccumuloOutputFormat myAOF = new AccumuloOutputFormat() {
+      @Override
+      public void checkOutputSpecs(JobContext job) throws IOException {
+        BatchWriterConfig bwOpts = getBatchWriterOptions(job);
+        
+        // passive check
+        assertEquals(bwConfig.getMaxLatency(TimeUnit.MILLISECONDS), bwOpts.getMaxLatency(TimeUnit.MILLISECONDS));
+        assertEquals(bwConfig.getTimeout(TimeUnit.MILLISECONDS), bwOpts.getTimeout(TimeUnit.MILLISECONDS));
+        assertEquals(bwConfig.getMaxWriteThreads(), bwOpts.getMaxWriteThreads());
+        assertEquals(bwConfig.getMaxMemory(), bwOpts.getMaxMemory());
+        
+        // explicit check
+        assertEquals(7654321l, bwOpts.getMaxLatency(TimeUnit.MILLISECONDS));
+        assertEquals(9898989l, bwOpts.getTimeout(TimeUnit.MILLISECONDS));
+        assertEquals(42, bwOpts.getMaxWriteThreads());
+        assertEquals(1123581321l, bwOpts.getMaxMemory());
+        
+      }
+    };
+    myAOF.checkOutputSpecs(job);
+  }
+  
+  @Test
   public void testMR() throws Exception {
     MockInstance mockInstance = new MockInstance("testmrinstance");
     Connector c = mockInstance.getConnector(new UserPassToken("root", ""));
@@ -88,34 +179,8 @@ public class AccumuloOutputFormatTest {
     }
     bw.close();
     
-    Job job = new Job();
-    job.setInputFormatClass(AccumuloInputFormat.class);
-    job.setMapperClass(TestMapper.class);
-    job.setOutputFormatClass(AccumuloOutputFormat.class);
-    job.setOutputKeyClass(Text.class);
-    job.setOutputValueClass(Mutation.class);
-    job.setNumReduceTasks(0);
-    AccumuloInputFormat.setInputInfo(job.getConfiguration(), "root", "".getBytes(), "testtable1", new Authorizations());
-    AccumuloInputFormat.setMockInstance(job.getConfiguration(), "testmrinstance");
-    AccumuloOutputFormat.setOutputInfo(job.getConfiguration(), "root", "".getBytes(), false, "testtable2");
-    AccumuloOutputFormat.setMockInstance(job.getConfiguration(), "testmrinstance");
-    
-    AccumuloInputFormat input = new AccumuloInputFormat();
-    List<InputSplit> splits = input.getSplits(job);
-    assertEquals(splits.size(), 1);
-    
-    AccumuloOutputFormat output = new AccumuloOutputFormat();
-    
-    TestMapper mapper = (TestMapper) job.getMapperClass().newInstance();
-    for (InputSplit split : splits) {
-      TaskAttemptContext tac = ContextFactory.createTaskAttemptContext(job);
-      RecordReader<Key,Value> reader = input.createRecordReader(split, tac);
-      RecordWriter<Text,Mutation> writer = output.getRecordWriter(tac);
-      Mapper<Key,Value,Text,Mutation>.Context context = ContextFactory.createMapContext(mapper, tac, reader, writer, split);
-      reader.initialize(split, context);
-      mapper.run(context);
-      writer.close(context);
-    }
+    MRTester.main(new String[] {"root", "", "testtable1", "testtable2"});
+    assertNull(e1);
     
     Scanner scanner = c.createScanner("testtable2", new Authorizations());
     Iterator<Entry<Key,Value>> iter = scanner.iterator();

Modified: accumulo/branches/ACCUMULO-259/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloRowInputFormatTest.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloRowInputFormatTest.java?rev=1437726&r1=1437725&r2=1437726&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloRowInputFormatTest.java (original)
+++ accumulo/branches/ACCUMULO-259/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloRowInputFormatTest.java Wed Jan 23 20:51:59 2013
@@ -17,7 +17,7 @@
 package org.apache.accumulo.core.client.mapreduce;
 
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
@@ -26,29 +26,26 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map.Entry;
 
-import org.apache.accumulo.core.client.AccumuloException;
-import org.apache.accumulo.core.client.AccumuloSecurityException;
 import org.apache.accumulo.core.client.BatchWriter;
 import org.apache.accumulo.core.client.BatchWriterConfig;
 import org.apache.accumulo.core.client.Connector;
 import org.apache.accumulo.core.client.MutationsRejectedException;
-import org.apache.accumulo.core.client.TableExistsException;
-import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.core.client.mapreduce.InputFormatBase.RangeInputSplit;
 import org.apache.accumulo.core.client.mock.MockInstance;
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.KeyValue;
 import org.apache.accumulo.core.data.Mutation;
 import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.security.Authorizations;
 import org.apache.accumulo.core.security.ColumnVisibility;
 import org.apache.accumulo.core.security.tokens.UserPassToken;
-import org.apache.accumulo.core.util.ContextFactory;
+import org.apache.accumulo.core.util.CachedConfiguration;
 import org.apache.accumulo.core.util.PeekingIterator;
+import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.RecordReader;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
 import org.junit.Test;
 
 public class AccumuloRowInputFormatTest {
@@ -56,9 +53,11 @@ public class AccumuloRowInputFormatTest 
   private static final String ROW2 = "row2";
   private static final String ROW3 = "row3";
   private static final String COLF1 = "colf1";
-  private transient final List<Entry<Key,Value>> row1;
-  private transient final List<Entry<Key,Value>> row2;
-  private transient final List<Entry<Key,Value>> row3;
+  private static List<Entry<Key,Value>> row1;
+  private static List<Entry<Key,Value>> row2;
+  private static List<Entry<Key,Value>> row3;
+  private static AssertionError e1 = null;
+  private static AssertionError e2 = null;
   
   public AccumuloRowInputFormatTest() {
     row1 = new ArrayList<Entry<Key,Value>>();
@@ -70,7 +69,7 @@ public class AccumuloRowInputFormatTest 
     row3 = new ArrayList<Entry<Key,Value>>();
     row3.add(new KeyValue(new Key(ROW3, COLF1, "colq5"), "v5".getBytes()));
   }
-
+  
   public static void checkLists(final List<Entry<Key,Value>> first, final List<Entry<Key,Value>> second) {
     assertEquals("Sizes should be the same.", first.size(), second.size());
     for (int i = 0; i < first.size(); i++) {
@@ -80,59 +79,119 @@ public class AccumuloRowInputFormatTest 
   }
   
   public static void checkLists(final List<Entry<Key,Value>> first, final Iterator<Entry<Key,Value>> second) {
-    int entryIndex = 0; // NOPMD
+    int entryIndex = 0;
     while (second.hasNext()) {
       final Entry<Key,Value> entry = second.next();
       assertEquals("Keys should be equal", first.get(entryIndex).getKey(), entry.getKey());
       assertEquals("Values should be equal", first.get(entryIndex).getValue(), entry.getValue());
-      entryIndex++; // NOPMD
+      entryIndex++;
     }
   }
   
   public static void insertList(final BatchWriter writer, final List<Entry<Key,Value>> list) throws MutationsRejectedException {
     for (Entry<Key,Value> e : list) {
       final Key key = e.getKey();
-      final Mutation mutation = new Mutation(key.getRow());  // NOPMD
-      ColumnVisibility colVisibility = new ColumnVisibility(key.getColumnVisibility()); // NOPMD
+      final Mutation mutation = new Mutation(key.getRow());
+      ColumnVisibility colVisibility = new ColumnVisibility(key.getColumnVisibility());
       mutation.put(key.getColumnFamily(), key.getColumnQualifier(), colVisibility, key.getTimestamp(), e.getValue());
       writer.addMutation(mutation);
     }
   }
   
+  private static class MRTester extends Configured implements Tool {
+    private static class TestMapper extends Mapper<Text,PeekingIterator<Entry<Key,Value>>,Key,Value> {
+      int count = 0;
+      
+      @Override
+      protected void map(Text k, PeekingIterator<Entry<Key,Value>> v, Context context) throws IOException, InterruptedException {
+        try {
+          switch (count) {
+            case 0:
+              assertEquals("Current key should be " + ROW1, new Text(ROW1), k);
+              checkLists(row1, v);
+              break;
+            case 1:
+              assertEquals("Current key should be " + ROW2, new Text(ROW2), k);
+              checkLists(row2, v);
+              break;
+            case 2:
+              assertEquals("Current key should be " + ROW3, new Text(ROW3), k);
+              checkLists(row3, v);
+              break;
+            default:
+              assertTrue(false);
+          }
+        } catch (AssertionError e) {
+          e1 = e;
+        }
+        count++;
+      }
+      
+      @Override
+      protected void cleanup(Context context) throws IOException, InterruptedException {
+        try {
+          assertEquals(3, count);
+        } catch (AssertionError e) {
+          e2 = e;
+        }
+      }
+    }
+    
+    @Override
+    public int run(String[] args) throws Exception {
+      
+      if (args.length != 3) {
+        throw new IllegalArgumentException("Usage : " + MRTester.class.getName() + " <user> <pass> <table>");
+      }
+      
+      String user = args[0];
+      String pass = args[1];
+      String table = args[2];
+      
+      Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
+      job.setJarByClass(this.getClass());
+      
+      job.setInputFormatClass(AccumuloRowInputFormat.class);
+      
+      AccumuloInputFormat.setConnectorInfo(job, new UserPassToken(user, pass));
+      AccumuloInputFormat.setInputTableName(job, table);
+      AccumuloRowInputFormat.setMockInstance(job, "instance1");
+      
+      job.setMapperClass(TestMapper.class);
+      job.setMapOutputKeyClass(Key.class);
+      job.setMapOutputValueClass(Value.class);
+      job.setOutputFormatClass(NullOutputFormat.class);
+      
+      job.setNumReduceTasks(0);
+      
+      job.waitForCompletion(true);
+      
+      return job.isSuccessful() ? 0 : 1;
+    }
+    
+    public static void main(String[] args) throws Exception {
+      assertEquals(0, ToolRunner.run(CachedConfiguration.getInstance(), new MRTester(), args));
+    }
+  }
+  
   @Test
-  public void test() throws AccumuloException, AccumuloSecurityException, TableExistsException, TableNotFoundException, IOException, InterruptedException {
+  public void test() throws Exception {
     final MockInstance instance = new MockInstance("instance1");
     final Connector conn = instance.getConnector(new UserPassToken("root", ""));
     conn.tableOperations().create("test");
-    BatchWriter writer = null; // NOPMD
+    BatchWriter writer = null;
     try {
       writer = conn.createBatchWriter("test", new BatchWriterConfig());
-        insertList(writer, row1);
-        insertList(writer, row2);
-        insertList(writer, row3);
+      insertList(writer, row1);
+      insertList(writer, row2);
+      insertList(writer, row3);
     } finally {
       if (writer != null) {
-    	  writer.close();
+        writer.close();
       }
     }
-    final JobContext job = ContextFactory.createJobContext();
-    AccumuloRowInputFormat.setInputInfo(job.getConfiguration(), "root", "".getBytes(), "test", new Authorizations());
-    AccumuloRowInputFormat.setMockInstance(job.getConfiguration(), "instance1");
-    final AccumuloRowInputFormat crif = new AccumuloRowInputFormat();
-    final RangeInputSplit ris = new RangeInputSplit();
-    final TaskAttemptContext tac = ContextFactory.createTaskAttemptContext(job);
-    final RecordReader<Text,PeekingIterator<Entry<Key,Value>>> recReader = crif.createRecordReader(ris, tac);
-    recReader.initialize(ris, tac);
-    
-    assertTrue("Next key value should be true.", recReader.nextKeyValue());
-    assertEquals("Current key should be " + ROW1, new Text(ROW1), recReader.getCurrentKey());
-    checkLists(row1, recReader.getCurrentValue());
-    assertTrue("Next key value should be true.", recReader.nextKeyValue());
-    assertEquals("Current key should be " + ROW2, new Text(ROW2), recReader.getCurrentKey());
-    checkLists(row2, recReader.getCurrentValue());
-    assertTrue("Next key value should be true.", recReader.nextKeyValue());
-    assertEquals("Current key should be " + ROW3, new Text(ROW3), recReader.getCurrentKey());
-    checkLists(row3, recReader.getCurrentValue());
-    assertFalse("Next key value should be false.", recReader.nextKeyValue());
+    MRTester.main(new String[] {"root", "", "test"});
+    assertNull(e1);
+    assertNull(e2);
   }
 }

Modified: accumulo/branches/ACCUMULO-259/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java?rev=1437726&r1=1437725&r2=1437726&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java (original)
+++ accumulo/branches/ACCUMULO-259/core/src/test/java/org/apache/accumulo/core/file/rfile/RFileTest.java Wed Jan 23 20:51:59 2013
@@ -28,6 +28,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Random;
@@ -1442,6 +1443,63 @@ public class RFileTest {
     trf.closeReader();
   }
   
+  @Test
+  public void testReseekUnconsumed() throws Exception {
+    TestRFile trf = new TestRFile();
+    
+    trf.openWriter();
+    
+    for (int i = 0; i < 2500; i++) {
+      trf.writer.append(nk(nf("r_", i), "cf1", "cq1", "L1", 42), nv("foo" + i));
+    }
+    
+    trf.closeWriter();
+    trf.openReader();
+    
+    Set<ByteSequence> cfs = Collections.emptySet();
+
+    Random rand = new Random();
+
+    for (int count = 0; count < 100; count++) {
+      
+      int start = rand.nextInt(2300);
+      Range range = new Range(nk(nf("r_", start), "cf1", "cq1", "L1", 42), nk(nf("r_", start + 100), "cf1", "cq1", "L1", 42));
+
+      trf.reader.seek(range, cfs, false);
+      
+      int numToScan = rand.nextInt(100);
+      
+      for (int j = 0; j < numToScan; j++) {
+        assertTrue(trf.reader.hasTop());
+        assertEquals(nk(nf("r_", start + j), "cf1", "cq1", "L1", 42), trf.reader.getTopKey());
+        trf.reader.next();
+      }
+      
+      assertTrue(trf.reader.hasTop());
+      assertEquals(nk(nf("r_", start + numToScan), "cf1", "cq1", "L1", 42), trf.reader.getTopKey());
+
+      // seek a little forward from the last range and read a few keys within the unconsumed portion of the last range
+
+      int start2 = start + numToScan + rand.nextInt(3);
+      int end2 = start2 + rand.nextInt(3);
+
+      range = new Range(nk(nf("r_", start2), "cf1", "cq1", "L1", 42), nk(nf("r_", end2), "cf1", "cq1", "L1", 42));
+      trf.reader.seek(range, cfs, false);
+      
+      for (int j = start2; j <= end2; j++) {
+        assertTrue(trf.reader.hasTop());
+        assertEquals(nk(nf("r_", j), "cf1", "cq1", "L1", 42), trf.reader.getTopKey());
+        trf.reader.next();
+      }
+      
+      assertFalse(trf.reader.hasTop());
+
+    }
+    
+    trf.closeReader();
+  }
+
+
   @Test(expected = NullPointerException.class)
   public void testMissingUnreleasedVersions() throws Exception {
     runVersionTest(5);

Modified: accumulo/branches/ACCUMULO-259/core/src/test/java/org/apache/accumulo/core/iterators/user/FilterTest.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/core/src/test/java/org/apache/accumulo/core/iterators/user/FilterTest.java?rev=1437726&r1=1437725&r2=1437726&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/core/src/test/java/org/apache/accumulo/core/iterators/user/FilterTest.java (original)
+++ accumulo/branches/ACCUMULO-259/core/src/test/java/org/apache/accumulo/core/iterators/user/FilterTest.java Wed Jan 23 20:51:59 2013
@@ -17,7 +17,6 @@
 package org.apache.accumulo.core.iterators.user;
 
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
@@ -199,7 +198,10 @@ public class FilterTest {
     AgeOffFilter.setCurrentTime(is, 1001l);
     AgeOffFilter.setNegate(is, true);
     assertTrue(((AgeOffFilter) a).validateOptions(is.getOptions()));
-    assertFalse(((AgeOffFilter) a).validateOptions(EMPTY_OPTS));
+    try {
+      ((AgeOffFilter) a).validateOptions(EMPTY_OPTS);
+      assertTrue(false);
+    } catch (IllegalArgumentException e) {}
     a.init(new SortedMapIterator(tm), is.getOptions(), null);
     a = a.deepCopy(null);
     SortedKeyValueIterator<Key,Value> copy = a.deepCopy(null);
@@ -467,7 +469,10 @@ public class FilterTest {
     a.seek(new Range(), EMPTY_COL_FAMS, false);
     assertEquals(size(a), 32);
     
-    assertFalse(a.validateOptions(EMPTY_OPTS));
+    try {
+      a.validateOptions(EMPTY_OPTS);
+      assertTrue(false);
+    } catch (IllegalArgumentException e) {}
   }
   
   @Test

Modified: accumulo/branches/ACCUMULO-259/docs/examples/README.helloworld
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/docs/examples/README.helloworld?rev=1437726&r1=1437725&r2=1437726&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/docs/examples/README.helloworld (original)
+++ accumulo/branches/ACCUMULO-259/docs/examples/README.helloworld Wed Jan 23 20:51:59 2013
@@ -19,7 +19,6 @@ Notice:    Licensed to the Apache Softwa
 This tutorial uses the following Java classes, which can be found in org.apache.accumulo.examples.simple.helloworld in the simple-examples module: 
 
  * InsertWithBatchWriter.java - Inserts 10K rows (50K entries) into accumulo with each row having 5 entries
- * InsertWithOutputFormat.java - Example of inserting data in MapReduce
  * ReadData.java - Reads all data between two rows
 
 Log into the accumulo shell:
@@ -34,10 +33,6 @@ Launch a Java program that inserts data 
 
     $ ./bin/accumulo org.apache.accumulo.examples.simple.helloworld.InsertWithBatchWriter -i instance -z zookeepers -u username -p password -t hellotable 
 
-Alternatively, the same data can be inserted using MapReduce writers:
-
-    $ ./bin/accumulo org.apache.accumulo.examples.simple.helloworld.InsertWithOutputFormat -i instance -z zookeepers -u username -p password -t hellotable 
-
 On the accumulo status page at the URL below (where 'master' is replaced with the name or IP of your accumulo master), you should see 50K entries
 	
     http://master:50095/

Modified: accumulo/branches/ACCUMULO-259/examples/instamo/src/main/java/org/apache/accumulo/instamo/MapReduceExample.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/examples/instamo/src/main/java/org/apache/accumulo/instamo/MapReduceExample.java?rev=1437726&r1=1437725&r2=1437726&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/examples/instamo/src/main/java/org/apache/accumulo/instamo/MapReduceExample.java (original)
+++ accumulo/branches/ACCUMULO-259/examples/instamo/src/main/java/org/apache/accumulo/instamo/MapReduceExample.java Wed Jan 23 20:51:59 2013
@@ -21,7 +21,6 @@ import java.util.UUID;
 
 import org.apache.accumulo.server.test.continuous.ContinuousIngest;
 import org.apache.accumulo.server.test.continuous.ContinuousVerify;
-import org.apache.accumulo.test.MacConfig;
 import org.apache.accumulo.test.MiniAccumuloCluster;
 import org.apache.commons.io.FileUtils;
 
@@ -52,7 +51,7 @@ public class MapReduceExample {
     File tmpDir = new File(FileUtils.getTempDirectory(), "macc-" + UUID.randomUUID().toString());
     
     try {
-      MiniAccumuloCluster la = new MiniAccumuloCluster(new MacConfig(tmpDir, "pass1234"));
+      MiniAccumuloCluster la = new MiniAccumuloCluster(tmpDir, "pass1234");
       la.start();
       
       System.out.println("\n   ---- Running Mapred Against Accumulo\n");

Modified: accumulo/branches/ACCUMULO-259/examples/instamo/src/test/java/org/apache/accumulo/instamo/ExampleAccumuloUnitTest.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/examples/instamo/src/test/java/org/apache/accumulo/instamo/ExampleAccumuloUnitTest.java?rev=1437726&r1=1437725&r2=1437726&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/examples/instamo/src/test/java/org/apache/accumulo/instamo/ExampleAccumuloUnitTest.java (original)
+++ accumulo/branches/ACCUMULO-259/examples/instamo/src/test/java/org/apache/accumulo/instamo/ExampleAccumuloUnitTest.java Wed Jan 23 20:51:59 2013
@@ -17,7 +17,6 @@ package org.apache.accumulo.instamo;
  */
 
 
-import org.apache.accumulo.test.MacConfig;
 import org.apache.accumulo.test.MiniAccumuloCluster;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
@@ -39,7 +38,7 @@ public class ExampleAccumuloUnitTest {
 
     folder.create();
     
-    accumulo = new MiniAccumuloCluster(new MacConfig(folder.getRoot(), "superSecret"));
+    accumulo = new MiniAccumuloCluster(folder.getRoot(), "superSecret");
     
     accumulo.start();
     

Modified: accumulo/branches/ACCUMULO-259/examples/simple/pom.xml
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/examples/simple/pom.xml?rev=1437726&r1=1437725&r2=1437726&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/examples/simple/pom.xml (original)
+++ accumulo/branches/ACCUMULO-259/examples/simple/pom.xml Wed Jan 23 20:51:59 2013
@@ -59,6 +59,14 @@
           <groupId>org.apache.hadoop</groupId>
           <artifactId>hadoop-client</artifactId>
         </dependency>
+        <dependency>
+          <groupId>commons-httpclient</groupId>
+          <artifactId>commons-httpclient</artifactId>
+        </dependency>
+        <dependency>
+          <groupId>org.apache.avro</groupId>
+          <artifactId>avro</artifactId>
+        </dependency>
       </dependencies>
     </profile>
   </profiles>

Modified: accumulo/branches/ACCUMULO-259/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/ReadWriteExample.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/ReadWriteExample.java?rev=1437726&r1=1437725&r2=1437726&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/ReadWriteExample.java (original)
+++ accumulo/branches/ACCUMULO-259/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/ReadWriteExample.java Wed Jan 23 20:51:59 2013
@@ -31,6 +31,7 @@ import org.apache.accumulo.core.data.Mut
 import org.apache.accumulo.core.data.Value;
 import org.apache.accumulo.core.security.Authorizations;
 import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.accumulo.core.util.ByteArraySet;
 import org.apache.hadoop.io.Text;
 
 import com.beust.jcommander.Parameter;
@@ -43,19 +44,19 @@ public class ReadWriteExample {
   private Connector conn;
   
   static class Opts extends ClientOnDefaultTable {
-    @Parameter(names={"-C", "--createtable"}, description="create table before doing anything")
+    @Parameter(names = {"-C", "--createtable"}, description = "create table before doing anything")
     boolean createtable = false;
-    @Parameter(names={"-D", "--deletetable"}, description="delete table when finished")
+    @Parameter(names = {"-D", "--deletetable"}, description = "delete table when finished")
     boolean deletetable = false;
-    @Parameter(names={"-c", "--create"}, description="create entries before any deletes")
+    @Parameter(names = {"-c", "--create"}, description = "create entries before any deletes")
     boolean createEntries = false;
-    @Parameter(names={"-r", "--read"}, description="read entries after any creates/deletes")
+    @Parameter(names = {"-r", "--read"}, description = "read entries after any creates/deletes")
     boolean readEntries = false;
-    @Parameter(names={"-d", "--delete"}, description="delete entries after any creates")
+    @Parameter(names = {"-d", "--delete"}, description = "delete entries after any creates")
     boolean deleteEntries = false;
     
-    public Opts() { 
-      super(DEFAULT_TABLE_NAME); 
+    public Opts() {
+      super(DEFAULT_TABLE_NAME);
       auths = new Authorizations(DEFAULT_AUTHS.split(","));
     }
   }
@@ -65,6 +66,12 @@ public class ReadWriteExample {
   
   private void execute(Opts opts, ScannerOpts scanOpts) throws Exception {
     conn = opts.getConnector();
+    // add the authorizations to the user
+    Authorizations userAuthorizations = conn.securityOperations().getUserAuthorizations(opts.user);
+    ByteArraySet auths = new ByteArraySet(userAuthorizations.getAuthorizations());
+    auths.addAll(opts.auths.getAuthorizations());
+    if (!auths.isEmpty())
+      conn.securityOperations().changeUserAuthorizations(opts.user, new Authorizations(auths));
     // create table
     if (opts.createtable) {
       SortedSet<Text> partitionKeys = new TreeSet<Text>();
@@ -73,7 +80,7 @@ public class ReadWriteExample {
       conn.tableOperations().create(opts.getTableName());
       conn.tableOperations().addSplits(opts.getTableName(), partitionKeys);
     }
-
+    
     // send mutations
     createEntries(opts);
     

Modified: accumulo/branches/ACCUMULO-259/examples/simple/src/main/java/org/apache/accumulo/examples/simple/combiner/StatsCombiner.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/examples/simple/src/main/java/org/apache/accumulo/examples/simple/combiner/StatsCombiner.java?rev=1437726&r1=1437725&r2=1437726&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/examples/simple/src/main/java/org/apache/accumulo/examples/simple/combiner/StatsCombiner.java (original)
+++ accumulo/branches/ACCUMULO-259/examples/simple/src/main/java/org/apache/accumulo/examples/simple/combiner/StatsCombiner.java Wed Jan 23 20:51:59 2013
@@ -92,7 +92,7 @@ public class StatsCombiner extends Combi
       return false;
     
     if (options.containsKey(RADIX_OPTION) && !options.get(RADIX_OPTION).matches("\\d+"))
-      return false;
+      throw new IllegalArgumentException("invalid option " + RADIX_OPTION + ":" + options.get(RADIX_OPTION));
     
     return true;
   }

Modified: accumulo/branches/ACCUMULO-259/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/Viewer.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/Viewer.java?rev=1437726&r1=1437725&r2=1437726&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/Viewer.java (original)
+++ accumulo/branches/ACCUMULO-259/examples/simple/src/main/java/org/apache/accumulo/examples/simple/dirlist/Viewer.java Wed Jan 23 20:51:59 2013
@@ -103,7 +103,7 @@ public class Viewer extends JFrame imple
     setSize(1000, 800);
     setDefaultCloseOperation(EXIT_ON_CLOSE);
     q = new QueryUtil(opts);
-    fdq = new FileDataQuery(opts.instance, opts.zookeepers, opts.user, opts.getPassword(), opts.dataTable, opts.auths);
+    fdq = new FileDataQuery(opts.instance, opts.zookeepers, opts.getAccumuloToken(), opts.dataTable, opts.auths);
     this.topPath = opts.path;
   }
   

Modified: accumulo/branches/ACCUMULO-259/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/FileDataQuery.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/FileDataQuery.java?rev=1437726&r1=1437725&r2=1437726&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/FileDataQuery.java (original)
+++ accumulo/branches/ACCUMULO-259/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/FileDataQuery.java Wed Jan 23 20:51:59 2013
@@ -31,6 +31,7 @@ import org.apache.accumulo.core.data.Key
 import org.apache.accumulo.core.data.Range;
 import org.apache.accumulo.core.data.Value;
 import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.tokens.AccumuloToken;
 import org.apache.accumulo.core.security.tokens.UserPassToken;
 import org.apache.accumulo.core.util.PeekingIterator;
 
@@ -44,10 +45,10 @@ public class FileDataQuery {
   private ChunkInputStream cis;
   Scanner scanner;
   
-  public FileDataQuery(String instanceName, String zooKeepers, String user, byte[] password, String tableName, Authorizations auths) throws AccumuloException,
+  public FileDataQuery(String instanceName, String zooKeepers, AccumuloToken<?,?> token, String tableName, Authorizations auths) throws AccumuloException,
       AccumuloSecurityException, TableNotFoundException {
     ZooKeeperInstance instance = new ZooKeeperInstance(instanceName, zooKeepers);
-    conn = instance.getConnector(new UserPassToken(user, password));
+    conn = instance.getConnector(token);
     lastRefs = new ArrayList<Entry<Key,Value>>();
     cis = new ChunkInputStream();
     scanner = conn.createScanner(tableName, auths);

Modified: accumulo/branches/ACCUMULO-259/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/RegexExample.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/RegexExample.java?rev=1437726&r1=1437725&r2=1437726&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/RegexExample.java (original)
+++ accumulo/branches/ACCUMULO-259/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/RegexExample.java Wed Jan 23 20:51:59 2013
@@ -37,24 +37,26 @@ import com.beust.jcommander.Parameter;
 
 public class RegexExample extends Configured implements Tool {
   public static class RegexMapper extends Mapper<Key,Value,Key,Value> {
+    @Override
     public void map(Key row, Value data, Context context) throws IOException, InterruptedException {
       context.write(row, data);
     }
   }
   
   static class Opts extends ClientOnRequiredTable {
-    @Parameter(names="--rowRegex")
+    @Parameter(names = "--rowRegex")
     String rowRegex;
-    @Parameter(names="--columnFamilyRegex")
+    @Parameter(names = "--columnFamilyRegex")
     String columnFamilyRegex;
-    @Parameter(names="--columnQualifierRegex")
+    @Parameter(names = "--columnQualifierRegex")
     String columnQualifierRegex;
-    @Parameter(names="--valueRegex")
+    @Parameter(names = "--valueRegex")
     String valueRegex;
-    @Parameter(names="--output", required=true)
+    @Parameter(names = "--output", required = true)
     String destination;
   }
   
+  @Override
   public int run(String[] args) throws Exception {
     Opts opts = new Opts();
     opts.parseArgs(getClass().getName(), args);
@@ -67,7 +69,7 @@ public class RegexExample extends Config
     
     IteratorSetting regex = new IteratorSetting(50, "regex", RegExFilter.class);
     RegExFilter.setRegexs(regex, opts.rowRegex, opts.columnFamilyRegex, opts.columnQualifierRegex, opts.valueRegex, false);
-    AccumuloInputFormat.addIterator(job.getConfiguration(), regex);
+    AccumuloInputFormat.addIterator(job, regex);
     
     job.setMapperClass(RegexMapper.class);
     job.setMapOutputKeyClass(Key.class);

Modified: accumulo/branches/ACCUMULO-259/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/RowHash.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/RowHash.java?rev=1437726&r1=1437725&r2=1437726&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/RowHash.java (original)
+++ accumulo/branches/ACCUMULO-259/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/RowHash.java Wed Jan 23 20:51:59 2013
@@ -43,6 +43,7 @@ public class RowHash extends Configured 
    * The Mapper class that given a row number, will generate the appropriate output line.
    */
   public static class HashDataMapper extends Mapper<Key,Value,Text,Mutation> {
+    @Override
     public void map(Key row, Value data, Context context) throws IOException, InterruptedException {
       Mutation m = new Mutation(row.getRow());
       m.put(new Text("cf-HASHTYPE"), new Text("cq-MD5BASE64"), new Value(Base64.encodeBase64(MD5Hash.digest(data.toString()).getDigest())));
@@ -55,7 +56,7 @@ public class RowHash extends Configured 
   }
   
   private static class Opts extends ClientOnRequiredTable {
-    @Parameter(names="--column", required=true)
+    @Parameter(names = "--column", required = true)
     String column = null;
   }
   
@@ -73,7 +74,7 @@ public class RowHash extends Configured 
     Text cf = new Text(idx < 0 ? col : col.substring(0, idx));
     Text cq = idx < 0 ? null : new Text(col.substring(idx + 1));
     if (cf.getLength() > 0)
-      AccumuloInputFormat.fetchColumns(job.getConfiguration(), Collections.singleton(new Pair<Text,Text>(cf, cq)));
+      AccumuloInputFormat.fetchColumns(job, Collections.singleton(new Pair<Text,Text>(cf, cq)));
     
     job.setMapperClass(HashDataMapper.class);
     job.setMapOutputKeyClass(Text.class);

Modified: accumulo/branches/ACCUMULO-259/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TableToFile.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-259/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TableToFile.java?rev=1437726&r1=1437725&r2=1437726&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-259/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TableToFile.java (original)
+++ accumulo/branches/ACCUMULO-259/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TableToFile.java Wed Jan 23 20:51:59 2013
@@ -46,9 +46,9 @@ import com.beust.jcommander.Parameter;
 public class TableToFile extends Configured implements Tool {
   
   static class Opts extends ClientOnRequiredTable {
-    @Parameter(names="--output", description="output directory", required=true)
+    @Parameter(names = "--output", description = "output directory", required = true)
     String output;
-    @Parameter(names="--columns", description="columns to extract, in cf:cq{,cf:cq,...} form")
+    @Parameter(names = "--columns", description = "columns to extract, in cf:cq{,cf:cq,...} form")
     String columns;
   }
   
@@ -56,6 +56,7 @@ public class TableToFile extends Configu
    * The Mapper class that given a row number, will generate the appropriate output line.
    */
   public static class TTFMapper extends Mapper<Key,Value,NullWritable,Text> {
+    @Override
     public void map(Key row, Value data, Context context) throws IOException, InterruptedException {
       final Key r = row;
       final Value v = data;
@@ -99,7 +100,7 @@ public class TableToFile extends Configu
         columnsToFetch.add(new Pair<Text,Text>(cf, cq));
     }
     if (!columnsToFetch.isEmpty())
-      AccumuloInputFormat.fetchColumns(job.getConfiguration(), columnsToFetch);
+      AccumuloInputFormat.fetchColumns(job, columnsToFetch);
     
     job.setMapperClass(TTFMapper.class);
     job.setMapOutputKeyClass(NullWritable.class);



Mime
View raw message