accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From els...@apache.org
Subject [27/31] git commit: ACCUMULO-1854 Update tests for hadoop-2 compatibility
Date Sun, 24 Nov 2013 00:33:58 GMT
ACCUMULO-1854 Update tests for hadoop-2 compatibility


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/c88d87aa
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/c88d87aa
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/c88d87aa

Branch: refs/heads/master
Commit: c88d87aa08819ed94fa9aae8583985b4fcf8e588
Parents: fae8282
Author: Josh Elser <elserj@apache.org>
Authored: Sat Nov 23 13:49:14 2013 -0500
Committer: Josh Elser <elserj@apache.org>
Committed: Sat Nov 23 13:49:14 2013 -0500

----------------------------------------------------------------------
 .../mapreduce/AccumuloInputFormatTest.java      | 198 ++++++++-----------
 .../BadPasswordSplitsAccumuloInputFormat.java   |  26 +++
 .../EmptySplitsAccumuloInputFormat.java         |  21 ++
 3 files changed, 129 insertions(+), 116 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/c88d87aa/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
index f9ccdf1..93dba65 100644
--- a/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
@@ -23,6 +23,7 @@ import static org.junit.Assert.assertTrue;
 import java.io.ByteArrayOutputStream;
 import java.io.DataOutputStream;
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
@@ -46,8 +47,10 @@ import org.apache.commons.codec.binary.Base64;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.InputFormat;
 import org.apache.hadoop.mapreduce.InputSplit;
 import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
 import org.apache.hadoop.mapreduce.Mapper;
 import org.apache.hadoop.mapreduce.RecordReader;
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
@@ -60,11 +63,9 @@ import org.junit.Assert;
 import org.junit.Test;
 
 public class AccumuloInputFormatTest {
-  
+
   private static final String PREFIX = AccumuloInputFormatTest.class.getSimpleName();
-  private static final String INSTANCE_NAME = PREFIX + "_mapreduce_instance";
-  private static final String TEST_TABLE_1 = PREFIX + "_mapreduce_table_1";
-  
+
   /**
    * Test basic setting & getting of max versions.
    * 
@@ -79,7 +80,7 @@ public class AccumuloInputFormatTest {
     int version = AccumuloInputFormat.getMaxVersions(job.getConfiguration());
     assertEquals(1, version);
   }
-  
+
   /**
    * Test max versions with an invalid value.
    * 
@@ -92,7 +93,7 @@ public class AccumuloInputFormatTest {
     Job job = new Job();
     AccumuloInputFormat.setMaxVersions(job.getConfiguration(), 0);
   }
-  
+
   /**
    * Test no max version configured.
    * 
@@ -104,7 +105,7 @@ public class AccumuloInputFormatTest {
     Job job = new Job();
     assertEquals(-1, AccumuloInputFormat.getMaxVersions(job.getConfiguration()));
   }
-  
+
   /**
    * Check that the iterator configuration is getting stored in the Job conf correctly.
    * 
@@ -113,7 +114,7 @@ public class AccumuloInputFormatTest {
   @Test
   public void testSetIterator() throws IOException {
     Job job = new Job();
-    
+
     IteratorSetting is = new IteratorSetting(1, "WholeRow", "org.apache.accumulo.core.iterators.WholeRowIterator");
     AccumuloInputFormat.addIterator(job, is);
     Configuration conf = job.getConfiguration();
@@ -122,36 +123,36 @@ public class AccumuloInputFormatTest {
     String iterators = conf.get("AccumuloInputFormat.ScanOpts.Iterators");
     assertEquals(new String(Base64.encodeBase64(baos.toByteArray())), iterators);
   }
-  
+
   @Test
   public void testAddIterator() throws IOException {
     Job job = new Job();
-    
+
     AccumuloInputFormat.addIterator(job, new IteratorSetting(1, "WholeRow", WholeRowIterator.class));
     AccumuloInputFormat.addIterator(job, new IteratorSetting(2, "Versions", "org.apache.accumulo.core.iterators.VersioningIterator"));
     IteratorSetting iter = new IteratorSetting(3, "Count", "org.apache.accumulo.core.iterators.CountingIterator");
     iter.addOption("v1", "1");
     iter.addOption("junk", "\0omg:!\\xyzzy");
     AccumuloInputFormat.addIterator(job, iter);
-    
+
     List<IteratorSetting> list = AccumuloInputFormat.getIterators(job);
-    
+
     // Check the list size
     assertTrue(list.size() == 3);
-    
+
     // Walk the list and make sure our settings are correct
     IteratorSetting setting = list.get(0);
     assertEquals(1, setting.getPriority());
     assertEquals("org.apache.accumulo.core.iterators.user.WholeRowIterator", setting.getIteratorClass());
     assertEquals("WholeRow", setting.getName());
     assertEquals(0, setting.getOptions().size());
-    
+
     setting = list.get(1);
     assertEquals(2, setting.getPriority());
     assertEquals("org.apache.accumulo.core.iterators.VersioningIterator", setting.getIteratorClass());
     assertEquals("Versions", setting.getName());
     assertEquals(0, setting.getOptions().size());
-    
+
     setting = list.get(2);
     assertEquals(3, setting.getPriority());
     assertEquals("org.apache.accumulo.core.iterators.CountingIterator", setting.getIteratorClass());
@@ -160,7 +161,7 @@ public class AccumuloInputFormatTest {
     assertEquals("1", setting.getOptions().get("v1"));
     assertEquals("\0omg:!\\xyzzy", setting.getOptions().get("junk"));
   }
-  
+
   /**
    * Test adding iterator options where the keys and values contain both the FIELD_SEPARATOR
character (':') and ITERATOR_SEPARATOR (',') characters. There
    * should be no exceptions thrown when trying to parse these types of option entries.
@@ -175,12 +176,12 @@ public class AccumuloInputFormatTest {
     someSetting.addOption(key, value);
     Job job = new Job();
     AccumuloInputFormat.addIterator(job, someSetting);
-    
+
     List<IteratorSetting> list = AccumuloInputFormat.getIterators(job);
     assertEquals(1, list.size());
     assertEquals(1, list.get(0).getOptions().size());
     assertEquals(list.get(0).getOptions().get(key), value);
-    
+
     someSetting.addOption(key + "2", value);
     someSetting.setPriority(2);
     someSetting.setName("it2");
@@ -193,7 +194,7 @@ public class AccumuloInputFormatTest {
     assertEquals(list.get(1).getOptions().get(key), value);
     assertEquals(list.get(1).getOptions().get(key + "2"), value);
   }
-  
+
   /**
    * Test getting iterator settings for multiple iterators set
    * 
@@ -202,55 +203,55 @@ public class AccumuloInputFormatTest {
   @Test
   public void testGetIteratorSettings() throws IOException {
     Job job = new Job();
-    
+
     AccumuloInputFormat.addIterator(job, new IteratorSetting(1, "WholeRow", "org.apache.accumulo.core.iterators.WholeRowIterator"));
     AccumuloInputFormat.addIterator(job, new IteratorSetting(2, "Versions", "org.apache.accumulo.core.iterators.VersioningIterator"));
     AccumuloInputFormat.addIterator(job, new IteratorSetting(3, "Count", "org.apache.accumulo.core.iterators.CountingIterator"));
-    
+
     List<IteratorSetting> list = AccumuloInputFormat.getIterators(job);
-    
+
     // Check the list size
     assertTrue(list.size() == 3);
-    
+
     // Walk the list and make sure our settings are correct
     IteratorSetting setting = list.get(0);
     assertEquals(1, setting.getPriority());
     assertEquals("org.apache.accumulo.core.iterators.WholeRowIterator", setting.getIteratorClass());
     assertEquals("WholeRow", setting.getName());
-    
+
     setting = list.get(1);
     assertEquals(2, setting.getPriority());
     assertEquals("org.apache.accumulo.core.iterators.VersioningIterator", setting.getIteratorClass());
     assertEquals("Versions", setting.getName());
-    
+
     setting = list.get(2);
     assertEquals(3, setting.getPriority());
     assertEquals("org.apache.accumulo.core.iterators.CountingIterator", setting.getIteratorClass());
     assertEquals("Count", setting.getName());
-    
+
   }
-  
+
   @Test
   public void testSetRegex() throws IOException {
     Job job = new Job();
-    
+
     String regex = ">\"*%<>\'\\";
-    
+
     IteratorSetting is = new IteratorSetting(50, regex, RegExFilter.class);
     RegExFilter.setRegexs(is, regex, null, null, null, false);
     AccumuloInputFormat.addIterator(job, is);
-    
+
     assertTrue(regex.equals(AccumuloInputFormat.getIterators(job).get(0).getName()));
   }
-  
+
   private static AssertionError e1 = null;
   private static AssertionError e2 = null;
-  
+
   private static class MRTester extends Configured implements Tool {
     private static class TestMapper extends Mapper<Key,Value,Key,Value> {
       Key key = null;
       int count = 0;
-      
+
       @Override
       protected void map(Key k, Value v, Context context) throws IOException, InterruptedException
{
         try {
@@ -264,7 +265,7 @@ public class AccumuloInputFormatTest {
         key = new Key(k);
         count++;
       }
-      
+
       @Override
       protected void cleanup(Context context) throws IOException, InterruptedException {
         try {
@@ -274,46 +275,52 @@ public class AccumuloInputFormatTest {
         }
       }
     }
-    
+
     @Override
     public int run(String[] args) throws Exception {
-      
-      if (args.length != 3) {
-        throw new IllegalArgumentException("Usage : " + MRTester.class.getName() + " <user>
<pass> <table>");
+
+      if (args.length != 5) {
+        throw new IllegalArgumentException("Usage : " + MRTester.class.getName() + " <user>
<pass> <table> <instanceName> <inputFormatClass>");
       }
-      
+
       String user = args[0];
       String pass = args[1];
       String table = args[2];
-      
+      String instanceName = args[3];
+      String inputFormatClassName = args[4];
+      Class<? extends InputFormat> inputFormatClass = (Class<? extends InputFormat>)
Class.forName(inputFormatClassName);
+
       Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
       job.setJarByClass(this.getClass());
-      
-      job.setInputFormatClass(AccumuloInputFormat.class);
-      
+
+      job.setInputFormatClass(inputFormatClass);
+
       AccumuloInputFormat.setConnectorInfo(job, user, new PasswordToken(pass));
       AccumuloInputFormat.setInputTableName(job, table);
-      AccumuloInputFormat.setMockInstance(job, INSTANCE_NAME);
-      
+      AccumuloInputFormat.setMockInstance(job, instanceName);
+
       job.setMapperClass(TestMapper.class);
       job.setMapOutputKeyClass(Key.class);
       job.setMapOutputValueClass(Value.class);
       job.setOutputFormatClass(NullOutputFormat.class);
-      
+
       job.setNumReduceTasks(0);
-      
+
       job.waitForCompletion(true);
-      
+
       return job.isSuccessful() ? 0 : 1;
     }
-    
-    public static void main(String[] args) throws Exception {
-      assertEquals(0, ToolRunner.run(CachedConfiguration.getInstance(), new MRTester(), args));
+
+    public static int main(String[] args) throws Exception {
+      return ToolRunner.run(CachedConfiguration.getInstance(), new MRTester(), args);
     }
   }
-  
+
   @Test
   public void testMap() throws Exception {
+    final String INSTANCE_NAME = PREFIX + "_mapreduce_instance";
+    final String TEST_TABLE_1 = PREFIX + "_mapreduce_table_1";
+
     MockInstance mockInstance = new MockInstance(INSTANCE_NAME);
     Connector c = mockInstance.getConnector("root", new PasswordToken(""));
     c.tableOperations().create(TEST_TABLE_1);
@@ -324,13 +331,12 @@ public class AccumuloInputFormatTest {
       bw.addMutation(m);
     }
     bw.close();
-    
-    MRTester.main(new String[] {"root", "", TEST_TABLE_1});
+
+    Assert.assertEquals(0, MRTester.main(new String[] {"root", "", TEST_TABLE_1, INSTANCE_NAME,
AccumuloInputFormat.class.getCanonicalName()}));
     assertNull(e1);
     assertNull(e2);
   }
-  
-  @SuppressWarnings("deprecation")
+
   @Test
   public void testCorrectRangeInputSplits() throws Exception {
     Job job = new Job(new Configuration(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
@@ -354,19 +360,19 @@ public class AccumuloInputFormatTest {
     AccumuloInputFormat.setLocalIterators(job, localIters);
     AccumuloInputFormat.fetchColumns(job, fetchColumns);
     AccumuloInputFormat.setLogLevel(job, level);
-    
+
     AccumuloInputFormat aif = new AccumuloInputFormat();
-    
+
     List<InputSplit> splits = aif.getSplits(job);
-    
+
     Assert.assertEquals(1, splits.size());
-    
+
     InputSplit split = splits.get(0);
-    
+
     Assert.assertEquals(RangeInputSplit.class, split.getClass());
-    
+
     RangeInputSplit risplit = (RangeInputSplit) split;
-    
+
     Assert.assertEquals(username, risplit.getPrincipal());
     Assert.assertEquals(table, risplit.getTable());
     Assert.assertEquals(password, risplit.getToken());
@@ -377,7 +383,7 @@ public class AccumuloInputFormatTest {
     Assert.assertEquals(fetchColumns, risplit.getFetchedColumns());
     Assert.assertEquals(level, risplit.getLogLevel());
   }
-  
+
   static class TestMapper extends Mapper<Key,Value,Key,Value> {
     Key key = null;
     int count = 0;
@@ -397,7 +403,7 @@ public class AccumuloInputFormatTest {
   public void testPartialInputSplitDelegationToConfiguration() throws Exception {
     String user = "testPartialInputSplitUser";
     PasswordToken password = new PasswordToken("");
-    
+
     MockInstance mockInstance = new MockInstance("testPartialInputSplitDelegationToConfiguration");
     Connector c = mockInstance.getConnector(user, password);
     c.tableOperations().create("testtable");
@@ -409,37 +415,17 @@ public class AccumuloInputFormatTest {
     }
     bw.close();
 
-    Job job = new Job(new Configuration());
-    job.setInputFormatClass(AccumuloInputFormat.class);
-    job.setMapperClass(TestMapper.class);
-    job.setNumReduceTasks(0);
-    AccumuloInputFormat.setConnectorInfo(job, user, password);
-    AccumuloInputFormat.setInputTableName(job, "testtable");
-    AccumuloInputFormat.setScanAuthorizations(job, new Authorizations());
-    AccumuloInputFormat.setMockInstance(job, "testPartialInputSplitDelegationToConfiguration");
-
-    AccumuloInputFormat input = new AccumuloInputFormat();
-    List<InputSplit> splits = input.getSplits(job);
-    assertEquals(splits.size(), 1);
-
-    TestMapper mapper = (TestMapper) job.getMapperClass().newInstance();
-    
-    RangeInputSplit emptySplit = new RangeInputSplit();
-    
-    // Using an empty split should fall back to the information in the Job's Configuration
-    TaskAttemptID id = new TaskAttemptID();
-    TaskAttemptContext attempt = new TaskAttemptContext(job.getConfiguration(), id);
-    RecordReader<Key,Value> reader = input.createRecordReader(emptySplit, attempt);
-    Mapper<Key,Value,Key,Value>.Context context = mapper.new Context(job.getConfiguration(),
id, reader, null, null, null, emptySplit);
-    reader.initialize(emptySplit, context);
-    mapper.run(context);
+    Assert.assertEquals(0, MRTester.main(new String[] {user, "", "testtable", "testPartialInputSplitDelegationToConfiguration",
+        EmptySplitsAccumuloInputFormat.class.getCanonicalName()}));
+    assertNull(e1);
+    assertNull(e2);
   }
 
-  @Test(expected = IOException.class)
+  @Test
   public void testPartialFailedInputSplitDelegationToConfiguration() throws Exception {
     String user = "testPartialFailedInputSplit";
     PasswordToken password = new PasswordToken("");
-    
+
     MockInstance mockInstance = new MockInstance("testPartialFailedInputSplitDelegationToConfiguration");
     Connector c = mockInstance.getConnector(user, password);
     c.tableOperations().create("testtable");
@@ -451,30 +437,10 @@ public class AccumuloInputFormatTest {
     }
     bw.close();
 
-    Job job = new Job(new Configuration());
-    job.setInputFormatClass(AccumuloInputFormat.class);
-    job.setMapperClass(TestMapper.class);
-    job.setNumReduceTasks(0);
-    AccumuloInputFormat.setConnectorInfo(job, user, password);
-    AccumuloInputFormat.setInputTableName(job, "testtable");
-    AccumuloInputFormat.setMockInstance(job, "testPartialFailedInputSplitDelegationToConfiguration");
-
-    AccumuloInputFormat input = new AccumuloInputFormat();
-    List<InputSplit> splits = input.getSplits(job);
-    assertEquals(splits.size(), 1);
-
-    TestMapper mapper = (TestMapper) job.getMapperClass().newInstance();
-    
-    RangeInputSplit emptySplit = new RangeInputSplit();
-    emptySplit.setPrincipal("root");
-    emptySplit.setToken(new PasswordToken("anythingelse"));
-    
-    // Using an empty split should fall back to the information in the Job's Configuration
-    TaskAttemptID id = new TaskAttemptID();
-    TaskAttemptContext attempt = new TaskAttemptContext(job.getConfiguration(), id);
-    RecordReader<Key,Value> reader = input.createRecordReader(emptySplit, attempt);
-    Mapper<Key,Value,Key,Value>.Context context = mapper.new Context(job.getConfiguration(),
id, reader, null, null, null, emptySplit);
-    reader.initialize(emptySplit, context);
-    mapper.run(context);
+    // We should fail before we even get into the Mapper because we can't make the RecordReader

+    Assert.assertEquals(1, MRTester.main(new String[] {user, "", "testtable", "testPartialFailedInputSplitDelegationToConfiguration",
+        BadPasswordSplitsAccumuloInputFormat.class.getCanonicalName()}));
+    assertNull(e1);
+    assertNull(e2);
   }
 }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/c88d87aa/core/src/test/java/org/apache/accumulo/core/client/mapreduce/BadPasswordSplitsAccumuloInputFormat.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/client/mapreduce/BadPasswordSplitsAccumuloInputFormat.java
b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/BadPasswordSplitsAccumuloInputFormat.java
new file mode 100644
index 0000000..846208b
--- /dev/null
+++ b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/BadPasswordSplitsAccumuloInputFormat.java
@@ -0,0 +1,26 @@
+package org.apache.accumulo.core.client.mapreduce;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.JobContext;
+
+/**
+ * AccumuloInputFormat which returns an "empty" RangeInputSplit
+ */
+public class BadPasswordSplitsAccumuloInputFormat extends AccumuloInputFormat {
+  
+  @Override
+  public List<InputSplit> getSplits(JobContext context) throws IOException {
+    List<InputSplit> splits = super.getSplits(context);
+    
+    for (InputSplit split : splits) {
+      RangeInputSplit rangeSplit = (RangeInputSplit) split;
+      rangeSplit.setToken(new PasswordToken("anythingelse"));
+    }
+    
+    return splits;
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/accumulo/blob/c88d87aa/core/src/test/java/org/apache/accumulo/core/client/mapreduce/EmptySplitsAccumuloInputFormat.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/client/mapreduce/EmptySplitsAccumuloInputFormat.java
b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/EmptySplitsAccumuloInputFormat.java
new file mode 100644
index 0000000..91db378
--- /dev/null
+++ b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/EmptySplitsAccumuloInputFormat.java
@@ -0,0 +1,21 @@
+package org.apache.accumulo.core.client.mapreduce;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.JobContext;
+
+/**
+ * AccumuloInputFormat which returns an "empty" RangeInputSplit
+ */
+public class EmptySplitsAccumuloInputFormat extends AccumuloInputFormat {
+  
+  @Override
+  public List<InputSplit> getSplits(JobContext context) throws IOException {
+    super.getSplits(context);
+    
+    return Arrays.<InputSplit> asList(new RangeInputSplit());
+  }
+}
\ No newline at end of file


Mime
View raw message