accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cjno...@apache.org
Subject [08/11] git commit: Cleaning up tests. Adding test for legacy input for base + new multi-table methods. ACCUMULO-391
Date Wed, 02 Oct 2013 03:22:44 GMT
Cleaning up tests. Adding test for legacy input for base + new multi-table methods. ACCUMULO-391


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/53bcc856
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/53bcc856
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/53bcc856

Branch: refs/heads/ACCUMULO-391
Commit: 53bcc85689510fc988c9e9f6aff0da0cb7091c6c
Parents: e4e05c8
Author: Corey J. Nolet <cjnolet@gmail.com>
Authored: Mon Sep 30 21:01:55 2013 -0400
Committer: Corey J. Nolet <cjnolet@gmail.com>
Committed: Tue Oct 1 21:46:17 2013 -0400

----------------------------------------------------------------------
 .../client/mapred/AccumuloInputFormatTest.java  |   9 +-
 .../mapreduce/AccumuloInputFormatTest.java      | 144 +++++++++++--------
 .../core/conf/TableQueryConfigTest.java         |  25 ++--
 3 files changed, 97 insertions(+), 81 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/53bcc856/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java
b/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java
index 96113b1..a94d80f 100644
--- a/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormatTest.java
@@ -16,7 +16,6 @@
  */
 package org.apache.accumulo.core.client.mapred;
 
-import static org.apache.commons.codec.binary.Base64.encodeBase64;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
@@ -70,13 +69,13 @@ public class AccumuloInputFormatTest {
     ByteArrayOutputStream baos = new ByteArrayOutputStream();
     is.write(new DataOutputStream(baos));
     String iterators = job.get("AccumuloInputFormat.ScanOpts.Iterators");
-    assertEquals(new String(encodeBase64 (baos.toByteArray ())), iterators);
+    assertEquals(new String(Base64.encodeBase64(baos.toByteArray())), iterators);
   }
   
   @Test
   public void testAddIterator() throws IOException {
     JobConf job = new JobConf();
-
+    
     AccumuloInputFormat.addIterator(job, new IteratorSetting(1, "WholeRow", WholeRowIterator.class));
     AccumuloInputFormat.addIterator(job, new IteratorSetting(2, "Versions", "org.apache.accumulo.core.iterators.VersioningIterator"));
     IteratorSetting iter = new IteratorSetting(3, "Count", "org.apache.accumulo.core.iterators.CountingIterator");
@@ -152,7 +151,7 @@ public class AccumuloInputFormatTest {
   @Test
   public void testGetIteratorSettings() throws IOException {
     JobConf job = new JobConf();
-
+    
     AccumuloInputFormat.addIterator(job, new IteratorSetting(1, "WholeRow", "org.apache.accumulo.core.iterators.WholeRowIterator"));
     AccumuloInputFormat.addIterator(job, new IteratorSetting(2, "Versions", "org.apache.accumulo.core.iterators.VersioningIterator"));
     AccumuloInputFormat.addIterator(job, new IteratorSetting(3, "Count", "org.apache.accumulo.core.iterators.CountingIterator"));
@@ -185,7 +184,7 @@ public class AccumuloInputFormatTest {
     JobConf job = new JobConf();
     
     String regex = ">\"*%<>\'\\";
-
+    
     IteratorSetting is = new IteratorSetting(50, regex, RegExFilter.class);
     RegExFilter.setRegexs(is, regex, null, null, null, false);
     AccumuloInputFormat.addIterator(job, is);

http://git-wip-us.apache.org/repos/asf/accumulo/blob/53bcc856/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
index f9dd9ec..c5361b9 100644
--- a/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java
@@ -16,11 +16,9 @@
  */
 package org.apache.accumulo.core.client.mapreduce;
 
-import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
 
 import java.io.ByteArrayOutputStream;
 import java.io.DataOutputStream;
@@ -55,12 +53,12 @@ import org.apache.hadoop.util.ToolRunner;
 import org.junit.Test;
 
 public class AccumuloInputFormatTest {
-
+  
   private static final String PREFIX = AccumuloInputFormatTest.class.getSimpleName();
   private static final String INSTANCE_NAME = PREFIX + "_mapreduce_instance";
   private static final String TEST_TABLE_1 = PREFIX + "_mapreduce_table_1";
   private static final String TEST_TABLE_2 = PREFIX + "_mapreduce_table_2";
-
+  
   /**
    * Check that the iterator configuration is getting stored in the Job conf correctly.
    * 
@@ -69,7 +67,7 @@ public class AccumuloInputFormatTest {
   @Test
   public void testSetIterator() throws IOException {
     Job job = new Job();
-
+    
     IteratorSetting is = new IteratorSetting(1, "WholeRow", "org.apache.accumulo.core.iterators.WholeRowIterator");
     AccumuloInputFormat.addIterator(job, is);
     Configuration conf = job.getConfiguration();
@@ -78,37 +76,36 @@ public class AccumuloInputFormatTest {
     String iterators = conf.get("AccumuloInputFormat.ScanOpts.Iterators");
     assertEquals(new String(Base64.encodeBase64(baos.toByteArray())), iterators);
   }
-
+  
   @Test
   public void testAddIterator() throws IOException {
     Job job = new Job();
-
-
+    
     AccumuloInputFormat.addIterator(job, new IteratorSetting(1, "WholeRow", WholeRowIterator.class));
     AccumuloInputFormat.addIterator(job, new IteratorSetting(2, "Versions", "org.apache.accumulo.core.iterators.VersioningIterator"));
     IteratorSetting iter = new IteratorSetting(3, "Count", "org.apache.accumulo.core.iterators.CountingIterator");
     iter.addOption("v1", "1");
     iter.addOption("junk", "\0omg:!\\xyzzy");
     AccumuloInputFormat.addIterator(job, iter);
-
+    
     List<IteratorSetting> list = AccumuloInputFormat.getIterators(job);
-
+    
     // Check the list size
     assertTrue(list.size() == 3);
-
+    
     // Walk the list and make sure our settings are correct
     IteratorSetting setting = list.get(0);
     assertEquals(1, setting.getPriority());
     assertEquals("org.apache.accumulo.core.iterators.user.WholeRowIterator", setting.getIteratorClass());
     assertEquals("WholeRow", setting.getName());
     assertEquals(0, setting.getOptions().size());
-
+    
     setting = list.get(1);
     assertEquals(2, setting.getPriority());
     assertEquals("org.apache.accumulo.core.iterators.VersioningIterator", setting.getIteratorClass());
     assertEquals("Versions", setting.getName());
     assertEquals(0, setting.getOptions().size());
-
+    
     setting = list.get(2);
     assertEquals(3, setting.getPriority());
     assertEquals("org.apache.accumulo.core.iterators.CountingIterator", setting.getIteratorClass());
@@ -117,7 +114,7 @@ public class AccumuloInputFormatTest {
     assertEquals("1", setting.getOptions().get("v1"));
     assertEquals("\0omg:!\\xyzzy", setting.getOptions().get("junk"));
   }
-
+  
   /**
    * Test adding iterator options where the keys and values contain both the FIELD_SEPARATOR
character (':') and ITERATOR_SEPARATOR (',') characters. There
    * should be no exceptions thrown when trying to parse these types of option entries.
@@ -132,12 +129,12 @@ public class AccumuloInputFormatTest {
     someSetting.addOption(key, value);
     Job job = new Job();
     AccumuloInputFormat.addIterator(job, someSetting);
-
+    
     List<IteratorSetting> list = AccumuloInputFormat.getIterators(job);
     assertEquals(1, list.size());
     assertEquals(1, list.get(0).getOptions().size());
     assertEquals(list.get(0).getOptions().get(key), value);
-
+    
     someSetting.addOption(key + "2", value);
     someSetting.setPriority(2);
     someSetting.setName("it2");
@@ -150,7 +147,7 @@ public class AccumuloInputFormatTest {
     assertEquals(list.get(1).getOptions().get(key), value);
     assertEquals(list.get(1).getOptions().get(key + "2"), value);
   }
-
+  
   /**
    * Test getting iterator settings for multiple iterators set
    * 
@@ -159,56 +156,56 @@ public class AccumuloInputFormatTest {
   @Test
   public void testGetIteratorSettings() throws IOException {
     Job job = new Job();
-
+    
     AccumuloInputFormat.addIterator(job, new IteratorSetting(1, "WholeRow", "org.apache.accumulo.core.iterators.WholeRowIterator"));
     AccumuloInputFormat.addIterator(job, new IteratorSetting(2, "Versions", "org.apache.accumulo.core.iterators.VersioningIterator"));
     AccumuloInputFormat.addIterator(job, new IteratorSetting(3, "Count", "org.apache.accumulo.core.iterators.CountingIterator"));
-
+    
     List<IteratorSetting> list = AccumuloInputFormat.getIterators(job);
-
+    
     // Check the list size
     assertTrue(list.size() == 3);
-
+    
     // Walk the list and make sure our settings are correct
     IteratorSetting setting = list.get(0);
     assertEquals(1, setting.getPriority());
     assertEquals("org.apache.accumulo.core.iterators.WholeRowIterator", setting.getIteratorClass());
     assertEquals("WholeRow", setting.getName());
-
+    
     setting = list.get(1);
     assertEquals(2, setting.getPriority());
     assertEquals("org.apache.accumulo.core.iterators.VersioningIterator", setting.getIteratorClass());
     assertEquals("Versions", setting.getName());
-
+    
     setting = list.get(2);
     assertEquals(3, setting.getPriority());
     assertEquals("org.apache.accumulo.core.iterators.CountingIterator", setting.getIteratorClass());
     assertEquals("Count", setting.getName());
-
+    
   }
-
+  
   @Test
   public void testSetRegex() throws IOException {
     Job job = new Job();
-
+    
     String regex = ">\"*%<>\'\\";
-
+    
     IteratorSetting is = new IteratorSetting(50, regex, RegExFilter.class);
     RegExFilter.setRegexs(is, regex, null, null, null, false);
     AccumuloInputFormat.addIterator(job, is);
-
+    
     assertTrue(regex.equals(AccumuloInputFormat.getIterators(job).get(0).getName()));
   }
-
+  
   private static AssertionError e1 = null;
   private static AssertionError e2 = null;
-
+  
   private static class MRTester extends Configured implements Tool {
-
+    
     private static class TestMapper extends Mapper<Key,Value,Key,Value> {
       Key key = null;
       int count = 0;
-
+      
       @Override
       protected void map(Key k, Value v, Context context) throws IOException, InterruptedException
{
         try {
@@ -223,7 +220,7 @@ public class AccumuloInputFormatTest {
         key = new Key(k);
         count++;
       }
-
+      
       @Override
       protected void cleanup(Context context) throws IOException, InterruptedException {
         try {
@@ -233,50 +230,50 @@ public class AccumuloInputFormatTest {
         }
       }
     }
-
+    
     @Override
     public int run(String[] args) throws Exception {
-
+      
       if (args.length != 4) {
         throw new IllegalArgumentException("Usage : " + MRTester.class.getName() + " <user>
<pass> <table>");
       }
-
+      
       String user = args[0];
       String pass = args[1];
       String table1 = args[2];
       String table2 = args[3];
-
+      
       Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
       job.setJarByClass(this.getClass());
-
+      
       job.setInputFormatClass(AccumuloInputFormat.class);
-
+      
       AccumuloInputFormat.setConnectorInfo(job, user, new PasswordToken(pass));
-
-//      AccumuloInputFormat.setInputTableNames(job, Arrays.asList(new String[]{table1, table2}));
+      
+      // AccumuloInputFormat.setInputTableNames(job, Arrays.asList(new String[]{table1, table2}));
       TableQueryConfig tableConfig1 = new TableQueryConfig(table1);
       TableQueryConfig tableConfig2 = new TableQueryConfig(table2);
-
-      AccumuloInputFormat.setTableQueryConfigs(job,tableConfig1,tableConfig2);
+      
+      AccumuloInputFormat.setTableQueryConfigs(job, tableConfig1, tableConfig2);
       AccumuloInputFormat.setMockInstance(job, INSTANCE_NAME);
-
+      
       job.setMapperClass(TestMapper.class);
       job.setMapOutputKeyClass(Key.class);
       job.setMapOutputValueClass(Value.class);
       job.setOutputFormatClass(NullOutputFormat.class);
-
+      
       job.setNumReduceTasks(0);
-
+      
       job.waitForCompletion(true);
-
+      
       return job.isSuccessful() ? 0 : 1;
     }
-
+    
     public static void main(String[] args) throws Exception {
       assertEquals(0, ToolRunner.run(CachedConfiguration.getInstance(), new MRTester(), args));
     }
   }
-
+  
   /**
    * Generate incrementing counts and attach table name to the key/value so that order and
multi-table data can be verified.
    */
@@ -298,30 +295,51 @@ public class AccumuloInputFormatTest {
     }
     bw.close();
     bw2.close();
-
+    
     MRTester.main(new String[] {"root", "", TEST_TABLE_1, TEST_TABLE_2});
     assertNull(e1);
     assertNull(e2);
   }
+  
+  @Test
+  public void testTableQueryConfigSerialization() throws IOException {
+    
+    Job job = new Job();
+    
+    TableQueryConfig table1 = new TableQueryConfig(TEST_TABLE_1).setRanges(Collections.singletonList(new
Range("a","b")))
+        .setColumns(Collections.singleton(new Pair<Text, Text>(new Text("CF1"),new
Text("CQ1"))))
+        .setIterators(Collections.singletonList(new IteratorSetting(50,"iter1","iterclass1")));
+    
+    TableQueryConfig table2 = new TableQueryConfig(TEST_TABLE_2).setRanges(Collections.singletonList(new
Range("a","b")))
+        .setColumns(Collections.singleton(new Pair<Text, Text>(new Text("CF1"),new
Text("CQ1"))))
+        .setIterators(Collections.singletonList(new IteratorSetting(50,"iter1","iterclass1")));
+    
+    AccumuloInputFormat.setTableQueryConfigs(job, table1, table2);
+    
+    assertEquals(table1, AccumuloInputFormat.getTableQueryConfig(job, TEST_TABLE_1));
+    assertEquals(table2, AccumuloInputFormat.getTableQueryConfig(job, TEST_TABLE_2));
+  }
 
   @Test
-  public void testTableQueryConfigSerialization() throws IOException{
+  public void testTableQueryConfigBackwardsCompatibility() throws IOException {
 
     Job job = new Job();
 
-    TableQueryConfig table1 = new TableQueryConfig(TEST_TABLE_1)
-            .setRanges(Collections.singletonList(new Range("a", "b")))
-            .setColumns(Collections.singleton(new Pair<Text,Text>(new Text("CF1"),
new Text("CQ1"))))
-            .setIterators(Collections.singletonList(new IteratorSetting(50, "iter1", "iterclass1")));
+    TableQueryConfig table1 = new TableQueryConfig(TEST_TABLE_1).setRanges(Collections.singletonList(new
Range("a","b")))
+            .setColumns(Collections.singleton(new Pair<Text, Text>(new Text("CF1"),new
Text("CQ1"))))
+            .setIterators(Collections.singletonList(new IteratorSetting(50,"iter1","iterclass1")));
 
-    TableQueryConfig table2 = new TableQueryConfig(TEST_TABLE_2)
-            .setRanges(Collections.singletonList(new Range("a", "b")))
-            .setColumns(Collections.singleton(new Pair<Text,Text>(new Text("CF1"),
new Text("CQ1"))))
-            .setIterators(Collections.singletonList(new IteratorSetting(50, "iter1", "iterclass1")));
+    TableQueryConfig table2 = new TableQueryConfig(TEST_TABLE_2).setRanges(Collections.singletonList(new
Range("a","b")))
+            .setColumns(Collections.singleton(new Pair<Text, Text>(new Text("CF1"),new
Text("CQ1"))))
+            .setIterators(Collections.singletonList(new IteratorSetting(50,"iter1","iterclass1")));
 
-    AccumuloInputFormat.setTableQueryConfigs(job,table1,table2);
+    AccumuloInputFormat.setTableQueryConfigs(job, table1);
+    AccumuloInputFormat.setInputTableName(job, table2.getTableName());
+    AccumuloInputFormat.setRanges(job, table2.getRanges());
+    AccumuloInputFormat.fetchColumns(job, table2.getColumns());
+    AccumuloInputFormat.addIterator(job, table2.getIterators().get(0));
 
-    assertEquals(table1, AccumuloInputFormat.getTableQueryConfig(job,TEST_TABLE_1));
-    assertEquals(table2, AccumuloInputFormat.getTableQueryConfig(job,TEST_TABLE_2));
+    assertEquals(table1, AccumuloInputFormat.getTableQueryConfig(job, TEST_TABLE_1));
+    assertEquals(table2, AccumuloInputFormat.getTableQueryConfig(job, TEST_TABLE_2));
   }
 }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/53bcc856/core/src/test/java/org/apache/accumulo/core/conf/TableQueryConfigTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/conf/TableQueryConfigTest.java b/core/src/test/java/org/apache/accumulo/core/conf/TableQueryConfigTest.java
index 1d5d187..d3fbd3c 100644
--- a/core/src/test/java/org/apache/accumulo/core/conf/TableQueryConfigTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/conf/TableQueryConfigTest.java
@@ -32,9 +32,7 @@ public class TableQueryConfigTest {
   @Test
   public void testSerialization_OnlyTable() throws IOException {
     byte[] serialized = serialize(tableQueryConfig);
-    ByteArrayInputStream bais = new ByteArrayInputStream(serialized);
-    TableQueryConfig actualConfig = new TableQueryConfig(new DataInputStream(bais));
-    bais.close();
+    TableQueryConfig actualConfig = deserialize(serialized);
     
     assertEquals(tableQueryConfig, actualConfig);
   }
@@ -47,9 +45,7 @@ public class TableQueryConfigTest {
     tableQueryConfig.setRanges(ranges);
     
     byte[] serialized = serialize(tableQueryConfig);
-    ByteArrayInputStream bais = new ByteArrayInputStream(serialized);
-    TableQueryConfig actualConfig = new TableQueryConfig(new DataInputStream(bais));
-    bais.close();
+    TableQueryConfig actualConfig = deserialize(serialized);
     
     assertEquals(ranges, actualConfig.getRanges());
   }
@@ -62,9 +58,7 @@ public class TableQueryConfigTest {
     tableQueryConfig.setColumns(columns);
     
     byte[] serialized = serialize(tableQueryConfig);
-    ByteArrayInputStream bais = new ByteArrayInputStream(serialized);
-    TableQueryConfig actualConfig = new TableQueryConfig(new DataInputStream(bais));
-    bais.close();
+    TableQueryConfig actualConfig = deserialize(serialized);
     
     assertEquals(actualConfig.getColumns(), columns);
   }
@@ -76,11 +70,9 @@ public class TableQueryConfigTest {
     settings.add(new IteratorSetting(55, "iter2", "iterclass2"));
     tableQueryConfig.setIterators(settings);
     byte[] serialized = serialize(tableQueryConfig);
-    ByteArrayInputStream bais = new ByteArrayInputStream(serialized);
-    TableQueryConfig actualConfig = new TableQueryConfig(new DataInputStream(bais));
-    bais.close();
-    
+    TableQueryConfig actualConfig = deserialize(serialized);
     assertEquals(actualConfig.getIterators(), settings);
+
   }
   
   private byte[] serialize(TableQueryConfig tableQueryConfig) throws IOException {
@@ -90,4 +82,11 @@ public class TableQueryConfigTest {
     
     return baos.toByteArray();
   }
+
+  private TableQueryConfig deserialize(byte[] bytes) throws IOException {
+    ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
+    TableQueryConfig actualConfig = new TableQueryConfig(new DataInputStream(bais));
+    bais.close();
+    return actualConfig;
+  }
 }


Mime
View raw message