Return-Path: X-Original-To: apmail-accumulo-commits-archive@www.apache.org Delivered-To: apmail-accumulo-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 8631C107E5 for ; Sat, 23 Nov 2013 23:51:55 +0000 (UTC) Received: (qmail 65204 invoked by uid 500); 23 Nov 2013 23:51:54 -0000 Delivered-To: apmail-accumulo-commits-archive@accumulo.apache.org Received: (qmail 64541 invoked by uid 500); 23 Nov 2013 23:51:51 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 64377 invoked by uid 99); 23 Nov 2013 23:51:50 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 23 Nov 2013 23:51:50 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id A0CD5904021; Sat, 23 Nov 2013 23:51:50 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: elserj@apache.org To: commits@accumulo.apache.org Date: Sat, 23 Nov 2013 23:52:16 -0000 Message-Id: In-Reply-To: <3a03c74a531946da9a279bed5eecbbd3@git.apache.org> References: <3a03c74a531946da9a279bed5eecbbd3@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [28/30] git commit: Merge branch '1.5.1-SNAPSHOT' into 1.6.0-SNAPSHOT Merge branch '1.5.1-SNAPSHOT' into 1.6.0-SNAPSHOT Conflicts: core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/0f3f93fd Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/0f3f93fd Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/0f3f93fd Branch: refs/heads/1.6.0-SNAPSHOT Commit: 0f3f93fdc9f66c49bde97019f8aea9086c9f0087 Parents: 10d2679 c88d87a Author: Josh Elser Authored: Sat Nov 23 14:21:36 2013 -0500 Committer: Josh Elser Committed: Sat Nov 23 14:21:36 2013 -0500 ---------------------------------------------------------------------- .../mapreduce/AccumuloInputFormatTest.java | 477 +++---------------- .../BadPasswordSplitsAccumuloInputFormat.java | 26 + .../EmptySplitsAccumuloInputFormat.java | 29 ++ 3 files changed, 131 insertions(+), 401 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/0f3f93fd/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java ---------------------------------------------------------------------- diff --cc core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java index 6a97d67,93dba65..397b203 --- a/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java +++ b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java @@@ -23,7 -23,7 +23,6 @@@ import static org.junit.Assert.assertTr import java.io.ByteArrayOutputStream; import java.io.DataOutputStream; import java.io.IOException; - import java.net.URI; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; @@@ -53,20 -50,12 +49,8 @@@ import org.apache.hadoop.io.Text import org.apache.hadoop.mapreduce.InputFormat; import org.apache.hadoop.mapreduce.InputSplit; import org.apache.hadoop.mapreduce.Job; - import org.apache.hadoop.mapreduce.JobID; -import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.Mapper; - import org.apache.hadoop.mapreduce.Mapper.Context; - import org.apache.hadoop.mapreduce.OutputCommitter; - import org.apache.hadoop.mapreduce.OutputFormat; - import org.apache.hadoop.mapreduce.Partitioner; --import org.apache.hadoop.mapreduce.RecordReader; - import org.apache.hadoop.mapreduce.Reducer; --import org.apache.hadoop.mapreduce.TaskAttemptContext; --import org.apache.hadoop.mapreduce.TaskAttemptID; import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat; - import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl; - import org.apache.hadoop.security.Credentials; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; import org.apache.log4j.Level; @@@ -74,21 -63,58 +58,19 @@@ import org.junit.Assert import org.junit.Test; public class AccumuloInputFormatTest { - + private static final String PREFIX = AccumuloInputFormatTest.class.getSimpleName(); - private static final String INSTANCE_NAME = PREFIX + "_mapreduce_instance"; - private static final String TEST_TABLE_1 = PREFIX + "_mapreduce_table_1"; - + /** - * Test basic setting & getting of max versions. - * - * @throws IOException - * Signals that an I/O exception has occurred. - */ - @Deprecated - @Test - public void testMaxVersions() throws IOException { - Job job = new Job(); - AccumuloInputFormat.setMaxVersions(job.getConfiguration(), 1); - int version = AccumuloInputFormat.getMaxVersions(job.getConfiguration()); - assertEquals(1, version); - } - - /** - * Test max versions with an invalid value. - * - * @throws IOException - * Signals that an I/O exception has occurred. - */ - @Deprecated - @Test(expected = IOException.class) - public void testMaxVersionsLessThan1() throws IOException { - Job job = new Job(); - AccumuloInputFormat.setMaxVersions(job.getConfiguration(), 0); - } - - /** - * Test no max version configured. - * - * @throws IOException - */ - @Deprecated - @Test - public void testNoMaxVersion() throws IOException { - Job job = new Job(); - assertEquals(-1, AccumuloInputFormat.getMaxVersions(job.getConfiguration())); - } - - /** * Check that the iterator configuration is getting stored in the Job conf correctly. * * @throws IOException */ @Test public void testSetIterator() throws IOException { + @SuppressWarnings("deprecation") Job job = new Job(); - + IteratorSetting is = new IteratorSetting(1, "WholeRow", "org.apache.accumulo.core.iterators.WholeRowIterator"); AccumuloInputFormat.addIterator(job, is); Configuration conf = job.getConfiguration(); @@@ -97,12 -123,11 +79,12 @@@ String iterators = conf.get("AccumuloInputFormat.ScanOpts.Iterators"); assertEquals(new String(Base64.encodeBase64(baos.toByteArray())), iterators); } - + @Test public void testAddIterator() throws IOException { + @SuppressWarnings("deprecation") Job job = new Job(); - + AccumuloInputFormat.addIterator(job, new IteratorSetting(1, "WholeRow", WholeRowIterator.class)); AccumuloInputFormat.addIterator(job, new IteratorSetting(2, "Versions", "org.apache.accumulo.core.iterators.VersioningIterator")); IteratorSetting iter = new IteratorSetting(3, "Count", "org.apache.accumulo.core.iterators.CountingIterator"); @@@ -149,10 -174,9 +131,10 @@@ String value = "comma,delimited,value"; IteratorSetting someSetting = new IteratorSetting(1, "iterator", "Iterator.class"); someSetting.addOption(key, value); + @SuppressWarnings("deprecation") Job job = new Job(); AccumuloInputFormat.addIterator(job, someSetting); - + List list = AccumuloInputFormat.getIterators(job); assertEquals(1, list.size()); assertEquals(1, list.get(0).getOptions().size()); @@@ -178,9 -202,8 +160,9 @@@ */ @Test public void testGetIteratorSettings() throws IOException { + @SuppressWarnings("deprecation") Job job = new Job(); - + AccumuloInputFormat.addIterator(job, new IteratorSetting(1, "WholeRow", "org.apache.accumulo.core.iterators.WholeRowIterator")); AccumuloInputFormat.addIterator(job, new IteratorSetting(2, "Versions", "org.apache.accumulo.core.iterators.VersioningIterator")); AccumuloInputFormat.addIterator(job, new IteratorSetting(3, "Count", "org.apache.accumulo.core.iterators.CountingIterator")); @@@ -205,16 -228,15 +187,16 @@@ assertEquals(3, setting.getPriority()); assertEquals("org.apache.accumulo.core.iterators.CountingIterator", setting.getIteratorClass()); assertEquals("Count", setting.getName()); - + } - + @Test public void testSetRegex() throws IOException { + @SuppressWarnings("deprecation") Job job = new Job(); - + String regex = ">\"*%<>\'\\"; - + IteratorSetting is = new IteratorSetting(50, regex, RegExFilter.class); RegExFilter.setRegexs(is, regex, null, null, null, false); AccumuloInputFormat.addIterator(job, is); @@@ -264,17 -286,19 +246,22 @@@ String user = args[0]; String pass = args[1]; String table = args[2]; + + String instanceName = args[3]; + String inputFormatClassName = args[4]; ++ @SuppressWarnings({"rawtypes", "unchecked"}) + Class inputFormatClass = (Class) Class.forName(inputFormatClassName); + + @SuppressWarnings("deprecation") Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis()); job.setJarByClass(this.getClass()); - - job.setInputFormatClass(AccumuloInputFormat.class); - + + job.setInputFormatClass(inputFormatClass); + AccumuloInputFormat.setConnectorInfo(job, user, new PasswordToken(pass)); AccumuloInputFormat.setInputTableName(job, table); - AccumuloInputFormat.setMockInstance(job, INSTANCE_NAME); - + AccumuloInputFormat.setMockInstance(job, instanceName); + job.setMapperClass(TestMapper.class); job.setMapOutputKeyClass(Key.class); job.setMapOutputValueClass(Value.class); @@@ -334,21 -360,21 +323,21 @@@ AccumuloInputFormat.setLocalIterators(job, localIters); AccumuloInputFormat.fetchColumns(job, fetchColumns); AccumuloInputFormat.setLogLevel(job, level); - + AccumuloInputFormat aif = new AccumuloInputFormat(); - + List splits = aif.getSplits(job); - + Assert.assertEquals(1, splits.size()); - + InputSplit split = splits.get(0); - + Assert.assertEquals(RangeInputSplit.class, split.getClass()); - + RangeInputSplit risplit = (RangeInputSplit) split; - + Assert.assertEquals(username, risplit.getPrincipal()); - Assert.assertEquals(table, risplit.getTable()); + Assert.assertEquals(table, risplit.getTableName()); Assert.assertEquals(password, risplit.getToken()); Assert.assertEquals(auths, risplit.getAuths()); Assert.assertEquals(instance, risplit.getInstanceName()); @@@ -357,22 -383,22 +346,7 @@@ Assert.assertEquals(fetchColumns, risplit.getFetchedColumns()); Assert.assertEquals(level, risplit.getLogLevel()); } - - static class TestMapper extends Mapper { - Key key = null; - int count = 0; - - @Override - protected void map(Key k, Value v, Context context) throws IOException, InterruptedException { - if (key != null) - assertEquals(key.getRow().toString(), new String(v.get())); - assertEquals(k.getRow(), new Text(String.format("%09x", count + 1))); - assertEquals(new String(v.get()), String.format("%09x", count)); - key = new Key(k); - count++; - } - } - static class TestMapper extends Mapper { - Key key = null; - int count = 0; - - @Override - protected void map(Key k, Value v, Context context) throws IOException, InterruptedException { - if (key != null) - assertEquals(key.getRow().toString(), new String(v.get())); - assertEquals(k.getRow(), new Text(String.format("%09x", count + 1))); - assertEquals(new String(v.get()), String.format("%09x", count)); - key = new Key(k); - count++; - } - } - @Test public void testPartialInputSplitDelegationToConfiguration() throws Exception { String user = "testPartialInputSplitUser"; @@@ -389,289 -415,13 +363,12 @@@ } bw.close(); - Job job = Job.getInstance(); - job.setInputFormatClass(AccumuloInputFormat.class); - job.setMapperClass(TestMapper.class); - job.setNumReduceTasks(0); - AccumuloInputFormat.setConnectorInfo(job, user, password); - AccumuloInputFormat.setInputTableName(job, "testtable"); - AccumuloInputFormat.setScanAuthorizations(job, new Authorizations()); - AccumuloInputFormat.setMockInstance(job, "testPartialInputSplitDelegationToConfiguration"); - - AccumuloInputFormat input = new AccumuloInputFormat(); - List splits = input.getSplits(job); - assertEquals(splits.size(), 1); - - TestMapper mapper = (TestMapper) job.getMapperClass().newInstance(); - - RangeInputSplit emptySplit = new RangeInputSplit(); - emptySplit.setTableName("testtable"); - emptySplit.setTableId(c.tableOperations().tableIdMap().get("testtable")); - - // Using an empty split should fall back to the information in the Job's Configuration - TaskAttemptID id = new TaskAttemptID(); - TaskAttemptContext attempt = new TaskAttemptContextImpl(job.getConfiguration(), id); - RecordReader reader = input.createRecordReader(emptySplit, attempt); - - reader.initialize(emptySplit, attempt); - Context nullContext = mapper.new Context() { - - @Override - public InputSplit getInputSplit() { - return null; - } - - @Override - public boolean nextKeyValue() throws IOException, InterruptedException { - return false; - } - - @Override - public Key getCurrentKey() throws IOException, InterruptedException { - return null; - } - - @Override - public Value getCurrentValue() throws IOException, InterruptedException { - return null; - } - - @Override - public void write(Key key, Value value) throws IOException, InterruptedException { - - } - - @Override - public OutputCommitter getOutputCommitter() { - return null; - } - - @Override - public TaskAttemptID getTaskAttemptID() { - return null; - } - - @Override - public void setStatus(String msg) { - - } - - @Override - public String getStatus() { - return null; - } - - @Override - public float getProgress() { - return 0; - } - - @Override - public Counter getCounter(Enum counterName) { - return null; - } - - @Override - public Counter getCounter(String groupName, String counterName) { - return null; - } - - @Override - public Configuration getConfiguration() { - return null; - } - - @Override - public Credentials getCredentials() { - return null; - } - - @Override - public JobID getJobID() { - return null; - } - - @Override - public int getNumReduceTasks() { - return 0; - } - - @Override - public Path getWorkingDirectory() throws IOException { - return null; - } - - @Override - public Class getOutputKeyClass() { - return null; - } - - @Override - public Class getOutputValueClass() { - return null; - } - - @Override - public Class getMapOutputKeyClass() { - return null; - } - - @Override - public Class getMapOutputValueClass() { - return null; - } - - @Override - public String getJobName() { - return null; - } - - @Override - public Class> getInputFormatClass() throws ClassNotFoundException { - return null; - } - - @Override - public Class> getMapperClass() throws ClassNotFoundException { - return null; - } - - @Override - public Class> getCombinerClass() throws ClassNotFoundException { - return null; - } - - @Override - public Class> getReducerClass() throws ClassNotFoundException { - return null; - } - - @Override - public Class> getOutputFormatClass() throws ClassNotFoundException { - return null; - } - - @Override - public Class> getPartitionerClass() throws ClassNotFoundException { - return null; - } - - @Override - public RawComparator getSortComparator() { - return null; - } - - @Override - public String getJar() { - return null; - } - - @Override - public RawComparator getGroupingComparator() { - return null; - } - - @Override - public boolean getJobSetupCleanupNeeded() { - return false; - } - - @Override - public boolean getTaskCleanupNeeded() { - return false; - } - - @Override - public boolean getProfileEnabled() { - return false; - } - - @Override - public String getProfileParams() { - return null; - } - - @Override - public IntegerRanges getProfileTaskRange(boolean isMap) { - return null; - } - - @Override - public String getUser() { - return null; - } - - @Override - public boolean getSymlink() { - return false; - } - - @Override - public Path[] getArchiveClassPaths() { - return null; - } - - @Override - public URI[] getCacheArchives() throws IOException { - return null; - } - - @Override - public URI[] getCacheFiles() throws IOException { - return null; - } - - @Override - public Path[] getLocalCacheArchives() throws IOException { - return null; - } - - @Override - public Path[] getLocalCacheFiles() throws IOException { - return null; - } - - @Override - public Path[] getFileClassPaths() { - return null; - } - - @Override - public String[] getArchiveTimestamps() { - return null; - } - - @Override - public String[] getFileTimestamps() { - return null; - } - - @Override - public int getMaxMapAttempts() { - return 0; - } - - @Override - public int getMaxReduceAttempts() { - return 0; - } - - @Override - public void progress() { - - } - - }; - - while (reader.nextKeyValue()) { - Key key = reader.getCurrentKey(); - Value value = reader.getCurrentValue(); - - mapper.map(key, value, nullContext); - } - } + Assert.assertEquals(0, MRTester.main(new String[] {user, "", "testtable", "testPartialInputSplitDelegationToConfiguration", + EmptySplitsAccumuloInputFormat.class.getCanonicalName()})); + assertNull(e1); - assertNull(e2); - } ++ assertNull(e2); } - @Test(expected = IOException.class) + @Test public void testPartialFailedInputSplitDelegationToConfiguration() throws Exception { String user = "testPartialFailedInputSplit"; PasswordToken password = new PasswordToken(""); http://git-wip-us.apache.org/repos/asf/accumulo/blob/0f3f93fd/core/src/test/java/org/apache/accumulo/core/client/mapreduce/EmptySplitsAccumuloInputFormat.java ---------------------------------------------------------------------- diff --cc core/src/test/java/org/apache/accumulo/core/client/mapreduce/EmptySplitsAccumuloInputFormat.java index 0000000,91db378..7130f24 mode 000000,100644..100644 --- a/core/src/test/java/org/apache/accumulo/core/client/mapreduce/EmptySplitsAccumuloInputFormat.java +++ b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/EmptySplitsAccumuloInputFormat.java @@@ -1,0 -1,21 +1,29 @@@ + package org.apache.accumulo.core.client.mapreduce; + + import java.io.IOException; -import java.util.Arrays; ++import java.util.ArrayList; + import java.util.List; + + import org.apache.hadoop.mapreduce.InputSplit; + import org.apache.hadoop.mapreduce.JobContext; + + /** + * AccumuloInputFormat which returns an "empty" RangeInputSplit + */ + public class EmptySplitsAccumuloInputFormat extends AccumuloInputFormat { + + @Override + public List getSplits(JobContext context) throws IOException { - super.getSplits(context); ++ List oldSplits = super.getSplits(context); ++ List newSplits = new ArrayList(oldSplits.size()); + - return Arrays. asList(new RangeInputSplit()); ++ // Copy only the necessary information ++ for (InputSplit oldSplit : oldSplits) { ++ RangeInputSplit newSplit = new RangeInputSplit((RangeInputSplit) oldSplit); ++ newSplits.add(newSplit); ++ } ++ ++ ++ return newSplits; + } + }