Return-Path: X-Original-To: apmail-accumulo-commits-archive@www.apache.org Delivered-To: apmail-accumulo-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id CF68D10D1F for ; Sun, 29 Sep 2013 03:30:43 +0000 (UTC) Received: (qmail 15992 invoked by uid 500); 29 Sep 2013 03:30:40 -0000 Delivered-To: apmail-accumulo-commits-archive@accumulo.apache.org Received: (qmail 15817 invoked by uid 500); 29 Sep 2013 03:30:29 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 15623 invoked by uid 99); 29 Sep 2013 03:30:22 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Sun, 29 Sep 2013 03:30:22 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 7AD2290D3E5; Sun, 29 Sep 2013 03:30:22 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: cjnolet@apache.org To: commits@accumulo.apache.org Date: Sun, 29 Sep 2013 03:30:25 -0000 Message-Id: <93baccc34243435b8ec0980a08f27d1d@git.apache.org> In-Reply-To: <3775283e380a4535b810b1329134c4c7@git.apache.org> References: <3775283e380a4535b810b1329134c4c7@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [5/7] git commit: Removing deprecation for now until we have some discussions. Updating/adding comments. ACCUMULO-391 Removing deprecation for now until we have some discussions. Updating/adding comments. ACCUMULO-391 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/793b5183 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/793b5183 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/793b5183 Branch: refs/heads/ACCUMULO-391 Commit: 793b5183b4caaa6b7da0cbbb53c26b4e1463eeaf Parents: c8a8583 Author: Corey J. Nolet Authored: Sat Sep 28 23:23:48 2013 -0400 Committer: Corey J. Nolet Committed: Sat Sep 28 23:29:37 2013 -0400 ---------------------------------------------------------------------- .../core/client/mapred/InputFormatBase.java | 2 + .../core/client/mapreduce/InputFormatBase.java | 71 +++++++++++++------- .../mapreduce/lib/util/InputConfigurator.java | 40 +++-------- .../mapreduce/AccumuloInputFormatTest.java | 15 ++--- 4 files changed, 64 insertions(+), 64 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/793b5183/core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java b/core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java index fb1b193..15c2dd2 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java +++ b/core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java @@ -314,6 +314,7 @@ public abstract class InputFormatBase implements InputFormat { * the ranges that will be mapped over * @since 1.5.0 */ + @Deprecated public static void setRanges(JobConf job, Collection ranges) { InputConfigurator.setRanges(CLASS, job, ranges); } @@ -329,6 +330,7 @@ public abstract class InputFormatBase implements InputFormat { * @since 1.5.0 * @see #setRanges(JobConf, Collection) */ + @Deprecated protected static List getRanges(JobConf job) throws IOException { return InputConfigurator.getRanges(CLASS,job); } http://git-wip-us.apache.org/repos/asf/accumulo/blob/793b5183/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java index d5208b4..5392459 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java +++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java @@ -157,7 +157,7 @@ public abstract class InputFormatBase extends InputFormat { * @see #setConnectorInfo(Job, String, AuthenticationToken) */ protected static String getPrincipal(JobContext context) { - return InputConfigurator.getPrincipal(CLASS, getConfiguration(context)); + return InputConfigurator.getPrincipal(CLASS,getConfiguration(context)); } /** @@ -170,7 +170,7 @@ public abstract class InputFormatBase extends InputFormat { * @see #setInputTableName(Job, String) */ protected static String getInputTableName(JobContext context) { - return InputConfigurator.getInputTableName(CLASS, getConfiguration(context)); + return InputConfigurator.getInputTableName(CLASS,getConfiguration(context)); } /** @@ -206,7 +206,7 @@ public abstract class InputFormatBase extends InputFormat { * @see #setConnectorInfo(Job, String, String) */ protected static AuthenticationToken getAuthenticationToken(JobContext context) { - return InputConfigurator.getAuthenticationToken(CLASS, getConfiguration(context)); + return InputConfigurator.getAuthenticationToken(CLASS,getConfiguration(context)); } /** @@ -234,7 +234,7 @@ public abstract class InputFormatBase extends InputFormat { * @since 1.5.0 */ public static void setMockInstance(Job job, String instanceName) { - InputConfigurator.setMockInstance(CLASS, job.getConfiguration(), instanceName); + InputConfigurator.setMockInstance(CLASS,job.getConfiguration(),instanceName); } /** @@ -286,7 +286,6 @@ public abstract class InputFormatBase extends InputFormat { * the table to use when the tablename is null in the write call * @since 1.5.0 */ - @Deprecated public static void setInputTableName(Job job, String tableName) { InputConfigurator.setInputTableName(CLASS, job.getConfiguration(), tableName); } @@ -301,7 +300,7 @@ public abstract class InputFormatBase extends InputFormat { * @since 1.5.0 */ public static void setScanAuthorizations(Job job, Authorizations auths) { - InputConfigurator.setScanAuthorizations(CLASS, job.getConfiguration(), auths); + InputConfigurator.setScanAuthorizations(CLASS,job.getConfiguration(),auths); } /** @@ -327,7 +326,6 @@ public abstract class InputFormatBase extends InputFormat { * the ranges that will be mapped over * @since 1.5.0 */ - @Deprecated public static void setRanges(Job job, Collection ranges) { InputConfigurator.setRanges(CLASS, job.getConfiguration(), ranges); } @@ -342,7 +340,6 @@ public abstract class InputFormatBase extends InputFormat { * @see #setRanges(Job, Collection) * @see #setRanges(org.apache.hadoop.mapreduce.Job, java.util.Collection) */ - @Deprecated protected static List getRanges(JobContext context) throws IOException { return InputConfigurator.getRanges(CLASS, getConfiguration(context)); } @@ -357,7 +354,6 @@ public abstract class InputFormatBase extends InputFormat { * selected. An empty set is the default and is equivalent to scanning the all columns. * @since 1.5.0 */ - @Deprecated public static void fetchColumns(Job job, Collection> columnFamilyColumnQualifierPairs) { InputConfigurator.fetchColumns(CLASS, job.getConfiguration(), columnFamilyColumnQualifierPairs); } @@ -384,14 +380,12 @@ public abstract class InputFormatBase extends InputFormat { * the configuration of the iterator * @since 1.5.0 */ - @Deprecated public static void addIterator(Job job, IteratorSetting cfg) { InputConfigurator.addIterator(CLASS, job.getConfiguration(), cfg); } /** - * Gets a list of the iterator settings (for iterators to apply to a scanner) from this configuration. This will only return iterators that have not been set - * for a specific table. + * Gets a list of the iterator settings (for iterators to apply to a scanner) from this configuration. * * @param context * the Hadoop context for the configured job @@ -399,7 +393,6 @@ public abstract class InputFormatBase extends InputFormat { * @since 1.5.0 * @see #addIterator(Job, IteratorSetting) */ - @Deprecated protected static List getIterators(JobContext context) { return InputConfigurator.getIterators(CLASS, getConfiguration(context)); } @@ -418,7 +411,6 @@ public abstract class InputFormatBase extends InputFormat { * @see #setRanges(Job, Collection) * @since 1.5.0 */ - @Deprecated public static void setAutoAdjustRanges(Job job, boolean enableFeature) { InputConfigurator.setAutoAdjustRanges(CLASS, job.getConfiguration(), enableFeature); } @@ -432,21 +424,52 @@ public abstract class InputFormatBase extends InputFormat { * @since 1.5.0 * @see #setAutoAdjustRanges(Job, boolean) */ - @Deprecated protected static boolean getAutoAdjustRanges(JobContext context) { return InputConfigurator.getAutoAdjustRanges(CLASS, getConfiguration(context)); } - - protected static void setTableQueryConfigurations(JobContext job, TableQueryConfig... configs) { + + /** + * Sets the {@link TableQueryConfig} objects on the given Hadoop configuration + * @param job + * the Hadoop job instance to be configured + * @param configs + * the table query configs to be set on the configuration. + */ + public static void setTableQueryConfigs(JobContext job, TableQueryConfig... configs) { checkNotNull(configs); InputConfigurator.setTableQueryConfigs(CLASS, getConfiguration(job), configs); } - - public static List getTableQueryConfigurations(JobContext job) { + + /** + * Fetches all {@link TableQueryConfig}s that have been set on the given Hadoop configuration. + * + *

+ * Note this also returns the {@link TableQueryConfig} representing the table configurations set through the single + * table input methods ({@link #setInputTableName(org.apache.hadoop.mapreduce.Job, String)}, + * {@link #setRanges(org.apache.hadoop.mapreduce.Job, java.util.Collection)}, + * {@link #fetchColumns(org.apache.hadoop.mapreduce.Job, java.util.Collection)}, + * {@link #addIterator(org.apache.hadoop.mapreduce.Job, org.apache.accumulo.core.client.IteratorSetting)}, etc...) + * + * @param job + * the Hadoop job instance to be configured + * @return + */ + public static List getTableQueryConfigs(JobContext job) { return InputConfigurator.getTableQueryConfigs(CLASS, getConfiguration(job)); } - - protected static TableQueryConfig getTableQueryConfiguration(JobContext job, String tableName) { + + /** + * Fetches a {@link TableQueryConfig} that has been set on the configuration for a specific table. + * + *

+ * null is returned in the event that the table doesn't exist. + * @param job + * the Hadoop job instance to be configured + * @param tableName + * the table name for which to grab the config object + * @return the {@link TableQueryConfig} for the given table + */ + public static TableQueryConfig getTableQueryConfig(JobContext job, String tableName) { return InputConfigurator.getTableQueryConfigs(CLASS, getConfiguration(job), tableName); } @@ -615,7 +638,7 @@ public abstract class InputFormatBase extends InputFormat { * the table name for which to set up the iterators */ protected void setupIterators(TaskAttemptContext context, Scanner scanner, String tableName) { - TableQueryConfig config = getTableQueryConfiguration(context, tableName); + TableQueryConfig config = getTableQueryConfig(context,tableName); List iterators = config.getIterators(); for (IteratorSetting iterator : iterators) scanner.addScanIterator(iterator); @@ -633,7 +656,7 @@ public abstract class InputFormatBase extends InputFormat { Instance instance = getInstance(attempt); String principal = getPrincipal(attempt); - TableQueryConfig tableConfig = getTableQueryConfiguration(attempt, split.getTableName()); + TableQueryConfig tableConfig = getTableQueryConfig(attempt,split.getTableName()); // in case the table name changed, we can still use the previous name for terms of configuration, // but for the scanner, we'll need to reference the new table name. @@ -828,7 +851,7 @@ public abstract class InputFormatBase extends InputFormat { validateOptions(conf); LinkedList splits = new LinkedList(); - List tableConfigs = getTableQueryConfigurations(conf); + List tableConfigs = getTableQueryConfigs(conf); for (TableQueryConfig tableConfig : tableConfigs) { boolean autoAdjust = tableConfig.shouldAutoAdjustRanges(); http://git-wip-us.apache.org/repos/asf/accumulo/blob/793b5183/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/InputConfigurator.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/InputConfigurator.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/InputConfigurator.java index 5e02177..82df3a1 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/InputConfigurator.java +++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/InputConfigurator.java @@ -89,9 +89,7 @@ public class InputConfigurator extends ConfiguratorBase { * @param tableName * the table to use when the tablename is null in the write call * @since 1.5.0 - * @deprecated since 1.6.0 */ - @Deprecated public static void setInputTableName(Class implementingClass, Configuration conf, String tableName) { notNull(tableName); conf.set(enumToConfKey(implementingClass, ScanOpts.TABLE), tableName); @@ -105,9 +103,7 @@ public class InputConfigurator extends ConfiguratorBase { * @param conf * the Hadoop configuration object to configure * @since 1.5.0 - * @deprecated since 1.6.0 */ - @Deprecated public static String getInputTableName(Class implementingClass, Configuration conf) { return conf.get(enumToConfKey(implementingClass, ScanOpts.TABLE)); } @@ -156,9 +152,7 @@ public class InputConfigurator extends ConfiguratorBase { * @throws IllegalArgumentException * if the ranges cannot be encoded into base 64 * @since 1.5.0 - * @deprecated since 1.6.0 */ - @Deprecated public static void setRanges(Class implementingClass, Configuration conf, Collection ranges) { notNull(ranges); @@ -186,10 +180,8 @@ public class InputConfigurator extends ConfiguratorBase { * @throws IOException * if the ranges have been encoded improperly * @since 1.6.0 - * @deprecated since 1.6.0 * @see #setRanges(Class, Configuration, Collection) */ - @Deprecated public static List getRanges(Class implementingClass, Configuration conf) throws IOException { Collection encodedRanges = conf.getStringCollection(enumToConfKey(implementingClass, ScanOpts.RANGES)); @@ -254,7 +246,6 @@ public class InputConfigurator extends ConfiguratorBase { * @since 1.5.0 * @deprecated since 1.6.0 */ - @Deprecated public static void fetchColumns(Class implementingClass, Configuration conf, Collection> columnFamilyColumnQualifierPairs) { notNull(columnFamilyColumnQualifierPairs); ArrayList columnStrings = new ArrayList(); @@ -306,9 +297,7 @@ public class InputConfigurator extends ConfiguratorBase { * @throws IllegalArgumentException * if the iterator can't be serialized into the configuration * @since 1.5.0 - * @deprecated since 1.6.0 */ - @Deprecated public static void addIterator(Class implementingClass, Configuration conf, IteratorSetting cfg) { ByteArrayOutputStream baos = new ByteArrayOutputStream(); String newIter; @@ -348,9 +337,7 @@ public class InputConfigurator extends ConfiguratorBase { * the feature is enabled if true, disabled otherwise * @see #setRanges(Class, Configuration, Collection) * @since 1.5.0 - * @deprecated since 1.6.0 */ - @Deprecated public static void setAutoAdjustRanges(Class implementingClass, Configuration conf, boolean enableFeature) { conf.setBoolean(enumToConfKey(implementingClass, Features.AUTO_ADJUST_RANGES), enableFeature); } @@ -365,9 +352,7 @@ public class InputConfigurator extends ConfiguratorBase { * @return false if the feature is disabled, true otherwise * @since 1.5.0 * @see #setAutoAdjustRanges(Class, Configuration, boolean) - * @deprecated since 1.6.0 */ - @Deprecated public static Boolean getAutoAdjustRanges(Class implementingClass, Configuration conf) { return conf.getBoolean(enumToConfKey(implementingClass, Features.AUTO_ADJUST_RANGES), true); } @@ -385,9 +370,7 @@ public class InputConfigurator extends ConfiguratorBase { * @param enableFeature * the feature is enabled if true, disabled otherwise * @since 1.5.0 - * @deprecated since 1.6.0 */ - @Deprecated public static void setScanIsolation(Class implementingClass, Configuration conf, boolean enableFeature) { conf.setBoolean(enumToConfKey(implementingClass, Features.SCAN_ISOLATION), enableFeature); } @@ -402,9 +385,7 @@ public class InputConfigurator extends ConfiguratorBase { * @return true if the feature is enabled, false otherwise * @since 1.5.0 * @see #setScanIsolation(Class, Configuration, boolean) - * @deprecated since 1.6.0 */ - @Deprecated public static Boolean isIsolated(Class implementingClass, Configuration conf) { return conf.getBoolean(enumToConfKey(implementingClass, Features.SCAN_ISOLATION), false); } @@ -423,9 +404,7 @@ public class InputConfigurator extends ConfiguratorBase { * @param enableFeature * the feature is enabled if true, disabled otherwise * @since 1.5.0 - * @deprecated since 1.6.0 */ - @Deprecated public static void setLocalIterators(Class implementingClass, Configuration conf, boolean enableFeature) { conf.setBoolean(enumToConfKey(implementingClass, Features.USE_LOCAL_ITERATORS), enableFeature); } @@ -440,9 +419,7 @@ public class InputConfigurator extends ConfiguratorBase { * @return true if the feature is enabled, false otherwise * @since 1.5.0 * @see #setLocalIterators(Class, Configuration, boolean) - * @deprecated since 1.6.0 */ - @Deprecated public static Boolean usesLocalIterators(Class implementingClass, Configuration conf) { return conf.getBoolean(enumToConfKey(implementingClass, Features.USE_LOCAL_ITERATORS), false); } @@ -615,9 +592,6 @@ public class InputConfigurator extends ConfiguratorBase { } } } - - // TODO: Check for the "default table case" - } catch (AccumuloException e) { throw new IOException(e); } catch (AccumuloSecurityException e) { @@ -627,7 +601,17 @@ public class InputConfigurator extends ConfiguratorBase { } } - + /** + * Returns the {@link TableQueryConfig} for the configuration based on the properties set using the single-table + * input methods. + * + * @param implementingClass + * the class whose name will be used as a prefix for the property configuration key + * @param conf + * the Hadoop instance for which to retrieve the configuration + * @return the config object built from the single input table properties set on the job + * @throws IOException + */ protected static TableQueryConfig getDefaultTableConfig(Class implementingClass, Configuration conf) throws IOException { String tableName = getInputTableName(implementingClass, conf); if(tableName != null) { @@ -650,6 +634,4 @@ public class InputConfigurator extends ConfiguratorBase { } return null; } - - } http://git-wip-us.apache.org/repos/asf/accumulo/blob/793b5183/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java index 1b4aa26..f9dd9ec 100644 --- a/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java +++ b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormatTest.java @@ -25,15 +25,9 @@ import static org.junit.Assert.fail; import java.io.ByteArrayOutputStream; import java.io.DataOutputStream; import java.io.IOException; -import java.util.Arrays; -import java.util.Collection; import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; import java.util.List; -import java.util.Map; -import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.BatchWriter; import org.apache.accumulo.core.client.BatchWriterConfig; import org.apache.accumulo.core.client.Connector; @@ -47,7 +41,6 @@ import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.iterators.user.RegExFilter; import org.apache.accumulo.core.iterators.user.WholeRowIterator; -import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.util.CachedConfiguration; import org.apache.accumulo.core.util.Pair; import org.apache.commons.codec.binary.Base64; @@ -264,7 +257,7 @@ public class AccumuloInputFormatTest { TableQueryConfig tableConfig1 = new TableQueryConfig(table1); TableQueryConfig tableConfig2 = new TableQueryConfig(table2); - AccumuloInputFormat.setTableQueryConfigurations(job, tableConfig1, tableConfig2); + AccumuloInputFormat.setTableQueryConfigs(job,tableConfig1,tableConfig2); AccumuloInputFormat.setMockInstance(job, INSTANCE_NAME); job.setMapperClass(TestMapper.class); @@ -326,9 +319,9 @@ public class AccumuloInputFormatTest { .setColumns(Collections.singleton(new Pair(new Text("CF1"), new Text("CQ1")))) .setIterators(Collections.singletonList(new IteratorSetting(50, "iter1", "iterclass1"))); - AccumuloInputFormat.setTableQueryConfigurations(job, table1, table2); + AccumuloInputFormat.setTableQueryConfigs(job,table1,table2); - assertEquals(table1, AccumuloInputFormat.getTableQueryConfiguration(job, TEST_TABLE_1)); - assertEquals(table2, AccumuloInputFormat.getTableQueryConfiguration(job, TEST_TABLE_2)); + assertEquals(table1, AccumuloInputFormat.getTableQueryConfig(job,TEST_TABLE_1)); + assertEquals(table2, AccumuloInputFormat.getTableQueryConfig(job,TEST_TABLE_2)); } }