accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mmil...@apache.org
Subject [accumulo] branch master updated: Make auths optional in new MR API (#789)
Date Fri, 30 Nov 2018 21:22:28 GMT
This is an automated email from the ASF dual-hosted git repository.

mmiller pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo.git


The following commit(s) were added to refs/heads/master by this push:
     new 2723e5d  Make auths optional in new MR API (#789)
2723e5d is described below

commit 2723e5de67e17a8ca2de4253019e0daf9490472f
Author: Mike Miller <mmiller@apache.org>
AuthorDate: Fri Nov 30 16:22:24 2018 -0500

    Make auths optional in new MR API (#789)
    
    * Get the users auths if non are set
---
 .../hadoop/mapreduce/InputFormatBuilder.java       | 33 ++++++++-----------
 .../hadoopImpl/mapred/AbstractInputFormat.java     |  2 +-
 .../mapreduce/InputFormatBuilderImpl.java          | 38 ++++++++++++++--------
 .../lib/MapReduceClientOnDefaultTable.java         |  7 +++-
 .../lib/MapReduceClientOnRequiredTable.java        |  7 +++-
 .../mapreduce/lib/MapReduceClientOpts.java         |  6 +++-
 .../hadoop/mapred/AccumuloInputFormatTest.java     | 11 +++----
 .../hadoop/mapreduce/AccumuloInputFormatTest.java  | 11 +++----
 8 files changed, 67 insertions(+), 48 deletions(-)

diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/InputFormatBuilder.java
b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/InputFormatBuilder.java
index 1751d9f..a1e6fd9 100644
--- a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/InputFormatBuilder.java
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/InputFormatBuilder.java
@@ -20,6 +20,8 @@ import java.util.Collection;
 import java.util.Map;
 import java.util.Properties;
 
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
 import org.apache.accumulo.core.client.ClientInfo;
 import org.apache.accumulo.core.client.ClientSideIteratorScanner;
 import org.apache.accumulo.core.client.IsolatedScanner;
@@ -65,23 +67,7 @@ public interface InputFormatBuilder {
      * @param tableName
      *          the table to use when the tablename is null in the write call
      */
-    AuthsParams<T> table(String tableName);
-  }
-
-  /**
-   * Required params for builder
-   *
-   * @since 2.0
-   */
-  interface AuthsParams<T> {
-    /**
-     * Sets the {@link Authorizations} used to scan. Must be a subset of the user's authorizations.
-     * If none present use {@link Authorizations#EMPTY}
-     *
-     * @param auths
-     *          the user's authorizations
-     */
-    InputFormatOptions<T> auths(Authorizations auths);
+    InputFormatOptions<T> table(String tableName);
   }
 
   /**
@@ -93,7 +79,7 @@ public interface InputFormatBuilder {
     /**
      * Finish configuring, verify and store options into the JobConf or Job
      */
-    void store(T t);
+    void store(T t) throws AccumuloException, AccumuloSecurityException;
   }
 
   /**
@@ -125,6 +111,15 @@ public interface InputFormatBuilder {
    */
   interface InputFormatOptions<T> {
     /**
+     * Sets the {@link Authorizations} used to scan. Must be a subset of the user's authorizations.
+     * By Default, all of the users auths are set.
+     *
+     * @param auths
+     *          the user's authorizations
+     */
+    InputFormatOptions<T> auths(Authorizations auths);
+
+    /**
      * Sets the name of the classloader context on this scanner
      *
      * @param context
@@ -260,6 +255,6 @@ public interface InputFormatBuilder {
     /**
      * Finish configuring, verify and serialize options into the JobConf or Job
      */
-    void store(T j);
+    void store(T j) throws AccumuloException, AccumuloSecurityException;
   }
 }
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapred/AbstractInputFormat.java
b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapred/AbstractInputFormat.java
index 9095d91..7b91397 100644
--- a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapred/AbstractInputFormat.java
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapred/AbstractInputFormat.java
@@ -181,7 +181,7 @@ public abstract class AbstractInputFormat {
    *           if the context is improperly configured
    * @since 1.5.0
    */
-  protected static void validateOptions(JobConf job) throws IOException {
+  public static void validateOptions(JobConf job) throws IOException {
     AccumuloClient client = InputConfigurator.getClient(CLASS, job);
     InputConfigurator.validatePermissions(CLASS, job, client);
   }
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/InputFormatBuilderImpl.java
b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/InputFormatBuilderImpl.java
index 79deca6..e6b3a51 100644
--- a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/InputFormatBuilderImpl.java
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/InputFormatBuilderImpl.java
@@ -23,6 +23,10 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
 import org.apache.accumulo.core.client.ClientInfo;
 import org.apache.accumulo.core.client.IteratorSetting;
 import org.apache.accumulo.core.client.sample.SamplerConfiguration;
@@ -36,9 +40,9 @@ import org.apache.hadoop.mapreduce.Job;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 
-public class InputFormatBuilderImpl<T> implements InputFormatBuilder,
-    InputFormatBuilder.ClientParams<T>, InputFormatBuilder.TableParams<T>,
-    InputFormatBuilder.AuthsParams<T>, InputFormatBuilder.InputFormatOptions<T>,
+public class InputFormatBuilderImpl<T>
+    implements InputFormatBuilder, InputFormatBuilder.ClientParams<T>,
+    InputFormatBuilder.TableParams<T>, InputFormatBuilder.InputFormatOptions<T>,
     InputFormatBuilder.ScanOptions<T>, InputFormatBuilder.BatchScanOptions<T>
{
 
   Class<?> callingClass;
@@ -65,7 +69,7 @@ public class InputFormatBuilderImpl<T> implements InputFormatBuilder,
   }
 
   @Override
-  public InputFormatBuilder.AuthsParams<T> table(String tableName) {
+  public InputFormatBuilder.InputFormatOptions<T> table(String tableName) {
     this.tableName = Objects.requireNonNull(tableName, "Table name must not be null");
     return this;
   }
@@ -159,7 +163,7 @@ public class InputFormatBuilderImpl<T> implements InputFormatBuilder,
   }
 
   @Override
-  public void store(T j) {
+  public void store(T j) throws AccumuloException, AccumuloSecurityException {
     if (j instanceof Job) {
       store((Job) j);
     } else if (j instanceof JobConf) {
@@ -172,13 +176,13 @@ public class InputFormatBuilderImpl<T> implements InputFormatBuilder,
   /**
    * Final builder method for mapreduce configuration
    */
-  private void store(Job job) {
-    // TODO validate params are set correctly, possibly call/modify
-    // AbstractInputFormat.validateOptions()
+  private void store(Job job) throws AccumuloException, AccumuloSecurityException {
     AbstractInputFormat.setClientInfo(job, clientInfo);
-    AbstractInputFormat.setScanAuthorizations(job, scanAuths);
     InputFormatBase.setInputTableName(job, tableName);
 
+    scanAuths = getUserAuths(scanAuths, clientInfo);
+    AbstractInputFormat.setScanAuthorizations(job, scanAuths);
+
     // all optional values
     if (context.isPresent())
       AbstractInputFormat.setClassLoaderContext(job, context.get());
@@ -203,13 +207,13 @@ public class InputFormatBuilderImpl<T> implements InputFormatBuilder,
   /**
    * Final builder method for legacy mapred configuration
    */
-  private void store(JobConf jobConf) {
-    // TODO validate params are set correctly, possibly call/modify
-    // AbstractInputFormat.validateOptions()
+  private void store(JobConf jobConf) throws AccumuloException, AccumuloSecurityException
{
     org.apache.accumulo.hadoopImpl.mapred.AbstractInputFormat.setClientInfo(jobConf, clientInfo);
+    org.apache.accumulo.hadoopImpl.mapred.InputFormatBase.setInputTableName(jobConf, tableName);
+
+    scanAuths = getUserAuths(scanAuths, clientInfo);
     org.apache.accumulo.hadoopImpl.mapred.AbstractInputFormat.setScanAuthorizations(jobConf,
         scanAuths);
-    org.apache.accumulo.hadoopImpl.mapred.InputFormatBase.setInputTableName(jobConf, tableName);
 
     // all optional values
     if (context.isPresent())
@@ -237,6 +241,14 @@ public class InputFormatBuilderImpl<T> implements InputFormatBuilder,
     org.apache.accumulo.hadoopImpl.mapred.InputFormatBase.setBatchScan(jobConf, bools.batchScan);
   }
 
+  private Authorizations getUserAuths(Authorizations scanAuths, ClientInfo clientInfo)
+      throws AccumuloSecurityException, AccumuloException {
+    if (scanAuths != null)
+      return scanAuths;
+    AccumuloClient c = Accumulo.newClient().from(clientInfo).build();
+    return c.securityOperations().getUserAuthorizations(clientInfo.getPrincipal());
+  }
+
   private static class BuilderBooleans {
     boolean autoAdjustRanges = true;
     boolean scanIsolation = false;
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/MapReduceClientOnDefaultTable.java
b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/MapReduceClientOnDefaultTable.java
index 6f4ff28..adfa62c 100644
--- a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/MapReduceClientOnDefaultTable.java
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/MapReduceClientOnDefaultTable.java
@@ -16,6 +16,10 @@
  */
 package org.apache.accumulo.hadoopImpl.mapreduce.lib;
 
+import java.io.IOException;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
 import org.apache.accumulo.core.client.ClientInfo;
 import org.apache.accumulo.hadoop.mapreduce.AccumuloInputFormat;
 import org.apache.accumulo.hadoop.mapreduce.AccumuloOutputFormat;
@@ -36,7 +40,8 @@ public class MapReduceClientOnDefaultTable extends MapReduceClientOpts {
   }
 
   @Override
-  public void setAccumuloConfigs(Job job) {
+  public void setAccumuloConfigs(Job job)
+      throws IOException, AccumuloException, AccumuloSecurityException {
     final String tableName = getTableName();
     final ClientInfo info = getClientInfo();
     AccumuloInputFormat.configure().clientInfo(info).table(tableName).auths(auths).store(job);
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/MapReduceClientOnRequiredTable.java
b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/MapReduceClientOnRequiredTable.java
index 9e48706..aed3382 100644
--- a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/MapReduceClientOnRequiredTable.java
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/MapReduceClientOnRequiredTable.java
@@ -16,6 +16,10 @@
  */
 package org.apache.accumulo.hadoopImpl.mapreduce.lib;
 
+import java.io.IOException;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
 import org.apache.accumulo.core.client.ClientInfo;
 import org.apache.accumulo.hadoop.mapreduce.AccumuloInputFormat;
 import org.apache.accumulo.hadoop.mapreduce.AccumuloOutputFormat;
@@ -29,7 +33,8 @@ public class MapReduceClientOnRequiredTable extends MapReduceClientOpts
{
   private String tableName;
 
   @Override
-  public void setAccumuloConfigs(Job job) {
+  public void setAccumuloConfigs(Job job)
+      throws IOException, AccumuloException, AccumuloSecurityException {
     final String tableName = getTableName();
     final ClientInfo info = getClientInfo();
     AccumuloInputFormat.configure().clientInfo(info).table(tableName).auths(auths).store(job);
diff --git a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/MapReduceClientOpts.java
b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/MapReduceClientOpts.java
index bbea6c3..96a516c 100644
--- a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/MapReduceClientOpts.java
+++ b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/lib/MapReduceClientOpts.java
@@ -16,9 +16,12 @@
  */
 package org.apache.accumulo.hadoopImpl.mapreduce.lib;
 
+import java.io.IOException;
+
 import org.apache.accumulo.core.cli.ClientOpts;
 import org.apache.accumulo.core.client.Accumulo;
 import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.AccumuloException;
 import org.apache.accumulo.core.client.AccumuloSecurityException;
 import org.apache.accumulo.core.client.admin.DelegationTokenConfig;
 import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
@@ -35,7 +38,8 @@ import org.slf4j.LoggerFactory;
 public abstract class MapReduceClientOpts extends ClientOpts {
   private static final Logger log = LoggerFactory.getLogger(MapReduceClientOpts.class);
 
-  public abstract void setAccumuloConfigs(Job job) throws AccumuloSecurityException;
+  public abstract void setAccumuloConfigs(Job job)
+      throws IOException, AccumuloException, AccumuloSecurityException;
 
   @Override
   public AuthenticationToken getToken() {
diff --git a/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/mapred/AccumuloInputFormatTest.java
b/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/mapred/AccumuloInputFormatTest.java
index 6831869..9f5b4fb 100644
--- a/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/mapred/AccumuloInputFormatTest.java
+++ b/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/mapred/AccumuloInputFormatTest.java
@@ -23,7 +23,6 @@ import static org.junit.Assert.assertEquals;
 
 import java.io.ByteArrayOutputStream;
 import java.io.DataOutputStream;
-import java.io.IOException;
 import java.util.Base64;
 import java.util.HashSet;
 import java.util.List;
@@ -76,7 +75,7 @@ public class AccumuloInputFormatTest {
    * Check that the iterator configuration is getting stored in the Job conf correctly.
    */
   @Test
-  public void testSetIterator() throws IOException {
+  public void testSetIterator() throws Exception {
     InputFormatOptions<JobConf> opts = AccumuloInputFormat.configure().clientInfo(clientInfo)
         .table("test").auths(Authorizations.EMPTY);
 
@@ -89,7 +88,7 @@ public class AccumuloInputFormatTest {
   }
 
   @Test
-  public void testAddIterator() {
+  public void testAddIterator() throws Exception {
     InputFormatOptions<JobConf> opts = AccumuloInputFormat.configure().clientInfo(clientInfo)
         .table("test").auths(Authorizations.EMPTY);
 
@@ -168,7 +167,7 @@ public class AccumuloInputFormatTest {
    * Test getting iterator settings for multiple iterators set
    */
   @Test
-  public void testGetIteratorSettings() {
+  public void testGetIteratorSettings() throws Exception {
     IteratorSetting iter1 = new IteratorSetting(1, "WholeRow", WholeRowIterator.class.getName());
     IteratorSetting iter2 = new IteratorSetting(2, "Versions", VersioningIterator.class.getName());
     IteratorSetting iter3 = new IteratorSetting(3, "Count", CountingIterator.class.getName());
@@ -199,7 +198,7 @@ public class AccumuloInputFormatTest {
   }
 
   @Test
-  public void testSetRegex() {
+  public void testSetRegex() throws Exception {
     String regex = ">\"*%<>\'\\";
 
     IteratorSetting is = new IteratorSetting(50, regex, RegExFilter.class);
@@ -212,7 +211,7 @@ public class AccumuloInputFormatTest {
   }
 
   @Test
-  public void testEmptyColumnFamily() throws IOException {
+  public void testEmptyColumnFamily() throws Exception {
     Set<IteratorSetting.Column> cols = new HashSet<>();
     cols.add(new IteratorSetting.Column(new Text(""), null));
     cols.add(new IteratorSetting.Column(new Text("foo"), new Text("bar")));
diff --git a/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/mapreduce/AccumuloInputFormatTest.java
b/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/mapreduce/AccumuloInputFormatTest.java
index dfac630..0d9ecac 100644
--- a/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/mapreduce/AccumuloInputFormatTest.java
+++ b/hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/mapreduce/AccumuloInputFormatTest.java
@@ -23,7 +23,6 @@ import static org.junit.Assert.assertEquals;
 
 import java.io.ByteArrayOutputStream;
 import java.io.DataOutputStream;
-import java.io.IOException;
 import java.util.Base64;
 import java.util.HashSet;
 import java.util.List;
@@ -63,7 +62,7 @@ public class AccumuloInputFormatTest {
    * Check that the iterator configuration is getting stored in the Job conf correctly.
    */
   @Test
-  public void testSetIterator() throws IOException {
+  public void testSetIterator() throws Exception {
     Job job = Job.getInstance();
 
     IteratorSetting is = new IteratorSetting(1, "WholeRow", WholeRowIterator.class);
@@ -77,7 +76,7 @@ public class AccumuloInputFormatTest {
   }
 
   @Test
-  public void testAddIterator() throws IOException {
+  public void testAddIterator() throws Exception {
     Job job = Job.getInstance();
 
     IteratorSetting iter1 = new IteratorSetting(1, "WholeRow", WholeRowIterator.class);
@@ -159,7 +158,7 @@ public class AccumuloInputFormatTest {
    * Test getting iterator settings for multiple iterators set
    */
   @Test
-  public void testGetIteratorSettings() throws IOException {
+  public void testGetIteratorSettings() throws Exception {
     Job job = Job.getInstance();
 
     IteratorSetting iter1 = new IteratorSetting(1, "WholeRow", WholeRowIterator.class.getName());
@@ -193,7 +192,7 @@ public class AccumuloInputFormatTest {
   }
 
   @Test
-  public void testSetRegex() throws IOException {
+  public void testSetRegex() throws Exception {
     Job job = Job.getInstance();
 
     String regex = ">\"*%<>\'\\";
@@ -208,7 +207,7 @@ public class AccumuloInputFormatTest {
   }
 
   @Test
-  public void testEmptyColumnFamily() throws IOException {
+  public void testEmptyColumnFamily() throws Exception {
     Job job = Job.getInstance();
     Set<IteratorSetting.Column> cols = new HashSet<>();
     cols.add(new IteratorSetting.Column(new Text(""), null));


Mime
View raw message