accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ctubb...@apache.org
Subject svn commit: r1485798 - in /accumulo/trunk: core/src/main/java/org/apache/accumulo/core/cli/ core/src/main/java/org/apache/accumulo/core/client/mapred/ core/src/main/java/org/apache/accumulo/core/client/mapreduce/ core/src/main/java/org/apache/accumulo/...
Date Thu, 23 May 2013 18:11:21 GMT
Author: ctubbsii
Date: Thu May 23 18:11:20 2013
New Revision: 1485798

URL: http://svn.apache.org/r1485798
Log:
ACCUMULO-829 Apply patch from Sean Hickey, with modification of moving utility class for creating
tokens to core instead of server

Added:
    accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/CreateToken.java
    accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapred/TokenFileTest.java
    accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapreduce/TokenFileTest.java
Modified:
    accumulo/trunk/core/src/main/java/org/apache/accumulo/core/cli/ClientOnRequiredTable.java
    accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormat.java
    accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloOutputFormat.java
    accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java
    accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormat.java
    accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java
    accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
    accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/ConfiguratorBase.java
    accumulo/trunk/start/src/main/java/org/apache/accumulo/start/Main.java

Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/cli/ClientOnRequiredTable.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/cli/ClientOnRequiredTable.java?rev=1485798&r1=1485797&r2=1485798&view=diff
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/cli/ClientOnRequiredTable.java
(original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/cli/ClientOnRequiredTable.java
Thu May 23 18:11:20 2013
@@ -28,13 +28,22 @@ public class ClientOnRequiredTable exten
   @Parameter(names = {"-t", "--table"}, required = true, description = "table to use")
   public String tableName = null;
   
+  @Parameter(names = {"-tf", "--tokenFile"}, description = "File in hdfs containing the user's
authentication token create with \"bin/accumulo create-token\"")
+  public String tokenFile = "";
+  
   @Override
   public void setAccumuloConfigs(Job job) throws AccumuloSecurityException {
     super.setAccumuloConfigs(job);
-    AccumuloInputFormat.setConnectorInfo(job, principal, getToken());
+    
+    if (tokenFile.isEmpty()) {
+      AccumuloInputFormat.setConnectorInfo(job, principal, getToken());
+      AccumuloOutputFormat.setConnectorInfo(job, principal, getToken());
+    } else {
+      AccumuloInputFormat.setConnectorInfo(job, principal, tokenFile);
+      AccumuloOutputFormat.setConnectorInfo(job, principal, tokenFile);
+    }
     AccumuloInputFormat.setInputTableName(job, tableName);
     AccumuloInputFormat.setScanAuthorizations(job, auths);
-    AccumuloOutputFormat.setConnectorInfo(job, principal, getToken());
     AccumuloOutputFormat.setCreateTables(job, true);
     AccumuloOutputFormat.setDefaultTableName(job, tableName);
   }

Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormat.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormat.java?rev=1485798&r1=1485797&r2=1485798&view=diff
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormat.java
(original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloInputFormat.java
Thu May 23 18:11:20 2013
@@ -38,6 +38,7 @@ import org.apache.hadoop.mapred.Reporter
  * 
  * <ul>
  * <li>{@link AccumuloInputFormat#setConnectorInfo(JobConf, String, AuthenticationToken)}
+ * <li>{@link AccumuloInputFormat#setConnectorInfo(JobConf, String, String)}
  * <li>{@link AccumuloInputFormat#setScanAuthorizations(JobConf, Authorizations)}
  * <li>{@link AccumuloInputFormat#setZooKeeperInstance(JobConf, String, String)} OR
{@link AccumuloInputFormat#setMockInstance(JobConf, String)}
  * </ul>

Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloOutputFormat.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloOutputFormat.java?rev=1485798&r1=1485797&r2=1485798&view=diff
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloOutputFormat.java
(original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloOutputFormat.java
Thu May 23 18:11:20 2013
@@ -60,6 +60,7 @@ import org.apache.log4j.Logger;
  * 
  * <ul>
  * <li>{@link AccumuloOutputFormat#setConnectorInfo(JobConf, String, AuthenticationToken)}
+ * <li>{@link AccumuloOutputFormat#setConnectorInfo(JobConf, String)}
  * <li>{@link AccumuloOutputFormat#setZooKeeperInstance(JobConf, String, String)} OR
{@link AccumuloOutputFormat#setMockInstance(JobConf, String)}
  * </ul>
  * 
@@ -91,6 +92,25 @@ public class AccumuloOutputFormat implem
   }
   
   /**
+   * Sets the connector information needed to communicate with Accumulo in this job.
+   * 
+   * <p>
+   * Stores the password in a file in HDFS and pulls that into the Distributed Cache in an
attempt to be more secure than storing it in the Configuration.
+   * 
+   * @param job
+   *          the Hadoop job instance to be configured
+   * @param principal
+   *          a valid Accumulo user name (user must have Table.CREATE permission if {@link
#setCreateTables(JobConf, boolean)} is set to true)
+   * @param tokenFile
+   *          the path to the password file
+   * @throws AccumuloSecurityException
+   * @since 1.6.0
+   */
+  public static void setConnectorInfo(JobConf job, String principal, String tokenFile) throws
AccumuloSecurityException {
+    OutputConfigurator.setConnectorInfo(CLASS, job, principal, tokenFile);
+  }
+  
+  /**
    * Determines if the connector has been configured.
    * 
    * @param job

Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java?rev=1485798&r1=1485797&r2=1485798&view=diff
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java
(original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java
Thu May 23 18:11:20 2013
@@ -104,6 +104,25 @@ public abstract class InputFormatBase<K,
   }
   
   /**
+   * Sets the connector information needed to communicate with Accumulo in this job.
+   * 
+   * <p>
+   * Stores the password in a file in HDFS and pulls that into the Distributed Cache in an
attempt to be more secure than storing it in the Configuration.
+   * 
+   * @param job
+   *          the Hadoop job instance to be configured
+   * @param principal
+   *          a valid Accumulo user name (user must have Table.CREATE permission)
+   * @param tokenFile
+   *          the path to the token file
+   * @throws AccumuloSecurityException
+   * @since 1.6.0
+   */
+  public static void setConnectorInfo(JobConf job, String principal, String tokenFile) throws
AccumuloSecurityException {
+    InputConfigurator.setConnectorInfo(CLASS, job, principal, tokenFile);
+  }
+  
+  /**
    * Determines if the connector has been configured.
    * 
    * @param job
@@ -157,6 +176,20 @@ public abstract class InputFormatBase<K,
   }
   
   /**
+   * Gets the password file from the configuration. It is BASE64 encoded to provide a charset
safe conversion to a string, and is not intended to be secure. If
+   * specified, the password will be stored in a file rather than in the Configuration.
+   * 
+   * @param job
+   *          the Hadoop context for the configured job
+   * @return path to the password file as a String
+   * @since 1.6.0
+   * @see #setConnectorInfo(JobConf, String, AuthenticationToken)
+   */
+  protected static String getTokenFile(JobConf job) {
+    return InputConfigurator.getTokenFile(CLASS, job);
+  }
+  
+  /**
    * Configures a {@link ZooKeeperInstance} for this job.
    * 
    * @param job

Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormat.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormat.java?rev=1485798&r1=1485797&r2=1485798&view=diff
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormat.java
(original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloInputFormat.java
Thu May 23 18:11:20 2013
@@ -38,6 +38,7 @@ import org.apache.hadoop.mapreduce.TaskA
  * 
  * <ul>
  * <li>{@link AccumuloInputFormat#setConnectorInfo(Job, String, AuthenticationToken)}
+ * <li>{@link AccumuloInputFormat#setConnectorInfo(Job, String, String)}
  * <li>{@link AccumuloInputFormat#setInputTableName(Job, String)}
  * <li>{@link AccumuloInputFormat#setScanAuthorizations(Job, Authorizations)}
  * <li>{@link AccumuloInputFormat#setZooKeeperInstance(Job, String, String)} OR {@link
AccumuloInputFormat#setMockInstance(Job, String)}

Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java?rev=1485798&r1=1485797&r2=1485798&view=diff
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java
(original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java
Thu May 23 18:11:20 2013
@@ -95,6 +95,25 @@ public class AccumuloOutputFormat extend
   }
   
   /**
+   * Sets the connector information needed to communicate with Accumulo in this job.
+   * 
+   * <p>
+   * Stores the password in a file in HDFS and pulls that into the Distributed Cache in an
attempt to be more secure than storing it in the Configuration.
+   * 
+   * @param job
+   *          the Hadoop job instance to be configured
+   * @param principal
+   *          a valid Accumulo user name (user must have Table.CREATE permission if {@link
#setCreateTables(Job, boolean)} is set to true)
+   * @param tokenFile
+   *          the path to the token file
+   * @throws AccumuloSecurityException
+   * @since 1.6.0
+   */
+  public static void setConnectorInfo(Job job, String principal, String tokenFile) throws
AccumuloSecurityException {
+    OutputConfigurator.setConnectorInfo(CLASS, job.getConfiguration(), principal, tokenFile);
+  }
+  
+  /**
    * Determines if the connector has been configured.
    * 
    * @param context

Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java?rev=1485798&r1=1485797&r2=1485798&view=diff
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
(original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
Thu May 23 18:11:20 2013
@@ -73,6 +73,7 @@ import org.apache.accumulo.core.util.Uti
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapreduce.InputFormat;
 import org.apache.hadoop.mapreduce.InputSplit;
 import org.apache.hadoop.mapreduce.Job;
@@ -118,6 +119,25 @@ public abstract class InputFormatBase<K,
   }
   
   /**
+   * Sets the connector information needed to communicate with Accumulo in this job.
+   * 
+   * <p>
+   * Stores the password in a file in HDFS and pulls that into the Distributed Cache in an
attempt to be more secure than storing it in the Configuration.
+   * 
+   * @param job
+   *          the Hadoop job instance to be configured
+   * @param principal
+   *          a valid Accumulo user name (user must have Table.CREATE permission)
+   * @param tokenFile
+   *          the path to the token file
+   * @throws AccumuloSecurityException
+   * @since 1.6.0
+   */
+  public static void setConnectorInfo(Job job, String principal, String tokenFile) throws
AccumuloSecurityException {
+    InputConfigurator.setConnectorInfo(CLASS, job.getConfiguration(), principal, tokenFile);
+  }
+  
+  /**
    * Determines if the connector has been configured.
    * 
    * @param context
@@ -171,6 +191,19 @@ public abstract class InputFormatBase<K,
   }
   
   /**
+   * Gets the password file from the configuration.
+   * 
+   * @param job
+   *          the Hadoop context for the configured job
+   * @return path to the password file as a String
+   * @since 1.6.0
+   * @see #setConnectorInfo(JobConf, String, AuthenticationToken)
+   */
+  protected static String getTokenFile(JobContext context) {
+    return InputConfigurator.getTokenFile(CLASS, getConfiguration(context));
+  }
+  
+  /**
    * Configures a {@link ZooKeeperInstance} for this job.
    * 
    * @param job

Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/ConfiguratorBase.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/ConfiguratorBase.java?rev=1485798&r1=1485797&r2=1485798&view=diff
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/ConfiguratorBase.java
(original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/ConfiguratorBase.java
Thu May 23 18:11:20 2013
@@ -16,7 +16,10 @@
  */
 package org.apache.accumulo.core.client.mapreduce.lib.util;
 
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.nio.charset.Charset;
+import java.io.IOException;
 
 import org.apache.accumulo.core.client.AccumuloSecurityException;
 import org.apache.accumulo.core.client.Instance;
@@ -28,6 +31,11 @@ import org.apache.accumulo.core.util.Arg
 import org.apache.commons.codec.binary.Base64;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.filecache.DistributedCache;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.mapred.JobConf;
 import org.apache.log4j.Level;
 import org.apache.log4j.Logger;
 
@@ -42,7 +50,7 @@ public class ConfiguratorBase {
    * @since 1.5.0
    */
   public static enum ConnectorInfo {
-    IS_CONFIGURED, PRINCIPAL, TOKEN, TOKEN_CLASS
+    IS_CONFIGURED, PRINCIPAL, TOKEN, TOKEN_CLASS, TOKEN_FILE
   }
   
   /**
@@ -108,6 +116,41 @@ public class ConfiguratorBase {
   }
   
   /**
+   * Sets the connector information needed to communicate with Accumulo in this job.
+   * 
+   * <p>
+   * Pulls a token file into the Distributed Cache that contains the authentication token
in an attempt to be more secure than storing the password in the
+   * Configuration. Token file created with "bin/accumulo create-token".
+   * 
+   * @param implementingClass
+   *          the class whose name will be used as a prefix for the property configuration
key
+   * @param conf
+   *          the Hadoop configuration object to configure
+   * @param principal
+   *          a valid Accumulo user name
+   * @param tokenFile
+   *          the path to the token file
+   * @throws AccumuloSecurityException
+   * @since 1.6.0
+   */
+  public static void setConnectorInfo(Class<?> implementingClass, Configuration conf,
String principal, String tokenFile) throws AccumuloSecurityException {
+    if (isConnectorInfoSet(implementingClass, conf))
+      throw new IllegalStateException("Connector info for " + implementingClass.getSimpleName()
+ " can only be set once per job");
+    
+    ArgumentChecker.notNull(principal, tokenFile);
+    
+    try {
+      DistributedCache.addCacheFile(new URI(tokenFile), conf);
+    } catch (URISyntaxException e) {
+      throw new IllegalStateException("Unable to add tokenFile \"" + tokenFile + "\" to distributed
cache.");
+    }
+    
+    conf.setBoolean(enumToConfKey(implementingClass, ConnectorInfo.IS_CONFIGURED), true);
+    conf.set(enumToConfKey(implementingClass, ConnectorInfo.TOKEN_FILE), tokenFile);
+    conf.set(enumToConfKey(implementingClass, ConnectorInfo.PRINCIPAL), principal);
+  }
+  
+  /**
    * Determines if the connector info has already been set for this instance.
    * 
    * @param implementingClass
@@ -138,7 +181,7 @@ public class ConfiguratorBase {
   }
   
   /**
-   * Gets the serialized token class from the configuration.
+   * Gets the serialized token class from either the configuration or the token file.
    * 
    * @param implementingClass
    *          the class whose name will be used as a prefix for the property configuration
key
@@ -149,12 +192,17 @@ public class ConfiguratorBase {
    * @see #setConnectorInfo(Class, Configuration, String, AuthenticationToken)
    */
   public static String getTokenClass(Class<?> implementingClass, Configuration conf)
{
-    return conf.get(enumToConfKey(implementingClass, ConnectorInfo.TOKEN_CLASS));
+    String tokenFile = getTokenFile(implementingClass, conf);
+    if (tokenFile.isEmpty()) {
+      return conf.get(enumToConfKey(implementingClass, ConnectorInfo.TOKEN_CLASS));
+    } else {
+      return readTokenFile(implementingClass, conf).split(":")[1];
+    }
   }
   
   /**
-   * Gets the password from the configuration. WARNING: The password is stored in the Configuration
and shared with all MapReduce tasks; It is BASE64 encoded to
-   * provide a charset safe conversion to a string, and is not intended to be secure.
+   * Gets the password from either the configuration or the token file. WARNING: If no token
file is specified, the password is stored in the Configuration and
+   * shared with all MapReduce tasks; It is BASE64 encoded to provide a charset safe conversion
to a string, and is not intended to be secure.
    * 
    * @param implementingClass
    *          the class whose name will be used as a prefix for the property configuration
key
@@ -165,7 +213,79 @@ public class ConfiguratorBase {
    * @see #setConnectorInfo(Class, Configuration, String, AuthenticationToken)
    */
   public static byte[] getToken(Class<?> implementingClass, Configuration conf) {
-    return Base64.decodeBase64(conf.get(enumToConfKey(implementingClass, ConnectorInfo.TOKEN),
"").getBytes(Charset.forName("UTF-8")));
+    String tokenFile = getTokenFile(implementingClass, conf);
+    String token = null;
+    if (tokenFile.isEmpty()) {
+      token = conf.get(enumToConfKey(implementingClass, ConnectorInfo.TOKEN));
+    } else {
+      token = readTokenFile(implementingClass, conf).split(":")[2];
+    }
+    return Base64.decodeBase64(token.getBytes(Charset.forName("UTF-8")));
+  }
+  
+  /**
+   * Grabs the token file's path out of the Configuration.
+   * 
+   * @param job
+   *          the Hadoop context for the configured job
+   * @return path to the token file as a String
+   * @since 1.6.0
+   * @see #setConnectorInfo(JobConf, String, AuthenticationToken)
+   */
+  public static String getTokenFile(Class<?> implementingClass, Configuration conf)
{
+    return conf.get(enumToConfKey(implementingClass, ConnectorInfo.TOKEN_FILE), "");
+  }
+  
+  /**
+   * Reads from the token file in distributed cache. Currently, the token file stores data
separated by colons e.g. principal:token_class:token
+   * 
+   * @param job
+   *          the Hadoop context for the configured job
+   * @return path to the token file as a String
+   * @since 1.6.0
+   * @see #setConnectorInfo(JobConf, String, AuthenticationToken)
+   */
+  public static String readTokenFile(Class<?> implementingClass, Configuration conf)
{
+    String tokenFile = getTokenFile(implementingClass, conf);
+    FSDataInputStream in = null;
+    try {
+      URI[] uris = DistributedCache.getCacheFiles(conf);
+      Path path = null;
+      for (URI u : uris) {
+        if (u.toString().equals(tokenFile)) {
+          path = new Path(u);
+        }
+      }
+      if (path == null) {
+        throw new IllegalArgumentException("Couldn't find password file called \"" + tokenFile
+ "\" in cache.");
+      }
+      FileSystem fs = FileSystem.get(conf);
+      in = fs.open(path);
+    } catch (IOException e) {
+      throw new IllegalArgumentException("Couldn't open password file called \"" + tokenFile
+ "\".");
+    }
+    java.util.Scanner fileScanner = new java.util.Scanner(in);
+    try {
+      String line = null;
+      boolean found = false;
+      String principal = getPrincipal(implementingClass, conf);
+      while (!found && fileScanner.hasNextLine()) {
+        line = fileScanner.nextLine();
+        if (line.startsWith(principal + ":")) {
+          found = true;
+          break;
+        }
+      }
+      if (found)
+        return line;
+      else
+        throw new IllegalArgumentException("Couldn't find token for user \"" + principal
+ "\" in file \"" + tokenFile + "\"");
+    } finally {
+      if (fileScanner != null && fileScanner.ioException() == null)
+        fileScanner.close();
+      else if (fileScanner.ioException() != null)
+        throw new RuntimeException(fileScanner.ioException());
+    }
   }
   
   /**

Added: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/CreateToken.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/CreateToken.java?rev=1485798&view=auto
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/CreateToken.java (added)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/util/CreateToken.java Thu May
23 18:11:20 2013
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.util;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+
+import jline.ConsoleReader;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.ClientOpts.Password;
+import org.apache.accumulo.core.cli.ClientOpts.PasswordConverter;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Properties;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.TokenProperty;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.security.CredentialHelper;
+
+import com.beust.jcommander.Parameter;
+
+public class CreateToken {
+  
+  private static ConsoleReader reader = null;
+  
+  private static ConsoleReader getConsoleReader() throws IOException {
+    if (reader == null)
+      reader = new ConsoleReader();
+    return reader;
+  }
+  
+  static class Opts extends Help {
+    @Parameter(names = {"-u", "--user"}, description = "Connection user")
+    public String principal = null;
+    
+    @Parameter(names = "-p", converter = PasswordConverter.class, description = "Connection
password")
+    public Password password = null;
+    
+    @Parameter(names = "--password", converter = PasswordConverter.class, description = "Enter
the connection password", password = true)
+    public Password securePassword = null;
+    
+    @Parameter(names = {"-tc", "--tokenClass"}, description = "The class of the authentication
token")
+    public String tokenClassName = PasswordToken.class.getName();
+    
+    @Parameter(names = {"-f", "--file"}, description = "The filename to save the auth token
to. Multiple tokens can be stored in the same file,"
+        + " but only the first for each user will be recognized.")
+    public String tokenFile = null;
+  }
+  
+  public static void main(String[] args) {
+    Opts opts = new Opts();
+    opts.parseArgs(CreateToken.class.getName(), args);
+    
+    Password pass = opts.password;
+    if (pass == null && opts.securePassword != null) {
+      pass = opts.securePassword;
+    }
+    
+    try {
+      String principal = opts.principal;
+      if (principal == null) {
+        principal = getConsoleReader().readLine("Username (aka principal): ");
+      }
+      
+      AuthenticationToken token = Class.forName(opts.tokenClassName).asSubclass(AuthenticationToken.class).newInstance();
+      Properties props = new Properties();
+      for (TokenProperty tp : token.getProperties()) {
+        String input;
+        if (pass != null && tp.getKey().equals("password")) {
+          input = pass.toString();
+        } else {
+          if (tp.getMask()) {
+            input = getConsoleReader().readLine(tp.getDescription() + ": ", '*');
+          } else {
+            input = getConsoleReader().readLine(tp.getDescription() + ": ");
+          }
+        }
+        props.put(tp.getKey(), input);
+        token.init(props);
+      }
+      String tokenBase64 = CredentialHelper.tokenAsBase64(token);
+      
+      String tokenFile = opts.tokenFile;
+      if (tokenFile == null) {
+        tokenFile = getConsoleReader().readLine("File to save auth token to: ");
+      }
+      File tf = new File(tokenFile);
+      if (!tf.exists()) {
+        tf.createNewFile();
+      }
+      PrintStream out = new PrintStream(new FileOutputStream(tf, true), true, Constants.UTF8.name());
+      String outString = principal + ":" + opts.tokenClassName + ":" + tokenBase64;
+      out.println(outString);
+      out.close();
+      System.out.println("Token written to " + tokenFile + ". Remember to upload it to hdfs.");
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+    return;
+  }
+}

Added: accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapred/TokenFileTest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapred/TokenFileTest.java?rev=1485798&view=auto
==============================================================================
--- accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapred/TokenFileTest.java
(added)
+++ accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapred/TokenFileTest.java
Thu May 23 18:11:20 2013
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapred;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.Iterator;
+import java.util.Map.Entry;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.CredentialHelper;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.junit.Test;
+
+/**
+ * 
+ */
+public class TokenFileTest {
+  private static AssertionError e1 = null;
+  private static final String PREFIX = TokenFileTest.class.getSimpleName();
+  private static final String INSTANCE_NAME = PREFIX + "_mapred_instance";
+  private static final String TEST_TABLE_1 = PREFIX + "_mapred_table_1";
+  private static final String TEST_TABLE_2 = PREFIX + "_mapred_table_2";
+  
+  private static class MRTokenFileTester extends Configured implements Tool {
+    private static class TestMapper implements Mapper<Key,Value,Text,Mutation> {
+      Key key = null;
+      int count = 0;
+      OutputCollector<Text,Mutation> finalOutput;
+      
+      @Override
+      public void map(Key k, Value v, OutputCollector<Text,Mutation> output, Reporter
reporter) throws IOException {
+        finalOutput = output;
+        try {
+          if (key != null)
+            assertEquals(key.getRow().toString(), new String(v.get()));
+          assertEquals(k.getRow(), new Text(String.format("%09x", count + 1)));
+          assertEquals(new String(v.get()), String.format("%09x", count));
+        } catch (AssertionError e) {
+          e1 = e;
+        }
+        key = new Key(k);
+        count++;
+      }
+      
+      @Override
+      public void configure(JobConf job) {}
+      
+      @Override
+      public void close() throws IOException {
+        Mutation m = new Mutation("total");
+        m.put("", "", Integer.toString(count));
+        finalOutput.collect(new Text(), m);
+      }
+      
+    }
+    
+    @Override
+    public int run(String[] args) throws Exception {
+      
+      if (args.length != 4) {
+        throw new IllegalArgumentException("Usage : " + MRTokenFileTester.class.getName()
+ " <user> <token file> <inputtable> <outputtable>");
+      }
+      
+      String user = args[0];
+      String tokenFile = args[1];
+      String table1 = args[2];
+      String table2 = args[3];
+      
+      JobConf job = new JobConf(getConf());
+      job.setJarByClass(this.getClass());
+      
+      job.setInputFormat(AccumuloInputFormat.class);
+      
+      AccumuloInputFormat.setConnectorInfo(job, user, tokenFile);
+      AccumuloInputFormat.setInputTableName(job, table1);
+      AccumuloInputFormat.setMockInstance(job, INSTANCE_NAME);
+      
+      job.setMapperClass(TestMapper.class);
+      job.setMapOutputKeyClass(Key.class);
+      job.setMapOutputValueClass(Value.class);
+      job.setOutputFormat(AccumuloOutputFormat.class);
+      job.setOutputKeyClass(Text.class);
+      job.setOutputValueClass(Mutation.class);
+      
+      AccumuloOutputFormat.setConnectorInfo(job, user, tokenFile);
+      AccumuloOutputFormat.setCreateTables(job, false);
+      AccumuloOutputFormat.setDefaultTableName(job, table2);
+      AccumuloOutputFormat.setMockInstance(job, INSTANCE_NAME);
+      
+      job.setNumReduceTasks(0);
+      
+      return JobClient.runJob(job).isSuccessful() ? 0 : 1;
+    }
+    
+    public static void main(String[] args) throws Exception {
+      assertEquals(0, ToolRunner.run(CachedConfiguration.getInstance(), new MRTokenFileTester(),
args));
+    }
+  }
+  
+  @Test
+  public void testMR() throws Exception {
+    MockInstance mockInstance = new MockInstance(INSTANCE_NAME);
+    Connector c = mockInstance.getConnector("root", new PasswordToken(""));
+    c.tableOperations().create(TEST_TABLE_1);
+    c.tableOperations().create(TEST_TABLE_2);
+    BatchWriter bw = c.createBatchWriter(TEST_TABLE_1, new BatchWriterConfig());
+    for (int i = 0; i < 100; i++) {
+      Mutation m = new Mutation(new Text(String.format("%09x", i + 1)));
+      m.put(new Text(), new Text(), new Value(String.format("%09x", i).getBytes()));
+      bw.addMutation(m);
+    }
+    bw.close();
+    
+    String tokenFile = "root_test.pw";
+    File tf = File.createTempFile(tokenFile, "");
+    PrintStream out = new PrintStream(tf);
+    String outString = "root:" + PasswordToken.class.getName() + ":" + CredentialHelper.tokenAsBase64(new
PasswordToken(""));
+    out.println(outString);
+    out.close();
+    
+    MRTokenFileTester.main(new String[] {"root", tf.getAbsolutePath(), TEST_TABLE_1, TEST_TABLE_2});
+    assertNull(e1);
+    
+    Scanner scanner = c.createScanner(TEST_TABLE_2, new Authorizations());
+    Iterator<Entry<Key,Value>> iter = scanner.iterator();
+    assertTrue(iter.hasNext());
+    Entry<Key,Value> entry = iter.next();
+    assertEquals(Integer.parseInt(new String(entry.getValue().get())), 100);
+    assertFalse(iter.hasNext());
+  }
+}

Added: accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapreduce/TokenFileTest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapreduce/TokenFileTest.java?rev=1485798&view=auto
==============================================================================
--- accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapreduce/TokenFileTest.java
(added)
+++ accumulo/trunk/core/src/test/java/org/apache/accumulo/core/client/mapreduce/TokenFileTest.java
Thu May 23 18:11:20 2013
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.mapreduce;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.Iterator;
+import java.util.Map.Entry;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.CredentialHelper;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.junit.Test;
+
+/**
+ * 
+ */
+public class TokenFileTest {
+  private static AssertionError e1 = null;
+  private static final String PREFIX = TokenFileTest.class.getSimpleName();
+  private static final String INSTANCE_NAME = PREFIX + "_mapreduce_instance";
+  private static final String TEST_TABLE_1 = PREFIX + "_mapreduce_table_1";
+  private static final String TEST_TABLE_2 = PREFIX + "_mapreduce_table_2";
+  
+  private static class MRTokenFileTester extends Configured implements Tool {
+    private static class TestMapper extends Mapper<Key,Value,Text,Mutation> {
+      Key key = null;
+      int count = 0;
+      
+      @Override
+      protected void map(Key k, Value v, Context context) throws IOException, InterruptedException
{
+        try {
+          if (key != null)
+            assertEquals(key.getRow().toString(), new String(v.get()));
+          assertEquals(k.getRow(), new Text(String.format("%09x", count + 1)));
+          assertEquals(new String(v.get()), String.format("%09x", count));
+        } catch (AssertionError e) {
+          e1 = e;
+        }
+        key = new Key(k);
+        count++;
+      }
+      
+      @Override
+      protected void cleanup(Context context) throws IOException, InterruptedException {
+        Mutation m = new Mutation("total");
+        m.put("", "", Integer.toString(count));
+        context.write(new Text(), m);
+      }
+    }
+    
+    @Override
+    public int run(String[] args) throws Exception {
+      
+      if (args.length != 4) {
+        throw new IllegalArgumentException("Usage : " + MRTokenFileTester.class.getName()
+ " <user> <token file> <inputtable> <outputtable>");
+      }
+      
+      String user = args[0];
+      String tokenFile = args[1];
+      String table1 = args[2];
+      String table2 = args[3];
+      
+      Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
+      job.setJarByClass(this.getClass());
+      
+      job.setInputFormatClass(AccumuloInputFormat.class);
+      
+      AccumuloInputFormat.setConnectorInfo(job, user, tokenFile);
+      AccumuloInputFormat.setInputTableName(job, table1);
+      AccumuloInputFormat.setMockInstance(job, INSTANCE_NAME);
+      
+      job.setMapperClass(TestMapper.class);
+      job.setMapOutputKeyClass(Key.class);
+      job.setMapOutputValueClass(Value.class);
+      job.setOutputFormatClass(AccumuloOutputFormat.class);
+      job.setOutputKeyClass(Text.class);
+      job.setOutputValueClass(Mutation.class);
+      
+      AccumuloOutputFormat.setConnectorInfo(job, user, tokenFile);
+      AccumuloOutputFormat.setCreateTables(job, false);
+      AccumuloOutputFormat.setDefaultTableName(job, table2);
+      AccumuloOutputFormat.setMockInstance(job, INSTANCE_NAME);
+      
+      job.setNumReduceTasks(0);
+      
+      job.waitForCompletion(true);
+      
+      return job.isSuccessful() ? 0 : 1;
+    }
+    
+    public static void main(String[] args) throws Exception {
+      assertEquals(0, ToolRunner.run(CachedConfiguration.getInstance(), new MRTokenFileTester(),
args));
+    }
+  }
+  
+  @Test
+  public void testMR() throws Exception {
+    MockInstance mockInstance = new MockInstance(INSTANCE_NAME);
+    Connector c = mockInstance.getConnector("root", new PasswordToken(""));
+    c.tableOperations().create(TEST_TABLE_1);
+    c.tableOperations().create(TEST_TABLE_2);
+    BatchWriter bw = c.createBatchWriter(TEST_TABLE_1, new BatchWriterConfig());
+    for (int i = 0; i < 100; i++) {
+      Mutation m = new Mutation(new Text(String.format("%09x", i + 1)));
+      m.put(new Text(), new Text(), new Value(String.format("%09x", i).getBytes()));
+      bw.addMutation(m);
+    }
+    bw.close();
+    
+    String tokenFile = "root_test.pw";
+    File tf = File.createTempFile(tokenFile, "");
+    PrintStream out = new PrintStream(tf);
+    String outString = "root:" + PasswordToken.class.getName() + ":" + CredentialHelper.tokenAsBase64(new
PasswordToken(""));
+    out.println(outString);
+    out.close();
+    
+    MRTokenFileTester.main(new String[] {"root", tf.getAbsolutePath(), TEST_TABLE_1, TEST_TABLE_2});
+    assertNull(e1);
+    
+    Scanner scanner = c.createScanner(TEST_TABLE_2, new Authorizations());
+    Iterator<Entry<Key,Value>> iter = scanner.iterator();
+    assertTrue(iter.hasNext());
+    Entry<Key,Value> entry = iter.next();
+    assertEquals(Integer.parseInt(new String(entry.getValue().get())), 100);
+    assertFalse(iter.hasNext());
+  }
+}

Modified: accumulo/trunk/start/src/main/java/org/apache/accumulo/start/Main.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/start/src/main/java/org/apache/accumulo/start/Main.java?rev=1485798&r1=1485797&r2=1485798&view=diff
==============================================================================
--- accumulo/trunk/start/src/main/java/org/apache/accumulo/start/Main.java (original)
+++ accumulo/trunk/start/src/main/java/org/apache/accumulo/start/Main.java Thu May 23 18:11:20
2013
@@ -77,6 +77,8 @@ public class Main {
         runTMP = cl.loadClass("org.apache.accumulo.core.util.LoginProperties");
       } else if (args[0].equals("zookeeper")) {
         runTMP = cl.loadClass("org.apache.accumulo.server.util.ZooKeeperMain");
+      } else if (args[0].equals("create-token")) {
+        runTMP = cl.loadClass("org.apache.accumulo.core.util.CreateToken");
       } else {
         try {
           runTMP = cl.loadClass(args[0]);
@@ -119,6 +121,6 @@ public class Main {
   }
   
   private static void printUsage() {
-    System.out.println("accumulo init | master | tserver | monitor | shell | admin | gc |
classpath | rfile-info | login-info | tracer | mini | proxy | zookeeper | <accumulo class>
args");
+    System.out.println("accumulo init | master | tserver | monitor | shell | admin | gc |
classpath | rfile-info | login-info | tracer | mini | proxy | zookeeper | create-token | <accumulo
class> args");
   }
 }



Mime
View raw message