accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From els...@apache.org
Subject [05/26] git commit: Fix an InputFormatBase blunder, add defensive copies, and lots of UTF8
Date Sat, 01 Feb 2014 03:23:04 GMT
Fix an InputFormatBase blunder, add defensive copies, and lots of UTF8


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/4ec34115
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/4ec34115
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/4ec34115

Branch: refs/heads/2292-findbugs
Commit: 4ec341156930af7805540954366f7d2ca75d2461
Parents: c4781bb
Author: Josh Elser <elserj@apache.org>
Authored: Thu Jan 30 21:45:30 2014 -0500
Committer: Josh Elser <elserj@apache.org>
Committed: Fri Jan 31 22:18:52 2014 -0500

----------------------------------------------------------------------
 .../accumulo/core/client/mapred/InputFormatBase.java  |  2 +-
 .../core/client/mapreduce/InputFormatBase.java        | 14 +++++++-------
 .../core/client/mapreduce/RangeInputSplit.java        |  6 +++---
 .../mapreduce/lib/partition/RangePartitioner.java     |  7 +++++--
 .../client/mapreduce/lib/util/InputConfigurator.java  | 12 ++++++------
 .../apache/accumulo/core/client/mock/MockShell.java   |  5 +++--
 .../core/client/security/tokens/PasswordToken.java    |  2 +-
 .../accumulo/core/conf/DefaultConfiguration.java      |  8 +++++---
 8 files changed, 31 insertions(+), 25 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/4ec34115/core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java
b/core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java
index 8cc4d18..ae361de 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java
@@ -619,7 +619,7 @@ public abstract class InputFormatBase<K,V> implements InputFormat<K,V>
{
         Connector conn = instance.getConnector(principal, token);
         log.debug("Creating scanner for table: " + table);
         log.debug("Authorizations are: " + authorizations);
-        if (isOfflineScan(job)) {
+        if (isOffline) {
           String tokenClass = token.getClass().getCanonicalName();
           ByteBuffer tokenBuffer = ByteBuffer.wrap(CredentialHelper.toBytes(token));
           scanner = new OfflineScanner(instance, new TCredentials(principal, tokenClass,
tokenBuffer, instance.getInstanceID()), Tables.getTableId(instance,

http://git-wip-us.apache.org/repos/asf/accumulo/blob/4ec34115/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
index 4c57d1d..4516176 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
@@ -631,7 +631,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V>
{
         Connector conn = instance.getConnector(principal, token);
         log.debug("Creating scanner for table: " + table);
         log.debug("Authorizations are: " + authorizations);
-        if (isOfflineScan(attempt)) {
+        if (isOffline) {
           String tokenClass = token.getClass().getCanonicalName();
           ByteBuffer tokenBuffer = ByteBuffer.wrap(CredentialHelper.toBytes(token));
           scanner = new OfflineScanner(instance, new TCredentials(principal, tokenClass,
tokenBuffer, instance.getInstanceID()), Tables.getTableId(instance,
@@ -1180,8 +1180,8 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V>
{
     List<IteratorSetting> iteratorSettings = InputConfigurator.getIterators(CLASS,
conf);
     List<AccumuloIterator> deprecatedIterators = new ArrayList<AccumuloIterator>(iteratorSettings.size());
     for (IteratorSetting setting : iteratorSettings) {
-      AccumuloIterator deprecatedIter = new AccumuloIterator(new String(setting.getPriority()
+ AccumuloIterator.FIELD_SEP + setting.getIteratorClass()
-          + AccumuloIterator.FIELD_SEP + setting.getName()));
+      AccumuloIterator deprecatedIter = new AccumuloIterator(setting.getPriority() + AccumuloIterator.FIELD_SEP
+ setting.getIteratorClass()
+          + AccumuloIterator.FIELD_SEP + setting.getName());
       deprecatedIterators.add(deprecatedIter);
     }
     return deprecatedIterators;
@@ -1198,8 +1198,8 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V>
{
       for (Entry<String,String> opt : setting.getOptions().entrySet()) {
         String deprecatedOption;
         try {
-          deprecatedOption = new String(setting.getName() + AccumuloIteratorOption.FIELD_SEP
+ URLEncoder.encode(opt.getKey(), "UTF-8")
-              + AccumuloIteratorOption.FIELD_SEP + URLEncoder.encode(opt.getValue(), "UTF-8"));
+          deprecatedOption = setting.getName() + AccumuloIteratorOption.FIELD_SEP + URLEncoder.encode(opt.getKey(),
"UTF-8")
+              + AccumuloIteratorOption.FIELD_SEP + URLEncoder.encode(opt.getValue(), "UTF-8");
         } catch (UnsupportedEncodingException e) {
           throw new RuntimeException(e);
         }
@@ -1250,7 +1250,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V>
{
 
     @Override
     public String toString() {
-      return new String(priority + FIELD_SEP + iteratorClass + FIELD_SEP + iteratorName);
+      return priority + FIELD_SEP + iteratorClass + FIELD_SEP + iteratorName;
     }
 
   }
@@ -1299,7 +1299,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V>
{
     @Override
     public String toString() {
       try {
-        return new String(iteratorName + FIELD_SEP + URLEncoder.encode(key, "UTF-8") + FIELD_SEP
+ URLEncoder.encode(value, "UTF-8"));
+        return iteratorName + FIELD_SEP + URLEncoder.encode(key, "UTF-8") + FIELD_SEP + URLEncoder.encode(value,
"UTF-8");
       } catch (UnsupportedEncodingException e) {
         throw new RuntimeException(e);
       }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/4ec34115/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
index 75f140b..73c9b59 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
@@ -67,7 +67,7 @@ public class RangeInputSplit extends InputSplit implements Writable {
 
   public RangeInputSplit(Range range, String[] locations) {
     this.range = range;
-    this.locations = locations;
+    setLocations(locations);
   }
 
   public Range getRange() {
@@ -138,7 +138,7 @@ public class RangeInputSplit extends InputSplit implements Writable {
 
   @Override
   public String[] getLocations() throws IOException {
-    return locations;
+    return Arrays.copyOf(locations, locations.length);
   }
 
   @Override
@@ -368,7 +368,7 @@ public class RangeInputSplit extends InputSplit implements Writable {
   }
 
   public void setLocations(String[] locations) {
-    this.locations = locations;
+    this.locations = Arrays.copyOf(locations, locations.length);
   }
 
   public Boolean isMockInstance() {

http://git-wip-us.apache.org/repos/asf/accumulo/blob/4ec34115/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitioner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitioner.java
b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitioner.java
index 920bd5b..e29c791 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitioner.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitioner.java
@@ -17,14 +17,17 @@
 package org.apache.accumulo.core.client.mapreduce.lib.partition;
 
 import java.io.BufferedReader;
+import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.FileReader;
 import java.io.IOException;
+import java.io.InputStreamReader;
 import java.net.URI;
 import java.util.Arrays;
 import java.util.Scanner;
 import java.util.TreeSet;
 
+import org.apache.accumulo.core.Constants;
 import org.apache.commons.codec.binary.Base64;
 import org.apache.hadoop.conf.Configurable;
 import org.apache.hadoop.conf.Configuration;
@@ -87,10 +90,10 @@ public class RangePartitioner extends Partitioner<Text,Writable>
implements Conf
         for (Path path : cf) {
           if (path.toUri().getPath().endsWith(cutFileName.substring(cutFileName.lastIndexOf('/'))))
{
             TreeSet<Text> cutPoints = new TreeSet<Text>();
-            Scanner in = new Scanner(new BufferedReader(new FileReader(path.toString())));
+            Scanner in = new Scanner(new BufferedReader(new InputStreamReader(new FileInputStream(path.toString()),
Constants.UTF8)));
             try {
               while (in.hasNextLine())
-                cutPoints.add(new Text(Base64.decodeBase64(in.nextLine().getBytes())));
+                cutPoints.add(new Text(Base64.decodeBase64(in.nextLine().getBytes(Constants.UTF8))));
             } finally {
               in.close();
             }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/4ec34115/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/InputConfigurator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/InputConfigurator.java
b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/InputConfigurator.java
index b0e649b..d9e313c 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/InputConfigurator.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/InputConfigurator.java
@@ -138,7 +138,7 @@ public class InputConfigurator extends ConfiguratorBase {
    */
   public static Authorizations getScanAuthorizations(Class<?> implementingClass, Configuration
conf) {
     String authString = conf.get(enumToConfKey(implementingClass, ScanOpts.AUTHORIZATIONS));
-    return authString == null ? Constants.NO_AUTHS : new Authorizations(authString.getBytes());
+    return authString == null ? Constants.NO_AUTHS : new Authorizations(authString.getBytes(Constants.UTF8));
   }
   
   /**
@@ -216,9 +216,9 @@ public class InputConfigurator extends ConfiguratorBase {
       if (column.getFirst() == null)
         throw new IllegalArgumentException("Column family can not be null");
 
-      String col = new String(Base64.encodeBase64(TextUtil.getBytes(column.getFirst())));
+      String col = new String(Base64.encodeBase64(TextUtil.getBytes(column.getFirst())),
Constants.UTF8);
       if (column.getSecond() != null)
-        col += ":" + new String(Base64.encodeBase64(TextUtil.getBytes(column.getSecond())));
+        col += ":" + new String(Base64.encodeBase64(TextUtil.getBytes(column.getSecond())),
Constants.UTF8);
       columnStrings.add(col);
     }
 
@@ -258,8 +258,8 @@ public class InputConfigurator extends ConfiguratorBase {
 
     for (String col : serialized) {
       int idx = col.indexOf(":");
-      Text cf = new Text(idx < 0 ? Base64.decodeBase64(col.getBytes(Charset.forName("UTF-8")))
: Base64.decodeBase64(col.substring(0, idx).getBytes(Charset.forName("UTF-8"))));
-      Text cq = idx < 0 ? null : new Text(Base64.decodeBase64(col.substring(idx + 1).getBytes()));
+      Text cf = new Text(idx < 0 ? Base64.decodeBase64(col.getBytes(Constants.UTF8)) :
Base64.decodeBase64(col.substring(0, idx).getBytes(Constants.UTF8)));
+      Text cq = idx < 0 ? null : new Text(Base64.decodeBase64(col.substring(idx + 1).getBytes(Constants.UTF8)));
       columns.add(new Pair<Text,Text>(cf, cq));
     }
     return columns;
@@ -323,7 +323,7 @@ public class InputConfigurator extends ConfiguratorBase {
     try {
       while (tokens.hasMoreTokens()) {
         String itstring = tokens.nextToken();
-        ByteArrayInputStream bais = new ByteArrayInputStream(Base64.decodeBase64(itstring.getBytes()));
+        ByteArrayInputStream bais = new ByteArrayInputStream(Base64.decodeBase64(itstring.getBytes(Constants.UTF8)));
         list.add(new IteratorSetting(new DataInputStream(bais)));
         bais.close();
       }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/4ec34115/core/src/main/java/org/apache/accumulo/core/client/mock/MockShell.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/mock/MockShell.java b/core/src/main/java/org/apache/accumulo/core/client/mock/MockShell.java
index 4469d5c..c97e2b4 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mock/MockShell.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mock/MockShell.java
@@ -24,6 +24,7 @@ import java.io.Writer;
 
 import jline.ConsoleReader;
 
+import org.apache.accumulo.core.Constants;
 import org.apache.accumulo.core.util.shell.Shell;
 import org.apache.commons.cli.CommandLine;
 
@@ -77,7 +78,7 @@ public class MockShell extends Shell {
       printInfo();
     
     if (execFile != null) {
-      java.util.Scanner scanner = new java.util.Scanner(new File(execFile));
+      java.util.Scanner scanner = new java.util.Scanner(new File(execFile), Constants.UTF8.name());
       try {
         while (scanner.hasNextLine() && !hasExited()) {
           execCommand(scanner.nextLine(), true, isVerbose());
@@ -137,6 +138,6 @@ public class MockShell extends Shell {
       sb.append(command).append(NEWLINE);
     }
     
-    return new ByteArrayInputStream(sb.toString().getBytes());
+    return new ByteArrayInputStream(sb.toString().getBytes(Constants.UTF8));
   }
 }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/4ec34115/core/src/main/java/org/apache/accumulo/core/client/security/tokens/PasswordToken.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/security/tokens/PasswordToken.java
b/core/src/main/java/org/apache/accumulo/core/client/security/tokens/PasswordToken.java
index 50d6938..9386b56 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/security/tokens/PasswordToken.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/security/tokens/PasswordToken.java
@@ -40,7 +40,7 @@ public class PasswordToken implements AuthenticationToken {
   private byte[] password = null;
   
   public byte[] getPassword() {
-    return password;
+    return Arrays.copyOf(password, password.length);
   }
   
   /**

http://git-wip-us.apache.org/repos/asf/accumulo/blob/4ec34115/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java b/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java
index 58f446d..d653274 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java
@@ -20,11 +20,13 @@ import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.PrintStream;
+import java.io.UnsupportedEncodingException;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.Map.Entry;
 import java.util.TreeMap;
 
+import org.apache.accumulo.core.Constants;
 import org.apache.log4j.Logger;
 
 public class DefaultConfiguration extends AccumuloConfiguration {
@@ -62,7 +64,7 @@ public class DefaultConfiguration extends AccumuloConfiguration {
       int n;
       try {
         while ((n = data.read(buffer)) > 0)
-          doc.print(new String(buffer, 0, n));
+          doc.print(new String(buffer, 0, n, Constants.UTF8));
       } catch (IOException e) {
         e.printStackTrace();
         return;
@@ -176,9 +178,9 @@ public class DefaultConfiguration extends AccumuloConfiguration {
   /*
    * Generate documentation for conf/accumulo-site.xml file usage
    */
-  public static void main(String[] args) throws FileNotFoundException {
+  public static void main(String[] args) throws FileNotFoundException, UnsupportedEncodingException
{
     if (args.length == 2 && args[0].equals("--generate-doc")) {
-      generateDocumentation(new PrintStream(args[1]));
+      generateDocumentation(new PrintStream(args[1], Constants.UTF8.name()));
     } else {
       throw new IllegalArgumentException("Usage: " + DefaultConfiguration.class.getName()
+ " --generate-doc <filename>");
     }


Mime
View raw message