Return-Path: X-Original-To: apmail-accumulo-commits-archive@www.apache.org Delivered-To: apmail-accumulo-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 659AC10264 for ; Sat, 1 Feb 2014 03:23:33 +0000 (UTC) Received: (qmail 69439 invoked by uid 500); 1 Feb 2014 03:23:32 -0000 Delivered-To: apmail-accumulo-commits-archive@accumulo.apache.org Received: (qmail 69342 invoked by uid 500); 1 Feb 2014 03:23:31 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 67740 invoked by uid 99); 1 Feb 2014 03:23:03 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 01 Feb 2014 03:23:03 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 545D78C09BE; Sat, 1 Feb 2014 03:23:00 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: elserj@apache.org To: commits@accumulo.apache.org Date: Sat, 01 Feb 2014 03:23:04 -0000 Message-Id: <57ceac26bf3d4bad8f546b5e6bf5648a@git.apache.org> In-Reply-To: <40fa21bd2f504f478a68fbe0a4f7e4a7@git.apache.org> References: <40fa21bd2f504f478a68fbe0a4f7e4a7@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [05/26] git commit: Fix an InputFormatBase blunder, add defensive copies, and lots of UTF8 Fix an InputFormatBase blunder, add defensive copies, and lots of UTF8 Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/4ec34115 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/4ec34115 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/4ec34115 Branch: refs/heads/2292-findbugs Commit: 4ec341156930af7805540954366f7d2ca75d2461 Parents: c4781bb Author: Josh Elser Authored: Thu Jan 30 21:45:30 2014 -0500 Committer: Josh Elser Committed: Fri Jan 31 22:18:52 2014 -0500 ---------------------------------------------------------------------- .../accumulo/core/client/mapred/InputFormatBase.java | 2 +- .../core/client/mapreduce/InputFormatBase.java | 14 +++++++------- .../core/client/mapreduce/RangeInputSplit.java | 6 +++--- .../mapreduce/lib/partition/RangePartitioner.java | 7 +++++-- .../client/mapreduce/lib/util/InputConfigurator.java | 12 ++++++------ .../apache/accumulo/core/client/mock/MockShell.java | 5 +++-- .../core/client/security/tokens/PasswordToken.java | 2 +- .../accumulo/core/conf/DefaultConfiguration.java | 8 +++++--- 8 files changed, 31 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/4ec34115/core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java b/core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java index 8cc4d18..ae361de 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java +++ b/core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java @@ -619,7 +619,7 @@ public abstract class InputFormatBase implements InputFormat { Connector conn = instance.getConnector(principal, token); log.debug("Creating scanner for table: " + table); log.debug("Authorizations are: " + authorizations); - if (isOfflineScan(job)) { + if (isOffline) { String tokenClass = token.getClass().getCanonicalName(); ByteBuffer tokenBuffer = ByteBuffer.wrap(CredentialHelper.toBytes(token)); scanner = new OfflineScanner(instance, new TCredentials(principal, tokenClass, tokenBuffer, instance.getInstanceID()), Tables.getTableId(instance, http://git-wip-us.apache.org/repos/asf/accumulo/blob/4ec34115/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java index 4c57d1d..4516176 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java +++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java @@ -631,7 +631,7 @@ public abstract class InputFormatBase extends InputFormat { Connector conn = instance.getConnector(principal, token); log.debug("Creating scanner for table: " + table); log.debug("Authorizations are: " + authorizations); - if (isOfflineScan(attempt)) { + if (isOffline) { String tokenClass = token.getClass().getCanonicalName(); ByteBuffer tokenBuffer = ByteBuffer.wrap(CredentialHelper.toBytes(token)); scanner = new OfflineScanner(instance, new TCredentials(principal, tokenClass, tokenBuffer, instance.getInstanceID()), Tables.getTableId(instance, @@ -1180,8 +1180,8 @@ public abstract class InputFormatBase extends InputFormat { List iteratorSettings = InputConfigurator.getIterators(CLASS, conf); List deprecatedIterators = new ArrayList(iteratorSettings.size()); for (IteratorSetting setting : iteratorSettings) { - AccumuloIterator deprecatedIter = new AccumuloIterator(new String(setting.getPriority() + AccumuloIterator.FIELD_SEP + setting.getIteratorClass() - + AccumuloIterator.FIELD_SEP + setting.getName())); + AccumuloIterator deprecatedIter = new AccumuloIterator(setting.getPriority() + AccumuloIterator.FIELD_SEP + setting.getIteratorClass() + + AccumuloIterator.FIELD_SEP + setting.getName()); deprecatedIterators.add(deprecatedIter); } return deprecatedIterators; @@ -1198,8 +1198,8 @@ public abstract class InputFormatBase extends InputFormat { for (Entry opt : setting.getOptions().entrySet()) { String deprecatedOption; try { - deprecatedOption = new String(setting.getName() + AccumuloIteratorOption.FIELD_SEP + URLEncoder.encode(opt.getKey(), "UTF-8") - + AccumuloIteratorOption.FIELD_SEP + URLEncoder.encode(opt.getValue(), "UTF-8")); + deprecatedOption = setting.getName() + AccumuloIteratorOption.FIELD_SEP + URLEncoder.encode(opt.getKey(), "UTF-8") + + AccumuloIteratorOption.FIELD_SEP + URLEncoder.encode(opt.getValue(), "UTF-8"); } catch (UnsupportedEncodingException e) { throw new RuntimeException(e); } @@ -1250,7 +1250,7 @@ public abstract class InputFormatBase extends InputFormat { @Override public String toString() { - return new String(priority + FIELD_SEP + iteratorClass + FIELD_SEP + iteratorName); + return priority + FIELD_SEP + iteratorClass + FIELD_SEP + iteratorName; } } @@ -1299,7 +1299,7 @@ public abstract class InputFormatBase extends InputFormat { @Override public String toString() { try { - return new String(iteratorName + FIELD_SEP + URLEncoder.encode(key, "UTF-8") + FIELD_SEP + URLEncoder.encode(value, "UTF-8")); + return iteratorName + FIELD_SEP + URLEncoder.encode(key, "UTF-8") + FIELD_SEP + URLEncoder.encode(value, "UTF-8"); } catch (UnsupportedEncodingException e) { throw new RuntimeException(e); } http://git-wip-us.apache.org/repos/asf/accumulo/blob/4ec34115/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java index 75f140b..73c9b59 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java +++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java @@ -67,7 +67,7 @@ public class RangeInputSplit extends InputSplit implements Writable { public RangeInputSplit(Range range, String[] locations) { this.range = range; - this.locations = locations; + setLocations(locations); } public Range getRange() { @@ -138,7 +138,7 @@ public class RangeInputSplit extends InputSplit implements Writable { @Override public String[] getLocations() throws IOException { - return locations; + return Arrays.copyOf(locations, locations.length); } @Override @@ -368,7 +368,7 @@ public class RangeInputSplit extends InputSplit implements Writable { } public void setLocations(String[] locations) { - this.locations = locations; + this.locations = Arrays.copyOf(locations, locations.length); } public Boolean isMockInstance() { http://git-wip-us.apache.org/repos/asf/accumulo/blob/4ec34115/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitioner.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitioner.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitioner.java index 920bd5b..e29c791 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitioner.java +++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/partition/RangePartitioner.java @@ -17,14 +17,17 @@ package org.apache.accumulo.core.client.mapreduce.lib.partition; import java.io.BufferedReader; +import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileReader; import java.io.IOException; +import java.io.InputStreamReader; import java.net.URI; import java.util.Arrays; import java.util.Scanner; import java.util.TreeSet; +import org.apache.accumulo.core.Constants; import org.apache.commons.codec.binary.Base64; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; @@ -87,10 +90,10 @@ public class RangePartitioner extends Partitioner implements Conf for (Path path : cf) { if (path.toUri().getPath().endsWith(cutFileName.substring(cutFileName.lastIndexOf('/')))) { TreeSet cutPoints = new TreeSet(); - Scanner in = new Scanner(new BufferedReader(new FileReader(path.toString()))); + Scanner in = new Scanner(new BufferedReader(new InputStreamReader(new FileInputStream(path.toString()), Constants.UTF8))); try { while (in.hasNextLine()) - cutPoints.add(new Text(Base64.decodeBase64(in.nextLine().getBytes()))); + cutPoints.add(new Text(Base64.decodeBase64(in.nextLine().getBytes(Constants.UTF8)))); } finally { in.close(); } http://git-wip-us.apache.org/repos/asf/accumulo/blob/4ec34115/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/InputConfigurator.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/InputConfigurator.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/InputConfigurator.java index b0e649b..d9e313c 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/InputConfigurator.java +++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/InputConfigurator.java @@ -138,7 +138,7 @@ public class InputConfigurator extends ConfiguratorBase { */ public static Authorizations getScanAuthorizations(Class implementingClass, Configuration conf) { String authString = conf.get(enumToConfKey(implementingClass, ScanOpts.AUTHORIZATIONS)); - return authString == null ? Constants.NO_AUTHS : new Authorizations(authString.getBytes()); + return authString == null ? Constants.NO_AUTHS : new Authorizations(authString.getBytes(Constants.UTF8)); } /** @@ -216,9 +216,9 @@ public class InputConfigurator extends ConfiguratorBase { if (column.getFirst() == null) throw new IllegalArgumentException("Column family can not be null"); - String col = new String(Base64.encodeBase64(TextUtil.getBytes(column.getFirst()))); + String col = new String(Base64.encodeBase64(TextUtil.getBytes(column.getFirst())), Constants.UTF8); if (column.getSecond() != null) - col += ":" + new String(Base64.encodeBase64(TextUtil.getBytes(column.getSecond()))); + col += ":" + new String(Base64.encodeBase64(TextUtil.getBytes(column.getSecond())), Constants.UTF8); columnStrings.add(col); } @@ -258,8 +258,8 @@ public class InputConfigurator extends ConfiguratorBase { for (String col : serialized) { int idx = col.indexOf(":"); - Text cf = new Text(idx < 0 ? Base64.decodeBase64(col.getBytes(Charset.forName("UTF-8"))) : Base64.decodeBase64(col.substring(0, idx).getBytes(Charset.forName("UTF-8")))); - Text cq = idx < 0 ? null : new Text(Base64.decodeBase64(col.substring(idx + 1).getBytes())); + Text cf = new Text(idx < 0 ? Base64.decodeBase64(col.getBytes(Constants.UTF8)) : Base64.decodeBase64(col.substring(0, idx).getBytes(Constants.UTF8))); + Text cq = idx < 0 ? null : new Text(Base64.decodeBase64(col.substring(idx + 1).getBytes(Constants.UTF8))); columns.add(new Pair(cf, cq)); } return columns; @@ -323,7 +323,7 @@ public class InputConfigurator extends ConfiguratorBase { try { while (tokens.hasMoreTokens()) { String itstring = tokens.nextToken(); - ByteArrayInputStream bais = new ByteArrayInputStream(Base64.decodeBase64(itstring.getBytes())); + ByteArrayInputStream bais = new ByteArrayInputStream(Base64.decodeBase64(itstring.getBytes(Constants.UTF8))); list.add(new IteratorSetting(new DataInputStream(bais))); bais.close(); } http://git-wip-us.apache.org/repos/asf/accumulo/blob/4ec34115/core/src/main/java/org/apache/accumulo/core/client/mock/MockShell.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/client/mock/MockShell.java b/core/src/main/java/org/apache/accumulo/core/client/mock/MockShell.java index 4469d5c..c97e2b4 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/mock/MockShell.java +++ b/core/src/main/java/org/apache/accumulo/core/client/mock/MockShell.java @@ -24,6 +24,7 @@ import java.io.Writer; import jline.ConsoleReader; +import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.util.shell.Shell; import org.apache.commons.cli.CommandLine; @@ -77,7 +78,7 @@ public class MockShell extends Shell { printInfo(); if (execFile != null) { - java.util.Scanner scanner = new java.util.Scanner(new File(execFile)); + java.util.Scanner scanner = new java.util.Scanner(new File(execFile), Constants.UTF8.name()); try { while (scanner.hasNextLine() && !hasExited()) { execCommand(scanner.nextLine(), true, isVerbose()); @@ -137,6 +138,6 @@ public class MockShell extends Shell { sb.append(command).append(NEWLINE); } - return new ByteArrayInputStream(sb.toString().getBytes()); + return new ByteArrayInputStream(sb.toString().getBytes(Constants.UTF8)); } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/4ec34115/core/src/main/java/org/apache/accumulo/core/client/security/tokens/PasswordToken.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/client/security/tokens/PasswordToken.java b/core/src/main/java/org/apache/accumulo/core/client/security/tokens/PasswordToken.java index 50d6938..9386b56 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/security/tokens/PasswordToken.java +++ b/core/src/main/java/org/apache/accumulo/core/client/security/tokens/PasswordToken.java @@ -40,7 +40,7 @@ public class PasswordToken implements AuthenticationToken { private byte[] password = null; public byte[] getPassword() { - return password; + return Arrays.copyOf(password, password.length); } /** http://git-wip-us.apache.org/repos/asf/accumulo/blob/4ec34115/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java b/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java index 58f446d..d653274 100644 --- a/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java +++ b/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java @@ -20,11 +20,13 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.PrintStream; +import java.io.UnsupportedEncodingException; import java.util.ArrayList; import java.util.Iterator; import java.util.Map.Entry; import java.util.TreeMap; +import org.apache.accumulo.core.Constants; import org.apache.log4j.Logger; public class DefaultConfiguration extends AccumuloConfiguration { @@ -62,7 +64,7 @@ public class DefaultConfiguration extends AccumuloConfiguration { int n; try { while ((n = data.read(buffer)) > 0) - doc.print(new String(buffer, 0, n)); + doc.print(new String(buffer, 0, n, Constants.UTF8)); } catch (IOException e) { e.printStackTrace(); return; @@ -176,9 +178,9 @@ public class DefaultConfiguration extends AccumuloConfiguration { /* * Generate documentation for conf/accumulo-site.xml file usage */ - public static void main(String[] args) throws FileNotFoundException { + public static void main(String[] args) throws FileNotFoundException, UnsupportedEncodingException { if (args.length == 2 && args[0].equals("--generate-doc")) { - generateDocumentation(new PrintStream(args[1])); + generateDocumentation(new PrintStream(args[1], Constants.UTF8.name())); } else { throw new IllegalArgumentException("Usage: " + DefaultConfiguration.class.getName() + " --generate-doc "); }