Return-Path: X-Original-To: apmail-tez-commits-archive@minotaur.apache.org Delivered-To: apmail-tez-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 8A739102BD for ; Wed, 7 Jan 2015 00:36:10 +0000 (UTC) Received: (qmail 85005 invoked by uid 500); 7 Jan 2015 00:36:11 -0000 Delivered-To: apmail-tez-commits-archive@tez.apache.org Received: (qmail 84964 invoked by uid 500); 7 Jan 2015 00:36:11 -0000 Mailing-List: contact commits-help@tez.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@tez.apache.org Delivered-To: mailing list commits@tez.apache.org Received: (qmail 84955 invoked by uid 99); 7 Jan 2015 00:36:11 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 07 Jan 2015 00:36:11 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 42419A25365; Wed, 7 Jan 2015 00:36:11 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: hitesh@apache.org To: commits@tez.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: tez git commit: TEZ-1898. Fix findbugs warnings in tez-api module. (hitesh) Date: Wed, 7 Jan 2015 00:36:11 +0000 (UTC) Repository: tez Updated Branches: refs/heads/master 9fb86963e -> 393e40b8d TEZ-1898. Fix findbugs warnings in tez-api module. (hitesh) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/393e40b8 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/393e40b8 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/393e40b8 Branch: refs/heads/master Commit: 393e40b8d65951b7488eae8dc97b9a99b77577a0 Parents: 9fb8696 Author: Hitesh Shah Authored: Tue Jan 6 16:35:51 2015 -0800 Committer: Hitesh Shah Committed: Tue Jan 6 16:35:51 2015 -0800 ---------------------------------------------------------------------- CHANGES.txt | 1 + pom.xml | 15 ++++ tez-api/findbugs-exclude.xml | 72 ++++++++++++++++++++ .../org/apache/tez/client/AMConfiguration.java | 5 -- .../org/apache/tez/client/TezClientUtils.java | 14 +--- .../org/apache/tez/common/TezCommonUtils.java | 19 ++++-- .../tez/common/security/DAGAccessControls.java | 10 +-- .../main/java/org/apache/tez/dag/api/DAG.java | 2 +- .../apache/tez/dag/api/DagTypeConverters.java | 6 +- .../apache/tez/dag/api/TaskLocationHint.java | 2 +- .../apache/tez/dag/api/TezConfiguration.java | 4 +- .../dag/api/client/DAGClientTimelineImpl.java | 6 +- .../apache/tez/dag/api/client/DAGStatus.java | 26 ++++++- .../org/apache/tez/dag/api/client/Progress.java | 22 +++++- .../apache/tez/dag/api/client/VertexStatus.java | 22 ++++++ .../dag/api/client/rpc/DAGClientRPCImpl.java | 2 +- .../tez/dag/api/TestDagTypeConverters.java | 4 ++ 17 files changed, 193 insertions(+), 39 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/393e40b8/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 114784b..b9cd1d1 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -7,6 +7,7 @@ Release 0.7.0: Unreleased INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-1898. Fix findbugs warnings in tez-api module. TEZ-1906. Fix findbugs warnings in tez-yarn-timeline-history-with-acls. TEZ-1767. Enable RLE in reducer side merge codepath. TEZ-1837. Restrict usage of Environment variables to main methods. http://git-wip-us.apache.org/repos/asf/tez/blob/393e40b8/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 6bff3fb..bd03cdd 100644 --- a/pom.xml +++ b/pom.xml @@ -846,6 +846,13 @@ org.codehaus.mojo build-helper-maven-plugin + + org.codehaus.mojo + findbugs-maven-plugin + + findbugs-exclude.xml + + @@ -964,6 +971,7 @@ html true + findbugs-exclude.xml @@ -1069,6 +1077,13 @@ + + org.codehaus.mojo + findbugs-maven-plugin + + findbugs-exclude.xml + + http://git-wip-us.apache.org/repos/asf/tez/blob/393e40b8/tez-api/findbugs-exclude.xml ---------------------------------------------------------------------- diff --git a/tez-api/findbugs-exclude.xml b/tez-api/findbugs-exclude.xml index 5b11308..b928a44 100644 --- a/tez-api/findbugs-exclude.xml +++ b/tez-api/findbugs-exclude.xml @@ -13,4 +13,76 @@ --> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + http://git-wip-us.apache.org/repos/asf/tez/blob/393e40b8/tez-api/src/main/java/org/apache/tez/client/AMConfiguration.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/client/AMConfiguration.java b/tez-api/src/main/java/org/apache/tez/client/AMConfiguration.java index d8cc2bf..238b155 100644 --- a/tez-api/src/main/java/org/apache/tez/client/AMConfiguration.java +++ b/tez-api/src/main/java/org/apache/tez/client/AMConfiguration.java @@ -35,7 +35,6 @@ class AMConfiguration { private TezConfiguration tezConf; private Credentials credentials; private YarnConfiguration yarnConfig; - private Map env; private LocalResource binaryConfLRsrc; AMConfiguration(TezConfiguration tezConf, Map localResources, @@ -90,10 +89,6 @@ class AMConfiguration { return credentials; } - Map getEnv() { - return env; - } - void setBinaryConfLR(LocalResource binaryConfLRsrc) { this.binaryConfLRsrc = binaryConfLRsrc; } http://git-wip-us.apache.org/repos/asf/tez/blob/393e40b8/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java index 16e85d9..597789c 100644 --- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java +++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java @@ -502,15 +502,6 @@ public class TezClientUtils { TezYARNUtils.setupDefaultEnv(environment, conf, TezConfiguration.TEZ_AM_LAUNCH_ENV, TezConfiguration.TEZ_AM_LAUNCH_ENV_DEFAULT, tezLrsAsArchive); - // finally apply env set in the code. This could potentially be removed in - // TEZ-692 - if (amConfig.getEnv() != null) { - for (Map.Entry entry : amConfig.getEnv().entrySet()) { - TezYARNUtils.addToEnvironment(environment, entry.getKey(), - entry.getValue(), File.pathSeparator); - } - } - addVersionInfoToEnv(environment, apiVersionInfo); addLogParamsToEnv(environment, amLogParams); @@ -676,10 +667,11 @@ public class TezClientUtils { } static void maybeAddDefaultLoggingJavaOpts(String logLevel, List vargs) { - if (vargs != null && !vargs.isEmpty()) { + Preconditions.checkNotNull(vargs); + if (!vargs.isEmpty()) { for (String arg : vargs) { if (arg.contains(TezConstants.TEZ_ROOT_LOGGER_NAME)) { - return ; + return; } } } http://git-wip-us.apache.org/repos/asf/tez/blob/393e40b8/tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java b/tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java index c5d5ebc..b7a402d 100644 --- a/tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java +++ b/tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java @@ -344,12 +344,19 @@ public class TezCommonUtils { @Private public static ByteString compressByteArrayToByteString(byte[] inBytes) throws IOException { ByteString.Output os = ByteString.newOutput(); - DeflaterOutputStream compressOs = new DeflaterOutputStream(os, new Deflater( - Deflater.BEST_COMPRESSION)); - compressOs.write(inBytes); - compressOs.finish(); - ByteString byteString = os.toByteString(); - return byteString; + DeflaterOutputStream compressOs = null; + try { + compressOs = new DeflaterOutputStream(os, new Deflater( + Deflater.BEST_COMPRESSION)); + compressOs.write(inBytes); + compressOs.finish(); + ByteString byteString = os.toByteString(); + return byteString; + } finally { + if (compressOs != null) { + compressOs.close(); + } + } } @Private http://git-wip-us.apache.org/repos/asf/tez/blob/393e40b8/tez-api/src/main/java/org/apache/tez/common/security/DAGAccessControls.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/common/security/DAGAccessControls.java b/tez-api/src/main/java/org/apache/tez/common/security/DAGAccessControls.java index 87e08c3..5fe352a 100644 --- a/tez-api/src/main/java/org/apache/tez/common/security/DAGAccessControls.java +++ b/tez-api/src/main/java/org/apache/tez/common/security/DAGAccessControls.java @@ -131,27 +131,27 @@ public class DAGAccessControls { } @Private - public Set getUsersWithViewACLs() { + public synchronized Set getUsersWithViewACLs() { return Collections.unmodifiableSet(usersWithViewACLs); } @Private - public Set getUsersWithModifyACLs() { + public synchronized Set getUsersWithModifyACLs() { return Collections.unmodifiableSet(usersWithModifyACLs); } @Private - public Set getGroupsWithViewACLs() { + public synchronized Set getGroupsWithViewACLs() { return Collections.unmodifiableSet(groupsWithViewACLs); } @Private - public Set getGroupsWithModifyACLs() { + public synchronized Set getGroupsWithModifyACLs() { return Collections.unmodifiableSet(groupsWithModifyACLs); } @Private - public void serializeToConfiguration(Configuration conf) { + public synchronized void serializeToConfiguration(Configuration conf) { if (usersWithViewACLs.contains(ACLManager.WILDCARD_ACL_VALUE)) { conf.set(TezConstants.TEZ_DAG_VIEW_ACLS, ACLManager.WILDCARD_ACL_VALUE); } else { http://git-wip-us.apache.org/repos/asf/tez/blob/393e40b8/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java b/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java index b4fdbd2..106d95e 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java @@ -621,7 +621,7 @@ public class DAG { // create protobuf message describing DAG @Private - public DAGPlan createDag(Configuration dagConf, Credentials extraCredentials, + public synchronized DAGPlan createDag(Configuration dagConf, Credentials extraCredentials, Map tezJarResources, LocalResource binaryConfig, boolean tezLrsAsArchive, Map additionalConfigs) { verify(true); http://git-wip-us.apache.org/repos/asf/tez/blob/393e40b8/tez-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java b/tez-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java index 17807d3..227897f 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java @@ -20,6 +20,7 @@ package org.apache.tez.dag.api; import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.charset.Charset; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -317,7 +318,7 @@ public class DagTypeConverters { if (descriptor.getHistoryText() != null) { try { builder.setHistoryText(TezCommonUtils.compressByteArrayToByteString( - descriptor.getHistoryText().getBytes())); + descriptor.getHistoryText().getBytes("UTF-8"))); } catch (IOException e) { throw new TezUncheckedException(e); } @@ -330,7 +331,8 @@ public class DagTypeConverters { return null; } try { - return new String(TezCommonUtils.decompressByteStringToByteArray(proto.getHistoryText())); + return new String(TezCommonUtils.decompressByteStringToByteArray(proto.getHistoryText()), + "UTF-8"); } catch (IOException e) { throw new TezUncheckedException(e); } http://git-wip-us.apache.org/repos/asf/tez/blob/393e40b8/tez-api/src/main/java/org/apache/tez/dag/api/TaskLocationHint.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TaskLocationHint.java b/tez-api/src/main/java/org/apache/tez/dag/api/TaskLocationHint.java index fef99f3..d1a1359 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/TaskLocationHint.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/TaskLocationHint.java @@ -40,7 +40,7 @@ public class TaskLocationHint { /** * Specifies location affinity to the given vertex and given task in that vertex */ - public class TaskBasedLocationAffinity { + public static class TaskBasedLocationAffinity { private String vertexName; private int taskIndex; public TaskBasedLocationAffinity(String vertexName, int taskIndex) { http://git-wip-us.apache.org/repos/asf/tez/blob/393e40b8/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java index 9a90209..d52a33d 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java @@ -115,7 +115,7 @@ public class TezConfiguration extends Configuration { * mode is recommended. If session mode is enabled then container reuse is recommended. */ public static final String TEZ_AM_SESSION_MODE = TEZ_AM_PREFIX + "mode.session"; - public static boolean TEZ_AM_SESSION_MODE_DEFAULT = false; + public static final boolean TEZ_AM_SESSION_MODE_DEFAULT = false; /** * Root Logging level passed to the Tez app master. @@ -341,7 +341,7 @@ public class TezConfiguration extends Configuration { */ public static final String TEZ_AM_MAX_APP_ATTEMPTS = TEZ_AM_PREFIX + "max.app.attempts"; - public static int TEZ_AM_MAX_APP_ATTEMPTS_DEFAULT = 2; + public static final int TEZ_AM_MAX_APP_ATTEMPTS_DEFAULT = 2; /** * Int value. The maximum number of attempts that can fail for a particular task before the task is failed. http://git-wip-us.apache.org/repos/asf/tez/blob/393e40b8/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientTimelineImpl.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientTimelineImpl.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientTimelineImpl.java index 6cbab80..ce88c93 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientTimelineImpl.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientTimelineImpl.java @@ -426,8 +426,8 @@ public class DAGClientTimelineImpl extends DAGClient { } } - @VisibleForTesting - protected class VertexTaskStats { + @VisibleForTesting + protected static class VertexTaskStats { final int numTaskCount; final int completedTaskCount; final int succeededTaskCount; @@ -496,7 +496,7 @@ public class DAGClientTimelineImpl extends DAGClient { }}); - class PseudoAuthenticatedURLConnectionFactory implements HttpURLConnectionFactory { + static class PseudoAuthenticatedURLConnectionFactory implements HttpURLConnectionFactory { @Override public HttpURLConnection getHttpURLConnection(URL url) throws IOException { String tokenString = (url.getQuery() == null ? "?" : "&") + "user.name=" + http://git-wip-us.apache.org/repos/asf/tez/blob/393e40b8/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGStatus.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGStatus.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGStatus.java index 7c51c42..d079da2 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGStatus.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGStatus.java @@ -161,7 +161,31 @@ public class DAGStatus { } return false; } - + + @Override + public int hashCode() { + final int prime = 44017; + int result = 1; + result = prime + + getState().hashCode(); + + List diagnostics = getDiagnostics(); + Progress dagProgress = getDAGProgress(); + Map vProgress = getVertexProgress(); + TezCounters counters = getDAGCounters(); + + result = prime * result + + ((diagnostics == null)? 0 : diagnostics.hashCode()); + result = prime * result + + ((dagProgress == null)? 0 : dagProgress.hashCode()); + result = prime * result + + ((vProgress == null)? 0 : vProgress.hashCode()); + result = prime * result + + ((counters == null)? 0 : counters.hashCode()); + + return result; + } + @Override public String toString() { StringBuilder sb = new StringBuilder(); http://git-wip-us.apache.org/repos/asf/tez/blob/393e40b8/tez-api/src/main/java/org/apache/tez/dag/api/client/Progress.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/Progress.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/Progress.java index b6a1bfc..110ac90 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/client/Progress.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/Progress.java @@ -77,7 +77,27 @@ public class Progress { } return false; } - + + @Override + public int hashCode() { + final int prime = 45007; + int result = prime + getTotalTaskCount(); + result = prime * result + + getSucceededTaskCount(); + result = prime * result + + getRunningTaskCount(); + result = prime * result + + getFailedTaskCount(); + result = prime * result + + getKilledTaskCount(); + result = prime * result + + getFailedTaskAttemptCount(); + result = prime * result + + getKilledTaskAttemptCount(); + + return result; + } + @Override public String toString() { StringBuilder sb = new StringBuilder(); http://git-wip-us.apache.org/repos/asf/tez/blob/393e40b8/tez-api/src/main/java/org/apache/tez/dag/api/client/VertexStatus.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/VertexStatus.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/VertexStatus.java index 378cd35..e22d6c0 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/client/VertexStatus.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/VertexStatus.java @@ -19,6 +19,7 @@ package org.apache.tez.dag.api.client; import java.util.List; +import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; import com.google.common.annotations.VisibleForTesting; @@ -127,6 +128,27 @@ public class VertexStatus { } @Override + public int hashCode() { + final int prime = 46021; + int result = 1; + result = prime + + getState().hashCode(); + + List diagnostics = getDiagnostics(); + Progress vProgress = getProgress(); + TezCounters counters = getVertexCounters(); + + result = prime * result + + ((diagnostics == null)? 0 : diagnostics.hashCode()); + result = prime * result + + ((vProgress == null)? 0 : vProgress.hashCode()); + result = prime * result + + ((counters == null)? 0 : counters.hashCode()); + + return result; + } + + @Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append("status=" + getState() http://git-wip-us.apache.org/repos/asf/tez/blob/393e40b8/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java index 09755fd..2ac89d5 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java @@ -75,7 +75,7 @@ public class DAGClientRPCImpl extends DAGClient { @Override public String getExecutionContext() { - return new String("Executing on YARN cluster with App id " + appId); + return "Executing on YARN cluster with App id " + appId; } @Override http://git-wip-us.apache.org/repos/asf/tez/blob/393e40b8/tez-api/src/test/java/org/apache/tez/dag/api/TestDagTypeConverters.java ---------------------------------------------------------------------- diff --git a/tez-api/src/test/java/org/apache/tez/dag/api/TestDagTypeConverters.java b/tez-api/src/test/java/org/apache/tez/dag/api/TestDagTypeConverters.java index 41d92fe..5d07eea 100644 --- a/tez-api/src/test/java/org/apache/tez/dag/api/TestDagTypeConverters.java +++ b/tez-api/src/test/java/org/apache/tez/dag/api/TestDagTypeConverters.java @@ -48,6 +48,10 @@ public class TestDagTypeConverters { InputDescriptor inputDescriptor = DagTypeConverters.convertInputDescriptorFromDAGPlan(proto); Assert.assertNull(inputDescriptor.getHistoryText()); + + // Check history text value + String actualHistoryText = DagTypeConverters.getHistoryTextFromProto(proto); + Assert.assertEquals(historytext, actualHistoryText); } }