accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e..@apache.org
Subject [08/34] accumulo git commit: ACCUMULO-3638 merge master branch
Date Fri, 24 Apr 2015 23:20:54 GMT
ACCUMULO-3638 merge master branch


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/4635de86
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/4635de86
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/4635de86

Branch: refs/heads/master
Commit: 4635de8671a62f0a412da775db05519f6831daaa
Parents: 98c3cef 9f108c0
Author: Eric C. Newton <eric.newton@gmail.com>
Authored: Tue Mar 10 16:16:53 2015 -0400
Committer: Eric C. Newton <eric.newton@gmail.com>
Committed: Tue Mar 10 16:16:53 2015 -0400

----------------------------------------------------------------------
 core/pom.xml                                    |  7 --
 core/src/main/findbugs/exclude-filter.xml       | 67 +++++++++++++-----
 .../client/admin/CompactionStrategyConfig.java  |  5 ++
 .../client/impl/MultiTableBatchWriterImpl.java  | 13 ++--
 .../core/client/impl/TabletLocatorImpl.java     | 41 +++++++----
 .../client/impl/TabletServerBatchWriter.java    | 45 ++++++++----
 .../core/client/mock/MockNamespace.java         |  2 +-
 .../client/security/tokens/DelegationToken.java | 16 ++---
 .../client/security/tokens/PasswordToken.java   | 13 ++--
 .../apache/accumulo/core/conf/PropertyType.java |  2 +-
 .../org/apache/accumulo/core/data/Value.java    |  1 +
 .../apache/accumulo/core/volume/VolumeImpl.java |  6 ++
 .../accumulo/core/cli/TestClientOpts.java       |  2 +-
 .../core/file/rfile/RFileMetricsTest.java       | 73 +++++++++-----------
 .../iterators/user/ColumnSliceFilterTest.java   |  2 +-
 .../core/iterators/user/RowFilterTest.java      |  2 +-
 .../simple/src/main/findbugs/exclude-filter.xml | 18 +++++
 .../examples/simple/mapreduce/RowHash.java      |  2 +-
 fate/src/main/findbugs/exclude-filter.xml       | 18 +++++
 .../accumulo/fate/zookeeper/ZooReader.java      | 14 ++--
 .../zookeeper/DistributedReadWriteLockTest.java | 17 +++--
 .../src/main/findbugs/exclude-filter.xml        | 18 +++++
 .../src/main/findbugs/exclude-filter.xml        | 30 ++++++++
 pom.xml                                         |  3 +-
 proxy/pom.xml                                   | 13 ----
 proxy/src/main/findbugs/exclude-filter.xml      |  7 +-
 .../base/src/main/findbugs/exclude-filter.xml   | 26 +++++++
 .../server/master/balancer/GroupBalancer.java   |  6 ++
 .../master/state/TabletLocationState.java       |  3 +-
 .../server/metrics/MetricsConfiguration.java    |  7 +-
 .../TCredentialsUpdatingInvocationHandler.java  |  5 +-
 .../server/security/SecurityOperation.java      |  7 --
 .../security/delegation/AuthenticationKey.java  | 11 +--
 .../security/handler/KerberosAuthenticator.java |  4 +-
 .../server/watcher/Log4jConfiguration.java      |  5 +-
 .../accumulo/server/ServerConstantsTest.java    | 12 +++-
 .../AuthenticationTokenKeyManagerTest.java      | 34 +++++----
 server/gc/src/main/findbugs/exclude-filter.xml  | 18 +++++
 .../gc/GarbageCollectWriteAheadLogs.java        |  2 +-
 .../master/src/main/findbugs/exclude-filter.xml | 23 ++++++
 .../java/org/apache/accumulo/master/Master.java | 24 ++++---
 .../RemoveCompleteReplicationRecords.java       |  3 +
 .../src/main/findbugs/exclude-filter.xml        | 18 +++++
 .../monitor/servlets/OperationServlet.java      | 53 +++++++++-----
 server/tracer/pom.xml                           | 13 ----
 .../tracer/src/main/findbugs/exclude-filter.xml |  7 +-
 .../src/main/findbugs/exclude-filter.xml        | 24 +++++++
 .../accumulo/server/logger/LogFileKey.java      | 25 -------
 .../accumulo/server/logger/LogFileValue.java    | 25 -------
 .../accumulo/tserver/log/LocalWALRecovery.java  |  4 +-
 .../replication/AccumuloReplicaSystem.java      | 19 ++++-
 .../apache/accumulo/tserver/scan/ScanTask.java  | 30 ++++++--
 .../apache/accumulo/tserver/tablet/Tablet.java  | 14 ++--
 .../tserver/tablet/TabletCommitter.java         |  3 +
 shell/src/main/findbugs/exclude-filter.xml      | 18 +++++
 .../apache/accumulo/shell/ShellConfigTest.java  | 29 ++++++--
 .../org/apache/accumulo/shell/ShellTest.java    | 11 ++-
 .../shell/commands/FormatterCommandTest.java    |  6 +-
 start/src/main/findbugs/exclude-filter.xml      | 18 +++++
 .../start/classloader/vfs/MiniDFSUtil.java      |  4 ++
 test/src/main/findbugs/exclude-filter.xml       | 30 ++++++++
 .../server/security/SystemCredentialsIT.java    |  2 +
 .../test/MasterRepairsDualAssignmentIT.java     |  2 +-
 .../accumulo/test/MultiTableRecoveryIT.java     |  3 +-
 .../java/org/apache/accumulo/test/UsersIT.java  | 60 ++++++++++++++++
 .../apache/accumulo/test/functional/BulkIT.java | 13 ++--
 .../test/functional/ConfigurableMacIT.java      |  4 +-
 .../accumulo/test/functional/MonitorSslIT.java  | 40 ++++++-----
 .../accumulo/test/functional/PermissionsIT.java | 43 ++++++++----
 .../apache/accumulo/test/functional/SslIT.java  |  5 +-
 trace/src/main/findbugs/exclude-filter.xml      | 26 +++++++
 71 files changed, 810 insertions(+), 366 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/4635de86/server/base/src/main/java/org/apache/accumulo/server/master/state/TabletLocationState.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/master/state/TabletLocationState.java
index 3ece3c9,b24b562..ebad2c8
--- a/server/base/src/main/java/org/apache/accumulo/server/master/state/TabletLocationState.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/state/TabletLocationState.java
@@@ -33,8 -32,6 +32,8 @@@ import org.apache.hadoop.io.Text
   */
  public class TabletLocationState {
  
-   private static final Logger log = Logger.getLogger(TabletLocationState.class);
++  // private static final Logger log = Logger.getLogger(TabletLocationState.class);
 +
    static public class BadLocationStateException extends Exception {
      private static final long serialVersionUID = 1L;
      private Text metadataTableEntry;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/4635de86/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
----------------------------------------------------------------------
diff --cc server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
index 444789b,35c60d6..f44a9d1
--- a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
@@@ -214,39 -268,76 +214,39 @@@ public class GarbageCollectWriteAheadLo
          }
        }
      }
 -
 -    return 0;
 +    return status.currentLog.deleted;
    }
  
 -  /**
 -   * Converts a list of paths to their corresponding strings.
 -   *
 -   * @param paths
 -   *          list of paths
 -   * @return string forms of paths
 -   */
 -  static List<String> paths2strings(List<Path> paths) {
 -    List<String> result = new ArrayList<String>(paths.size());
 -    for (Path path : paths)
 -      result.add(path.toString());
 -    return result;
 -  }
 +  private long removeMetadataEntries(Map<TServerInstance, Set<Path> > candidates,
GCStatus status, Set<TServerInstance> liveServers) throws IOException, KeeperException,
 +      InterruptedException {
  
 -  /**
 -   * Reverses the given mapping of file paths to servers. The returned map provides a list
of file paths for each server. Any path whose name is not in the
 -   * mapping of file names to paths is skipped.
 -   *
 -   * @param fileToServerMap
 -   *          map of file paths to servers
 -   * @param nameToFileMap
 -   *          map of file names to paths
 -   * @return map of servers to lists of file paths
 -   */
 -  static Map<String,ArrayList<Path>> mapServersToFiles(Map<Path,String>
fileToServerMap, Map<String,Path> nameToFileMap) {
 -    Map<String,ArrayList<Path>> result = new HashMap<String,ArrayList<Path>>();
 -    for (Entry<Path,String> fileServer : fileToServerMap.entrySet()) {
 -      if (!nameToFileMap.containsKey(fileServer.getKey().getName()))
 -        continue;
 -      ArrayList<Path> files = result.get(fileServer.getValue());
 -      if (files == null) {
 -        files = new ArrayList<Path>();
 -        result.put(fileServer.getValue(), files);
 +    // remove any entries if there's a log reference, or a tablet is still assigned to the
dead server
 +
 +    Map<Path, TServerInstance> walToDeadServer = new HashMap<>();
 +    for (Entry<TServerInstance,Set<Path>> entry : candidates.entrySet()) {
 +      for (Path file : entry.getValue()) {
 +        walToDeadServer.put(file, entry.getKey());
        }
 -      files.add(fileServer.getKey());
      }
 -    return result;
 -  }
 -
 -  protected int removeMetadataEntries(Map<String,Path> nameToFileMap, Map<String,Path>
sortedWALogs, GCStatus status) throws IOException, KeeperException,
 -      InterruptedException {
 -    int count = 0;
 -    Iterator<LogEntry> iterator = MetadataTableUtil.getLogEntries(context);
 -
 -    // For each WAL reference in the metadata table
 -    while (iterator.hasNext()) {
 -      // Each metadata reference has at least one WAL file
 -      for (String entry : iterator.next().logSet) {
 -        // old style WALs will have the IP:Port of their logger and new style will either
be a Path either absolute or relative, in all cases
 -        // the last "/" will mark a UUID file name.
 -        String uuid = entry.substring(entry.lastIndexOf("/") + 1);
 -        if (!isUUID(uuid)) {
 -          // fully expect this to be a uuid, if its not then something is wrong and walog
GC should not proceed!
 -          throw new IllegalArgumentException("Expected uuid, but got " + uuid + " from "
+ entry);
 -        }
 -
 -        Path pathFromNN = nameToFileMap.remove(uuid);
 -        if (pathFromNN != null) {
 -          status.currentLog.inUse++;
 -          sortedWALogs.remove(uuid);
 +    long count = 0;
 +    RootTabletStateStore root = new RootTabletStateStore(context);
 +    MetaDataStateStore meta = new MetaDataStateStore(context);
 +    Iterator<TabletLocationState> states = Iterators.concat(root.iterator(), meta.iterator());
 +    while (states.hasNext()) {
 +      count++;
 +      TabletLocationState state = states.next();
 +      if (state.getState(liveServers) == TabletState.ASSIGNED_TO_DEAD_SERVER) {
 +        candidates.remove(state.current);
 +      }
 +      for (Collection<String> wals : state.walogs) {
 +        for (String wal : wals) {
-           TServerInstance dead = walToDeadServer.get(wal);
++          TServerInstance dead = walToDeadServer.get(new Path(wal));
 +          if (dead != null) {
 +            candidates.get(dead).remove(wal);
 +          }
          }
 -
 -        count++;
        }
      }
 -
      return count;
    }
  

http://git-wip-us.apache.org/repos/asf/accumulo/blob/4635de86/server/master/src/main/java/org/apache/accumulo/master/Master.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/4635de86/server/tserver/src/main/findbugs/exclude-filter.xml
----------------------------------------------------------------------
diff --cc server/tserver/src/main/findbugs/exclude-filter.xml
index 0000000,aec413a..45f6a78
mode 000000,100644..100644
--- a/server/tserver/src/main/findbugs/exclude-filter.xml
+++ b/server/tserver/src/main/findbugs/exclude-filter.xml
@@@ -1,0 -1,24 +1,24 @@@
+ <!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+ 
+       http://www.apache.org/licenses/LICENSE-2.0
+ 
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+ -->
+ <FindBugsFilter>
+   <Match>
+     <!-- locking is confusing, but probably correct -->
+     <Class name="org.apache.accumulo.tserver.tablet.Tablet" />
 -    <Method name="beginUpdatingLogsUsed" params="org.apache.accumulo.tserver.InMemoryMap,java.util.Collection,boolean"
returns="boolean" />
++    <Method name="beginUpdatingLogsUsed" params="org.apache.accumulo.tserver.InMemoryMap,org.apache.accumulo.tserver.log.DfsLogger,boolean"
returns="boolean" />
+     <Bug code="UL" pattern="UL_UNRELEASED_LOCK" />
+   </Match>
+ </FindBugsFilter>

http://git-wip-us.apache.org/repos/asf/accumulo/blob/4635de86/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index 0c1edfa,95fe24a..e964719
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@@ -2442,8 -2441,9 +2440,9 @@@ public class Tablet implements TabletCo
      return currentLogs.size();
    }
  
+   /* don't release the lock if this method returns true for success; instead, the caller
should clean up by calling finishUpdatingLogsUsed() */
    @Override
 -  public boolean beginUpdatingLogsUsed(InMemoryMap memTable, Collection<DfsLogger>
more, boolean mincFinish) {
 +  public boolean beginUpdatingLogsUsed(InMemoryMap memTable, DfsLogger more, boolean mincFinish)
{
  
      boolean releaseLock = true;
  

http://git-wip-us.apache.org/repos/asf/accumulo/blob/4635de86/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletCommitter.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletCommitter.java
index 39bde5c,4bc05a6..91a955e
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletCommitter.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletCommitter.java
@@@ -34,7 -35,10 +34,10 @@@ public interface TabletCommitter 
  
    void commit(CommitSession commitSession, List<Mutation> mutations);
  
+   /**
+    * If this method returns true, the caller must call {@link #finishUpdatingLogsUsed()}
to clean up
+    */
 -  boolean beginUpdatingLogsUsed(InMemoryMap memTable, Collection<DfsLogger> copy,
boolean mincFinish);
 +  boolean beginUpdatingLogsUsed(InMemoryMap memTable, DfsLogger copy, boolean mincFinish);
  
    void finishUpdatingLogsUsed();
  


Mime
View raw message