accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bhava...@apache.org
Subject [09/12] git commit: Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT
Date Thu, 16 Jan 2014 20:16:58 GMT
Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT

Conflicts:
	start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileAttributes.java
	test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/CheckBalance.java

The conflict for HdfsFileAttributes above was Git erroneously merging updates for
ConcurrentFixture into it. Also, the State class merged cleanly but required the
addition of getInteger(String).


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9aa4b70e
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9aa4b70e
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9aa4b70e

Branch: refs/heads/master
Commit: 9aa4b70e260dd82d5da9bdb5381ff9d32bf0ed89
Parents: fe1348f cd4eac0
Author: Bill Havanki <bhavanki@cloudera.com>
Authored: Thu Jan 16 15:08:51 2014 -0500
Committer: Bill Havanki <bhavanki@cloudera.com>
Committed: Thu Jan 16 15:08:51 2014 -0500

----------------------------------------------------------------------
 .../apache/accumulo/test/randomwalk/State.java  |  8 +++++
 .../randomwalk/concurrent/CheckBalance.java     | 31 ++++++++++++--------
 .../concurrent/ConcurrentFixture.java           |  5 +++-
 3 files changed, 31 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/9aa4b70e/test/src/main/java/org/apache/accumulo/test/randomwalk/State.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/State.java
index 65d26c6,0000000..b799d04
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/State.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/State.java
@@@ -1,190 -1,0 +1,198 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test.randomwalk;
 +
 +import java.io.File;
 +import java.lang.management.ManagementFactory;
 +import java.util.HashMap;
 +import java.util.Properties;
 +import java.util.concurrent.TimeUnit;
 +
 +import org.apache.accumulo.core.client.AccumuloException;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.BatchWriterConfig;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Instance;
 +import org.apache.accumulo.core.client.MultiTableBatchWriter;
 +import org.apache.accumulo.core.client.MutationsRejectedException;
 +import org.apache.accumulo.core.client.ZooKeeperInstance;
 +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
 +import org.apache.accumulo.core.client.security.tokens.PasswordToken;
 +import org.apache.accumulo.core.security.CredentialHelper;
 +import org.apache.accumulo.core.security.thrift.TCredentials;
 +import org.apache.log4j.Logger;
 +
 +public class State {
 +  
 +  private static final Logger log = Logger.getLogger(State.class);
 +  private HashMap<String,Object> stateMap = new HashMap<String,Object>();
 +  private Properties props;
 +  private int numVisits = 0;
 +  private int maxVisits = Integer.MAX_VALUE;
 +  
 +  private MultiTableBatchWriter mtbw = null;
 +  private Connector connector = null;
 +  private Instance instance = null;
 +  
 +  State(Properties props) {
 +    this.props = props;
 +  }
 +  
 +  public void setMaxVisits(int num) {
 +    maxVisits = num;
 +  }
 +  
 +  public void visitedNode() throws Exception {
 +    numVisits++;
 +    if (numVisits > maxVisits) {
 +      log.debug("Visited max number (" + maxVisits + ") of nodes");
 +      throw new Exception("Visited max number (" + maxVisits + ") of nodes");
 +    }
 +  }
 +  
 +  public String getPid() {
 +    return ManagementFactory.getRuntimeMXBean().getName().split("@")[0];
 +  }
 +  
 +  public void set(String key, Object value) {
 +    stateMap.put(key, value);
 +  }
++
++  public void remove(String key) {
++    stateMap.remove(key);
++  }
 +  
 +  public Object get(String key) {
 +    if (stateMap.containsKey(key) == false) {
 +      throw new RuntimeException("State does not contain " + key);
 +    }
 +    return stateMap.get(key);
 +  }
 +  
 +  public HashMap<String,Object> getMap() {
 +    return stateMap;
 +  }
 +  
 +  /**
 +   * 
 +   * @return a copy of Properties, so accidental changes don't affect the framework
 +   */
 +  public Properties getProperties() {
 +    return new Properties(props);
 +  }
 +  
 +  public String getString(String key) {
 +    return (String) stateMap.get(key);
 +  }
 +  
++  public Integer getInteger(String key) {
++    return (Integer) stateMap.get(key);
++  }
++
 +  public Long getLong(String key) {
 +    return (Long) stateMap.get(key);
 +  }
 +  
 +  public String getProperty(String key) {
 +    return props.getProperty(key);
 +  }
 +  
 +  public Connector getConnector() throws AccumuloException, AccumuloSecurityException {
 +    if (connector == null) {
 +      connector = getInstance().getConnector(getUserName(), getToken());
 +    }
 +    return connector;
 +  }
 +  
 +  public TCredentials getCredentials() {
 +    String username = getUserName();
 +    AuthenticationToken password = getToken();
 +    return CredentialHelper.createSquelchError(username, password, getInstance().getInstanceID());
 +  }
 +  
 +  public String getUserName() {
 +    return props.getProperty("USERNAME");
 +  }
 +  
 +  public AuthenticationToken getToken() {
 +    return new PasswordToken(props.getProperty("PASSWORD"));
 +  }
 +  
 +  public Instance getInstance() {
 +    if (instance == null) {
 +      String instance = props.getProperty("INSTANCE");
 +      String zookeepers = props.getProperty("ZOOKEEPERS");
 +      this.instance = new ZooKeeperInstance(instance, zookeepers);
 +    }
 +    return instance;
 +  }
 +  
 +  public MultiTableBatchWriter getMultiTableBatchWriter() {
 +    if (mtbw == null) {
 +      long maxMem = Long.parseLong(props.getProperty("MAX_MEM"));
 +      long maxLatency = Long.parseLong(props.getProperty("MAX_LATENCY"));
 +      int numThreads = Integer.parseInt(props.getProperty("NUM_THREADS"));
 +      mtbw = connector.createMultiTableBatchWriter(new BatchWriterConfig().setMaxMemory(maxMem).setMaxLatency(maxLatency,
TimeUnit.MILLISECONDS)
 +          .setMaxWriteThreads(numThreads));
 +    }
 +    return mtbw;
 +  }
 +  
 +  public boolean isMultiTableBatchWriterInitialized() {
 +    return mtbw != null;
 +  }
 +
 +  public void resetMultiTableBatchWriter() {
 +    if (!mtbw.isClosed()) {
 +      log.warn("Setting non-closed MultiTableBatchWriter to null (leaking resources)");
 +    }
 +
 +    mtbw = null;
 +  }
 +
 +  public String getMapReduceJars() {
 +    
 +    String acuHome = System.getenv("ACCUMULO_HOME");
 +    String zkHome = System.getenv("ZOOKEEPER_HOME");
 +    
 +    if (acuHome == null || zkHome == null) {
 +      throw new RuntimeException("ACCUMULO or ZOOKEEPER home not set!");
 +    }
 +    
 +    String retval = null;
 +    
 +    File zkLib = new File(zkHome);
 +    String[] files = zkLib.list();
 +    for (int i = 0; i < files.length; i++) {
 +      String f = files[i];
 +      if (f.matches("^zookeeper-.+jar$")) {
 +        if (retval == null) {
 +          retval = String.format("%s/%s", zkLib.getAbsolutePath(), f);
 +        } else {
 +          retval += String.format(",%s/%s", zkLib.getAbsolutePath(), f);
 +        }
 +      }
 +    }
 +    
 +    File libdir = new File(acuHome + "/lib");
 +    for (String jar : "accumulo-core accumulo-server accumulo-fate accumulo-trace libthrift".split("
")) {
 +      retval += String.format(",%s/%s.jar", libdir.getAbsolutePath(), jar);      
 +    }
 +    
 +    return retval;
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9aa4b70e/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/CheckBalance.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/CheckBalance.java
index 3017d73,0000000..8278843
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/CheckBalance.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/CheckBalance.java
@@@ -1,91 -1,0 +1,98 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test.randomwalk.concurrent;
 +
 +import java.util.HashMap;
 +import java.util.Map;
 +import java.util.Map.Entry;
 +import java.util.Properties;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.test.randomwalk.State;
 +import org.apache.accumulo.test.randomwalk.Test;
 +
 +/**
 + * 
 + */
 +public class CheckBalance extends Test {
 +  
-   private static final String LAST_UNBALANCED_TIME = "lastUnbalancedTime";
-   private static final String UNBALANCED_COUNT = "unbalancedCount";
++  static final String LAST_UNBALANCED_TIME = "lastUnbalancedTime";
++  static final String UNBALANCED_COUNT = "unbalancedCount";
 +
 +  /* (non-Javadoc)
 +   * @see org.apache.accumulo.test.randomwalk.Node#visit(org.apache.accumulo.test.randomwalk.State,
java.util.Properties)
 +   */
 +  @Override
 +  public void visit(State state, Properties props) throws Exception {
 +    log.debug("checking balance");
 +    Map<String,Long> counts = new HashMap<String,Long>();
 +    Scanner scanner = state.getConnector().createScanner(Constants.METADATA_TABLE_NAME,
Constants.NO_AUTHS);
 +    scanner.fetchColumnFamily(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY);
 +    for (Entry<Key,Value> entry : scanner) {
 +      String location = entry.getKey().getColumnQualifier().toString();
 +      Long count = counts.get(location);
 +      if (count == null)
 +        count = new Long(0);
 +      counts.put(location, count + 1);
 +    }
 +    double total = 0.;
 +    for (Long count : counts.values()) {
 +      total += count.longValue();
 +    }
 +    final double average = total / counts.size();
 +    
 +    // Check for even # of tablets on each node
++    double maxDifference = Math.max(1, average / 5);
++    String unbalancedLocation = null;
++    long lastCount = 0L;
 +    boolean balanced = true;
 +    for (Entry<String,Long> entry : counts.entrySet()) {
-       if (Math.abs(entry.getValue().longValue() - average) > Math.max(1, average / 5))
{
++      long thisCount = entry.getValue().longValue();
++      if (Math.abs(thisCount - average) > maxDifference) {
 +        balanced = false;
 +        log.debug("unbalanced: " + entry.getKey() + " has " + entry.getValue() + " tablets
and the average is " + average);
++        unbalancedLocation = entry.getKey();
++        lastCount = thisCount;
 +      }
 +    }
 +    
 +    // It is expected that the number of tablets will be uneven for short
 +    // periods of time. Don't complain unless we've seen it only unbalanced
 +    // over a 15 minute period and it's been at least three checks.
 +    if (!balanced) {
-       String last = props.getProperty(LAST_UNBALANCED_TIME);
-       if (last != null && System.currentTimeMillis() - Long.parseLong(last) >
15 * 60 * 1000) {
-         String countString = props.getProperty(UNBALANCED_COUNT, "0");
-         int count = Integer.parseInt(countString);
++      Long last = state.getLong(LAST_UNBALANCED_TIME);
++      if (last != null && System.currentTimeMillis() - last > 15 * 60 * 1000)
{
++        Integer count = state.getInteger(UNBALANCED_COUNT);
++        if (count == null)
++          count = Integer.valueOf(0);
 +        if (count > 3)
-           throw new Exception("servers are unbalanced!");
++          throw new Exception("servers are unbalanced! location " + unbalancedLocation +
" count " + lastCount + " too far from average " + average);
 +        count++;
-         props.setProperty(UNBALANCED_COUNT, "" + count);
++        state.set(UNBALANCED_COUNT, count);
 +      }
 +      if (last == null)
-         props.setProperty(LAST_UNBALANCED_TIME, Long.toString(System.currentTimeMillis()));
++        state.set(LAST_UNBALANCED_TIME, System.currentTimeMillis());
 +    } else {
-       props.remove(LAST_UNBALANCED_TIME);
-       props.remove(UNBALANCED_COUNT);
++      state.remove(LAST_UNBALANCED_TIME);
++      state.remove(UNBALANCED_COUNT);
 +    }
 +  }
 +  
 +}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9aa4b70e/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/ConcurrentFixture.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/ConcurrentFixture.java
index 7438ce6,0000000..df7ddd7
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/ConcurrentFixture.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/ConcurrentFixture.java
@@@ -1,74 -1,0 +1,77 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test.randomwalk.concurrent;
 +
 +import java.util.ArrayList;
 +import java.util.List;
 +import java.util.Random;
 +
 +import org.apache.accumulo.test.randomwalk.Fixture;
 +import org.apache.accumulo.test.randomwalk.State;
 +import org.apache.hadoop.io.Text;
 +
 +/**
 + * When multiple instance of this test suite are run, all instances will operate on the
same set of table names.
 + * 
 + * 
 + */
 +
 +public class ConcurrentFixture extends Fixture {
 +  
 +  @Override
 +  public void setUp(State state) throws Exception {}
 +  
 +  @Override
-   public void tearDown(State state) throws Exception {}
++  public void tearDown(State state) throws Exception {
++    state.remove(CheckBalance.LAST_UNBALANCED_TIME);
++    state.remove(CheckBalance.UNBALANCED_COUNT);
++  }
 +  
 +  /**
 +   * 
 +   * @param rand
 +   *  A Random to use
 +   * @return
 +   *  A two element list with first being smaller than the second, but either value (or
both) can be null
 +   */
 +  public static List<Text> generateRange(Random rand) {
 +    ArrayList<Text> toRet = new ArrayList<Text>(2);
 +
 +    long firstLong = rand.nextLong();
 +    
 +    
 +    long secondLong = rand.nextLong();
 +    Text first = null, second = null;
 +    
 +    // Having all negative values = null might be too frequent
 +    if (firstLong >= 0)
 +      first = new Text(String.format("%016x", firstLong & 0x7fffffffffffffffl));
 +    if (secondLong >= 0)
 +      second = new Text(String.format("%016x", secondLong & 0x7fffffffffffffffl));
 +
 +    if (first != null && second != null && first.compareTo(second) >
0) {
 +      Text swap = first;
 +      first = second;
 +      second = swap;
 +    }
 +    
 +    toRet.add(first);
 +    toRet.add(second);
 +    
 +    return toRet;
 +  }
 +}


Mime
View raw message