Return-Path: X-Original-To: apmail-accumulo-commits-archive@www.apache.org Delivered-To: apmail-accumulo-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 425BA10093 for ; Thu, 16 Jan 2014 20:17:05 +0000 (UTC) Received: (qmail 68379 invoked by uid 500); 16 Jan 2014 20:16:57 -0000 Delivered-To: apmail-accumulo-commits-archive@accumulo.apache.org Received: (qmail 68308 invoked by uid 500); 16 Jan 2014 20:16:55 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 68092 invoked by uid 99); 16 Jan 2014 20:16:51 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 16 Jan 2014 20:16:51 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id AED03524D7; Thu, 16 Jan 2014 20:16:50 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: bhavanki@apache.org To: commits@accumulo.apache.org Date: Thu, 16 Jan 2014 20:16:57 -0000 Message-Id: <61007c37ff6743739777bb9f2596be14@git.apache.org> In-Reply-To: <13cf46688a3548d68f02804c992093e4@git.apache.org> References: <13cf46688a3548d68f02804c992093e4@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [08/12] git commit: Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT Merge branch '1.4.5-SNAPSHOT' into 1.5.1-SNAPSHOT Conflicts: start/src/main/java/org/apache/accumulo/start/classloader/vfs/providers/HdfsFileAttributes.java test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/CheckBalance.java The conflict for HdfsFileAttributes above was Git erroneously merging updates for ConcurrentFixture into it. Also, the State class merged cleanly but required the addition of getInteger(String). Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9aa4b70e Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9aa4b70e Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9aa4b70e Branch: refs/heads/1.6.0-SNAPSHOT Commit: 9aa4b70e260dd82d5da9bdb5381ff9d32bf0ed89 Parents: fe1348f cd4eac0 Author: Bill Havanki Authored: Thu Jan 16 15:08:51 2014 -0500 Committer: Bill Havanki Committed: Thu Jan 16 15:08:51 2014 -0500 ---------------------------------------------------------------------- .../apache/accumulo/test/randomwalk/State.java | 8 +++++ .../randomwalk/concurrent/CheckBalance.java | 31 ++++++++++++-------- .../concurrent/ConcurrentFixture.java | 5 +++- 3 files changed, 31 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/9aa4b70e/test/src/main/java/org/apache/accumulo/test/randomwalk/State.java ---------------------------------------------------------------------- diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/State.java index 65d26c6,0000000..b799d04 mode 100644,000000..100644 --- a/test/src/main/java/org/apache/accumulo/test/randomwalk/State.java +++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/State.java @@@ -1,190 -1,0 +1,198 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.test.randomwalk; + +import java.io.File; +import java.lang.management.ManagementFactory; +import java.util.HashMap; +import java.util.Properties; +import java.util.concurrent.TimeUnit; + +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.BatchWriterConfig; +import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.client.Instance; +import org.apache.accumulo.core.client.MultiTableBatchWriter; +import org.apache.accumulo.core.client.MutationsRejectedException; +import org.apache.accumulo.core.client.ZooKeeperInstance; +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; +import org.apache.accumulo.core.client.security.tokens.PasswordToken; +import org.apache.accumulo.core.security.CredentialHelper; +import org.apache.accumulo.core.security.thrift.TCredentials; +import org.apache.log4j.Logger; + +public class State { + + private static final Logger log = Logger.getLogger(State.class); + private HashMap stateMap = new HashMap(); + private Properties props; + private int numVisits = 0; + private int maxVisits = Integer.MAX_VALUE; + + private MultiTableBatchWriter mtbw = null; + private Connector connector = null; + private Instance instance = null; + + State(Properties props) { + this.props = props; + } + + public void setMaxVisits(int num) { + maxVisits = num; + } + + public void visitedNode() throws Exception { + numVisits++; + if (numVisits > maxVisits) { + log.debug("Visited max number (" + maxVisits + ") of nodes"); + throw new Exception("Visited max number (" + maxVisits + ") of nodes"); + } + } + + public String getPid() { + return ManagementFactory.getRuntimeMXBean().getName().split("@")[0]; + } + + public void set(String key, Object value) { + stateMap.put(key, value); + } ++ ++ public void remove(String key) { ++ stateMap.remove(key); ++ } + + public Object get(String key) { + if (stateMap.containsKey(key) == false) { + throw new RuntimeException("State does not contain " + key); + } + return stateMap.get(key); + } + + public HashMap getMap() { + return stateMap; + } + + /** + * + * @return a copy of Properties, so accidental changes don't affect the framework + */ + public Properties getProperties() { + return new Properties(props); + } + + public String getString(String key) { + return (String) stateMap.get(key); + } + ++ public Integer getInteger(String key) { ++ return (Integer) stateMap.get(key); ++ } ++ + public Long getLong(String key) { + return (Long) stateMap.get(key); + } + + public String getProperty(String key) { + return props.getProperty(key); + } + + public Connector getConnector() throws AccumuloException, AccumuloSecurityException { + if (connector == null) { + connector = getInstance().getConnector(getUserName(), getToken()); + } + return connector; + } + + public TCredentials getCredentials() { + String username = getUserName(); + AuthenticationToken password = getToken(); + return CredentialHelper.createSquelchError(username, password, getInstance().getInstanceID()); + } + + public String getUserName() { + return props.getProperty("USERNAME"); + } + + public AuthenticationToken getToken() { + return new PasswordToken(props.getProperty("PASSWORD")); + } + + public Instance getInstance() { + if (instance == null) { + String instance = props.getProperty("INSTANCE"); + String zookeepers = props.getProperty("ZOOKEEPERS"); + this.instance = new ZooKeeperInstance(instance, zookeepers); + } + return instance; + } + + public MultiTableBatchWriter getMultiTableBatchWriter() { + if (mtbw == null) { + long maxMem = Long.parseLong(props.getProperty("MAX_MEM")); + long maxLatency = Long.parseLong(props.getProperty("MAX_LATENCY")); + int numThreads = Integer.parseInt(props.getProperty("NUM_THREADS")); + mtbw = connector.createMultiTableBatchWriter(new BatchWriterConfig().setMaxMemory(maxMem).setMaxLatency(maxLatency, TimeUnit.MILLISECONDS) + .setMaxWriteThreads(numThreads)); + } + return mtbw; + } + + public boolean isMultiTableBatchWriterInitialized() { + return mtbw != null; + } + + public void resetMultiTableBatchWriter() { + if (!mtbw.isClosed()) { + log.warn("Setting non-closed MultiTableBatchWriter to null (leaking resources)"); + } + + mtbw = null; + } + + public String getMapReduceJars() { + + String acuHome = System.getenv("ACCUMULO_HOME"); + String zkHome = System.getenv("ZOOKEEPER_HOME"); + + if (acuHome == null || zkHome == null) { + throw new RuntimeException("ACCUMULO or ZOOKEEPER home not set!"); + } + + String retval = null; + + File zkLib = new File(zkHome); + String[] files = zkLib.list(); + for (int i = 0; i < files.length; i++) { + String f = files[i]; + if (f.matches("^zookeeper-.+jar$")) { + if (retval == null) { + retval = String.format("%s/%s", zkLib.getAbsolutePath(), f); + } else { + retval += String.format(",%s/%s", zkLib.getAbsolutePath(), f); + } + } + } + + File libdir = new File(acuHome + "/lib"); + for (String jar : "accumulo-core accumulo-server accumulo-fate accumulo-trace libthrift".split(" ")) { + retval += String.format(",%s/%s.jar", libdir.getAbsolutePath(), jar); + } + + return retval; + } +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/9aa4b70e/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/CheckBalance.java ---------------------------------------------------------------------- diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/CheckBalance.java index 3017d73,0000000..8278843 mode 100644,000000..100644 --- a/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/CheckBalance.java +++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/CheckBalance.java @@@ -1,91 -1,0 +1,98 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.test.randomwalk.concurrent; + +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Properties; + +import org.apache.accumulo.core.Constants; +import org.apache.accumulo.core.client.Scanner; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.test.randomwalk.State; +import org.apache.accumulo.test.randomwalk.Test; + +/** + * + */ +public class CheckBalance extends Test { + - private static final String LAST_UNBALANCED_TIME = "lastUnbalancedTime"; - private static final String UNBALANCED_COUNT = "unbalancedCount"; ++ static final String LAST_UNBALANCED_TIME = "lastUnbalancedTime"; ++ static final String UNBALANCED_COUNT = "unbalancedCount"; + + /* (non-Javadoc) + * @see org.apache.accumulo.test.randomwalk.Node#visit(org.apache.accumulo.test.randomwalk.State, java.util.Properties) + */ + @Override + public void visit(State state, Properties props) throws Exception { + log.debug("checking balance"); + Map counts = new HashMap(); + Scanner scanner = state.getConnector().createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS); + scanner.fetchColumnFamily(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY); + for (Entry entry : scanner) { + String location = entry.getKey().getColumnQualifier().toString(); + Long count = counts.get(location); + if (count == null) + count = new Long(0); + counts.put(location, count + 1); + } + double total = 0.; + for (Long count : counts.values()) { + total += count.longValue(); + } + final double average = total / counts.size(); + + // Check for even # of tablets on each node ++ double maxDifference = Math.max(1, average / 5); ++ String unbalancedLocation = null; ++ long lastCount = 0L; + boolean balanced = true; + for (Entry entry : counts.entrySet()) { - if (Math.abs(entry.getValue().longValue() - average) > Math.max(1, average / 5)) { ++ long thisCount = entry.getValue().longValue(); ++ if (Math.abs(thisCount - average) > maxDifference) { + balanced = false; + log.debug("unbalanced: " + entry.getKey() + " has " + entry.getValue() + " tablets and the average is " + average); ++ unbalancedLocation = entry.getKey(); ++ lastCount = thisCount; + } + } + + // It is expected that the number of tablets will be uneven for short + // periods of time. Don't complain unless we've seen it only unbalanced + // over a 15 minute period and it's been at least three checks. + if (!balanced) { - String last = props.getProperty(LAST_UNBALANCED_TIME); - if (last != null && System.currentTimeMillis() - Long.parseLong(last) > 15 * 60 * 1000) { - String countString = props.getProperty(UNBALANCED_COUNT, "0"); - int count = Integer.parseInt(countString); ++ Long last = state.getLong(LAST_UNBALANCED_TIME); ++ if (last != null && System.currentTimeMillis() - last > 15 * 60 * 1000) { ++ Integer count = state.getInteger(UNBALANCED_COUNT); ++ if (count == null) ++ count = Integer.valueOf(0); + if (count > 3) - throw new Exception("servers are unbalanced!"); ++ throw new Exception("servers are unbalanced! location " + unbalancedLocation + " count " + lastCount + " too far from average " + average); + count++; - props.setProperty(UNBALANCED_COUNT, "" + count); ++ state.set(UNBALANCED_COUNT, count); + } + if (last == null) - props.setProperty(LAST_UNBALANCED_TIME, Long.toString(System.currentTimeMillis())); ++ state.set(LAST_UNBALANCED_TIME, System.currentTimeMillis()); + } else { - props.remove(LAST_UNBALANCED_TIME); - props.remove(UNBALANCED_COUNT); ++ state.remove(LAST_UNBALANCED_TIME); ++ state.remove(UNBALANCED_COUNT); + } + } + +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/9aa4b70e/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/ConcurrentFixture.java ---------------------------------------------------------------------- diff --cc test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/ConcurrentFixture.java index 7438ce6,0000000..df7ddd7 mode 100644,000000..100644 --- a/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/ConcurrentFixture.java +++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/ConcurrentFixture.java @@@ -1,74 -1,0 +1,77 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.test.randomwalk.concurrent; + +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +import org.apache.accumulo.test.randomwalk.Fixture; +import org.apache.accumulo.test.randomwalk.State; +import org.apache.hadoop.io.Text; + +/** + * When multiple instance of this test suite are run, all instances will operate on the same set of table names. + * + * + */ + +public class ConcurrentFixture extends Fixture { + + @Override + public void setUp(State state) throws Exception {} + + @Override - public void tearDown(State state) throws Exception {} ++ public void tearDown(State state) throws Exception { ++ state.remove(CheckBalance.LAST_UNBALANCED_TIME); ++ state.remove(CheckBalance.UNBALANCED_COUNT); ++ } + + /** + * + * @param rand + * A Random to use + * @return + * A two element list with first being smaller than the second, but either value (or both) can be null + */ + public static List generateRange(Random rand) { + ArrayList toRet = new ArrayList(2); + + long firstLong = rand.nextLong(); + + + long secondLong = rand.nextLong(); + Text first = null, second = null; + + // Having all negative values = null might be too frequent + if (firstLong >= 0) + first = new Text(String.format("%016x", firstLong & 0x7fffffffffffffffl)); + if (secondLong >= 0) + second = new Text(String.format("%016x", secondLong & 0x7fffffffffffffffl)); + + if (first != null && second != null && first.compareTo(second) > 0) { + Text swap = first; + first = second; + second = swap; + } + + toRet.add(first); + toRet.add(second); + + return toRet; + } +}