Return-Path: X-Original-To: apmail-accumulo-commits-archive@www.apache.org Delivered-To: apmail-accumulo-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 1787E1052F for ; Thu, 11 Apr 2013 17:57:24 +0000 (UTC) Received: (qmail 8547 invoked by uid 500); 11 Apr 2013 17:57:24 -0000 Delivered-To: apmail-accumulo-commits-archive@accumulo.apache.org Received: (qmail 8529 invoked by uid 500); 11 Apr 2013 17:57:24 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 8522 invoked by uid 99); 11 Apr 2013 17:57:23 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 11 Apr 2013 17:57:23 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 11 Apr 2013 17:57:20 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 69E7D2388847; Thu, 11 Apr 2013 17:56:58 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1467010 - in /accumulo/branches/1.5: server/src/main/java/org/apache/accumulo/server/master/ server/src/main/java/org/apache/accumulo/server/master/state/ test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/ test/src/test/jav... Date: Thu, 11 Apr 2013 17:56:58 -0000 To: commits@accumulo.apache.org From: ecn@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20130411175658.69E7D2388847@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: ecn Date: Thu Apr 11 17:56:57 2013 New Revision: 1467010 URL: http://svn.apache.org/r1467010 Log: ACCUMULO-1264 tweak the scan range when looking for !METADATA tablet consistency check; add !METADATA table merge/split to the randomwalk test Added: accumulo/branches/1.5/test/src/test/java/org/apache/accumulo/test/MetaSplitTest.java (with props) Modified: accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/master/Master.java accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/master/state/MergeStats.java accumulo/branches/1.5/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/AddSplits.java accumulo/branches/1.5/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/Merge.java Modified: accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/master/Master.java URL: http://svn.apache.org/viewvc/accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/master/Master.java?rev=1467010&r1=1467009&r2=1467010&view=diff ============================================================================== --- accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/master/Master.java (original) +++ accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/master/Master.java Thu Apr 11 17:56:57 2013 @@ -1203,6 +1203,7 @@ public class Master implements LiveTServ } // Handle merge transitions if (mergeInfo.getRange() != null) { + log.debug("mergeInfo overlaps: " + extent + " " + mergeInfo.overlaps(extent)); if (mergeInfo.overlaps(extent)) { switch (mergeInfo.getState()) { case NONE: @@ -2094,8 +2095,11 @@ public class Master implements LiveTServ }); TCredentials systemAuths = SecurityConstants.getSystemCredentials(); - final TabletStateStore stores[] = {new ZooTabletStateStore(new ZooStore(zroot)), new RootTabletStateStore(instance, systemAuths, this), - new MetaDataStateStore(instance, systemAuths, this)}; + final TabletStateStore stores[] = { + new ZooTabletStateStore(new ZooStore(zroot)), + new RootTabletStateStore(instance, systemAuths, this), + new MetaDataStateStore(instance, systemAuths, this) + }; watchers.add(new TabletGroupWatcher(stores[2], null)); watchers.add(new TabletGroupWatcher(stores[1], watchers.get(0))); watchers.add(new TabletGroupWatcher(stores[0], watchers.get(1))); Modified: accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/master/state/MergeStats.java URL: http://svn.apache.org/viewvc/accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/master/state/MergeStats.java?rev=1467010&r1=1467009&r2=1467010&view=diff ============================================================================== --- accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/master/state/MergeStats.java (original) +++ accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/master/state/MergeStats.java Thu Apr 11 17:56:57 2013 @@ -28,6 +28,7 @@ import org.apache.accumulo.core.client.S import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.KeyExtent; +import org.apache.accumulo.core.data.PartialKey; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.zookeeper.ZooUtil; @@ -182,11 +183,14 @@ public class MergeStats { Text tableId = extent.getTableId(); Text first = KeyExtent.getMetadataEntry(tableId, start); Range range = new Range(first, false, null, true); - if (extent.isMeta()) - range = new Range(Constants.METADATA_ROOT_TABLET_KEYSPACE); + if (extent.isMeta()) { + // don't go off the root tablet + range = new Range(new Key(first).followingKey(PartialKey.ROW), false, Constants.METADATA_ROOT_TABLET_KEYSPACE.getEndKey(), false); + } scanner.setRange(range); KeyExtent prevExtent = null; + log.debug("Scanning range " + range); for (Entry entry : scanner) { TabletLocationState tls; try { @@ -195,24 +199,30 @@ public class MergeStats { log.error(e, e); return false; } + log.debug("consistency check: " + tls + " walogs " + tls.walogs.size()); if (!tls.extent.getTableId().equals(tableId)) { break; } if (!tls.walogs.isEmpty() && verify.getMergeInfo().needsToBeChopped(tls.extent)) { + log.debug("failing consistency: needs to be chopped" + tls.extent); return false; } if (prevExtent == null) { // this is the first tablet observed, it must be offline and its prev row must be less than the start of the merge range if (tls.extent.getPrevEndRow() != null && tls.extent.getPrevEndRow().compareTo(start) > 0) { + log.debug("failing consistency: prev row is too high " + start); return false; } - if (tls.getState(master.onlineTabletServers()) != TabletState.UNASSIGNED) + if (tls.getState(master.onlineTabletServers()) != TabletState.UNASSIGNED) { + log.debug("failing consistency: assigned or hosted " + tls); return false; + } } else if (!tls.extent.isPreviousExtent(prevExtent)) { + log.debug("hole in !METADATA"); return false; } @@ -224,6 +234,9 @@ public class MergeStats { break; } } + log.debug("chopped " + chopped + " v.chopped " + verify.chopped + + " unassigned " + unassigned + " v.unassigned " + verify.unassigned + + " verify.total " + verify.total); return chopped == verify.chopped && unassigned == verify.unassigned && unassigned == verify.total; } Modified: accumulo/branches/1.5/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/AddSplits.java URL: http://svn.apache.org/viewvc/accumulo/branches/1.5/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/AddSplits.java?rev=1467010&r1=1467009&r2=1467010&view=diff ============================================================================== --- accumulo/branches/1.5/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/AddSplits.java (original) +++ accumulo/branches/1.5/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/AddSplits.java Thu Apr 11 17:56:57 2013 @@ -16,11 +16,13 @@ */ package org.apache.accumulo.test.randomwalk.concurrent; +import java.util.ArrayList; import java.util.List; import java.util.Properties; import java.util.Random; import java.util.TreeSet; +import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.client.TableOfflineException; @@ -38,7 +40,8 @@ public class AddSplits extends Test { @SuppressWarnings("unchecked") List tableNames = (List) state.get("tables"); - + tableNames = new ArrayList(tableNames); + tableNames.add(Constants.METADATA_TABLE_NAME); String tableName = tableNames.get(rand.nextInt(tableNames.size())); TreeSet splits = new TreeSet(); Modified: accumulo/branches/1.5/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/Merge.java URL: http://svn.apache.org/viewvc/accumulo/branches/1.5/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/Merge.java?rev=1467010&r1=1467009&r2=1467010&view=diff ============================================================================== --- accumulo/branches/1.5/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/Merge.java (original) +++ accumulo/branches/1.5/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/Merge.java Thu Apr 11 17:56:57 2013 @@ -16,11 +16,13 @@ */ package org.apache.accumulo.test.randomwalk.concurrent; +import java.util.ArrayList; import java.util.List; import java.util.Properties; import java.util.Random; import java.util.TreeSet; +import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.client.TableOfflineException; @@ -38,7 +40,8 @@ public class Merge extends Test { @SuppressWarnings("unchecked") List tableNames = (List) state.get("tables"); - + tableNames = new ArrayList(tableNames); + tableNames.add(Constants.METADATA_TABLE_NAME); String tableName = tableNames.get(rand.nextInt(tableNames.size())); // TODO need to sometimes do null start and end ranges Added: accumulo/branches/1.5/test/src/test/java/org/apache/accumulo/test/MetaSplitTest.java URL: http://svn.apache.org/viewvc/accumulo/branches/1.5/test/src/test/java/org/apache/accumulo/test/MetaSplitTest.java?rev=1467010&view=auto ============================================================================== --- accumulo/branches/1.5/test/src/test/java/org/apache/accumulo/test/MetaSplitTest.java (added) +++ accumulo/branches/1.5/test/src/test/java/org/apache/accumulo/test/MetaSplitTest.java Thu Apr 11 17:56:57 2013 @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.test; + +import static org.junit.Assert.assertEquals; + +import java.util.SortedSet; +import java.util.TreeSet; + +import org.apache.accumulo.core.Constants; +import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.client.Instance; +import org.apache.accumulo.core.client.ZooKeeperInstance; +import org.apache.accumulo.core.client.admin.TableOperations; +import org.apache.accumulo.core.client.security.tokens.PasswordToken; +import org.apache.hadoop.io.Text; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class MetaSplitTest { + + private static String secret = "superSecret"; + public static TemporaryFolder folder = new TemporaryFolder(); + public static MiniAccumuloCluster cluster; + + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + folder.create(); + MiniAccumuloConfig cfg = new MiniAccumuloConfig(folder.newFolder("miniAccumulo"), secret); + cluster = new MiniAccumuloCluster(cfg); + cluster.start(); + + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + cluster.stop(); + folder.delete(); + } + + private void addSplits(TableOperations opts, String ... points) throws Exception { + SortedSet splits = new TreeSet(); + for (String point : points) { + splits.add(new Text(point)); + } + opts.addSplits(Constants.METADATA_TABLE_NAME, splits); + } + + @Test(timeout = 60000) + public void testMetaSplit() throws Exception { + Instance instance = new ZooKeeperInstance(cluster.getInstanceName(), cluster.getZooKeepers()); + Connector connector = instance.getConnector("root", new PasswordToken(secret)); + TableOperations opts = connector.tableOperations(); + for (int i = 1; i <= 10; i++) { + opts.create("" + i); + } + assertEquals(2, opts.listSplits(Constants.METADATA_TABLE_NAME).size()); + addSplits(opts, "4 5 6 7 8".split(" ")); + assertEquals(7, opts.listSplits(Constants.METADATA_TABLE_NAME).size()); + opts.merge(Constants.METADATA_TABLE_NAME, new Text("6"), new Text("9")); + assertEquals(5, opts.listSplits(Constants.METADATA_TABLE_NAME).size()); + addSplits(opts, "44 55 66 77 88".split(" ")); + assertEquals(10, opts.listSplits(Constants.METADATA_TABLE_NAME).size()); + opts.merge(Constants.METADATA_TABLE_NAME, new Text("5"), new Text("7")); + assertEquals(7, opts.listSplits(Constants.METADATA_TABLE_NAME).size()); + opts.merge(Constants.METADATA_TABLE_NAME, null, null); + assertEquals(1, opts.listSplits(Constants.METADATA_TABLE_NAME).size()); + } + +} Propchange: accumulo/branches/1.5/test/src/test/java/org/apache/accumulo/test/MetaSplitTest.java ------------------------------------------------------------------------------ svn:eol-style = native