accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e..@apache.org
Subject [16/34] accumulo git commit: ACCUMULO-3423 merge in master from apache
Date Fri, 24 Apr 2015 23:21:02 GMT
ACCUMULO-3423 merge in master from apache


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/c71fc121
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/c71fc121
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/c71fc121

Branch: refs/heads/master
Commit: c71fc121e70848170f3a3a69c1bb4a0b1b77b24a
Parents: afa887b 94bd393
Author: Eric C. Newton <eric.newton@gmail.com>
Authored: Mon Mar 30 11:38:07 2015 -0400
Committer: Eric C. Newton <eric.newton@gmail.com>
Committed: Mon Mar 30 11:38:07 2015 -0400

----------------------------------------------------------------------
 .../accumulo/core/trace/DistributedTrace.java   |   1 +
 .../impl/MiniAccumuloClusterImpl.java           |  10 +-
 .../server/security/SecurityOperation.java      |  15 +-
 .../accumulo/master/TabletGroupWatcher.java     |   6 +
 .../master/tserverOps/ShutdownTServer.java      |  51 ++++---
 .../accumulo/tracer/AsyncSpanReceiver.java      |  13 ++
 .../accumulo/tracer/SendSpansViaThrift.java     |   5 +
 .../server/GarbageCollectionLogger.java         |   2 +-
 .../apache/accumulo/tserver/TabletServer.java   |   2 +-
 .../org/apache/accumulo/test/UnusedWALIT.java   | 144 -------------------
 .../apache/accumulo/proxy/SimpleProxyBase.java  |   8 +-
 .../accumulo/test/AssignmentThreadsIT.java      |  10 +-
 .../test/BalanceWithOfflineTableIT.java         |  90 ++++++++++++
 .../org/apache/accumulo/test/UnusedWALIT.java   | 144 +++++++++++++++++++
 .../accumulo/test/functional/ClassLoaderIT.java |   4 +-
 .../accumulo/test/functional/DurabilityIT.java  |   2 -
 .../test/functional/KerberosProxyIT.java        |  14 +-
 .../test/functional/WatchTheWatchCountIT.java   |   4 +-
 .../test/replication/ReplicationIT.java         |   4 +-
 19 files changed, 346 insertions(+), 183 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/c71fc121/server/master/src/main/java/org/apache/accumulo/master/TabletGroupWatcher.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/c71fc121/server/tserver/src/main/java/org/apache/accumulo/server/GarbageCollectionLogger.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/c71fc121/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/c71fc121/test/src/test/java/org/apache/accumulo/test/UnusedWALIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/UnusedWALIT.java
index 0000000,0000000..3684ee1
new file mode 100644
--- /dev/null
+++ b/test/src/test/java/org/apache/accumulo/test/UnusedWALIT.java
@@@ -1,0 -1,0 +1,144 @@@
++/*
++ * Licensed to the Apache Software Foundation (ASF) under one or more
++ * contributor license agreements.  See the NOTICE file distributed with
++ * this work for additional information regarding copyright ownership.
++ * The ASF licenses this file to You under the Apache License, Version 2.0
++ * (the "License"); you may not use this file except in compliance with
++ * the License.  You may obtain a copy of the License at
++ *
++ *     http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ */
++package org.apache.accumulo.test;
++
++import static org.junit.Assert.assertEquals;
++
++import java.util.Map.Entry;
++
++import org.apache.accumulo.core.client.BatchWriter;
++import org.apache.accumulo.core.client.BatchWriterConfig;
++import org.apache.accumulo.core.client.Connector;
++import org.apache.accumulo.core.client.Scanner;
++import org.apache.accumulo.core.conf.Property;
++import org.apache.accumulo.core.data.Key;
++import org.apache.accumulo.core.data.Mutation;
++import org.apache.accumulo.core.data.Range;
++import org.apache.accumulo.core.data.Value;
++import org.apache.accumulo.core.metadata.MetadataTable;
++import org.apache.accumulo.core.metadata.schema.MetadataSchema.CurrentLogsSection;
++import org.apache.accumulo.core.security.Authorizations;
++import org.apache.accumulo.minicluster.ServerType;
++import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
++import org.apache.accumulo.test.functional.ConfigurableMacIT;
++import org.apache.hadoop.conf.Configuration;
++import org.apache.hadoop.fs.RawLocalFileSystem;
++import org.junit.Test;
++
++import com.google.common.collect.Iterators;
++
++// When reviewing the changes for ACCUMULO-3423, kturner suggested
++// "tablets will now have log references that contain no data,
++// so it may be marked with 3 WALs, the first with data, the 2nd without, a 3rd with data.
++// It would be useful to have an IT that will test this situation.
++public class UnusedWALIT extends ConfigurableMacIT {
++
++  @Override
++  protected void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
++    final long logSize = 1024 * 1024 * 10;
++    cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s");
++    cfg.setProperty(Property.TSERV_WALOG_MAX_SIZE, Long.toString(logSize));
++    cfg.setNumTservers(1);
++    // use raw local file system so walogs sync and flush will work
++    hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
++    hadoopCoreSite.set("fs.namenode.fs-limits.min-block-size", Long.toString(logSize));
++  }
++
++  @Test(timeout = 2 * 60 * 1000)
++  public void test() throws Exception {
++    // don't want this bad boy cleaning up walog entries
++    getCluster().getClusterControl().stop(ServerType.GARBAGE_COLLECTOR);
++
++    // make two tables
++    String[] tableNames = getUniqueNames(2);
++    String bigTable = tableNames[0];
++    String lilTable = tableNames[1];
++    Connector c = getConnector();
++    c.tableOperations().create(bigTable);
++    c.tableOperations().create(lilTable);
++
++    // put some data in a log that should be replayed for both tables
++    writeSomeData(c, bigTable, 0, 10, 0, 10);
++    scanSomeData(c, bigTable, 0, 10, 0, 10);
++    writeSomeData(c, lilTable, 0, 1, 0, 1);
++    scanSomeData(c, lilTable, 0, 1, 0, 1);
++    assertEquals(1, getWALCount(c));
++
++    // roll the logs by pushing data into bigTable
++    writeSomeData(c, bigTable, 0, 3000, 0, 1000);
++    assertEquals(3, getWALCount(c));
++
++    // put some data in the latest log
++    writeSomeData(c, lilTable, 1, 10, 0, 10);
++    scanSomeData(c, lilTable, 1, 10, 0, 10);
++
++    // bounce the tserver
++    getCluster().getClusterControl().stop(ServerType.TABLET_SERVER);
++    getCluster().getClusterControl().start(ServerType.TABLET_SERVER);
++
++    // wait for the metadata table to be online
++    Iterators.size(c.createScanner(MetadataTable.NAME, Authorizations.EMPTY).iterator());
++
++    // check our two sets of data in different logs
++    scanSomeData(c, lilTable, 0, 1, 0, 1);
++    scanSomeData(c, lilTable, 1, 10, 0, 10);
++  }
++
++  private void scanSomeData(Connector c, String table, int startRow, int rowCount, int startCol,
int colCount) throws Exception {
++    Scanner s = c.createScanner(table, Authorizations.EMPTY);
++    s.setRange(new Range(Integer.toHexString(startRow), Integer.toHexString(startRow + rowCount)));
++    int row = startRow;
++    int col = startCol;
++    for (Entry<Key,Value> entry : s) {
++      assertEquals(row, Integer.parseInt(entry.getKey().getRow().toString(), 16));
++      assertEquals(col++, Integer.parseInt(entry.getKey().getColumnQualifier().toString(),
16));
++      if (col == startCol + colCount) {
++        col = startCol;
++        row++;
++        if (row == startRow + rowCount) {
++          break;
++        }
++      }
++    }
++    assertEquals(row, startRow + rowCount);
++  }
++
++  private int getWALCount(Connector c) throws Exception {
++    Scanner s = c.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
++    s.setRange(CurrentLogsSection.getRange());
++    try {
++      return Iterators.size(s.iterator());
++    } finally {
++      s.close();
++    }
++  }
++
++  private void writeSomeData(Connector conn, String table, int startRow, int rowCount, int
startCol, int colCount) throws Exception {
++    BatchWriterConfig config = new BatchWriterConfig();
++    config.setMaxMemory(10 * 1024 * 1024);
++    BatchWriter bw = conn.createBatchWriter(table, config);
++    for (int r = startRow; r < startRow + rowCount; r++) {
++      Mutation m = new Mutation(Integer.toHexString(r));
++      for (int c = startCol; c < startCol + colCount; c++) {
++        m.put("", Integer.toHexString(c), "");
++      }
++      bw.addMutation(m);
++    }
++    bw.close();
++  }
++
++}

http://git-wip-us.apache.org/repos/asf/accumulo/blob/c71fc121/test/src/test/java/org/apache/accumulo/test/functional/WatchTheWatchCountIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/WatchTheWatchCountIT.java
index 4c86172,3b1dd2f..140fd59
--- a/test/src/test/java/org/apache/accumulo/test/functional/WatchTheWatchCountIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/WatchTheWatchCountIT.java
@@@ -51,8 -51,8 +51,8 @@@ public class WatchTheWatchCountIT exten
        int n = socket.getInputStream().read(buffer);
        String response = new String(buffer, 0, n);
        long total = Long.parseLong(response.split(":")[1].trim());
-       assertTrue("Total watches was not greater than 600, but was " + total, total >
600);
-       assertTrue("Total watches was not less than 600, but was " + total, total < 675);
+       assertTrue("Total watches was not greater than 500, but was " + total, total >
500);
 -      assertTrue("Total watches was not less than 650, but was " + total, total < 600);
++      assertTrue("Total watches was not less than 675, but was " + total, total < 675);
      } finally {
        socket.close();
      }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/c71fc121/test/src/test/java/org/apache/accumulo/test/replication/ReplicationIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/replication/ReplicationIT.java
index ca58a59,46e3ac1..b7a518a
--- a/test/src/test/java/org/apache/accumulo/test/replication/ReplicationIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/replication/ReplicationIT.java
@@@ -993,7 -1085,18 +993,8 @@@ public class ReplicationIT extends Conf
      }
  
      // Write some data to table1
 -    BatchWriter bw = conn.createBatchWriter(table1, new BatchWriterConfig());
 -    for (int rows = 0; rows < 2000; rows++) {
 -      Mutation m = new Mutation(Integer.toString(rows));
 -      for (int cols = 0; cols < 50; cols++) {
 -        String value = Integer.toString(cols);
 -        m.put(value, "", value);
 -      }
 -      bw.addMutation(m);
 -    }
 -
 -    bw.close();
 +    writeSomeData(conn, table1, 2000, 50);
+     conn.tableOperations().flush(table1, null, null, true);
  
      String tableId = conn.tableOperations().tableIdMap().get(table1);
      Assert.assertNotNull("Table ID was null", tableId);
@@@ -1253,7 -1397,18 +1254,8 @@@
      Assert.assertNotNull("Could not determine table id for " + table1, tableId);
  
      // Write some data to table1
 -    BatchWriter bw = conn.createBatchWriter(table1, new BatchWriterConfig());
 -    for (int rows = 0; rows < 2000; rows++) {
 -      Mutation m = new Mutation(Integer.toString(rows));
 -      for (int cols = 0; cols < 50; cols++) {
 -        String value = Integer.toString(cols);
 -        m.put(value, "", value);
 -      }
 -      bw.addMutation(m);
 -    }
 -
 -    bw.close();
 +    writeSomeData(conn, table1, 2000, 50);
+     conn.tableOperations().flush(table1, null, null, true);
  
      // Make sure the replication table exists at this point
      boolean online = ReplicationTable.isOnline(conn);


Mime
View raw message