accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e..@apache.org
Subject [25/34] accumulo git commit: ACCUMULO-3423 finished up testing, went back to lazy metadata table references
Date Fri, 24 Apr 2015 23:21:11 GMT
ACCUMULO-3423 finished up testing, went back to lazy metadata table references


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/f43b2160
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/f43b2160
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/f43b2160

Branch: refs/heads/master
Commit: f43b216038157e471612eac4679696ce7d6f7835
Parents: ed7c4f6
Author: Eric C. Newton <eric.newton@gmail.com>
Authored: Wed Apr 22 10:34:02 2015 -0400
Committer: Eric C. Newton <eric.newton@gmail.com>
Committed: Wed Apr 22 10:34:02 2015 -0400

----------------------------------------------------------------------
 .../tserver/log/TabletServerLogger.java         | 24 +++++++++++++++-----
 .../accumulo/proxy/ProxyDurabilityIT.java       |  4 ++--
 .../org/apache/accumulo/test/UnusedWALIT.java   |  2 +-
 .../accumulo/test/functional/WALSunnyDayIT.java | 20 ++++++++--------
 4 files changed, 31 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/f43b2160/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java
index cdee51b..7d16aae 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java
@@ -78,7 +78,7 @@ public class TabletServerLogger {
 
   // The current logger
   private DfsLogger currentLog = null;
-  private final SynchronousQueue<DfsLogger> nextLog = new SynchronousQueue<>();
+  private final SynchronousQueue<Object> nextLog = new SynchronousQueue<>();
   private ThreadPoolExecutor nextLogMaker;
 
   // The current generation of logs.
@@ -200,11 +200,18 @@ public class TabletServerLogger {
 
     try {
       startLogMaker();
-      DfsLogger next = nextLog.take();
-      log.info("Using next log " + next.getFileName());
-      currentLog = next;
-      logId.incrementAndGet();
-      return;
+      Object next = nextLog.take();
+      if (next instanceof Exception) {
+        throw (Exception)next;
+      }
+      if (next instanceof DfsLogger) {
+        currentLog = (DfsLogger)next;
+        logId.incrementAndGet();
+        log.info("Using next log " + currentLog.getFileName());
+        return;
+      } else {
+        throw new RuntimeException("Error: unexpected type seen: " + next);
+      }
     } catch (Exception t) {
       walErrors.put(System.currentTimeMillis(), "");
       if (walErrors.size() >= HALT_AFTER_ERROR_COUNT) {
@@ -233,6 +240,11 @@ public class TabletServerLogger {
             }
           } catch (Exception t) {
             log.error("{}", t.getMessage(), t);
+            try {
+              nextLog.offer(t, 12, TimeUnit.HOURS);
+            } catch (InterruptedException ex) {
+              // ignore
+            }
           }
         }
       }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/f43b2160/test/src/test/java/org/apache/accumulo/proxy/ProxyDurabilityIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/proxy/ProxyDurabilityIT.java b/test/src/test/java/org/apache/accumulo/proxy/ProxyDurabilityIT.java
index 654d27d..81e25cc 100644
--- a/test/src/test/java/org/apache/accumulo/proxy/ProxyDurabilityIT.java
+++ b/test/src/test/java/org/apache/accumulo/proxy/ProxyDurabilityIT.java
@@ -116,7 +116,7 @@ public class ProxyDurabilityIT extends ConfigurableMacIT {
     assertEquals(0, count(tableName));
 
     ConditionalWriterOptions cfg = new ConditionalWriterOptions();
-    cfg.setDurability(Durability.LOG);
+    cfg.setDurability(Durability.SYNC);
     String cwriter = client.createConditionalWriter(login, tableName, cfg);
     ConditionalUpdates updates = new ConditionalUpdates();
     updates.addToConditions(new Condition(new Column(bytes("cf"), bytes("cq"), bytes(""))));
@@ -125,7 +125,7 @@ public class ProxyDurabilityIT extends ConfigurableMacIT {
     assertEquals(ConditionalStatus.ACCEPTED, status.get(bytes("row")));
     assertEquals(1, count(tableName));
     restartTServer();
-    assertEquals(0, count(tableName));
+    assertEquals(1, count(tableName));
 
     proxyServer.stop();
   }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/f43b2160/test/src/test/java/org/apache/accumulo/test/UnusedWALIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/UnusedWALIT.java b/test/src/test/java/org/apache/accumulo/test/UnusedWALIT.java
index 3684ee1..03d783c 100644
--- a/test/src/test/java/org/apache/accumulo/test/UnusedWALIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/UnusedWALIT.java
@@ -80,7 +80,7 @@ public class UnusedWALIT extends ConfigurableMacIT {
 
     // roll the logs by pushing data into bigTable
     writeSomeData(c, bigTable, 0, 3000, 0, 1000);
-    assertEquals(3, getWALCount(c));
+    assertEquals(2, getWALCount(c));
 
     // put some data in the latest log
     writeSomeData(c, lilTable, 1, 10, 0, 10);

http://git-wip-us.apache.org/repos/asf/accumulo/blob/f43b2160/test/src/test/java/org/apache/accumulo/test/functional/WALSunnyDayIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/WALSunnyDayIT.java b/test/src/test/java/org/apache/accumulo/test/functional/WALSunnyDayIT.java
index 3f51c8d..490bd7c 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/WALSunnyDayIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/WALSunnyDayIT.java
@@ -105,9 +105,9 @@ public class WALSunnyDayIT extends ConfigurableMacIT {
     c.tableOperations().create(tableName);
     writeSomeData(c, tableName, 1, 1);
 
-    // should have two markers: wal in use, and next wal
+    // wal markers are added lazily
     Map<String,Boolean> wals = getWals(c, zoo);
-    assertEquals(wals.toString(), 2, wals.size());
+    assertEquals(wals.toString(), 1, wals.size());
     for (Boolean b : wals.values()) {
       assertTrue("logs should be in use", b.booleanValue());
     }
@@ -115,9 +115,9 @@ public class WALSunnyDayIT extends ConfigurableMacIT {
     // roll log, get a new next
     writeSomeData(c, tableName, 1000, 50);
     Map<String,Boolean> walsAfterRoll = getWals(c, zoo);
-    assertEquals("should have 3 WALs after roll", 3, walsAfterRoll.size());
+    assertEquals("should have 3 WALs after roll", 2, walsAfterRoll.size());
     assertTrue("new WALs should be a superset of the old WALs", walsAfterRoll.keySet().containsAll(wals.keySet()));
-    assertEquals("all WALs should be in use", 3, countTrue(walsAfterRoll.values()));
+    assertEquals("all WALs should be in use", 2, countTrue(walsAfterRoll.values()));
 
     // flush the tables
     for (String table: new String[] { tableName, MetadataTable.NAME, RootTable.NAME} ) {
@@ -126,15 +126,15 @@ public class WALSunnyDayIT extends ConfigurableMacIT {
     UtilWaitThread.sleep(1000);
     // rolled WAL is no longer in use, but needs to be GC'd
     Map<String,Boolean> walsAfterflush = getWals(c, zoo);
-    assertEquals(walsAfterflush.toString(), 3, walsAfterflush.size());
-    assertEquals("inUse should be 2", 2, countTrue(walsAfterflush.values()));
+    assertEquals(walsAfterflush.toString(), 2, walsAfterflush.size());
+    assertEquals("inUse should be 1", 1, countTrue(walsAfterflush.values()));
 
     // let the GC run for a little bit
     control.start(GARBAGE_COLLECTOR);
     UtilWaitThread.sleep(5 * 1000);
     // make sure the unused WAL goes away
     Map<String,Boolean> walsAfterGC = getWals(c, zoo);
-    assertEquals(walsAfterGC.toString(), 2, walsAfterGC.size());
+    assertEquals(walsAfterGC.toString(), 1, walsAfterGC.size());
     control.stop(GARBAGE_COLLECTOR);
     // restart the tserver, but don't run recovery on all tablets
     control.stop(TABLET_SERVER);
@@ -158,12 +158,12 @@ public class WALSunnyDayIT extends ConfigurableMacIT {
 
     Map<String,Boolean> walsAfterRestart = getWals(c, zoo);
     //log.debug("wals after " + walsAfterRestart);
-    assertEquals("used WALs after restart should be 2", 2, countTrue(walsAfterRestart.values()));
+    assertEquals("used WALs after restart should be 1", 1, countTrue(walsAfterRestart.values()));
     control.start(GARBAGE_COLLECTOR);
     UtilWaitThread.sleep(5 * 1000);
     Map<String,Boolean> walsAfterRestartAndGC = getWals(c, zoo);
-    assertEquals("wals left should be 2", 2, walsAfterRestartAndGC.size());
-    assertEquals("logs in use should be 2", 2, countTrue(walsAfterRestartAndGC.values()));
+    assertEquals("wals left should be 1", 1, walsAfterRestartAndGC.size());
+    assertEquals("logs in use should be 1", 1, countTrue(walsAfterRestartAndGC.values()));
   }
 
   private void verifySomeData(Connector c, String tableName, int expected) throws Exception
{


Mime
View raw message