aurora-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From wfar...@apache.org
Subject git commit: Disable log and initiate shutdown upon log storage failures.
Date Fri, 21 Mar 2014 02:56:12 GMT
Repository: incubator-aurora
Updated Branches:
  refs/heads/master 930553ad3 -> d41987b43


Disable log and initiate shutdown upon log storage failures.

Bugs closed: AURORA-221

Reviewed at https://reviews.apache.org/r/18714/


Project: http://git-wip-us.apache.org/repos/asf/incubator-aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-aurora/commit/d41987b4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-aurora/tree/d41987b4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-aurora/diff/d41987b4

Branch: refs/heads/master
Commit: d41987b43d3cb0f2a2f5fbbe4674921800515a97
Parents: 930553a
Author: Bill Farner <wfarner@apache.org>
Authored: Thu Mar 20 19:55:41 2014 -0700
Committer: Bill Farner <wfarner@apache.org>
Committed: Thu Mar 20 19:55:41 2014 -0700

----------------------------------------------------------------------
 .../aurora/scheduler/log/mesos/MesosLog.java    | 78 +++++++++++++++-----
 .../scheduler/log/mesos/MesosLogTest.java       | 56 ++++++++++----
 2 files changed, 100 insertions(+), 34 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/d41987b4/src/main/java/org/apache/aurora/scheduler/log/mesos/MesosLog.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/log/mesos/MesosLog.java b/src/main/java/org/apache/aurora/scheduler/log/mesos/MesosLog.java
index 071c4fc..d86e773 100644
--- a/src/main/java/org/apache/aurora/scheduler/log/mesos/MesosLog.java
+++ b/src/main/java/org/apache/aurora/scheduler/log/mesos/MesosLog.java
@@ -26,15 +26,18 @@ import java.util.concurrent.atomic.AtomicLong;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
+import javax.annotation.Nullable;
 import javax.inject.Inject;
 import javax.inject.Provider;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Optional;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.UnmodifiableIterator;
 import com.google.common.primitives.Longs;
 import com.google.inject.BindingAnnotation;
+import com.twitter.common.application.Lifecycle;
 import com.twitter.common.base.Function;
 import com.twitter.common.base.MorePreconditions;
 import com.twitter.common.inject.TimedInterceptor.Timed;
@@ -94,6 +97,8 @@ public class MesosLog implements org.apache.aurora.scheduler.log.Log {
 
   private final byte[] noopEntry;
 
+  private final Lifecycle lifecycle;
+
   /**
    * Creates a new mesos log.
    *
@@ -103,6 +108,7 @@ public class MesosLog implements org.apache.aurora.scheduler.log.Log {
    * @param writerFactory Factory to provide access to log writers.
    * @param writeTimeout Log write timeout.
    * @param noopEntry A no-op log entry blob.
+   * @param lifecycle Lifecycle to use for initiating application teardown.
    */
   @Inject
   public MesosLog(
@@ -111,7 +117,8 @@ public class MesosLog implements org.apache.aurora.scheduler.log.Log {
       @ReadTimeout Amount<Long, Time> readTimeout,
       Provider<WriterInterface> writerFactory,
       @WriteTimeout Amount<Long, Time> writeTimeout,
-      @NoopEntry byte[] noopEntry) {
+      @NoopEntry byte[] noopEntry,
+      Lifecycle lifecycle) {
 
     this.logFactory = checkNotNull(logFactory);
 
@@ -122,12 +129,20 @@ public class MesosLog implements org.apache.aurora.scheduler.log.Log
{
     this.writeTimeout = checkNotNull(writeTimeout);
 
     this.noopEntry = checkNotNull(noopEntry);
+
+    this.lifecycle = checkNotNull(lifecycle);
   }
 
   @Override
   public Stream open() {
     return new LogStream(
-        logFactory.get(), readerFactory.get(), readTimeout, writerFactory, writeTimeout,
noopEntry);
+        logFactory.get(),
+        readerFactory.get(),
+        readTimeout,
+        writerFactory,
+        writeTimeout,
+        noopEntry,
+        lifecycle);
   }
 
   @VisibleForTesting
@@ -177,11 +192,29 @@ public class MesosLog implements org.apache.aurora.scheduler.log.Log
{
 
     private final byte[] noopEntry;
 
-    private WriterInterface writer;
-
-    LogStream(LogInterface log, ReaderInterface reader, Amount<Long, Time> readTimeout,
-        Provider<WriterInterface> writerFactory, Amount<Long, Time> writeTimeout,
-        byte[] noopEntry) {
+    private final Lifecycle lifecycle;
+
+    /**
+     * The underlying writer to use for mutation operations.  This field has three states:
+     * <ul>
+     *   <li>present: the writer is active and available for use</li>
+     *   <li>absent: the writer has not yet been initialized (initialization is lazy)</li>
+     *   <li>{@code null}: the writer has suffered a fatal error and no further operations
may
+     *       be performed.</li>
+     * </ul>
+     * When {@code true}, indicates that the log has suffered a fatal error and no further
+     * operations may be performed.
+     */
+    @Nullable private Optional<WriterInterface> writer = Optional.absent();
+
+    LogStream(
+        LogInterface log,
+        ReaderInterface reader,
+        Amount<Long, Time> readTimeout,
+        Provider<WriterInterface> writerFactory,
+        Amount<Long, Time> writeTimeout,
+        byte[] noopEntry,
+        Lifecycle lifecycle) {
 
       this.log = log;
 
@@ -194,6 +227,8 @@ public class MesosLog implements org.apache.aurora.scheduler.log.Log {
       this.writeTimeUnit = writeTimeout.getUnit().getTimeUnit();
 
       this.noopEntry = noopEntry;
+
+      this.lifecycle = lifecycle;
     }
 
     @Override
@@ -313,25 +348,30 @@ public class MesosLog implements org.apache.aurora.scheduler.log.Log
{
       T apply(WriterInterface writer) throws TimeoutException, Log.WriterFailedException;
     }
 
+    private StreamAccessException disableLog(AtomicLong stat, String message, Throwable cause)
{
+      stat.incrementAndGet();
+      writer = null;
+      lifecycle.shutdown();
+
+      throw new StreamAccessException(message, cause);
+    }
+
     @VisibleForTesting
     synchronized <T> T mutate(OpStats stats, Mutation<T> mutation) {
-      long start = System.nanoTime();
       if (writer == null) {
-        writer = writerFactory.get();
+        throw new IllegalStateException("The log has encountered an error and cannot be used.");
+      }
+
+      long start = System.nanoTime();
+      if (!writer.isPresent()) {
+        writer = Optional.of(writerFactory.get());
       }
       try {
-        return mutation.apply(writer);
+        return mutation.apply(writer.get());
       } catch (TimeoutException e) {
-        stats.timeouts.getAndIncrement();
-        throw new StreamAccessException("Timeout performing log " + stats.opName, e);
+        throw disableLog(stats.timeouts, "Timeout performing log " + stats.opName, e);
       } catch (Log.WriterFailedException e) {
-        stats.failures.getAndIncrement();
-
-        // We must throw away a writer on any write failure - this could be because of a
coordinator
-        // election in which case we must trigger a new election.
-        writer = null;
-
-        throw new StreamAccessException("Problem performing log" + stats.opName, e);
+        throw disableLog(stats.failures, "Problem performing log" + stats.opName, e);
       } finally {
         stats.timing.accumulate(System.nanoTime() - start);
       }

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/d41987b4/src/test/java/org/apache/aurora/scheduler/log/mesos/MesosLogTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/log/mesos/MesosLogTest.java b/src/test/java/org/apache/aurora/scheduler/log/mesos/MesosLogTest.java
index 359f5d4..7c7db45 100644
--- a/src/test/java/org/apache/aurora/scheduler/log/mesos/MesosLogTest.java
+++ b/src/test/java/org/apache/aurora/scheduler/log/mesos/MesosLogTest.java
@@ -17,9 +17,9 @@ package org.apache.aurora.scheduler.log.mesos;
 
 import java.util.concurrent.TimeoutException;
 
-import javax.inject.Provider;
-
 import com.google.inject.util.Providers;
+import com.twitter.common.application.Lifecycle;
+import com.twitter.common.base.Command;
 import com.twitter.common.quantity.Amount;
 import com.twitter.common.quantity.Time;
 import com.twitter.common.testing.easymock.EasyMockTest;
@@ -33,6 +33,7 @@ import org.junit.Before;
 import org.junit.Test;
 
 import static org.easymock.EasyMock.expect;
+import static org.junit.Assert.fail;
 
 public class MesosLogTest extends EasyMockTest {
 
@@ -40,39 +41,64 @@ public class MesosLogTest extends EasyMockTest {
   private static final Amount<Long, Time> WRITE_TIMEOUT = Amount.of(3L, Time.SECONDS);
   private static final byte[] DUMMY_CONTENT = "test data".getBytes();
 
-  private LogInterface logInterface;
-  private ReaderInterface reader;
-  private Provider<WriterInterface> writerFactory;
+  private Command shutdownHooks;
   private MesosLog.LogStream logStream;
   private MesosLog.LogStream.Mutation<String> dummyMutation;
   private MesosLog.LogStream.OpStats stats;
 
   @Before
   public void setUp() {
-    logInterface = createMock(LogInterface.class);
-    reader = createMock(ReaderInterface.class);
-    writerFactory = Providers.of(createMock(WriterInterface.class));
-
+    shutdownHooks = createMock(Command.class);
     dummyMutation = createMock(new Clazz<MesosLog.LogStream.Mutation<String>>()
{ });
     stats = new MesosLog.LogStream.OpStats("test");
-    logStream = new MesosLog.LogStream(logInterface, reader, READ_TIMEOUT,
-        writerFactory, WRITE_TIMEOUT, DUMMY_CONTENT);
+    logStream = new MesosLog.LogStream(
+        createMock(LogInterface.class),
+        createMock(ReaderInterface.class),
+        READ_TIMEOUT,
+        Providers.of(createMock(WriterInterface.class)),
+        WRITE_TIMEOUT,
+        DUMMY_CONTENT,
+        new Lifecycle(shutdownHooks, null));
   }
 
-  @Test(expected = StreamAccessException.class)
+  @Test
   public void testLogStreamTimeout() throws TimeoutException, Log.WriterFailedException {
-    testMutationFailure(new TimeoutException("Task timed out"));
+    try {
+      testMutationFailure(new TimeoutException("Task timed out"));
+      fail();
+    } catch (StreamAccessException e) {
+      // Expected.
+    }
+
+    expectStreamUnusable();
   }
 
-  @Test(expected = StreamAccessException.class)
+  @Test
   public void testLogStreamWriteFailure() throws TimeoutException, Log.WriterFailedException
{
-    testMutationFailure(new Log.WriterFailedException("Failed to write to log"));
+    try {
+      testMutationFailure(new Log.WriterFailedException("Failed to write to log"));
+      fail();
+    } catch (StreamAccessException e) {
+      // Expected.
+    }
+
+    expectStreamUnusable();
   }
 
   private void testMutationFailure(Exception e) throws TimeoutException, Log.WriterFailedException
{
+    shutdownHooks.execute();
     expect(dummyMutation.apply(EasyMock.<WriterInterface>anyObject())).andThrow(e);
 
     control.replay();
     logStream.mutate(stats, dummyMutation);
   }
+
+  private void expectStreamUnusable() {
+    try {
+      logStream.mutate(stats, dummyMutation);
+      fail();
+    } catch (IllegalStateException e) {
+      // Expected.
+    }
+  }
 }


Mime
View raw message