aurora-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From wfar...@apache.org
Subject aurora git commit: Remove several scheduler command line arguments.
Date Sun, 03 Jan 2016 17:39:48 GMT
Repository: aurora
Updated Branches:
  refs/heads/master 297ebc157 -> 6b4045397


Remove several scheduler command line arguments.

Reviewed at https://reviews.apache.org/r/41786/


Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/6b404539
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/6b404539
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/6b404539

Branch: refs/heads/master
Commit: 6b4045397305040616affdf1989c73d0f78b667a
Parents: 297ebc1
Author: Bill Farner <wfarner@apache.org>
Authored: Sun Jan 3 09:39:44 2016 -0800
Committer: Bill Farner <wfarner@apache.org>
Committed: Sun Jan 3 09:39:44 2016 -0800

----------------------------------------------------------------------
 NEWS                                            |   5 +
 docs/scheduler-storage.md                       |  47 --------
 .../aurora/scheduler/http/api/ApiModule.java    |   7 +-
 .../aurora/scheduler/storage/log/Entries.java   |   4 +-
 .../scheduler/storage/log/LogManager.java       |  16 ---
 .../scheduler/storage/log/LogStorageModule.java |  12 --
 .../storage/log/StreamManagerImpl.java          |  23 +---
 .../aurora/scheduler/thrift/aop/AopModule.java  |  91 +---------------
 .../thrift/aop/FeatureToggleInterceptor.java    |  46 --------
 .../scheduler/storage/log/LogManagerTest.java   |  19 ++--
 .../scheduler/storage/log/LogStorageTest.java   |  23 ++--
 .../storage/log/testing/LogOpMatcher.java       |  14 ++-
 .../scheduler/thrift/aop/AopModuleTest.java     | 109 -------------------
 .../aop/FeatureToggleInterceptorTest.java       |  84 --------------
 14 files changed, 43 insertions(+), 457 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/aurora/blob/6b404539/NEWS
----------------------------------------------------------------------
diff --git a/NEWS b/NEWS
index 394b31c..965fbf4 100644
--- a/NEWS
+++ b/NEWS
@@ -4,6 +4,11 @@
 - Upgraded Mesos to 0.25.0.
 - Env variables can be passed through to task processes by passing `--preserve_env`
   to thermos.
+- Removed scheduler command line arguments:
+  - `-enable_cors_support`.  Enabling CORS is now implicit by setting the argument
+    `-enable_cors_for`.
+  - `-deduplicate_snapshots` and `-deflate_snapshots`.  These features are good to always
enable.
+  - `-enable_job_updates` and `-enable_job_creation`
 
 0.11.0
 ------

http://git-wip-us.apache.org/repos/asf/aurora/blob/6b404539/docs/scheduler-storage.md
----------------------------------------------------------------------
diff --git a/docs/scheduler-storage.md b/docs/scheduler-storage.md
deleted file mode 100644
index 1cd02f8..0000000
--- a/docs/scheduler-storage.md
+++ /dev/null
@@ -1,47 +0,0 @@
-# Snapshot Performance
-
-Periodically the scheduler writes a full snapshot of its state to the replicated log. To
do this
-it needs to hold a global storage write lock while it writes out this data. In large clusters
-this has been observed to take up to 40 seconds. Long pauses can cause issues in the system,
-including delays in scheduling new tasks.
-
-The scheduler has two optimizations to reduce the size of snapshots and thus improve snapshot
-performance: compression and deduplication. Most users will want to enable both compression
-and deduplication.
-
-## Compression
-
-To reduce the size of the snapshot the DEFLATE algorithm can be applied to the serialized
bytes
-of the snapshot as they are written to the stream. This reduces the total number of bytes
that
-need to be written to the replicated log at the cost of CPU and generally reduces the amount
-of time a snapshot takes. Most users will want to enable both compression and deduplication.
-
-### Enabling Compression
-
-Snapshot compression is enabled via the `-deflate_snapshots` flag. This is the default since
-Aurora 0.5.0. All released versions of Aurora can read both compressed and uncompressed snapshots,
-so there are no backwards compatibility concerns associated with changing this flag.
-
-### Disabling compression
-
-Disable compression by passing `-deflate_snapshots=false`.
-
-## Deduplication
-
-In Aurora 0.6.0 a new snapshot format was introduced. Rather than write one configuration
blob
-per Mesos task this format stores each configuration blob once, and each Mesos task with
a
-pointer to its blob. This format is not backwards compatible with earlier versions of Aurora.
-
-### Enabling Deduplication
-
-After upgrading Aurora to 0.6.0, enable deduplication with the `-deduplicate_snapshots` flag.
-After the first snapshot the cluster will be using the deduplicated format to write to the
-replicated log. Snapshots are created periodically by the scheduler (according to
-the `-dlog_snapshot_interval` flag). An administrator can also force a snapshot operation
with
-`aurora_admin snapshot`.
-
-### Disabling Deduplication
-
-To disable deduplication, for example to rollback to Aurora, restart all of the cluster's
-schedulers with `-deduplicate_snapshots=false` and either wait for a snapshot or force one
-using `aurora_admin snapshot`.

http://git-wip-us.apache.org/repos/asf/aurora/blob/6b404539/src/main/java/org/apache/aurora/scheduler/http/api/ApiModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/http/api/ApiModule.java b/src/main/java/org/apache/aurora/scheduler/http/api/ApiModule.java
index da6894e..cd5adf9 100644
--- a/src/main/java/org/apache/aurora/scheduler/http/api/ApiModule.java
+++ b/src/main/java/org/apache/aurora/scheduler/http/api/ApiModule.java
@@ -35,16 +35,13 @@ import org.eclipse.jetty.util.resource.Resource;
 public class ApiModule extends ServletModule {
   public static final String API_PATH = "/api";
 
-  @CmdLine(name = "enable_cors_support", help = "Enable CORS support for thrift end points.")
-  private static final Arg<Boolean> ENABLE_CORS_SUPPORT = Arg.create(false);
-
   /**
    * Set the {@code Access-Control-Allow-Origin} header for API requests. See
    * http://www.w3.org/TR/cors/
    */
   @CmdLine(name = "enable_cors_for",
       help = "List of domains for which CORS support should be enabled.")
-  private static final Arg<String> ENABLE_CORS_FOR = Arg.create("*");
+  private static final Arg<String> ENABLE_CORS_FOR = Arg.create(null);
 
   private static final String API_CLIENT_ROOT = Resource
       .newClassPathResource("org/apache/aurora/scheduler/gen/client")
@@ -52,7 +49,7 @@ public class ApiModule extends ServletModule {
 
   @Override
   protected void configureServlets() {
-    if (ENABLE_CORS_SUPPORT.get()) {
+    if (ENABLE_CORS_FOR.get() != null) {
       filter(API_PATH).through(new CorsFilter(ENABLE_CORS_FOR.get()));
     }
     serve(API_PATH).with(TServlet.class);

http://git-wip-us.apache.org/repos/asf/aurora/blob/6b404539/src/main/java/org/apache/aurora/scheduler/storage/log/Entries.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/log/Entries.java b/src/main/java/org/apache/aurora/scheduler/storage/log/Entries.java
index 548b5e7..413d59a 100644
--- a/src/main/java/org/apache/aurora/scheduler/storage/log/Entries.java
+++ b/src/main/java/org/apache/aurora/scheduler/storage/log/Entries.java
@@ -25,7 +25,7 @@ import org.apache.aurora.gen.storage.LogEntry._Fields;
 /**
  * Utility class for working with log entries.
  */
-final class Entries {
+public final class Entries {
 
   private static final Logger LOG = Logger.getLogger(Entries.class.getName());
 
@@ -46,7 +46,7 @@ final class Entries {
    *         of the original entry.
    * @throws CodingException If the value could not be encoded or deflated.
    */
-  static LogEntry deflate(LogEntry entry) throws CodingException {
+  public static LogEntry deflate(LogEntry entry) throws CodingException {
     return LogEntry.deflatedEntry(ThriftBinaryCodec.deflateNonNull(entry));
   }
 

http://git-wip-us.apache.org/repos/asf/aurora/blob/6b404539/src/main/java/org/apache/aurora/scheduler/storage/log/LogManager.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/log/LogManager.java b/src/main/java/org/apache/aurora/scheduler/storage/log/LogManager.java
index 4099503..cfa9c56 100644
--- a/src/main/java/org/apache/aurora/scheduler/storage/log/LogManager.java
+++ b/src/main/java/org/apache/aurora/scheduler/storage/log/LogManager.java
@@ -39,22 +39,6 @@ public class LogManager {
   public @interface MaxEntrySize { }
 
   /**
-   * When true, enable snapshot deflation.
-   */
-  @Retention(RetentionPolicy.RUNTIME)
-  @Target({ElementType.METHOD, ElementType.PARAMETER})
-  @Qualifier
-  public @interface DeflateSnapshots { }
-
-  /**
-   * When true, enable snapshot deduplication.
-   */
-  @Retention(RetentionPolicy.RUNTIME)
-  @Target({ElementType.METHOD, ElementType.PARAMETER})
-  @Qualifier
-  public @interface DeduplicateSnapshots { }
-
-  /**
    * Hash function used to verify log entries.
    */
   @Retention(RetentionPolicy.RUNTIME)

http://git-wip-us.apache.org/repos/asf/aurora/blob/6b404539/src/main/java/org/apache/aurora/scheduler/storage/log/LogStorageModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/log/LogStorageModule.java b/src/main/java/org/apache/aurora/scheduler/storage/log/LogStorageModule.java
index 561c70b..ed63a74 100644
--- a/src/main/java/org/apache/aurora/scheduler/storage/log/LogStorageModule.java
+++ b/src/main/java/org/apache/aurora/scheduler/storage/log/LogStorageModule.java
@@ -31,12 +31,10 @@ import org.apache.aurora.scheduler.storage.CallOrderEnforcingStorage;
 import org.apache.aurora.scheduler.storage.DistributedSnapshotStore;
 import org.apache.aurora.scheduler.storage.Storage;
 import org.apache.aurora.scheduler.storage.Storage.NonVolatileStorage;
-import org.apache.aurora.scheduler.storage.log.LogManager.DeduplicateSnapshots;
 import org.apache.aurora.scheduler.storage.log.LogManager.MaxEntrySize;
 import org.apache.aurora.scheduler.storage.log.LogStorage.Settings;
 
 import static org.apache.aurora.scheduler.storage.log.EntrySerializer.EntrySerializerImpl;
-import static org.apache.aurora.scheduler.storage.log.LogManager.DeflateSnapshots;
 import static org.apache.aurora.scheduler.storage.log.LogManager.LogEntryHashFunction;
 import static org.apache.aurora.scheduler.storage.log.SnapshotDeduplicator.SnapshotDeduplicatorImpl;
 
@@ -64,14 +62,6 @@ public class LogStorageModule extends PrivateModule {
   public static final Arg<Amount<Integer, Data>> MAX_LOG_ENTRY_SIZE =
       Arg.create(Amount.of(512, Data.KB));
 
-  @CmdLine(name = "deduplicate_snapshots",
-      help = "Write snapshots in deduplicated format. For details and backwards compatibility
"
-          + "concerns see docs/scheduler-storage.md.")
-  private static final Arg<Boolean> DEDUPLICATE_SNAPSHOTS = Arg.create(false);
-
-  @CmdLine(name = "deflate_snapshots", help = "Whether snapshots should be deflate-compressed.")
-  private static final Arg<Boolean> DEFLATE_SNAPSHOTS = Arg.create(true);
-
   @Override
   protected void configure() {
     bind(Settings.class)
@@ -80,8 +70,6 @@ public class LogStorageModule extends PrivateModule {
     bind(new TypeLiteral<Amount<Integer, Data>>() { }).annotatedWith(MaxEntrySize.class)
         .toInstance(MAX_LOG_ENTRY_SIZE.get());
     bind(LogManager.class).in(Singleton.class);
-    bindConstant().annotatedWith(DeduplicateSnapshots.class).to(DEDUPLICATE_SNAPSHOTS.get());
-    bindConstant().annotatedWith(DeflateSnapshots.class).to(DEFLATE_SNAPSHOTS.get());
     bind(LogStorage.class).in(Singleton.class);
 
     install(CallOrderEnforcingStorage.wrappingModule(LogStorage.class));

http://git-wip-us.apache.org/repos/asf/aurora/blob/6b404539/src/main/java/org/apache/aurora/scheduler/storage/log/StreamManagerImpl.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/log/StreamManagerImpl.java
b/src/main/java/org/apache/aurora/scheduler/storage/log/StreamManagerImpl.java
index b252468..7a63515 100644
--- a/src/main/java/org/apache/aurora/scheduler/storage/log/StreamManagerImpl.java
+++ b/src/main/java/org/apache/aurora/scheduler/storage/log/StreamManagerImpl.java
@@ -55,8 +55,6 @@ import static org.apache.aurora.codec.ThriftBinaryCodec.CodingException;
 import static org.apache.aurora.common.inject.TimedInterceptor.Timed;
 import static org.apache.aurora.scheduler.log.Log.Stream.InvalidPositionException;
 import static org.apache.aurora.scheduler.log.Log.Stream.StreamAccessException;
-import static org.apache.aurora.scheduler.storage.log.LogManager.DeduplicateSnapshots;
-import static org.apache.aurora.scheduler.storage.log.LogManager.DeflateSnapshots;
 import static org.apache.aurora.scheduler.storage.log.LogManager.LogEntryHashFunction;
 
 class StreamManagerImpl implements StreamManager {
@@ -79,26 +77,20 @@ class StreamManagerImpl implements StreamManager {
   private final Object writeMutex = new Object();
   private final Log.Stream stream;
   private final EntrySerializer entrySerializer;
-  private final boolean deflateSnapshots;
   private final HashFunction hashFunction;
   private final SnapshotDeduplicator snapshotDeduplicator;
-  private final boolean deduplicateSnapshots;
 
   @Inject
   StreamManagerImpl(
       @Assisted Stream stream,
       EntrySerializer entrySerializer,
-      @DeflateSnapshots boolean deflateSnapshots,
       @LogEntryHashFunction HashFunction hashFunction,
-      SnapshotDeduplicator snapshotDeduplicator,
-      @DeduplicateSnapshots boolean deduplicateSnapshots) {
+      SnapshotDeduplicator snapshotDeduplicator) {
 
     this.stream = requireNonNull(stream);
     this.entrySerializer = requireNonNull(entrySerializer);
-    this.deflateSnapshots = deflateSnapshots;
     this.hashFunction = requireNonNull(hashFunction);
     this.snapshotDeduplicator = requireNonNull(snapshotDeduplicator);
-    this.deduplicateSnapshots = deduplicateSnapshots;
   }
 
   @Override
@@ -203,17 +195,8 @@ class StreamManagerImpl implements StreamManager {
   public void snapshot(Snapshot snapshot)
       throws CodingException, InvalidPositionException, StreamAccessException {
 
-    LogEntry entry;
-    if (deduplicateSnapshots) {
-      entry = LogEntry.deduplicatedSnapshot(snapshotDeduplicator.deduplicate(snapshot));
-    } else {
-      entry = LogEntry.snapshot(snapshot);
-    }
-
-    if (deflateSnapshots) {
-      entry = deflate(entry);
-    }
-
+    LogEntry entry =
+        deflate(LogEntry.deduplicatedSnapshot(snapshotDeduplicator.deduplicate(snapshot)));
     Log.Position position = appendAndGetPosition(entry);
     vars.snapshots.incrementAndGet();
     vars.unSnapshottedTransactions.set(0);

http://git-wip-us.apache.org/repos/asf/aurora/blob/6b404539/src/main/java/org/apache/aurora/scheduler/thrift/aop/AopModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/thrift/aop/AopModule.java b/src/main/java/org/apache/aurora/scheduler/thrift/aop/AopModule.java
index 7f29b79..f59ee1a 100644
--- a/src/main/java/org/apache/aurora/scheduler/thrift/aop/AopModule.java
+++ b/src/main/java/org/apache/aurora/scheduler/thrift/aop/AopModule.java
@@ -13,33 +13,13 @@
  */
 package org.apache.aurora.scheduler.thrift.aop;
 
-import java.lang.reflect.Method;
-import java.util.List;
-import java.util.Map;
-
-import javax.inject.Inject;
-import javax.inject.Singleton;
-
 import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Function;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Predicate;
-import com.google.common.base.Predicates;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Iterables;
 import com.google.inject.AbstractModule;
 import com.google.inject.Binder;
-import com.google.inject.Key;
-import com.google.inject.PrivateModule;
-import com.google.inject.TypeLiteral;
 import com.google.inject.matcher.Matcher;
 import com.google.inject.matcher.Matchers;
 
 import org.aopalliance.intercept.MethodInterceptor;
-import org.apache.aurora.common.args.Arg;
-import org.apache.aurora.common.args.CmdLine;
-import org.apache.aurora.gen.AuroraSchedulerManager;
 import org.apache.aurora.gen.Response;
 import org.apache.aurora.scheduler.thrift.auth.DecoratedThrift;
 
@@ -48,66 +28,20 @@ import org.apache.aurora.scheduler.thrift.auth.DecoratedThrift;
  */
 public class AopModule extends AbstractModule {
 
-  @CmdLine(name = "enable_job_updates", help = "Whether new job updates should be accepted.")
-  private static final Arg<Boolean> ENABLE_UPDATES = Arg.create(true);
-
-  @CmdLine(name = "enable_job_creation",
-      help = "Allow new jobs to be created, if false all job creation requests will be denied.")
-  private static final Arg<Boolean> ENABLE_JOB_CREATION = Arg.create(true);
-
   private static final Matcher<? super Class<?>> THRIFT_IFACE_MATCHER =
       Matchers.subclassesOf(AnnotatedAuroraAdmin.class)
           .and(Matchers.annotatedWith(DecoratedThrift.class));
 
-  private final Map<String, Boolean> toggledMethods;
-
-  public AopModule() {
-    this(ImmutableMap.of(
-        "createJob", ENABLE_JOB_CREATION.get(),
-        "acquireLock", ENABLE_UPDATES.get()));
-  }
-
-  @VisibleForTesting
-  AopModule(Map<String, Boolean> toggledMethods) {
-    this.toggledMethods = ImmutableMap.copyOf(toggledMethods);
-  }
-
-  private static final Function<Method, String> GET_NAME = Method::getName;
-
   @Override
   protected void configure() {
     // Layer ordering:
-    // APIVersion -> Log -> FeatureToggle -> StatsExporter -> SchedulerThriftInterface
+    // APIVersion -> Log -> StatsExporter -> SchedulerThriftInterface
 
     // It's important for this interceptor to be registered first to ensure it's at the 'top'
of
     // the stack and the standard message is always applied.
     bindThriftDecorator(new ServerInfoInterceptor());
 
     bindThriftDecorator(new LoggingInterceptor());
-
-    install(new PrivateModule() {
-      @Override
-      protected void configure() {
-        // Ensure that the provided methods exist on the decorated interface.
-        List<Method> methods =
-            ImmutableList.copyOf(AuroraSchedulerManager.Iface.class.getMethods());
-        for (String toggledMethod : toggledMethods.keySet()) {
-          Preconditions.checkArgument(
-              Iterables.any(methods,
-                  Predicates.compose(Predicates.equalTo(toggledMethod), GET_NAME)),
-              String.format("Method %s was not found in class %s",
-                  toggledMethod,
-                  AuroraSchedulerManager.Iface.class));
-        }
-
-        bind(new TypeLiteral<Map<String, Boolean>>() { }).toInstance(toggledMethods);
-        bind(IsFeatureEnabled.class).in(Singleton.class);
-        Key<Predicate<Method>> predicateKey = Key.get(new TypeLiteral<Predicate<Method>>()
{ });
-        bind(predicateKey).to(IsFeatureEnabled.class);
-        expose(predicateKey);
-      }
-    });
-    bindThriftDecorator(new FeatureToggleInterceptor());
     bindThriftDecorator(new ThriftStatsExporterInterceptor());
   }
 
@@ -127,27 +61,4 @@ public class AopModule extends AbstractModule {
         interceptor);
     binder.requestInjection(interceptor);
   }
-
-  private static class IsFeatureEnabled implements Predicate<Method> {
-    private final Predicate<String> methodEnabled;
-
-    @Inject
-    IsFeatureEnabled(Map<String, Boolean> toggleMethods) {
-      Predicate<String> builder = Predicates.alwaysTrue();
-      for (Map.Entry<String, Boolean> toggleMethod : toggleMethods.entrySet()) {
-        Predicate<String> enableMethod = Predicates.or(
-            toggleMethod.getValue()
-                ? Predicates.alwaysTrue()
-                : Predicates.alwaysFalse(),
-            Predicates.not(Predicates.equalTo(toggleMethod.getKey())));
-        builder = Predicates.and(builder, enableMethod);
-      }
-      methodEnabled = builder;
-    }
-
-    @Override
-    public boolean apply(Method method) {
-      return methodEnabled.apply(method.getName());
-    }
-  }
 }

http://git-wip-us.apache.org/repos/asf/aurora/blob/6b404539/src/main/java/org/apache/aurora/scheduler/thrift/aop/FeatureToggleInterceptor.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/thrift/aop/FeatureToggleInterceptor.java
b/src/main/java/org/apache/aurora/scheduler/thrift/aop/FeatureToggleInterceptor.java
deleted file mode 100644
index 2492796..0000000
--- a/src/main/java/org/apache/aurora/scheduler/thrift/aop/FeatureToggleInterceptor.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/**
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.aurora.scheduler.thrift.aop;
-
-import java.lang.reflect.Method;
-
-import javax.inject.Inject;
-
-import com.google.common.base.Predicate;
-
-import org.aopalliance.intercept.MethodInterceptor;
-import org.aopalliance.intercept.MethodInvocation;
-import org.apache.aurora.gen.ResponseCode;
-import org.apache.aurora.scheduler.thrift.Responses;
-
-/**
- * A method interceptor that blocks access to features based on a supplied predicate.
- */
-public class FeatureToggleInterceptor implements MethodInterceptor {
-
-  @Inject private Predicate<Method> allowMethod;
-
-  @Override
-  public Object invoke(MethodInvocation invocation) throws Throwable {
-    Method method = invocation.getMethod();
-    if (allowMethod.apply(method)) {
-      return invocation.proceed();
-    } else {
-      return Responses.addMessage(
-          Responses.empty(),
-          ResponseCode.ERROR,
-          "The " + method.getName() + " feature is currently disabled on this scheduler.");
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/aurora/blob/6b404539/src/test/java/org/apache/aurora/scheduler/storage/log/LogManagerTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/storage/log/LogManagerTest.java b/src/test/java/org/apache/aurora/scheduler/storage/log/LogManagerTest.java
index 692ace0..0443bb3 100644
--- a/src/test/java/org/apache/aurora/scheduler/storage/log/LogManagerTest.java
+++ b/src/test/java/org/apache/aurora/scheduler/storage/log/LogManagerTest.java
@@ -42,6 +42,7 @@ import org.apache.aurora.gen.HostAttributes;
 import org.apache.aurora.gen.ScheduleStatus;
 import org.apache.aurora.gen.ScheduledTask;
 import org.apache.aurora.gen.TaskConfig;
+import org.apache.aurora.gen.storage.DeduplicatedSnapshot;
 import org.apache.aurora.gen.storage.Frame;
 import org.apache.aurora.gen.storage.FrameChunk;
 import org.apache.aurora.gen.storage.FrameHeader;
@@ -103,10 +104,8 @@ public class LogManagerTest extends EasyMockTest {
     return new StreamManagerImpl(
         stream,
         new EntrySerializer.EntrySerializerImpl(maxEntrySize, Hashing.md5()),
-        false,
         Hashing.md5(),
-        new SnapshotDeduplicatorImpl(),
-        false);
+        new SnapshotDeduplicatorImpl());
   }
 
   @Test
@@ -261,7 +260,8 @@ public class LogManagerTest extends EasyMockTest {
   @Test
   public void testTransactionSnapshot() throws CodingException {
     Snapshot snapshot = createSnapshot();
-    expectAppend(position1, LogEntry.snapshot(snapshot));
+    DeduplicatedSnapshot deduplicated = new SnapshotDeduplicatorImpl().deduplicate(snapshot);
+    expectAppend(position1, Entries.deflate(LogEntry.deduplicatedSnapshot(deduplicated)));
     stream.truncateBefore(position1);
 
     control.replay();
@@ -393,10 +393,8 @@ public class LogManagerTest extends EasyMockTest {
     final StreamManagerImpl streamManager = new StreamManagerImpl(
         mockStream,
         new EntrySerializer.EntrySerializerImpl(message1.chunkSize, Hashing.md5()),
-        false,
         Hashing.md5(),
-        new SnapshotDeduplicatorImpl(),
-        false);
+        new SnapshotDeduplicatorImpl());
     StreamTransaction tr1 = streamManager.startTransaction();
     tr1.add(op1);
 
@@ -485,7 +483,8 @@ public class LogManagerTest extends EasyMockTest {
   public void testWriteAndReadDeflatedEntry() throws Exception {
     Snapshot snapshot = createSnapshot();
     LogEntry snapshotLogEntry = LogEntry.snapshot(snapshot);
-    LogEntry deflatedSnapshotEntry = Entries.deflate(snapshotLogEntry);
+    LogEntry deflatedSnapshotEntry = Entries.deflate(
+        LogEntry.deduplicatedSnapshot(new SnapshotDeduplicatorImpl().deduplicate(snapshot)));
 
     Entry snapshotEntry = createMock(Entry.class);
     expect(stream.append(entryEq(deflatedSnapshotEntry))).andReturn(position1);
@@ -504,10 +503,8 @@ public class LogManagerTest extends EasyMockTest {
     StreamManagerImpl streamManager = new StreamManagerImpl(
         stream,
         new EntrySerializer.EntrySerializerImpl(NO_FRAMES_EVER_SIZE, md5),
-        true,
         md5,
-        new SnapshotDeduplicatorImpl(),
-        false);
+        new SnapshotDeduplicatorImpl());
     streamManager.snapshot(snapshot);
     streamManager.readFromBeginning(reader);
   }

http://git-wip-us.apache.org/repos/asf/aurora/blob/6b404539/src/test/java/org/apache/aurora/scheduler/storage/log/LogStorageTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/storage/log/LogStorageTest.java b/src/test/java/org/apache/aurora/scheduler/storage/log/LogStorageTest.java
index 216f92f..4305270 100644
--- a/src/test/java/org/apache/aurora/scheduler/storage/log/LogStorageTest.java
+++ b/src/test/java/org/apache/aurora/scheduler/storage/log/LogStorageTest.java
@@ -56,6 +56,7 @@ import org.apache.aurora.gen.ResourceAggregate;
 import org.apache.aurora.gen.ScheduleStatus;
 import org.apache.aurora.gen.ScheduledTask;
 import org.apache.aurora.gen.TaskConfig;
+import org.apache.aurora.gen.storage.DeduplicatedSnapshot;
 import org.apache.aurora.gen.storage.LogEntry;
 import org.apache.aurora.gen.storage.Op;
 import org.apache.aurora.gen.storage.PruneJobUpdateHistory;
@@ -78,6 +79,7 @@ import org.apache.aurora.gen.storage.Transaction;
 import org.apache.aurora.gen.storage.storageConstants;
 import org.apache.aurora.scheduler.base.JobKeys;
 import org.apache.aurora.scheduler.base.Query;
+import org.apache.aurora.scheduler.base.TaskTestUtil;
 import org.apache.aurora.scheduler.base.Tasks;
 import org.apache.aurora.scheduler.events.EventSink;
 import org.apache.aurora.scheduler.events.PubsubEvent;
@@ -104,6 +106,7 @@ import org.apache.aurora.scheduler.storage.entities.IResourceAggregate;
 import org.apache.aurora.scheduler.storage.entities.IScheduledTask;
 import org.apache.aurora.scheduler.storage.entities.ITaskConfig;
 import org.apache.aurora.scheduler.storage.log.LogStorage.SchedulingService;
+import org.apache.aurora.scheduler.storage.log.SnapshotDeduplicator.SnapshotDeduplicatorImpl;
 import org.apache.aurora.scheduler.storage.log.testing.LogOpMatcher;
 import org.apache.aurora.scheduler.storage.log.testing.LogOpMatcher.StreamMatcher;
 import org.apache.aurora.scheduler.storage.testing.StorageTestUtil;
@@ -131,6 +134,7 @@ public class LogStorageTest extends EasyMockTest {
 
   private LogStorage logStorage;
   private Log log;
+  private SnapshotDeduplicator deduplicator;
   private Stream stream;
   private Position position;
   private StreamMatcher streamMatcher;
@@ -142,19 +146,15 @@ public class LogStorageTest extends EasyMockTest {
   @Before
   public void setUp() {
     log = createMock(Log.class);
-    SnapshotDeduplicator snapshotDeduplicator = createMock(SnapshotDeduplicator.class);
+    deduplicator = createMock(SnapshotDeduplicator.class);
 
     StreamManagerFactory streamManagerFactory = logStream -> {
       HashFunction md5 = Hashing.md5();
       return new StreamManagerImpl(
           logStream,
-          new EntrySerializer.EntrySerializerImpl(
-              Amount.of(1, Data.GB),
-              md5),
-          false,
+          new EntrySerializer.EntrySerializerImpl(Amount.of(1, Data.GB), md5),
           md5,
-          snapshotDeduplicator,
-          false);
+          deduplicator);
     };
     LogManager logManager = new LogManager(log, streamManagerFactory);
 
@@ -227,11 +227,12 @@ public class LogStorageTest extends EasyMockTest {
     Snapshot snapshotContents = new Snapshot()
         .setTimestamp(NOW)
         .setTasks(ImmutableSet.of(
-            new ScheduledTask()
-                .setStatus(ScheduleStatus.RUNNING)
-                .setAssignedTask(new AssignedTask().setTaskId("task_id"))));
+            TaskTestUtil.makeTask("task_id", TaskTestUtil.JOB).newBuilder()));
     expect(snapshotStore.createSnapshot()).andReturn(snapshotContents);
-    streamMatcher.expectSnapshot(snapshotContents).andReturn(position);
+    DeduplicatedSnapshot deduplicated =
+        new SnapshotDeduplicatorImpl().deduplicate(snapshotContents);
+    expect(deduplicator.deduplicate(snapshotContents)).andReturn(deduplicated);
+    streamMatcher.expectSnapshot(deduplicated).andReturn(position);
     stream.truncateBefore(position);
     Capture<MutateWork<Void, RuntimeException>> snapshotWork = createCapture();
     expect(storageUtil.storage.write(capture(snapshotWork))).andAnswer(

http://git-wip-us.apache.org/repos/asf/aurora/blob/6b404539/src/test/java/org/apache/aurora/scheduler/storage/log/testing/LogOpMatcher.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/storage/log/testing/LogOpMatcher.java
b/src/test/java/org/apache/aurora/scheduler/storage/log/testing/LogOpMatcher.java
index 9abdbbb..5a2524d 100644
--- a/src/test/java/org/apache/aurora/scheduler/storage/log/testing/LogOpMatcher.java
+++ b/src/test/java/org/apache/aurora/scheduler/storage/log/testing/LogOpMatcher.java
@@ -15,17 +15,19 @@ package org.apache.aurora.scheduler.storage.log.testing;
 
 import java.util.Objects;
 
+import com.google.common.base.Throwables;
 import com.google.common.collect.ImmutableList;
 
 import org.apache.aurora.codec.ThriftBinaryCodec;
 import org.apache.aurora.codec.ThriftBinaryCodec.CodingException;
+import org.apache.aurora.gen.storage.DeduplicatedSnapshot;
 import org.apache.aurora.gen.storage.LogEntry;
 import org.apache.aurora.gen.storage.Op;
-import org.apache.aurora.gen.storage.Snapshot;
 import org.apache.aurora.gen.storage.Transaction;
 import org.apache.aurora.gen.storage.storageConstants;
 import org.apache.aurora.scheduler.log.Log.Position;
 import org.apache.aurora.scheduler.log.Log.Stream;
+import org.apache.aurora.scheduler.storage.log.Entries;
 import org.easymock.EasyMock;
 import org.easymock.IArgumentMatcher;
 import org.easymock.IExpectationSetters;
@@ -92,9 +94,13 @@ public class LogOpMatcher implements IArgumentMatcher {
      * @param snapshot Expected snapshot.
      * @return An expectation setter.
      */
-    public IExpectationSetters<Position> expectSnapshot(Snapshot snapshot) {
-      LogEntry entry = LogEntry.snapshot(snapshot);
-      return expect(stream.append(sameEntry(entry)));
+    public IExpectationSetters<Position> expectSnapshot(DeduplicatedSnapshot snapshot)
{
+      try {
+        LogEntry entry = Entries.deflate(LogEntry.deduplicatedSnapshot(snapshot));
+        return expect(stream.append(sameEntry(entry)));
+      } catch (CodingException e) {
+        throw Throwables.propagate(e);
+      }
     }
   }
 

http://git-wip-us.apache.org/repos/asf/aurora/blob/6b404539/src/test/java/org/apache/aurora/scheduler/thrift/aop/AopModuleTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/thrift/aop/AopModuleTest.java b/src/test/java/org/apache/aurora/scheduler/thrift/aop/AopModuleTest.java
deleted file mode 100644
index 19b7415..0000000
--- a/src/test/java/org/apache/aurora/scheduler/thrift/aop/AopModuleTest.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/**
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.aurora.scheduler.thrift.aop;
-
-import java.util.Map;
-
-import com.google.common.collect.ImmutableMap;
-import com.google.inject.AbstractModule;
-import com.google.inject.CreationException;
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-
-import org.apache.aurora.common.testing.easymock.EasyMockTest;
-import org.apache.aurora.gen.AuroraAdmin.Iface;
-import org.apache.aurora.gen.JobConfiguration;
-import org.apache.aurora.gen.Response;
-import org.apache.aurora.gen.ResponseCode;
-import org.apache.aurora.gen.ServerInfo;
-import org.apache.aurora.scheduler.storage.entities.IServerInfo;
-import org.junit.Before;
-import org.junit.Test;
-
-import static org.easymock.EasyMock.expect;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertSame;
-
-public class AopModuleTest extends EasyMockTest {
-  private AnnotatedAuroraAdmin mockThrift;
-
-  @Before
-  public void setUp() throws Exception {
-    mockThrift = createMock(AnnotatedAuroraAdmin.class);
-  }
-
-  private Iface getIface(Map<String, Boolean> toggledMethods) {
-    Injector injector = Guice.createInjector(
-        new AbstractModule() {
-          @Override
-          protected void configure() {
-            bind(IServerInfo.class).toInstance(IServerInfo.build(new ServerInfo()));
-            MockDecoratedThrift.bindForwardedMock(binder(), mockThrift);
-          }
-        },
-        new AopModule(toggledMethods));
-    return injector.getInstance(AnnotatedAuroraAdmin.class);
-  }
-
-  @Test
-  public void testNonFlaggedMethod() throws Exception {
-    assertCreateAllowed(ImmutableMap.of("acquireLock", false));
-  }
-
-  @Test
-  public void testNoFlaggedMethods() throws Exception {
-    assertCreateAllowed(ImmutableMap.of());
-  }
-
-  @Test
-  public void testFlaggedMethodEnabled() throws Exception {
-    assertCreateAllowed(ImmutableMap.of("createJob", true));
-  }
-
-  @Test
-  public void testFlaggedMethodDisabled() throws Exception {
-    JobConfiguration job = new JobConfiguration();
-
-    control.replay();
-
-    Iface thrift = getIface(ImmutableMap.of("createJob", false));
-    assertEquals(ResponseCode.ERROR, thrift.createJob(job, null).getResponseCode());
-  }
-
-  @Test(expected = CreationException.class)
-  public void testMissingMethod() {
-    control.replay();
-    getIface(ImmutableMap.of("notamethod", true));
-  }
-
-  private void assertCreateAllowed(Map<String, Boolean> toggledMethods) throws Exception
{
-    JobConfiguration job = new JobConfiguration();
-    Response response = new Response();
-    expect(mockThrift.createJob(job, null)).andReturn(response);
-
-    control.replay();
-
-    Iface thrift = getIface(toggledMethods);
-    assertSame(response, thrift.createJob(job, null));
-  }
-
-  @Test
-  public void assertToStringNotIntercepted() {
-    control.replay();
-
-    Iface thrift = getIface(ImmutableMap.of());
-    assertNotNull(thrift.toString());
-  }
-}

http://git-wip-us.apache.org/repos/asf/aurora/blob/6b404539/src/test/java/org/apache/aurora/scheduler/thrift/aop/FeatureToggleInterceptorTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/thrift/aop/FeatureToggleInterceptorTest.java
b/src/test/java/org/apache/aurora/scheduler/thrift/aop/FeatureToggleInterceptorTest.java
deleted file mode 100644
index 0edb315..0000000
--- a/src/test/java/org/apache/aurora/scheduler/thrift/aop/FeatureToggleInterceptorTest.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/**
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.aurora.scheduler.thrift.aop;
-
-import java.lang.reflect.Method;
-
-import com.google.common.base.Predicate;
-import com.google.inject.AbstractModule;
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import com.google.inject.TypeLiteral;
-import com.google.inject.matcher.Matchers;
-
-import org.apache.aurora.common.testing.easymock.EasyMockTest;
-import org.apache.aurora.gen.Response;
-import org.apache.aurora.gen.ResponseCode;
-import org.apache.aurora.gen.TaskQuery;
-import org.apache.aurora.scheduler.thrift.auth.DecoratedThrift;
-import org.easymock.EasyMock;
-import org.junit.Before;
-import org.junit.Test;
-
-import static org.easymock.EasyMock.expect;
-import static org.junit.Assert.assertSame;
-
-public class FeatureToggleInterceptorTest extends EasyMockTest {
-
-  private AnnotatedAuroraAdmin realThrift;
-  private AnnotatedAuroraAdmin decoratedThrift;
-  private Predicate<Method> predicate;
-
-  @Before
-  public void setUp() {
-    realThrift = createMock(AnnotatedAuroraAdmin.class);
-    predicate = createMock(new Clazz<Predicate<Method>>() { });
-    Injector injector = Guice.createInjector(new AbstractModule() {
-      @Override
-      protected void configure() {
-        MockDecoratedThrift.bindForwardedMock(binder(), realThrift);
-        bind(new TypeLiteral<Predicate<Method>>() { }).toInstance(predicate);
-        AopModule.bindThriftDecorator(
-            binder(),
-            Matchers.annotatedWith(DecoratedThrift.class),
-            new FeatureToggleInterceptor());
-      }
-    });
-    decoratedThrift = injector.getInstance(AnnotatedAuroraAdmin.class);
-  }
-
-  @Test
-  public void testPredicatePass() throws Exception {
-    TaskQuery query = new TaskQuery();
-    Response response = new Response()
-        .setResponseCode(ResponseCode.OK);
-
-    expect(predicate.apply(EasyMock.anyObject())).andReturn(true);
-    expect(realThrift.getTasksStatus(query)).andReturn(response);
-
-    control.replay();
-
-    assertSame(response, decoratedThrift.getTasksStatus(query));
-  }
-
-  @Test
-  public void testPredicateDeny() throws Exception {
-    TaskQuery query = new TaskQuery();
-    expect(predicate.apply(EasyMock.anyObject())).andReturn(false);
-
-    control.replay();
-
-    assertSame(ResponseCode.ERROR, decoratedThrift.getTasksStatus(query).getResponseCode());
-  }
-}


Mime
View raw message