aurora-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ma...@apache.org
Subject git commit: Dropping assert on empty desired instances set.
Date Mon, 29 Sep 2014 23:33:48 GMT
Repository: incubator-aurora
Updated Branches:
  refs/heads/master 67939c977 -> 60d25a76e


Dropping assert on empty desired instances set.

Bugs closed: AURORA-756

Reviewed at https://reviews.apache.org/r/26102/


Project: http://git-wip-us.apache.org/repos/asf/incubator-aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-aurora/commit/60d25a76
Tree: http://git-wip-us.apache.org/repos/asf/incubator-aurora/tree/60d25a76
Diff: http://git-wip-us.apache.org/repos/asf/incubator-aurora/diff/60d25a76

Branch: refs/heads/master
Commit: 60d25a76ec8bfef01821996e9f376ccb8fb34ad4
Parents: 67939c9
Author: Maxim Khutornenko <maxim@apache.org>
Authored: Mon Sep 29 16:33:29 2014 -0700
Committer: Maxim Khutornenko <maxim@apache.org>
Committed: Mon Sep 29 16:33:29 2014 -0700

----------------------------------------------------------------------
 .../aurora/scheduler/quota/QuotaManager.java    |  3 +-
 .../aurora/scheduler/quota/QuotaUtil.java       | 22 +++++--
 .../thrift/SchedulerThriftInterface.java        | 14 ++--
 .../updater/InstanceActionHandler.java          |  3 +-
 .../aurora/scheduler/updater/JobDiff.java       |  6 +-
 .../aurora/scheduler/updater/UpdateFactory.java | 16 ++---
 .../scheduler/quota/QuotaManagerImplTest.java   | 67 ++++++++++++++++----
 .../thrift/SchedulerThriftInterfaceTest.java    | 40 ++++++++++++
 .../aurora/scheduler/updater/JobUpdaterIT.java  | 43 +++++++++++--
 9 files changed, 172 insertions(+), 42 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/60d25a76/src/main/java/org/apache/aurora/scheduler/quota/QuotaManager.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/quota/QuotaManager.java b/src/main/java/org/apache/aurora/scheduler/quota/QuotaManager.java
index ea6b7d9..a0da417 100644
--- a/src/main/java/org/apache/aurora/scheduler/quota/QuotaManager.java
+++ b/src/main/java/org/apache/aurora/scheduler/quota/QuotaManager.java
@@ -200,7 +200,8 @@ public interface QuotaManager {
           Optional<IJobUpdate> update = Optional.fromNullable(
               roleJobUpdates.get(JobKeys.from(input.getAssignedTask().getTask())));
 
-          if (update.isPresent()) {
+          // TODO(maxim): Address AURORA-768 to avoid double counting.
+          if (update.isPresent() && update.get().getInstructions().isSetDesiredState())
{
             IInstanceTaskConfig configs =
                 update.get().getInstructions().getDesiredState();
             RangeSet<Integer> desiredInstances = rangesToRangeSet(configs.getInstances());

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/60d25a76/src/main/java/org/apache/aurora/scheduler/quota/QuotaUtil.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/quota/QuotaUtil.java b/src/main/java/org/apache/aurora/scheduler/quota/QuotaUtil.java
index 105426f..d197dd5 100644
--- a/src/main/java/org/apache/aurora/scheduler/quota/QuotaUtil.java
+++ b/src/main/java/org/apache/aurora/scheduler/quota/QuotaUtil.java
@@ -15,6 +15,8 @@ package org.apache.aurora.scheduler.quota;
 
 import java.util.Set;
 
+import com.google.common.base.Function;
+import com.google.common.base.Optional;
 import com.google.common.collect.FluentIterable;
 import com.google.common.collect.ImmutableSet;
 
@@ -81,12 +83,8 @@ public final class QuotaUtil {
     }
 
     // Calculate desired prod task consumption.
-    ITaskConfig desiredConfig = instructions.getDesiredState().getTask();
-    IResourceAggregate desired = desiredConfig.isProduction()
-        ? ResourceAggregates.scale(
-        prodResourcesFromTasks(ImmutableSet.of(desiredConfig)),
-        getUpdateInstanceCount(instructions.getDesiredState().getInstances()))
-        : ResourceAggregates.EMPTY;
+    IResourceAggregate desired = Optional.fromNullable(instructions.getDesiredState())
+        .transform(TO_PROD_RESOURCES).or(ResourceAggregates.EMPTY);
 
     // Calculate result as max(existing, desired) per resource.
     return IResourceAggregate.build(new ResourceAggregate()
@@ -117,6 +115,18 @@ public final class QuotaUtil {
         .setDiskMb(diskMb));
   }
 
+  private static final Function<IInstanceTaskConfig, IResourceAggregate> TO_PROD_RESOURCES
=
+      new Function<IInstanceTaskConfig, IResourceAggregate>() {
+        @Override
+        public IResourceAggregate apply(IInstanceTaskConfig input) {
+          return input.getTask().isProduction()
+              ? ResourceAggregates.scale(
+                  prodResourcesFromTasks(ImmutableSet.of(input.getTask())),
+                  getUpdateInstanceCount(input.getInstances()))
+              : ResourceAggregates.EMPTY;
+        }
+      };
+
   private static int getUpdateInstanceCount(Set<IRange> ranges) {
     int instanceCount = 0;
     for (IRange range : ranges) {

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/60d25a76/src/main/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterface.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterface.java
b/src/main/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterface.java
index fa54d5b..2847bdb 100644
--- a/src/main/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterface.java
+++ b/src/main/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterface.java
@@ -1354,6 +1354,15 @@ class SchedulerThriftInterface implements AuroraAdmin.Iface {
     return builder.build();
   }
 
+  private static Optional<InstanceTaskConfig> buildDesiredState(JobDiff diff, ITaskConfig
config) {
+    Set<Integer> desiredInstances = diff.getReplacementInstances();
+    return desiredInstances.isEmpty()
+        ? Optional.<InstanceTaskConfig>absent()
+        : Optional.of(new InstanceTaskConfig()
+            .setTask(config.newBuilder())
+            .setInstances(convertRanges(Numbers.toRanges(desiredInstances))));
+  }
+
   @Override
   public Response startJobUpdate(JobUpdateRequest mutableRequest, SessionKey session) {
     if (!isUpdaterEnabled) {
@@ -1428,10 +1437,7 @@ class SchedulerThriftInterface implements AuroraAdmin.Iface {
             IJobUpdateInstructions.build(new JobUpdateInstructions()
                 .setSettings(settings.newBuilder())
                 .setInitialState(buildInitialState(diff.getReplacedInstances()))
-                .setDesiredState(new InstanceTaskConfig()
-                    .setTask(request.getTaskConfig().newBuilder())
-                    .setInstances(convertRanges(
-                        Numbers.toRanges(diff.getReplacementInstances())))));
+                .setDesiredState(buildDesiredState(diff, request.getTaskConfig()).orNull()));
 
         IJobUpdate update = IJobUpdate.build(new JobUpdate()
             .setSummary(new JobUpdateSummary()

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/60d25a76/src/main/java/org/apache/aurora/scheduler/updater/InstanceActionHandler.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/updater/InstanceActionHandler.java
b/src/main/java/org/apache/aurora/scheduler/updater/InstanceActionHandler.java
index b271697..f4363aa 100644
--- a/src/main/java/org/apache/aurora/scheduler/updater/InstanceActionHandler.java
+++ b/src/main/java/org/apache/aurora/scheduler/updater/InstanceActionHandler.java
@@ -54,6 +54,7 @@ interface InstanceActionHandler {
         int instanceId) {
 
       if (rollingForward) {
+        // Desired state is assumed to be non-null when AddTask is used.
         return instructions.getDesiredState().getTask();
       } else {
         for (IInstanceTaskConfig config : instructions.getInitialState()) {
@@ -76,8 +77,6 @@ interface InstanceActionHandler {
         StateManager stateManager,
         JobUpdateStatus status) {
 
-      // TODO(wfarner): This skips quota validation.  Either check quota here, or augment
-      // quota checking to take updates into consideration (AURORA-686).
       LOG.info("Adding instance " + instance + " while " + status);
       ITaskConfig replacement = getTargetConfig(
           instructions,

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/60d25a76/src/main/java/org/apache/aurora/scheduler/updater/JobDiff.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/updater/JobDiff.java b/src/main/java/org/apache/aurora/scheduler/updater/JobDiff.java
index bc0b830..9d02474 100644
--- a/src/main/java/org/apache/aurora/scheduler/updater/JobDiff.java
+++ b/src/main/java/org/apache/aurora/scheduler/updater/JobDiff.java
@@ -193,7 +193,7 @@ public final class JobDiff {
    * @param config Instance task configuration to represent as an instance mapping.
    * @return An instance ID to task config mapping that is equivalent to {@code config}.
    */
-  public static Map<Integer, ITaskConfig> asMap(IInstanceTaskConfig config) {
+  private static Map<Integer, ITaskConfig> asMap(IInstanceTaskConfig config) {
     return Maps.asMap(
         Numbers.rangesToInstanceIds(config.getInstances()),
         Functions.constant(config.getTask()));
@@ -216,7 +216,9 @@ public final class JobDiff {
     JobDiff diff = JobDiff.compute(
         taskStore,
         job,
-        ImmutableMap.copyOf(asMap(instructions.getDesiredState())),
+        ImmutableMap.copyOf(instructions.isSetDesiredState()
+            ? asMap(instructions.getDesiredState())
+            : ImmutableMap.<Integer, ITaskConfig>of()),
         instructions.getSettings().getUpdateOnlyTheseInstances());
     return diff.getReplacedInstances().isEmpty() && diff.getReplacementInstances().isEmpty();
   }

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/60d25a76/src/main/java/org/apache/aurora/scheduler/updater/UpdateFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/updater/UpdateFactory.java b/src/main/java/org/apache/aurora/scheduler/updater/UpdateFactory.java
index 62d08ad..b530861 100644
--- a/src/main/java/org/apache/aurora/scheduler/updater/UpdateFactory.java
+++ b/src/main/java/org/apache/aurora/scheduler/updater/UpdateFactory.java
@@ -87,13 +87,11 @@ interface UpdateFactory {
       checkArgument(
           settings.getUpdateGroupSize() > 0,
           "Update group size must be positive.");
-      checkArgument(
-          !instructions.getDesiredState().getInstances().isEmpty(),
-          "Instance count must be positive.");
 
       Set<Integer> instances;
-      Set<Integer> desiredInstances =
-          expandInstanceIds(ImmutableSet.of(instructions.getDesiredState()));
+      Set<Integer> desiredInstances = instructions.isSetDesiredState()
+          ? expandInstanceIds(ImmutableSet.of(instructions.getDesiredState()))
+          : ImmutableSet.<Integer>of();
 
       if (settings.getUpdateOnlyTheseInstances().isEmpty()) {
         // In a full job update, the working set is the union of instance IDs before and
after.
@@ -106,19 +104,19 @@ interface UpdateFactory {
       ImmutableMap.Builder<Integer, StateEvaluator<Optional<IScheduledTask>>>
evaluators =
           ImmutableMap.builder();
       for (int instanceId : instances) {
-        Optional<ITaskConfig> desiredState;
+        Optional<ITaskConfig> desiredStateConfig;
         if (rollingForward) {
-          desiredState = desiredInstances.contains(instanceId)
+          desiredStateConfig = desiredInstances.contains(instanceId)
               ? Optional.of(instructions.getDesiredState().getTask())
               : Optional.<ITaskConfig>absent();
         } else {
-          desiredState = getConfig(instanceId, instructions.getInitialState());
+          desiredStateConfig = getConfig(instanceId, instructions.getInitialState());
         }
 
         evaluators.put(
             instanceId,
             new InstanceUpdater(
-                desiredState,
+                desiredStateConfig,
                 settings.getMaxPerInstanceFailures(),
                 Amount.of((long) settings.getMinWaitInInstanceRunningMs(), Time.MILLISECONDS),
                 Amount.of((long) settings.getMaxWaitToInstanceRunningMs(), Time.MILLISECONDS),

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/60d25a76/src/test/java/org/apache/aurora/scheduler/quota/QuotaManagerImplTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/quota/QuotaManagerImplTest.java b/src/test/java/org/apache/aurora/scheduler/quota/QuotaManagerImplTest.java
index 11cc2f6..f961494 100644
--- a/src/test/java/org/apache/aurora/scheduler/quota/QuotaManagerImplTest.java
+++ b/src/test/java/org/apache/aurora/scheduler/quota/QuotaManagerImplTest.java
@@ -38,6 +38,7 @@ import org.apache.aurora.scheduler.base.ResourceAggregates;
 import org.apache.aurora.scheduler.quota.QuotaManager.QuotaException;
 import org.apache.aurora.scheduler.quota.QuotaManager.QuotaManagerImpl;
 import org.apache.aurora.scheduler.storage.JobUpdateStore;
+import org.apache.aurora.scheduler.storage.entities.IJobKey;
 import org.apache.aurora.scheduler.storage.entities.IJobUpdate;
 import org.apache.aurora.scheduler.storage.entities.IJobUpdateSummary;
 import org.apache.aurora.scheduler.storage.entities.IResourceAggregate;
@@ -373,6 +374,33 @@ public class QuotaManagerImplTest extends EasyMockTest {
   }
 
   @Test
+  public void testCheckQuotaNoDesiredState() {
+    expectQuota(IResourceAggregate.build(new ResourceAggregate(6, 6, 6))).times(2);
+    expectTasks(createProdTask("foo", 2, 2, 2), createProdTask("bar", 2, 2, 2)).times(2);
+
+    String updateId = "u1";
+    ITaskConfig config = createTaskConfig(2, 2, 2, true);
+    List<IJobUpdateSummary> summaries = buildJobUpdateSummaries(updateId, JobKeys.from(config));
+    IJobUpdate update = buildJobUpdate(summaries.get(0), config, 1, config, 1);
+    JobUpdate builder = update.newBuilder();
+    builder.getInstructions().unsetDesiredState();
+
+    expect(jobUpdateStore.fetchJobUpdateSummaries(updateQuery(config.getOwner().getRole())))
+        .andReturn(summaries).times(2);
+
+    expect(jobUpdateStore.fetchJobUpdate(updateId))
+        .andReturn(Optional.of(IJobUpdate.build(builder))).times(2);
+
+    control.replay();
+
+    QuotaCheckResult checkQuota = quotaManager.checkQuota(ROLE, prodResource(1, 1, 1));
+    assertEquals(INSUFFICIENT_QUOTA, checkQuota.getResult());
+    assertEquals(
+        IResourceAggregate.build(new ResourceAggregate(6, 6, 6)),
+        quotaManager.getQuotaInfo(ROLE).getProdConsumption());
+  }
+
+  @Test
   public void testSaveQuotaPasses() throws Exception {
     storageUtil.quotaStore.saveQuota(ROLE, QUOTA);
 
@@ -412,13 +440,33 @@ public class QuotaManagerImplTest extends EasyMockTest {
       int times) {
 
     String updateId = "u1";
-    List<IJobUpdateSummary> summaries =
-        ImmutableList.of(IJobUpdateSummary.build(new JobUpdateSummary()
-            .setJobKey(JobKeys.from(initial).newBuilder())
-            .setUpdateId(updateId)));
+    List<IJobUpdateSummary> summaries = buildJobUpdateSummaries(updateId, JobKeys.from(initial));
+    IJobUpdate update =
+        buildJobUpdate(summaries.get(0), initial, intialInstances, desired, desiredInstances);
+
+    expect(jobUpdateStore.fetchJobUpdateSummaries(updateQuery(initial.getOwner().getRole())))
+        .andReturn(summaries)
+        .times(times);
+
+    expect(jobUpdateStore.fetchJobUpdate(updateId)).andReturn(Optional.of(update)).times(times);
+
+  }
+
+  private List<IJobUpdateSummary> buildJobUpdateSummaries(String updateId, IJobKey
jobKey) {
+    return ImmutableList.of(IJobUpdateSummary.build(new JobUpdateSummary()
+        .setJobKey(jobKey.newBuilder())
+        .setUpdateId(updateId)));
+  }
 
-    IJobUpdate update = IJobUpdate.build(new JobUpdate()
-        .setSummary(summaries.get(0).newBuilder())
+  private IJobUpdate buildJobUpdate(
+      IJobUpdateSummary summary,
+      ITaskConfig initial,
+      int intialInstances,
+      ITaskConfig desired,
+      int desiredInstances) {
+
+    return IJobUpdate.build(new JobUpdate()
+        .setSummary(summary.newBuilder())
         .setInstructions(new JobUpdateInstructions()
             .setDesiredState(new InstanceTaskConfig()
                 .setTask(desired.newBuilder())
@@ -426,13 +474,6 @@ public class QuotaManagerImplTest extends EasyMockTest {
             .setInitialState(ImmutableSet.of(new InstanceTaskConfig()
                 .setTask(initial.newBuilder())
                 .setInstances(ImmutableSet.of(new Range(0, intialInstances - 1)))))));
-
-    expect(jobUpdateStore.fetchJobUpdateSummaries(updateQuery(initial.getOwner().getRole())))
-        .andReturn(summaries)
-        .times(times);
-
-    expect(jobUpdateStore.fetchJobUpdate(updateId)).andReturn(Optional.of(update)).times(times);
-
   }
 
   private void expectNoJobUpdates() {

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/60d25a76/src/test/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterfaceTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterfaceTest.java
b/src/test/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterfaceTest.java
index 83630a3..12b19fd 100644
--- a/src/test/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterfaceTest.java
+++ b/src/test/java/org/apache/aurora/scheduler/thrift/SchedulerThriftInterfaceTest.java
@@ -2144,6 +2144,46 @@ public class SchedulerThriftInterfaceTest extends EasyMockTest {
   }
 
   @Test
+  public void testStartUpdateEmptyDesired() throws Exception {
+    expectAuth(ROLE, true);
+    expect(cronJobManager.hasJob(JOB_KEY)).andReturn(false);
+
+    ITaskConfig newTask = buildScheduledTask(0, 5).getAssignedTask().getTask();
+    expect(taskIdGenerator.generate(newTask, 1))
+        .andReturn(TASK_ID);
+    expect(quotaManager.checkQuota(
+        ROLE,
+        IResourceAggregate.build(new ResourceAggregate(1, 5, 1024)))).andReturn(ENOUGH_QUOTA);
+
+    IScheduledTask oldTask1 = buildScheduledTask(0, 5);
+    IScheduledTask oldTask2 = buildScheduledTask(1, 5);
+
+    // Set instance count to 1 to generate empty desired state in diff.
+    IJobUpdate update = buildJobUpdate(1, newTask, ImmutableMap.of(
+        oldTask1.getAssignedTask().getTask(), ImmutableSet.of(new Range(0, 1))));
+
+    // Set diff-adjusted IJobUpdate expectations.
+    JobUpdate expected = update.newBuilder();
+    expected.getInstructions().setInitialState(ImmutableSet.of(
+        new InstanceTaskConfig(newTask.newBuilder(), ImmutableSet.of(new Range(1, 1)))));
+    expected.getInstructions().unsetDesiredState();
+
+    expect(uuidGenerator.createNew()).andReturn(UU_ID);
+    storageUtil.expectTaskFetch(
+        Query.unscoped().byJob(JOB_KEY).active(),
+        oldTask1,
+        oldTask2);
+
+    jobUpdateController.start(IJobUpdate.build(expected), USER);
+
+    control.replay();
+
+    Response response =
+        assertOkResponse(thrift.startJobUpdate(buildJobUpdateRequest(update), SESSION));
+    assertEquals(UPDATE_ID, response.getResult().getStartJobUpdateResult().getUpdateId());
+  }
+
+  @Test
   public void testStartUpdateFailsNullRequest() throws Exception {
     control.replay();
     assertResponse(ERROR, thrift.startJobUpdate(null, SESSION));

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/60d25a76/src/test/java/org/apache/aurora/scheduler/updater/JobUpdaterIT.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/updater/JobUpdaterIT.java b/src/test/java/org/apache/aurora/scheduler/updater/JobUpdaterIT.java
index 4934cbb..7254588 100644
--- a/src/test/java/org/apache/aurora/scheduler/updater/JobUpdaterIT.java
+++ b/src/test/java/org/apache/aurora/scheduler/updater/JobUpdaterIT.java
@@ -665,11 +665,6 @@ public class JobUpdaterIT extends EasyMockTest {
     update = makeJobUpdate().newBuilder();
     update.getInstructions().getSettings().setMinWaitInInstanceRunningMs(0);
     expectInvalid(update);
-
-    update = makeJobUpdate().newBuilder();
-    update.getInstructions().setDesiredState(
-        new InstanceTaskConfig().setInstances(ImmutableSet.<Range>of()));
-    expectInvalid(update);
   }
 
   @Test
@@ -816,6 +811,17 @@ public class JobUpdaterIT extends EasyMockTest {
     updater.start(update, USER);
   }
 
+  @Test(expected = NoopUpdateStateException.class)
+  public void testNoopUpdateEmptyDiff() throws Exception {
+    control.replay();
+
+    final IJobUpdate update = makeJobUpdate();
+    JobUpdate builder = update.newBuilder();
+    builder.getInstructions().unsetDesiredState();
+
+    updater.start(IJobUpdate.build(builder), USER);
+  }
+
   @Test
   public void testStuckTask() throws Exception {
     expectTaskKilled().times(3);
@@ -888,6 +894,33 @@ public class JobUpdaterIT extends EasyMockTest {
   }
 
   @Test
+  public void testRemoveInstances() throws Exception {
+    expectTaskKilled();
+
+    control.replay();
+
+    // Set instance count such that instance 1 is removed.
+    IJobUpdate update = setInstanceCount(makeJobUpdate(makeInstanceConfig(0, 1, NEW_CONFIG)),
1);
+    insertInitialTasks(update);
+
+    changeState(JOB, 0, ASSIGNED, STARTING, RUNNING);
+    changeState(JOB, 1, ASSIGNED, STARTING, RUNNING);
+    clock.advance(WATCH_TIMEOUT);
+
+    ImmutableMultimap.Builder<Integer, JobUpdateAction> actions = ImmutableMultimap.builder();
+
+    // Instance 1 is removed.
+    updater.start(update, USER);
+    actions.putAll(1, INSTANCE_UPDATING);
+    changeState(JOB, 1, KILLED);
+    clock.advance(WATCH_TIMEOUT);
+
+    actions.put(1, INSTANCE_UPDATED);
+    assertState(ROLLED_FORWARD, actions.build());
+    assertJobState(JOB, ImmutableMap.of(0, NEW_CONFIG));
+  }
+
+  @Test
   public void testBadPubsubUpdate() {
     control.replay();
 


Mime
View raw message