Return-Path: X-Original-To: apmail-aurora-commits-archive@minotaur.apache.org Delivered-To: apmail-aurora-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id B883B10E31 for ; Tue, 11 Mar 2014 18:00:22 +0000 (UTC) Received: (qmail 91623 invoked by uid 500); 11 Mar 2014 18:00:22 -0000 Delivered-To: apmail-aurora-commits-archive@aurora.apache.org Received: (qmail 91603 invoked by uid 500); 11 Mar 2014 18:00:20 -0000 Mailing-List: contact commits-help@aurora.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@aurora.incubator.apache.org Delivered-To: mailing list commits@aurora.incubator.apache.org Received: (qmail 91596 invoked by uid 99); 11 Mar 2014 18:00:20 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 11 Mar 2014 18:00:19 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED,T_RP_MATCHES_RCVD X-Spam-Check-By: apache.org Received: from [140.211.11.3] (HELO mail.apache.org) (140.211.11.3) by apache.org (qpsmtpd/0.29) with SMTP; Tue, 11 Mar 2014 18:00:16 +0000 Received: (qmail 88937 invoked by uid 99); 11 Mar 2014 17:59:51 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 11 Mar 2014 17:59:51 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 0F8409404BF; Tue, 11 Mar 2014 17:59:51 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: maxim@apache.org To: commits@aurora.incubator.apache.org Message-Id: <5a2d4027df7240c6bb9f5c6061019a3e@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: git commit: Part 1 of converting from Packages to Metadata. Date: Tue, 11 Mar 2014 17:59:51 +0000 (UTC) X-Virus-Checked: Checked by ClamAV on apache.org Repository: incubator-aurora Updated Branches: refs/heads/master 6fd220423 -> c90c87464 Part 1 of converting from Packages to Metadata. Bugs closed: AURORA-158 Reviewed at https://reviews.apache.org/r/18935/ Project: http://git-wip-us.apache.org/repos/asf/incubator-aurora/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-aurora/commit/c90c8746 Tree: http://git-wip-us.apache.org/repos/asf/incubator-aurora/tree/c90c8746 Diff: http://git-wip-us.apache.org/repos/asf/incubator-aurora/diff/c90c8746 Branch: refs/heads/master Commit: c90c87464a3bf326fe6863e111238de480f56a59 Parents: 6fd2204 Author: Maxim Khutornenko Authored: Tue Mar 11 10:59:24 2014 -0700 Committer: Maxim Khutornenko Committed: Tue Mar 11 10:59:24 2014 -0700 ---------------------------------------------------------------------- .../aurora/scheduler/http/SchedulerzJob.java | 9 ++---- .../aurora/scheduler/http/SchedulerzRole.java | 20 +++--------- .../scheduler/http/TransformationUtils.java | 32 ++++++++++++++++++++ .../local/IsolatedSchedulerModule.java | 6 ++-- .../aurora/scheduler/http/schedulerzrole.st | 4 +-- .../thrift/org/apache/aurora/gen/api.thrift | 10 ++++-- .../configuration/ConfigurationManagerTest.java | 2 +- .../org/apache/aurora/gen/api.thrift.md5 | 2 +- 8 files changed, 55 insertions(+), 30 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/c90c8746/src/main/java/org/apache/aurora/scheduler/http/SchedulerzJob.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/http/SchedulerzJob.java b/src/main/java/org/apache/aurora/scheduler/http/SchedulerzJob.java index c14e195..2ccc6f3 100644 --- a/src/main/java/org/apache/aurora/scheduler/http/SchedulerzJob.java +++ b/src/main/java/org/apache/aurora/scheduler/http/SchedulerzJob.java @@ -268,12 +268,9 @@ public class SchedulerzJob extends JerseyTemplateServlet { details.put("ports", Joiner.on(", ").join(ImmutableSortedSet.copyOf(task.getRequestedPorts()))); } - if (!task.getPackages().isEmpty()) { - List packages = Ordering.natural().sortedCopy( - Iterables.transform(task.getPackages(), TransformationUtils.PACKAGE_TOSTRING)); - details.put( - "packages", - Joiner.on(',').join(packages)); + Optional metadata = TransformationUtils.getMetadata(task); + if (metadata.isPresent()) { + details.put("metadata", metadata.get()); } details.put("contact", task.isSetContactEmail() ? task.getContactEmail() : "none"); return new SchedulingDetails(details.build()); http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/c90c8746/src/main/java/org/apache/aurora/scheduler/http/SchedulerzRole.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/http/SchedulerzRole.java b/src/main/java/org/apache/aurora/scheduler/http/SchedulerzRole.java index 1151b03..b3f3124 100644 --- a/src/main/java/org/apache/aurora/scheduler/http/SchedulerzRole.java +++ b/src/main/java/org/apache/aurora/scheduler/http/SchedulerzRole.java @@ -18,7 +18,6 @@ package org.apache.aurora.scheduler.http; import java.util.Collection; import java.util.List; import java.util.Map; -import java.util.Set; import javax.inject.Inject; import javax.ws.rs.GET; @@ -31,16 +30,12 @@ import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; import com.google.common.base.Function; -import com.google.common.base.Joiner; import com.google.common.base.Optional; import com.google.common.base.Predicate; import com.google.common.collect.FluentIterable; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Iterables; -import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Multimap; -import com.google.common.collect.Sets; import com.twitter.common.base.Closure; import org.antlr.stringtemplate.StringTemplate; @@ -184,7 +179,7 @@ public class SchedulerzRole extends JerseyTemplateServlet { .put("cronSchedule", job.getCronSchedule()) .put("nextRun", cronPredictor.predictNextRun(job.getCronSchedule()).getTime()) .put("cronCollisionPolicy", cronCollisionPolicy(job)) - .put("packages", getPackages(job)) + .put("metadata", getMetadata(job)) .build(); } }); @@ -194,16 +189,9 @@ public class SchedulerzRole extends JerseyTemplateServlet { return CronJobManager.orDefault(jobConfiguration.getCronCollisionPolicy()); } - private static String getPackages(IJobConfiguration job) { - Set packages = Sets.newHashSet(); - - // Insert all packages for all tasks in the set to eliminate duplicates - ITaskConfig task = job.getTaskConfig(); - if (!task.getPackages().isEmpty()) { - packages.addAll(Lists.newArrayList( - Iterables.transform(task.getPackages(), TransformationUtils.PACKAGE_TOSTRING))); - } - return Joiner.on(',').join(packages); + private static String getMetadata(IJobConfiguration job) { + Optional metadata = TransformationUtils.getMetadata(job.getTaskConfig()); + return metadata.isPresent() ? metadata.get() : ""; } private List fetchJobsBy( http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/c90c8746/src/main/java/org/apache/aurora/scheduler/http/TransformationUtils.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/http/TransformationUtils.java b/src/main/java/org/apache/aurora/scheduler/http/TransformationUtils.java index b6ee04e..bfd28a0 100644 --- a/src/main/java/org/apache/aurora/scheduler/http/TransformationUtils.java +++ b/src/main/java/org/apache/aurora/scheduler/http/TransformationUtils.java @@ -19,11 +19,16 @@ import java.util.Collection; import com.google.common.base.Function; import com.google.common.base.Joiner; +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; +import com.google.common.collect.Ordering; import com.google.common.collect.Range; import org.apache.aurora.scheduler.base.Numbers; +import org.apache.aurora.scheduler.storage.entities.IMetadata; import org.apache.aurora.scheduler.storage.entities.IPackage; +import org.apache.aurora.scheduler.storage.entities.ITaskConfig; /** * Utility class to hold common object to string transformation helper functions. @@ -37,6 +42,14 @@ final class TransformationUtils { } }; + public static final Function METADATA_TOSTRING = + new com.twitter.common.base.Function() { + @Override + public String apply(IMetadata item) { + return item.getKey() + ": " + item.getValue(); + } + }; + public static final Function, String> RANGE_TOSTRING = new Function, String>() { @Override @@ -59,4 +72,23 @@ final class TransformationUtils { private TransformationUtils() { // Utility class } + + /** + * Gets an optional task metadata. + * + * @param task Task to get metadata from. + * @return Present if task metadata exists, absent otherwise. + */ + public static Optional getMetadata(ITaskConfig task) { + if (task.isSetPackagesDEPRECATED()) { + Iterable packages = ImmutableSet.copyOf( + Iterables.transform(task.getPackagesDEPRECATED(), TransformationUtils.PACKAGE_TOSTRING)); + return Optional.of(Joiner.on(", ").join(Ordering.natural().sortedCopy(packages))); + } else if (task.isSetMetadata()) { + Iterable metadata = ImmutableSet.copyOf( + Iterables.transform(task.getMetadata(), TransformationUtils.METADATA_TOSTRING)); + return Optional.of(Joiner.on(", ").join(Ordering.natural().sortedCopy(metadata))); + } + return Optional.absent(); + } } http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/c90c8746/src/main/java/org/apache/aurora/scheduler/local/IsolatedSchedulerModule.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/local/IsolatedSchedulerModule.java b/src/main/java/org/apache/aurora/scheduler/local/IsolatedSchedulerModule.java index fe2965d..b931baa 100644 --- a/src/main/java/org/apache/aurora/scheduler/local/IsolatedSchedulerModule.java +++ b/src/main/java/org/apache/aurora/scheduler/local/IsolatedSchedulerModule.java @@ -49,7 +49,7 @@ import org.apache.aurora.gen.AuroraAdmin; import org.apache.aurora.gen.ExecutorConfig; import org.apache.aurora.gen.Identity; import org.apache.aurora.gen.JobConfiguration; -import org.apache.aurora.gen.Package; +import org.apache.aurora.gen.Metadata; import org.apache.aurora.gen.ResourceAggregate; import org.apache.aurora.gen.Response; import org.apache.aurora.gen.SessionKey; @@ -277,7 +277,9 @@ public class IsolatedSchedulerModule extends AbstractModule { .setNumCpus(1.0) .setDiskMb(1024) .setRamMb(1024) - .setPackages(ImmutableSet.of(new Package(owner.getRole(), "package", 15))) + .setMetadata(ImmutableSet.of( + new Metadata("role", owner.getRole()), + new Metadata("package", "15"))) .setExecutorConfig(new ExecutorConfig("aurora", "opaque"))); } http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/c90c8746/src/main/resources/org/apache/aurora/scheduler/http/schedulerzrole.st ---------------------------------------------------------------------- diff --git a/src/main/resources/org/apache/aurora/scheduler/http/schedulerzrole.st b/src/main/resources/org/apache/aurora/scheduler/http/schedulerzrole.st index d568a7e..39c1d95 100644 --- a/src/main/resources/org/apache/aurora/scheduler/http/schedulerzrole.st +++ b/src/main/resources/org/apache/aurora/scheduler/http/schedulerzrole.st @@ -134,7 +134,7 @@ Schedule Next Run Collision policy - Packages + Metadata $cronJobs:{ job | @@ -145,7 +145,7 @@ $job.cronSchedule$ $job.cronCollisionPolicy$ - $job.packages$ + $job.metadata$ }$ http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/c90c8746/src/main/thrift/org/apache/aurora/gen/api.thrift ---------------------------------------------------------------------- diff --git a/src/main/thrift/org/apache/aurora/gen/api.thrift b/src/main/thrift/org/apache/aurora/gen/api.thrift index 5c5142c..f6c444d 100644 --- a/src/main/thrift/org/apache/aurora/gen/api.thrift +++ b/src/main/thrift/org/apache/aurora/gen/api.thrift @@ -110,6 +110,12 @@ struct Package { 3: i32 version } +// Arbitrary key-value metadata to be included into TaskConfig. +struct Metadata { + 1: string key + 2: string value +} + // A unique identifier for a Job. struct JobKey { 1: string role // Mesos role (Unix service account), for example "mesos" @@ -170,9 +176,9 @@ struct TaskConfig { // Wildcards are supported for dynamic link // crafting based on host, ports, instance, etc. 23: optional string contactEmail - 24: optional set packages // Used only to display package information in the - // scheduler UI. + 24: optional set packagesDEPRECATED // TODO(maxim): Drop when fully migrated to metadata. 25: optional ExecutorConfig executorConfig // Executor configuration + 27: optional set metadata // Used to display additional details in the UI. } // Defines the policy for launching a new cron job when one is already running. http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/c90c8746/src/test/java/org/apache/aurora/scheduler/configuration/ConfigurationManagerTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/configuration/ConfigurationManagerTest.java b/src/test/java/org/apache/aurora/scheduler/configuration/ConfigurationManagerTest.java index 4beb2cf..05e80e8 100644 --- a/src/test/java/org/apache/aurora/scheduler/configuration/ConfigurationManagerTest.java +++ b/src/test/java/org/apache/aurora/scheduler/configuration/ConfigurationManagerTest.java @@ -58,7 +58,7 @@ public class ConfigurationManagerTest { .setContactEmail("foo@twitter.com") .setProduction(false) .setDiskMb(1) - .setPackages(null) + .setMetadata(null) .setNumCpus(1.0) .setRamMb(1) .setMaxTaskFailures(0) http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/c90c8746/src/test/resources/org/apache/aurora/gen/api.thrift.md5 ---------------------------------------------------------------------- diff --git a/src/test/resources/org/apache/aurora/gen/api.thrift.md5 b/src/test/resources/org/apache/aurora/gen/api.thrift.md5 index 250dd6b..8b6f0a1 100644 --- a/src/test/resources/org/apache/aurora/gen/api.thrift.md5 +++ b/src/test/resources/org/apache/aurora/gen/api.thrift.md5 @@ -1 +1 @@ -0f9ef34166894f7e8099b1c355bb81f9 +81e3efd49d0b177759414564a4903a3d