aurora-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ma...@apache.org
Subject aurora git commit: Revocable: schema changes.
Date Tue, 25 Aug 2015 22:57:43 GMT
Repository: aurora
Updated Branches:
  refs/heads/master f58cbf934 -> 506d0cdbc


Revocable: schema changes.

Bugs closed: AURORA-1414

Reviewed at https://reviews.apache.org/r/37719/


Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/506d0cdb
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/506d0cdb
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/506d0cdb

Branch: refs/heads/master
Commit: 506d0cdbcb558fe43cab60ced8b613cfddc54b7f
Parents: f58cbf9
Author: Maxim Khutornenko <maxim@apache.org>
Authored: Tue Aug 25 15:55:20 2015 -0700
Committer: Maxim Khutornenko <maxim@apache.org>
Committed: Tue Aug 25 15:55:20 2015 -0700

----------------------------------------------------------------------
 .../thrift/org/apache/aurora/gen/api.thrift     |  2 ++
 .../apache/aurora/scheduler/TierManager.java    |  4 +---
 .../aurora/scheduler/base/TaskTestUtil.java     |  1 +
 .../configuration/ConfigurationManager.java     |  4 ++++
 .../storage/db/views/DbTaskConfig.java          |  2 ++
 src/main/python/apache/aurora/client/config.py  | 25 ++++++++++++++++----
 .../python/apache/aurora/config/schema/base.py  |  1 +
 src/main/python/apache/aurora/config/thrift.py  |  1 +
 .../scheduler/storage/db/TaskConfigMapper.xml   |  3 +++
 .../aurora/scheduler/storage/db/schema.sql      |  3 ++-
 .../aurora/scheduler/TierManagerTest.java       |  2 +-
 .../configuration/ConfigurationManagerTest.java | 15 ++++++++++--
 .../apache/aurora/client/cli/test_create.py     |  5 +---
 .../python/apache/aurora/client/test_config.py  | 25 ++++++++++++++++----
 .../python/apache/aurora/config/test_thrift.py  |  7 ++++++
 15 files changed, 80 insertions(+), 20 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/aurora/blob/506d0cdb/api/src/main/thrift/org/apache/aurora/gen/api.thrift
----------------------------------------------------------------------
diff --git a/api/src/main/thrift/org/apache/aurora/gen/api.thrift b/api/src/main/thrift/org/apache/aurora/gen/api.thrift
index f792be0..21137bb 100644
--- a/api/src/main/thrift/org/apache/aurora/gen/api.thrift
+++ b/api/src/main/thrift/org/apache/aurora/gen/api.thrift
@@ -251,6 +251,8 @@ struct TaskConfig {
  13: i32 maxTaskFailures
  /** Whether this is a production task, which can preempt. */
  18: optional bool production
+ /** Task tier type. */
+ 30: optional string tier
 
  20: set<Constraint> constraints
  /** a list of named ports this task requests */

http://git-wip-us.apache.org/repos/asf/aurora/blob/506d0cdb/src/main/java/org/apache/aurora/scheduler/TierManager.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/TierManager.java b/src/main/java/org/apache/aurora/scheduler/TierManager.java
index ebfad97..652afec 100644
--- a/src/main/java/org/apache/aurora/scheduler/TierManager.java
+++ b/src/main/java/org/apache/aurora/scheduler/TierManager.java
@@ -17,8 +17,7 @@ import org.apache.aurora.scheduler.storage.entities.ITaskConfig;
 
 /**
  * Translates job tier configuration into a set of task traits/attributes.
- * TODO(maxim): Implement external configuration support defined here:
- * https://docs.google.com/document/d/1gexe2uM_9gjsV62cMmX0VjH85Uokko21vEoENY2jjF0
+ * TODO(maxim): Implement external configuration support: AURORA-1443.
  */
 public interface TierManager {
 
@@ -34,7 +33,6 @@ public interface TierManager {
 
     @Override
     public TierInfo getTier(ITaskConfig taskConfig) {
-      // TODO(maxim): Implement when schema changes are defined.
       return new TierInfo(false);
     }
   }

http://git-wip-us.apache.org/repos/asf/aurora/blob/506d0cdb/src/main/java/org/apache/aurora/scheduler/base/TaskTestUtil.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/base/TaskTestUtil.java b/src/main/java/org/apache/aurora/scheduler/base/TaskTestUtil.java
index 1903315..d7f3c60 100644
--- a/src/main/java/org/apache/aurora/scheduler/base/TaskTestUtil.java
+++ b/src/main/java/org/apache/aurora/scheduler/base/TaskTestUtil.java
@@ -63,6 +63,7 @@ public final class TaskTestUtil {
         .setPriority(1)
         .setMaxTaskFailures(-1)
         .setProduction(true)
+        .setTier("tier-" + job.getEnvironment())
         .setConstraints(ImmutableSet.of(
             new Constraint(
                 "valueConstraint",

http://git-wip-us.apache.org/repos/asf/aurora/blob/506d0cdb/src/main/java/org/apache/aurora/scheduler/configuration/ConfigurationManager.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/configuration/ConfigurationManager.java
b/src/main/java/org/apache/aurora/scheduler/configuration/ConfigurationManager.java
index d103d19..05e8b71 100644
--- a/src/main/java/org/apache/aurora/scheduler/configuration/ConfigurationManager.java
+++ b/src/main/java/org/apache/aurora/scheduler/configuration/ConfigurationManager.java
@@ -287,6 +287,10 @@ public final class ConfigurationManager {
           "Environment contains illegal characters: " + config.getEnvironment());
     }
 
+    if (config.isSetTier() && !isGoodIdentifier(config.getTier())) {
+      throw new TaskDescriptionException("Tier contains illegal characters: " + config.getTier());
+    }
+
     if (config.isSetJob()) {
       if (!JobKeys.isValid(config.getJob())) {
         // Job key is set but invalid

http://git-wip-us.apache.org/repos/asf/aurora/blob/506d0cdb/src/main/java/org/apache/aurora/scheduler/storage/db/views/DbTaskConfig.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/db/views/DbTaskConfig.java
b/src/main/java/org/apache/aurora/scheduler/storage/db/views/DbTaskConfig.java
index 956c508..9879836 100644
--- a/src/main/java/org/apache/aurora/scheduler/storage/db/views/DbTaskConfig.java
+++ b/src/main/java/org/apache/aurora/scheduler/storage/db/views/DbTaskConfig.java
@@ -48,6 +48,7 @@ public final class DbTaskConfig {
   private ExecutorConfig executorConfig;
   private List<Metadata> metadata;
   private DbContainer container;
+  private String tier;
 
   private DbTaskConfig() {
   }
@@ -69,6 +70,7 @@ public final class DbTaskConfig {
         .setPriority(priority)
         .setMaxTaskFailures(maxTaskFailures)
         .setProduction(production)
+        .setTier(tier)
         .setConstraints(constraints.stream()
             .map(DbConstraint::toThrift)
             .collect(GuavaUtils.toImmutableSet()))

http://git-wip-us.apache.org/repos/asf/aurora/blob/506d0cdb/src/main/python/apache/aurora/client/config.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/config.py b/src/main/python/apache/aurora/client/config.py
index 59703ef..f5ac084 100644
--- a/src/main/python/apache/aurora/client/config.py
+++ b/src/main/python/apache/aurora/client/config.py
@@ -22,6 +22,8 @@ import math
 import re
 import sys
 
+from pystachio import Empty
+
 from apache.aurora.client import binding_helper
 from apache.aurora.client.base import die
 from apache.aurora.config import AuroraConfig
@@ -55,13 +57,25 @@ def _validate_announce_configuration(config):
 STAGING_RE = re.compile(r'^staging\d*$')
 
 
+#TODO(maxim): Merge env and tier and move definitions to scheduler: AURORA-1443.
+def __validate_env(name, config_name):
+  if STAGING_RE.match(name):
+    return
+  if name not in ('prod', 'devel', 'test'):
+    raise ValueError('%s should be one of "prod", "devel", "test" or '
+                     'staging<number>!  Got %s' % (config_name, name))
+
+
 def _validate_environment_name(config):
   env_name = str(config.raw().environment())
-  if STAGING_RE.match(env_name):
-    return
-  if env_name not in ('prod', 'devel', 'test'):
-    raise ValueError('Environment name should be one of "prod", "devel", "test" or '
-                     'staging<number>!  Got %s' % env_name)
+  __validate_env(env_name, 'Environment')
+
+
+def _validate_tier(config):
+  tier_raw = config.raw().tier()
+  tier_name = str(tier_raw) if tier_raw is not Empty else None
+  if tier_name is not None:
+    __validate_env(tier_name, 'Tier')
 
 
 UPDATE_CONFIG_MAX_FAILURES_ERROR = '''
@@ -114,6 +128,7 @@ def validate_config(config, env=None):
   _validate_update_config(config)
   _validate_announce_configuration(config)
   _validate_environment_name(config)
+  _validate_tier(config)
 
 
 class GlobalHookRegistry(object):

http://git-wip-us.apache.org/repos/asf/aurora/blob/506d0cdb/src/main/python/apache/aurora/config/schema/base.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/config/schema/base.py b/src/main/python/apache/aurora/config/schema/base.py
index 214d559..f8a1f05 100644
--- a/src/main/python/apache/aurora/config/schema/base.py
+++ b/src/main/python/apache/aurora/config/schema/base.py
@@ -110,6 +110,7 @@ class MesosJob(Struct):
   instances     = Default(Integer, 1)
   task          = Required(Task)
   announce      = Announcer
+  tier          = String
 
   cron_schedule = String
   cron_collision_policy = Default(String, "KILL_EXISTING")

http://git-wip-us.apache.org/repos/asf/aurora/blob/506d0cdb/src/main/python/apache/aurora/config/thrift.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/config/thrift.py b/src/main/python/apache/aurora/config/thrift.py
index adf53bb..f69977c 100644
--- a/src/main/python/apache/aurora/config/thrift.py
+++ b/src/main/python/apache/aurora/config/thrift.py
@@ -197,6 +197,7 @@ def convert(job, metadata=frozenset(), ports=frozenset()):
   task.maxTaskFailures = fully_interpolated(job.max_task_failures())
   task.priority = fully_interpolated(job.priority())
   task.contactEmail = not_empty_or(job.contact(), None)
+  task.tier = not_empty_or(job.tier(), None)
 
   # Add metadata to a task, to display in the scheduler UI.
   task.metadata = frozenset(Metadata(key=str(key), value=str(value)) for key, value in metadata)

http://git-wip-us.apache.org/repos/asf/aurora/blob/506d0cdb/src/main/resources/org/apache/aurora/scheduler/storage/db/TaskConfigMapper.xml
----------------------------------------------------------------------
diff --git a/src/main/resources/org/apache/aurora/scheduler/storage/db/TaskConfigMapper.xml
b/src/main/resources/org/apache/aurora/scheduler/storage/db/TaskConfigMapper.xml
index bfad339..c70e859 100644
--- a/src/main/resources/org/apache/aurora/scheduler/storage/db/TaskConfigMapper.xml
+++ b/src/main/resources/org/apache/aurora/scheduler/storage/db/TaskConfigMapper.xml
@@ -32,6 +32,7 @@
       contact_email,
       executor_name,
       executor_data,
+      tier
     ) VALUES (
       (
         SELECT ID
@@ -51,6 +52,7 @@
       #{config.contactEmail},
       #{config.executorConfig.name},
       #{config.executorConfig.data},
+      #{config.tier}
     )
   </insert>
 
@@ -167,6 +169,7 @@
       c.contact_email AS contact_email,
       c.executor_name AS executor_name,
       c.executor_data AS executor_data,
+      c.tier AS tier,
       j.role AS j_role,
       j.environment AS j_environment,
       j.name AS j_name,

http://git-wip-us.apache.org/repos/asf/aurora/blob/506d0cdb/src/main/resources/org/apache/aurora/scheduler/storage/db/schema.sql
----------------------------------------------------------------------
diff --git a/src/main/resources/org/apache/aurora/scheduler/storage/db/schema.sql b/src/main/resources/org/apache/aurora/scheduler/storage/db/schema.sql
index 7634047..d971aa1 100644
--- a/src/main/resources/org/apache/aurora/scheduler/storage/db/schema.sql
+++ b/src/main/resources/org/apache/aurora/scheduler/storage/db/schema.sql
@@ -187,7 +187,8 @@ CREATE TABLE task_configs(
   production BOOLEAN NOT NULL,
   contact_email VARCHAR,
   executor_name VARCHAR NOT NULL,
-  executor_data VARCHAR NOT NULL
+  executor_data VARCHAR NOT NULL,
+  tier VARCHAR
 );
 
 CREATE TABLE task_constraints(

http://git-wip-us.apache.org/repos/asf/aurora/blob/506d0cdb/src/test/java/org/apache/aurora/scheduler/TierManagerTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/TierManagerTest.java b/src/test/java/org/apache/aurora/scheduler/TierManagerTest.java
index 37e19ac..eb02071 100644
--- a/src/test/java/org/apache/aurora/scheduler/TierManagerTest.java
+++ b/src/test/java/org/apache/aurora/scheduler/TierManagerTest.java
@@ -23,7 +23,7 @@ import static org.junit.Assert.assertEquals;
 public class TierManagerTest {
 
   @Test
-  public void testIsRevocable() {
+  public void testIsNotRevocable() {
     TierInfo expected = new TierInfo(false);
     assertEquals(expected, new TierManagerImpl().getTier(ITaskConfig.build(new TaskConfig())));
   }

http://git-wip-us.apache.org/repos/asf/aurora/blob/506d0cdb/src/test/java/org/apache/aurora/scheduler/configuration/ConfigurationManagerTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/configuration/ConfigurationManagerTest.java
b/src/test/java/org/apache/aurora/scheduler/configuration/ConfigurationManagerTest.java
index b231827..f3b62cc 100644
--- a/src/test/java/org/apache/aurora/scheduler/configuration/ConfigurationManagerTest.java
+++ b/src/test/java/org/apache/aurora/scheduler/configuration/ConfigurationManagerTest.java
@@ -28,6 +28,7 @@ import org.apache.aurora.gen.LimitConstraint;
 import org.apache.aurora.gen.TaskConfig;
 import org.apache.aurora.gen.TaskConstraint;
 import org.apache.aurora.gen.ValueConstraint;
+import org.apache.aurora.scheduler.configuration.ConfigurationManager.TaskDescriptionException;
 import org.apache.aurora.scheduler.storage.entities.ITaskConfig;
 import org.junit.Test;
 
@@ -110,11 +111,21 @@ public class ConfigurationManagerTest {
     assertTrue(copy.isSetKey());
   }
 
-  @Test(expected = ConfigurationManager.TaskDescriptionException.class)
-  public void testBadContainerConfig() throws ConfigurationManager.TaskDescriptionException
{
+  @Test(expected = TaskDescriptionException.class)
+  public void testBadContainerConfig() throws TaskDescriptionException {
     TaskConfig taskConfig = CONFIG_WITH_CONTAINER.deepCopy();
     taskConfig.getContainer().getDocker().setImage(null);
 
     ConfigurationManager.validateAndPopulate(ITaskConfig.build(taskConfig));
   }
+
+  @Test(expected = TaskDescriptionException.class)
+  public void testInvalidTier() throws TaskDescriptionException {
+    ITaskConfig config = ITaskConfig.build(UNSANITIZED_JOB_CONFIGURATION.deepCopy().getTaskConfig()
+        .setJobName("job")
+        .setEnvironment("env")
+        .setTier("pr/d"));
+
+    ConfigurationManager.validateAndPopulate(config);
+  }
 }

http://git-wip-us.apache.org/repos/asf/aurora/blob/506d0cdb/src/test/python/apache/aurora/client/cli/test_create.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/client/cli/test_create.py b/src/test/python/apache/aurora/client/cli/test_create.py
index 69039b6..8c27e2b 100644
--- a/src/test/python/apache/aurora/client/cli/test_create.py
+++ b/src/test/python/apache/aurora/client/cli/test_create.py
@@ -404,10 +404,7 @@ class TestClientCreateCommand(AuroraClientCommandTest):
             fp.name])
         assert result == EXIT_INVALID_CONFIGURATION
       assert mock_context.get_out() == []
-      assert mock_context.get_err() == [
-            "Error loading configuration: "
-            "TypeCheck(FAILED): MesosJob[update_config] failed: "
-            "UpdateConfig[batch_size] failed: u'{{TEST_BATCH}}' not an integer"]
+      assert "Error loading configuration: TypeCheck(FAILED):" in mock_context.get_err()[0]
 
   def test_create_cron_job_fails(self):
     """Test a cron job is not accepted."""

http://git-wip-us.apache.org/repos/asf/aurora/blob/506d0cdb/src/test/python/apache/aurora/client/test_config.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/client/test_config.py b/src/test/python/apache/aurora/client/test_config.py
index 986061b..37459f5 100644
--- a/src/test/python/apache/aurora/client/test_config.py
+++ b/src/test/python/apache/aurora/client/test_config.py
@@ -138,9 +138,11 @@ def test_include():
           get_aurora_config('hello_world', hello_include_fname_fp)
 
 
+BAD_ENV = ('Prod', ' prod', 'prod ', 'tEst', 'production', 'staging 2', 'stagingA')
+GOOD_ENV = ('prod', 'devel', 'test', 'staging', 'staging001', 'staging1', 'staging1234')
+
+
 def test_environment_names():
-  BAD = ('Prod', ' prod', 'prod ', 'tEst', 'production', 'staging 2', 'stagingA')
-  GOOD = ('prod', 'devel', 'test', 'staging', 'staging001', 'staging1', 'staging1234')
   base_job = Job(
       name='hello_world', role='john_doe', cluster='test-cluster',
       task=Task(name='main', processes=[],
@@ -148,13 +150,28 @@ def test_environment_names():
 
   with pytest.raises(ValueError):
     config._validate_environment_name(AuroraConfig(base_job))
-  for env_name in GOOD:
+  for env_name in GOOD_ENV:
     config._validate_environment_name(AuroraConfig(base_job(environment=env_name)))
-  for env_name in BAD:
+  for env_name in BAD_ENV:
     with pytest.raises(ValueError):
       config._validate_environment_name(AuroraConfig(base_job(environment=env_name)))
 
 
+def test_tier_names():
+  base_job = Job(
+      name='hello_world', role='john_doe', cluster='test-cluster',
+      task=Task(name='main', processes=[]))
+
+  # Make sure empty value does not raise.
+  config._validate_tier(AuroraConfig(base_job))
+
+  for tier in GOOD_ENV:
+    config._validate_tier(AuroraConfig(base_job(tier=tier)))
+  for tier in BAD_ENV:
+    with pytest.raises(ValueError):
+      config._validate_tier(AuroraConfig(base_job(tier=tier)))
+
+
 def test_dedicated_portmap():
   base_job = Job(
       name='hello_world', role='john_doe', cluster='test-cluster',

http://git-wip-us.apache.org/repos/asf/aurora/blob/506d0cdb/src/test/python/apache/aurora/config/test_thrift.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/config/test_thrift.py b/src/test/python/apache/aurora/config/test_thrift.py
index 061864e..0c0432c 100644
--- a/src/test/python/apache/aurora/config/test_thrift.py
+++ b/src/test/python/apache/aurora/config/test_thrift.py
@@ -71,6 +71,13 @@ def test_simple_config():
   assert tti.constraints == set()
   assert tti.metadata == set()
   assert tti.environment == HELLO_WORLD.environment().get()
+  assert tti.tier is None
+
+
+def test_config_with_tier():
+  config = HELLO_WORLD(tier='devel')
+  job = convert_pystachio_to_thrift(config)
+  assert job.taskConfig.tier == 'devel'
 
 
 def test_docker_with_parameters():


Mime
View raw message