aurora-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ma...@apache.org
Subject aurora git commit: Adding oversubscription e2e tests.
Date Thu, 03 Sep 2015 04:24:15 GMT
Repository: aurora
Updated Branches:
  refs/heads/master a45952785 -> f3cbc399e


Adding oversubscription e2e tests.

Bugs closed: AURORA-1464

Reviewed at https://reviews.apache.org/r/37926/


Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/f3cbc399
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/f3cbc399
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/f3cbc399

Branch: refs/heads/master
Commit: f3cbc399ef85f7976ab984c83a43b113a9db0e83
Parents: a459527
Author: Maxim Khutornenko <maxim@apache.org>
Authored: Wed Sep 2 21:23:51 2015 -0700
Committer: Maxim Khutornenko <maxim@apache.org>
Committed: Wed Sep 2 21:23:51 2015 -0700

----------------------------------------------------------------------
 examples/vagrant/upstart/aurora-scheduler.conf  |  3 +-
 examples/vagrant/upstart/mesos-slave.conf       | 15 ++++-
 .../apache/aurora/scheduler/http/Offers.java    |  3 +
 src/main/python/apache/aurora/client/config.py  | 11 ----
 .../python/apache/aurora/client/test_config.py  | 15 -----
 .../apache/aurora/e2e/http/http_example.aurora  | 25 +++++++-
 .../aurora/e2e/http/http_example_docker.aurora  | 53 ----------------
 .../e2e/http/http_example_docker_updated.aurora | 53 ----------------
 .../aurora/e2e/http/http_example_updated.aurora | 25 +++++++-
 .../sh/org/apache/aurora/e2e/test_end_to_end.sh | 65 ++++++++++++--------
 10 files changed, 102 insertions(+), 166 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/aurora/blob/f3cbc399/examples/vagrant/upstart/aurora-scheduler.conf
----------------------------------------------------------------------
diff --git a/examples/vagrant/upstart/aurora-scheduler.conf b/examples/vagrant/upstart/aurora-scheduler.conf
index e909451..f710295 100644
--- a/examples/vagrant/upstart/aurora-scheduler.conf
+++ b/examples/vagrant/upstart/aurora-scheduler.conf
@@ -45,4 +45,5 @@ exec bin/aurora-scheduler \
   -use_beta_db_task_store=true \
   -shiro_ini_path=etc/shiro.example.ini \
   -enable_h2_console=true \
-  -tier_config=/home/vagrant/aurora/src/test/resources/org/apache/aurora/scheduler/tiers-example.json
+  -tier_config=/home/vagrant/aurora/src/test/resources/org/apache/aurora/scheduler/tiers-example.json
\
+  -receive_revocable_resources=true
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/aurora/blob/f3cbc399/examples/vagrant/upstart/mesos-slave.conf
----------------------------------------------------------------------
diff --git a/examples/vagrant/upstart/mesos-slave.conf b/examples/vagrant/upstart/mesos-slave.conf
index 2b6a606..9af680e 100644
--- a/examples/vagrant/upstart/mesos-slave.conf
+++ b/examples/vagrant/upstart/mesos-slave.conf
@@ -29,4 +29,17 @@ exec /usr/sbin/mesos-slave --master=zk://$ZK_HOST:2181/mesos/master \
   --attributes="host:$MY_HOST;rack:a" \
   --resources="cpus:4;mem:1024;disk:20000" \
   --work_dir="/var/lib/mesos" \
-  --containerizers=docker,mesos
+  --containerizers=docker,mesos \
+  --resource_estimator="org_apache_mesos_FixedResourceEstimator" \
+  --modules='{
+    "libraries": {
+      "file": "/usr/lib/libfixed_resource_estimator.so",
+      "modules": {
+        "name": "org_apache_mesos_FixedResourceEstimator",
+        "parameters": {
+          "key": "resources",
+          "value": "cpus:3"
+        }
+      }
+    }
+  }'

http://git-wip-us.apache.org/repos/asf/aurora/blob/f3cbc399/src/main/java/org/apache/aurora/scheduler/http/Offers.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/http/Offers.java b/src/main/java/org/apache/aurora/scheduler/http/Offers.java
index 4329ce1..15f1582 100644
--- a/src/main/java/org/apache/aurora/scheduler/http/Offers.java
+++ b/src/main/java/org/apache/aurora/scheduler/http/Offers.java
@@ -113,6 +113,9 @@ public class Offers {
           if (resource.hasSet()) {
             builder.put("set", resource.getSet().getItemList());
           }
+          if (resource.hasRevocable()) {
+            builder.put("revocable", "true");
+          }
           return builder.build();
         }
       };

http://git-wip-us.apache.org/repos/asf/aurora/blob/f3cbc399/src/main/python/apache/aurora/client/config.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/client/config.py b/src/main/python/apache/aurora/client/config.py
index f5ac084..2fc1255 100644
--- a/src/main/python/apache/aurora/client/config.py
+++ b/src/main/python/apache/aurora/client/config.py
@@ -22,8 +22,6 @@ import math
 import re
 import sys
 
-from pystachio import Empty
-
 from apache.aurora.client import binding_helper
 from apache.aurora.client.base import die
 from apache.aurora.config import AuroraConfig
@@ -57,7 +55,6 @@ def _validate_announce_configuration(config):
 STAGING_RE = re.compile(r'^staging\d*$')
 
 
-#TODO(maxim): Merge env and tier and move definitions to scheduler: AURORA-1443.
 def __validate_env(name, config_name):
   if STAGING_RE.match(name):
     return
@@ -71,13 +68,6 @@ def _validate_environment_name(config):
   __validate_env(env_name, 'Environment')
 
 
-def _validate_tier(config):
-  tier_raw = config.raw().tier()
-  tier_name = str(tier_raw) if tier_raw is not Empty else None
-  if tier_name is not None:
-    __validate_env(tier_name, 'Tier')
-
-
 UPDATE_CONFIG_MAX_FAILURES_ERROR = '''
 max_total_failures in update_config must be lesser than the job size.
 Based on your job size (%s) you should use max_total_failures <= %s.
@@ -128,7 +118,6 @@ def validate_config(config, env=None):
   _validate_update_config(config)
   _validate_announce_configuration(config)
   _validate_environment_name(config)
-  _validate_tier(config)
 
 
 class GlobalHookRegistry(object):

http://git-wip-us.apache.org/repos/asf/aurora/blob/f3cbc399/src/test/python/apache/aurora/client/test_config.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/client/test_config.py b/src/test/python/apache/aurora/client/test_config.py
index 37459f5..b1a3c18 100644
--- a/src/test/python/apache/aurora/client/test_config.py
+++ b/src/test/python/apache/aurora/client/test_config.py
@@ -157,21 +157,6 @@ def test_environment_names():
       config._validate_environment_name(AuroraConfig(base_job(environment=env_name)))
 
 
-def test_tier_names():
-  base_job = Job(
-      name='hello_world', role='john_doe', cluster='test-cluster',
-      task=Task(name='main', processes=[]))
-
-  # Make sure empty value does not raise.
-  config._validate_tier(AuroraConfig(base_job))
-
-  for tier in GOOD_ENV:
-    config._validate_tier(AuroraConfig(base_job(tier=tier)))
-  for tier in BAD_ENV:
-    with pytest.raises(ValueError):
-      config._validate_tier(AuroraConfig(base_job(tier=tier)))
-
-
 def test_dedicated_portmap():
   base_job = Job(
       name='hello_world', role='john_doe', cluster='test-cluster',

http://git-wip-us.apache.org/repos/asf/aurora/blob/f3cbc399/src/test/sh/org/apache/aurora/e2e/http/http_example.aurora
----------------------------------------------------------------------
diff --git a/src/test/sh/org/apache/aurora/e2e/http/http_example.aurora b/src/test/sh/org/apache/aurora/e2e/http/http_example.aurora
index c1a10d8..d7bf108 100644
--- a/src/test/sh/org/apache/aurora/e2e/http/http_example.aurora
+++ b/src/test/sh/org/apache/aurora/e2e/http/http_example.aurora
@@ -14,13 +14,15 @@
 
 import getpass
 
+DEFAULT_CMD = 'cp /vagrant/src/test/sh/org/apache/aurora/e2e/http_example.py .'
+
 run_server = Process(
   name = 'run_server',
   cmdline = 'python http_example.py {{thermos.ports[http]}}')
 
 stage_server = Process(
   name = 'stage_server',
-  cmdline = "cp /vagrant/src/test/sh/org/apache/aurora/e2e/http_example.py ."
+  cmdline = '{{cmd}}'
 )
 
 test_task = Task(
@@ -33,7 +35,6 @@ update_config = UpdateConfig(watch_secs=10, batch_size=2)
 health_check_config = HealthCheckConfig(initial_interval_secs=5, interval_secs=1)
 
 job = Service(
-  name = 'http_example',
   cluster = 'devcluster',
   instances = 2,
   update_config = update_config,
@@ -49,4 +50,22 @@ job = Service(
   announce = Announcer(),
 )
 
-jobs = [ job ]
+jobs = [
+  job(
+    name = 'http_example'
+  ).bind(
+    cmd = DEFAULT_CMD
+  ),
+  job(
+    name = 'http_example_revocable',
+    tier = 'revocable'
+  ).bind(
+    cmd = DEFAULT_CMD
+  ),
+  job(
+    name = 'http_example_docker',
+    container = Container(docker=Docker(image = 'http_example'))
+  ).bind(
+    cmd = 'cp /tmp/http_example.py .'
+  )
+]

http://git-wip-us.apache.org/repos/asf/aurora/blob/f3cbc399/src/test/sh/org/apache/aurora/e2e/http/http_example_docker.aurora
----------------------------------------------------------------------
diff --git a/src/test/sh/org/apache/aurora/e2e/http/http_example_docker.aurora b/src/test/sh/org/apache/aurora/e2e/http/http_example_docker.aurora
deleted file mode 100644
index 870b3e6..0000000
--- a/src/test/sh/org/apache/aurora/e2e/http/http_example_docker.aurora
+++ /dev/null
@@ -1,53 +0,0 @@
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import getpass
-
-run_server = Process(
-  name = 'run_server',
-  cmdline = 'python http_example.py {{thermos.ports[http]}}')
-
-stage_server = Process(
-  name = 'stage_server',
-  cmdline = "cp /tmp/http_example.py ."
-)
-
-test_task = Task(
-  name = 'http_example',
-  resources = Resources(cpu=0.5, ram=32*MB, disk=64*MB),
-  processes = [stage_server, run_server],
-  constraints = order(stage_server, run_server))
-
-update_config = UpdateConfig(watch_secs=10, batch_size=2)
-health_check_config = HealthCheckConfig(initial_interval_secs=5, interval_secs=1)
-
-job = Service(
-  name = 'http_example_docker',
-  cluster = 'devcluster',
-  instances = 2,
-  update_config = update_config,
-  health_check_config = health_check_config,
-  task = test_task,
-  role = getpass.getuser(),
-  environment = 'test',
-  contact = '{{role}}@localhost',
-  # Since there is only one slave in devcluster allow all instances to run there.
-  constraints = {
-    'host': 'limit:2',
-  },
-  announce = Announcer(),
-  container = Container(docker=Docker(image = 'http_example')),
-)
-
-jobs = [ job ]

http://git-wip-us.apache.org/repos/asf/aurora/blob/f3cbc399/src/test/sh/org/apache/aurora/e2e/http/http_example_docker_updated.aurora
----------------------------------------------------------------------
diff --git a/src/test/sh/org/apache/aurora/e2e/http/http_example_docker_updated.aurora b/src/test/sh/org/apache/aurora/e2e/http/http_example_docker_updated.aurora
deleted file mode 100644
index e55aad3..0000000
--- a/src/test/sh/org/apache/aurora/e2e/http/http_example_docker_updated.aurora
+++ /dev/null
@@ -1,53 +0,0 @@
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import getpass
-
-run_server = Process(
-  name = 'run_server',
-  cmdline = 'python http_example.py {{thermos.ports[http]}}')
-
-stage_server = Process(
-  name = 'stage_server',
-  cmdline = "cp /tmp/http_example.py ."
-)
-
-test_task = Task(
-  name = 'http_example',
-  resources = Resources(cpu=0.5, ram=34*MB, disk=64*MB),
-  processes = [stage_server, run_server],
-  constraints = order(stage_server, run_server))
-
-update_config = UpdateConfig(watch_secs=10, batch_size=3)
-health_check_config = HealthCheckConfig(initial_interval_secs=5, interval_secs=1)
-
-job = Service(
-  name = 'http_example_docker',
-  cluster = 'devcluster',
-  instances = 3,
-  update_config = update_config,
-  health_check_config = health_check_config,
-  task = test_task,
-  role = getpass.getuser(),
-  environment = 'test',
-  contact = '{{role}}@localhost',
-  # Since there is only one slave in devcluster allow all instances to run there.
-  constraints = {
-    'host': 'limit:4',
-  },
-  announce = Announcer(),
-  container = Container(docker=Docker(image = 'http_example')),
-)
-
-jobs = [ job ]

http://git-wip-us.apache.org/repos/asf/aurora/blob/f3cbc399/src/test/sh/org/apache/aurora/e2e/http/http_example_updated.aurora
----------------------------------------------------------------------
diff --git a/src/test/sh/org/apache/aurora/e2e/http/http_example_updated.aurora b/src/test/sh/org/apache/aurora/e2e/http/http_example_updated.aurora
index 423dd4d..c973966 100644
--- a/src/test/sh/org/apache/aurora/e2e/http/http_example_updated.aurora
+++ b/src/test/sh/org/apache/aurora/e2e/http/http_example_updated.aurora
@@ -14,13 +14,15 @@
 
 import getpass
 
+DEFAULT_CMD = 'cp /vagrant/src/test/sh/org/apache/aurora/e2e/http_example.py .'
+
 run_server = Process(
   name = 'run_server',
   cmdline = 'python http_example.py {{thermos.ports[http]}}')
 
 stage_server = Process(
   name = 'stage_server',
-  cmdline = "cp /vagrant/src/test/sh/org/apache/aurora/e2e/http_example.py ."
+  cmdline = '{{cmd}}'
 )
 
 test_task = Task(
@@ -33,7 +35,6 @@ update_config = UpdateConfig(watch_secs=10, batch_size=3)
 health_check_config = HealthCheckConfig(initial_interval_secs=5, interval_secs=1)
 
 job = Service(
-  name = 'http_example',
   cluster = 'devcluster',
   instances = 3,
   update_config = update_config,
@@ -49,4 +50,22 @@ job = Service(
   announce = Announcer(),
 )
 
-jobs = [ job ]
+jobs = [
+  job(
+    name = 'http_example'
+  ).bind(
+    cmd = DEFAULT_CMD
+  ),
+  job(
+    name = 'http_example_revocable',
+    tier = 'revocable'
+  ).bind(
+    cmd = DEFAULT_CMD
+  ),
+  job(
+    name = 'http_example_docker',
+    container = Container(docker=Docker(image = 'http_example'))
+  ).bind(
+    cmd = 'cp /tmp/http_example.py .'
+  )
+]

http://git-wip-us.apache.org/repos/asf/aurora/blob/f3cbc399/src/test/sh/org/apache/aurora/e2e/test_end_to_end.sh
----------------------------------------------------------------------
diff --git a/src/test/sh/org/apache/aurora/e2e/test_end_to_end.sh b/src/test/sh/org/apache/aurora/e2e/test_end_to_end.sh
index 018efbe..1eec20c 100755
--- a/src/test/sh/org/apache/aurora/e2e/test_end_to_end.sh
+++ b/src/test/sh/org/apache/aurora/e2e/test_end_to_end.sh
@@ -33,7 +33,7 @@ _curl() { curl --silent --fail --retry 4 --retry-delay 10 "$@" ; }
 tear_down() {
   set +x  # Disable command echo, as this makes it more difficult see which command failed.
 
-  for job in http_example http_example_docker; do
+  for job in http_example http_example_revocable http_example_docker; do
     aurora job cancel-update devcluster/vagrant/test/$job >/dev/null 2>&1
     aurora update abort devcluster/vagrant/test/$job || true >/dev/null 2>&1
     aurora job killall --no-batching devcluster/vagrant/test/$job >/dev/null 2>&1
@@ -65,7 +65,7 @@ test_config() {
   local _config=$1 _jobkey=$2
 
   joblist=$(aurora config list $_config | tr -dc '[[:print:]]')
-  [[ "$joblist" = "jobs=[$_jobkey]" ]]
+  [[ "$joblist" = *"$_jobkey"* ]]
 }
 
 test_inspect() {
@@ -225,8 +225,9 @@ test_quota() {
 }
 
 test_http_example() {
-  local _cluster=$1 _role=$2 _env=$3 _job=$4
-  local _base_config=$5 _updated_config=$6
+  local _cluster=$1 _role=$2 _env=$3
+  local _base_config=$4 _updated_config=$5
+  local _job=$6
   local _jobkey="$_cluster/$_role/$_env/$_job"
 
   test_config $_base_config $_jobkey
@@ -244,6 +245,17 @@ test_http_example() {
   test_quota $_cluster $_role
 }
 
+test_http_revocable_example() {
+  local _cluster=$1 _role=$2 _env=$3
+  local _base_config=$4
+  local _job=$6
+  local _jobkey="$_cluster/$_role/$_env/$_job"
+
+  test_create $_jobkey $_base_config
+  test_observer_ui $_cluster $_role $_job
+  test_kill $_jobkey
+}
+
 test_admin() {
   local _cluster=$1
   echo '== Testing admin commands'
@@ -256,8 +268,9 @@ restore_netrc() {
 }
 
 test_basic_auth_unauthenticated() {
-  local _cluster=$1 _role=$2 _env=$3 _job=$4
-  local _config=$5
+  local _cluster=$1 _role=$2 _env=$3
+  local _config=$4
+  local _job=$6
   local _jobkey="$_cluster/$_role/$_env/$_job"
 
   mv ~/.netrc ~/.netrc.bak
@@ -280,41 +293,41 @@ TEST_CLUSTER=devcluster
 TEST_ROLE=vagrant
 TEST_ENV=test
 TEST_JOB=http_example
-TEST_DOCKER_JOB=http_example_docker
-TEST_ARGS=(
-  $TEST_CLUSTER
-  $TEST_ROLE
-  $TEST_ENV
-  $TEST_JOB
-  $EXAMPLE_DIR/http_example.aurora
-  $EXAMPLE_DIR/http_example_updated.aurora
-  )
-
-TEST_ADMIN_ARGS=(
-  $TEST_CLUSTER
-)
+TEST_JOB_REVOCABLE=http_example_revocable
+TEST_JOB_DOCKER=http_example_docker
+TEST_CONFIG_FILE=$EXAMPLE_DIR/http_example.aurora
+TEST_CONFIG_UPDATED_FILE=$EXAMPLE_DIR/http_example_updated.aurora
 
-TEST_DOCKER_ARGS=(
+BASE_ARGS=(
   $TEST_CLUSTER
   $TEST_ROLE
   $TEST_ENV
-  $TEST_DOCKER_JOB
-  $EXAMPLE_DIR/http_example_docker.aurora
-  $EXAMPLE_DIR/http_example_docker_updated.aurora
+  $TEST_CONFIG_FILE
+  $TEST_CONFIG_UPDATED_FILE
 )
 
+TEST_JOB_ARGS=("${BASE_ARGS[@]}" "$TEST_JOB")
+
+TEST_JOB_REVOCABLE_ARGS=("${BASE_ARGS[@]}" "$TEST_JOB_REVOCABLE")
+
+TEST_JOB_DOCKER_ARGS=("${BASE_ARGS[@]}" "$TEST_JOB_DOCKER")
+
+TEST_ADMIN_ARGS=($TEST_CLUSTER)
+
 trap collect_result EXIT
 
 aurorabuild all
 test_version
-test_http_example "${TEST_ARGS[@]}"
+test_http_example "${TEST_JOB_ARGS[@]}"
+
+test_http_revocable_example "${TEST_JOB_REVOCABLE_ARGS[@]}"
 
 # build the test docker image
 sudo docker build -t http_example ${TEST_ROOT}
-test_http_example "${TEST_DOCKER_ARGS[@]}"
+test_http_example "${TEST_JOB_DOCKER_ARGS[@]}"
 
 test_admin "${TEST_ADMIN_ARGS[@]}"
-test_basic_auth_unauthenticated  "${TEST_ARGS[@]}"
+test_basic_auth_unauthenticated  "${TEST_JOB_ARGS[@]}"
 
 /vagrant/src/test/sh/org/apache/aurora/e2e/test_kerberos_end_to_end.sh
 RETCODE=0


Mime
View raw message