aurora-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From wfar...@apache.org
Subject aurora git commit: Introduce a utility class to read executor configurations in JSON.
Date Thu, 19 Nov 2015 04:09:51 GMT
Repository: aurora
Updated Branches:
  refs/heads/master 7b1e8484a -> 6ba06ae84


Introduce a utility class to read executor configurations in JSON.

Reviewed at https://reviews.apache.org/r/40391/


Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/6ba06ae8
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/6ba06ae8
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/6ba06ae8

Branch: refs/heads/master
Commit: 6ba06ae84440021b5821e666ac5a628ca094af61
Parents: 7b1e848
Author: Bill Farner <wfarner@apache.org>
Authored: Wed Nov 18 20:09:46 2015 -0800
Committer: Bill Farner <wfarner@apache.org>
Committed: Wed Nov 18 20:09:46 2015 -0800

----------------------------------------------------------------------
 build.gradle                                    |   4 +
 .../executor/ExecutorSettingsLoader.java        | 164 +++++++++++++++++++
 .../scheduler/mesos/TestExecutorSettings.java   |   1 +
 .../executor/ExecutorSettingsLoaderTest.java    |  71 ++++++++
 .../executor/test-missing-field.json            |   4 +
 .../executor/test-thermos-executor.json         |  49 ++++++
 6 files changed, 293 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/aurora/blob/6ba06ae8/build.gradle
----------------------------------------------------------------------
diff --git a/build.gradle b/build.gradle
index 17dbd25..32295e6 100644
--- a/build.gradle
+++ b/build.gradle
@@ -110,6 +110,8 @@ For more details, please see https://issues.apache.org/jira/browse/AURORA-1169
         force "org.hamcrest:hamcrest-core:1.3"
         force "org.apache.thrift:libthrift:${thriftRev}"
         force "org.slf4j:slf4j-jdk14:${slf4jRev}"
+        force 'com.fasterxml.jackson.core:jackson-annotations:2.5.1'
+        force 'com.fasterxml.jackson.core:jackson-core:2.5.1'
       }
     }
   }
@@ -347,6 +349,8 @@ dependencies {
   compile "com.google.inject.extensions:guice-assistedinject:${guiceRev}"
   compile 'com.google.protobuf:protobuf-java:2.5.0'
   compile 'com.h2database:h2:1.4.187'
+  compile 'com.hubspot.jackson:jackson-datatype-protobuf:0.9.2'
+  compile 'com.fasterxml.jackson.core:jackson-core:2.5.1'
   compile "com.sun.jersey:jersey-core:${jerseyRev}"
   compile "com.sun.jersey:jersey-json:${jerseyRev}"
   compile "com.sun.jersey:jersey-server:${jerseyRev}"

http://git-wip-us.apache.org/repos/asf/aurora/blob/6ba06ae8/src/main/java/org/apache/aurora/scheduler/configuration/executor/ExecutorSettingsLoader.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/configuration/executor/ExecutorSettingsLoader.java
b/src/main/java/org/apache/aurora/scheduler/configuration/executor/ExecutorSettingsLoader.java
new file mode 100644
index 0000000..fafd90d
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/configuration/executor/ExecutorSettingsLoader.java
@@ -0,0 +1,164 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.aurora.scheduler.configuration.executor;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Objects;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.MoreObjects;
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableList;
+import com.google.common.io.CharStreams;
+import com.google.protobuf.UninitializedMessageException;
+import com.hubspot.jackson.datatype.protobuf.ProtobufModule;
+
+import org.apache.mesos.Protos.ExecutorID;
+import org.apache.mesos.Protos.ExecutorInfo;
+import org.apache.mesos.Protos.Volume;
+
+import static java.util.Objects.requireNonNull;
+
+import static com.fasterxml.jackson.databind.PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES;
+
+/**
+ * A utility class to read JSON-formatted executor configurations.
+ */
+public final class ExecutorSettingsLoader {
+  public static final ExecutorID PLACEHOLDER_EXECUTOR_ID = ExecutorID.newBuilder()
+      .setValue("PLACEHOLDER")
+      .build();
+
+  private ExecutorSettingsLoader()  {
+    // Utility class
+  }
+
+  /**
+   * Thrown when an executor configuration could not be read.
+   */
+  public static class ExecutorConfigException extends Exception {
+    public ExecutorConfigException(Throwable cause) {
+      super(cause);
+    }
+  }
+
+  /**
+   * Reads an executor configuration from a JSON-encoded source.
+   *
+   * @param input The configuration data source.
+   * @return An executor configuration.
+   * @throws ExecutorConfigException If the input cannot be read or is not properly formatted.
+   */
+  public static ExecutorConfig read(Readable input) throws ExecutorConfigException {
+    String configContents;
+    try {
+      configContents = CharStreams.toString(input);
+    } catch (IOException e) {
+      throw new ExecutorConfigException(e);
+    }
+
+    ObjectMapper mapper = new ObjectMapper()
+        .registerModule(new ProtobufModule())
+        .setPropertyNamingStrategy(CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES);
+    Schema parsed;
+    try {
+      parsed = mapper.readValue(configContents, Schema.class);
+    } catch (IOException e) {
+      throw new ExecutorConfigException(e);
+    }
+
+    ExecutorInfo executorInfo;
+    try {
+      // We apply a placeholder value for the executor ID so that we can construct and validate
+      // the protobuf schema.  This allows us to catch many validation errors here rather
than
+      // later on when launching tasks.
+      executorInfo = parsed.executor.setExecutorId(PLACEHOLDER_EXECUTOR_ID).build();
+    } catch (UninitializedMessageException e) {
+      throw new ExecutorConfigException(e);
+    }
+
+    return new ExecutorConfig(
+        executorInfo,
+        Optional.fromNullable(parsed.volumeMounts).or(ImmutableList.of()));
+  }
+
+  /**
+   * The JSON schema.  This is separated from the public {@link ExecutorConfig} so we can
read
+   * objects that do not have all fields required by the protobuf set in the JSON config.
+   */
+  private static class Schema {
+    private ExecutorInfo.Builder executor;
+    private List<Volume> volumeMounts;
+
+    ExecutorInfo.Builder getExecutor() {
+      return executor;
+    }
+
+    void setExecutor(ExecutorInfo.Builder executor) {
+      this.executor = executor;
+    }
+
+    List<Volume> getVolumeMounts() {
+      return volumeMounts;
+    }
+
+    void setVolumeMounts(List<Volume> volumeMounts) {
+      this.volumeMounts = volumeMounts;
+    }
+  }
+
+  public static class ExecutorConfig {
+    private final ExecutorInfo executor;
+    private final List<Volume> volumeMounts;
+
+    public ExecutorConfig(ExecutorInfo executor, List<Volume> volumeMounts) {
+      this.executor = requireNonNull(executor);
+      this.volumeMounts = requireNonNull(volumeMounts);
+    }
+
+    public ExecutorInfo getExecutor() {
+      return executor;
+    }
+
+    public List<Volume> getVolumeMounts() {
+      return volumeMounts;
+    }
+
+    @Override
+    public String toString() {
+      return MoreObjects.toStringHelper(this)
+          .add("executor", executor)
+          .add("volumeMounts", volumeMounts)
+          .toString();
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+      if (!(obj instanceof ExecutorConfig)) {
+        return false;
+      }
+
+      ExecutorConfig other = (ExecutorConfig) obj;
+      return Objects.equals(executor, other.executor)
+          && Objects.equals(volumeMounts, other.volumeMounts);
+    }
+
+    @Override
+    public int hashCode() {
+      return Objects.hash(executor, volumeMounts);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/aurora/blob/6ba06ae8/src/main/java/org/apache/aurora/scheduler/mesos/TestExecutorSettings.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/mesos/TestExecutorSettings.java b/src/main/java/org/apache/aurora/scheduler/mesos/TestExecutorSettings.java
index 556a3f8..6214882 100644
--- a/src/main/java/org/apache/aurora/scheduler/mesos/TestExecutorSettings.java
+++ b/src/main/java/org/apache/aurora/scheduler/mesos/TestExecutorSettings.java
@@ -37,6 +37,7 @@ public final class TestExecutorSettings {
   }
 
   public static final ExecutorInfo THERMOS_EXECUTOR_INFO = ExecutorInfo.newBuilder()
+      .setName("thermos")
       .setExecutorId(Executors.PLACEHOLDER_EXECUTOR_ID)
       .setCommand(CommandInfo.newBuilder().setValue("thermos_executor.pex")
           .addAllArguments(ImmutableList.of(

http://git-wip-us.apache.org/repos/asf/aurora/blob/6ba06ae8/src/test/java/org/apache/aurora/scheduler/configuration/executor/ExecutorSettingsLoaderTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/configuration/executor/ExecutorSettingsLoaderTest.java
b/src/test/java/org/apache/aurora/scheduler/configuration/executor/ExecutorSettingsLoaderTest.java
new file mode 100644
index 0000000..90986a5
--- /dev/null
+++ b/src/test/java/org/apache/aurora/scheduler/configuration/executor/ExecutorSettingsLoaderTest.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.aurora.scheduler.configuration.executor;
+
+import java.io.InputStreamReader;
+import java.io.StringReader;
+
+import com.google.common.base.Charsets;
+import com.google.common.collect.ImmutableList;
+
+import org.apache.aurora.scheduler.configuration.executor.ExecutorSettingsLoader.ExecutorConfig;
+import org.apache.aurora.scheduler.configuration.executor.ExecutorSettingsLoader.ExecutorConfigException;
+import org.apache.aurora.scheduler.mesos.TestExecutorSettings;
+import org.apache.mesos.Protos.Volume;
+import org.apache.mesos.Protos.Volume.Mode;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class ExecutorSettingsLoaderTest {
+  private static final ExecutorConfig THERMOS_CONFIG = new ExecutorConfig(
+      TestExecutorSettings.THERMOS_CONFIG.getExecutor(),
+      ImmutableList.of(
+          Volume.newBuilder()
+              .setHostPath("/path/to/observer_root")
+              .setContainerPath("/path/to/observer_root")
+              .setMode(Mode.RO)
+              .build(),
+          Volume.newBuilder()
+              .setHostPath("/host")
+              .setContainerPath("/container")
+              .setMode(Mode.RW)
+              .build()));
+
+  private ExecutorConfig loadResource(String name) throws ExecutorConfigException {
+    return ExecutorSettingsLoader.read(
+        new InputStreamReader(getClass().getResourceAsStream(name), Charsets.UTF_8));
+  }
+
+  private void assertParsedResult(ExecutorConfig expected, String file)
+      throws ExecutorConfigException {
+
+    assertEquals(expected, loadResource(file));
+  }
+
+  @Test
+  public void testParse() throws Exception {
+    assertParsedResult(THERMOS_CONFIG, "test-thermos-executor.json");
+  }
+
+  @Test(expected = ExecutorConfigException.class)
+  public void testInvalidJson() throws Exception {
+    ExecutorSettingsLoader.read(new StringReader("this is not json"));
+  }
+
+  @Test(expected = ExecutorConfigException.class)
+  public void testMissingField() throws Exception {
+    loadResource("test-missing-field.json");
+  }
+}

http://git-wip-us.apache.org/repos/asf/aurora/blob/6ba06ae8/src/test/resources/org/apache/aurora/scheduler/configuration/executor/test-missing-field.json
----------------------------------------------------------------------
diff --git a/src/test/resources/org/apache/aurora/scheduler/configuration/executor/test-missing-field.json
b/src/test/resources/org/apache/aurora/scheduler/configuration/executor/test-missing-field.json
new file mode 100644
index 0000000..4646170
--- /dev/null
+++ b/src/test/resources/org/apache/aurora/scheduler/configuration/executor/test-missing-field.json
@@ -0,0 +1,4 @@
+{
+  "executor": {
+  }
+}

http://git-wip-us.apache.org/repos/asf/aurora/blob/6ba06ae8/src/test/resources/org/apache/aurora/scheduler/configuration/executor/test-thermos-executor.json
----------------------------------------------------------------------
diff --git a/src/test/resources/org/apache/aurora/scheduler/configuration/executor/test-thermos-executor.json
b/src/test/resources/org/apache/aurora/scheduler/configuration/executor/test-thermos-executor.json
new file mode 100644
index 0000000..8d4d90b
--- /dev/null
+++ b/src/test/resources/org/apache/aurora/scheduler/configuration/executor/test-thermos-executor.json
@@ -0,0 +1,49 @@
+{
+  "executor": {
+    "name": "thermos",
+    "command": {
+      "value": "thermos_executor.pex",
+      "arguments": [
+        "--announcer-enable",
+        "--announcer-ensemble",
+        "localhost:2181"
+      ],
+      "uris": [
+        {
+          "value": "/home/vagrant/aurora/dist/thermos_executor.pex",
+          "executable": true,
+          "extract": false,
+          "cache": false
+        }
+      ]
+    },
+    "resources": [
+      {
+        "name": "cpus",
+        "type": "SCALAR",
+        "scalar": {
+          "value": 0.25
+        }
+      },
+      {
+        "name": "mem",
+        "type": "SCALAR",
+        "scalar": {
+          "value": 128
+        }
+      }
+    ]
+  },
+  "volume_mounts": [
+    {
+      "mode": "RO",
+      "container_path": "/path/to/observer_root",
+      "host_path": "/path/to/observer_root"
+    },
+    {
+      "mode": "RW",
+      "container_path": "/container",
+      "host_path": "/host"
+    }
+  ]
+}


Mime
View raw message