myriad-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jpgilabe...@apache.org
Subject [15/26] incubator-myriad git commit: Upgrade mesos driver to Mesos 1.5 with protobuf 2.5
Date Wed, 12 Sep 2018 15:52:11 GMT
Upgrade mesos driver to Mesos 1.5 with protobuf 2.5


Project: http://git-wip-us.apache.org/repos/asf/incubator-myriad/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-myriad/commit/b5b468b9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-myriad/tree/b5b468b9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-myriad/diff/b5b468b9

Branch: refs/heads/upgrademesos1.5
Commit: b5b468b96f3cab34e8f8b87d87e22f31130ec72b
Parents: 6af9d37
Author: jpgilaberte <jpgilaberte@stratio.com>
Authored: Thu Sep 6 15:42:31 2018 +0200
Committer: jpgilaberte <jpgilaberte@stratio.com>
Committed: Thu Sep 6 15:42:31 2018 +0200

----------------------------------------------------------------------
 build.gradle                                    |      6 +-
 gradle/quality.gradle                           |     16 +
 myriad-commons/build.gradle                     |     22 +
 myriad-commons/proto/mesos/executor.proto       |    212 +
 myriad-commons/proto/mesos/mesos.proto          |   3190 +
 myriad-commons/proto/mesos/scheduler.proto      |    433 +
 myriad-commons/proto/mesos/v1/executor.proto    |    214 +
 myriad-commons/proto/mesos/v1/mesos.proto       |   3173 +
 myriad-commons/proto/mesos/v1/scheduler.proto   |    420 +
 .../main/java/org/apache/mesos/Executor.java    |    149 +
 .../java/org/apache/mesos/ExecutorDriver.java   |    112 +
 .../src/main/java/org/apache/mesos/Log.java     |    468 +
 .../org/apache/mesos/MesosExecutorDriver.java   |     97 +
 .../org/apache/mesos/MesosNativeLibrary.java    |    245 +
 .../org/apache/mesos/MesosSchedulerDriver.java  |    307 +
 .../src/main/java/org/apache/mesos/Protos.java  | 162923 ++++++++++++++++
 .../main/java/org/apache/mesos/Scheduler.java   |    211 +
 .../java/org/apache/mesos/SchedulerDriver.java  |    323 +
 .../java/org/apache/mesos/executor/Protos.java  |  10759 +
 .../java/org/apache/mesos/scheduler/Protos.java |  24780 +++
 .../org/apache/mesos/state/AbstractState.java   |    403 +
 .../org/apache/mesos/state/InMemoryState.java   |    143 +
 .../org/apache/mesos/state/LevelDBState.java    |     36 +
 .../java/org/apache/mesos/state/LogState.java   |     79 +
 .../main/java/org/apache/mesos/state/State.java |     95 +
 .../java/org/apache/mesos/state/Variable.java   |     48 +
 .../org/apache/mesos/state/ZooKeeperState.java  |     75 +
 .../main/java/org/apache/mesos/v1/Protos.java   | 162577 +++++++++++++++
 .../org/apache/mesos/v1/executor/Protos.java    |  10785 +
 .../org/apache/mesos/v1/scheduler/Mesos.java    |     58 +
 .../org/apache/mesos/v1/scheduler/Protos.java   |  24570 +++
 .../apache/mesos/v1/scheduler/Scheduler.java    |     53 +
 .../org/apache/mesos/v1/scheduler/V0Mesos.java  |     84 +
 .../org/apache/mesos/v1/scheduler/V1Mesos.java  |     84 +
 34 files changed, 407148 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/b5b468b9/build.gradle
----------------------------------------------------------------------
diff --git a/build.gradle b/build.gradle
index 075b768..575f1b6 100644
--- a/build.gradle
+++ b/build.gradle
@@ -52,6 +52,7 @@ idea {
     }
 }
 
+
 subprojects {
 
     apply plugin: 'java'
@@ -69,7 +70,7 @@ subprojects {
     version = "0.2.0"
 
     ext {
-        mesosVer = "0.28.1"
+        //mesosVer = "1.5.0"
         //Allows passing -PhadoopVer=2.6.0 from command line
         if(!project.hasProperty('hadoopVer')) {
             hadoopVer = "2.7.0"
@@ -102,7 +103,7 @@ subprojects {
         capsule "co.paralleluniverse:capsule:0.7.1"
     	myriadExecutorConf 'org.slf4j:slf4j-log4j12:1.7.10'
 
-        compile "org.apache.mesos:mesos:${mesosVer}"
+        //compile "org.apache.mesos:mesos:${mesosVer}"
         compile 'com.google.code.gson:gson:2.3.1'        // marshalling between the scheduler
and executor
 
         testCompile 'org.apache.zookeeper:zookeeper:3.4.6' // to resolve temporary mavenlocal
issue
@@ -115,4 +116,5 @@ subprojects {
     run {
         systemProperty "myriad.config", "build/resources/main/myriad-config-default.yml"
     }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/b5b468b9/gradle/quality.gradle
----------------------------------------------------------------------
diff --git a/gradle/quality.gradle b/gradle/quality.gradle
index c013d7d..e58d9ef 100644
--- a/gradle/quality.gradle
+++ b/gradle/quality.gradle
@@ -25,6 +25,22 @@ tasks.withType(FindBugs) {
   excludeFilter = file("$rootProject.projectDir/config/findbugs/excludeFilter.xml")
 }
 
+tasks.withType(FindBugs) {
+    exclude '**/org/apache/mesos/*'
+    classes = classes.filter {
+        !it.path.contains(new File("org/apache/mesos/").path)
+    }
+}
+
+tasks.withType(Pmd) {
+    exclude 'org/apache/mesos/**'
+}
+
+tasks.withType(Checkstyle) {
+    exclude 'org/apache/mesos/**'
+}
+
+
 pmd {
     ruleSets = [
 //            'java-basic',         // need brandon's http config to add in

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/b5b468b9/myriad-commons/build.gradle
----------------------------------------------------------------------
diff --git a/myriad-commons/build.gradle b/myriad-commons/build.gradle
new file mode 100644
index 0000000..c66e8f7
--- /dev/null
+++ b/myriad-commons/build.gradle
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+dependencies {
+    compile group: 'com.google.protobuf', name: 'protobuf-java', version: '2.5.0'
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/b5b468b9/myriad-commons/proto/mesos/executor.proto
----------------------------------------------------------------------
diff --git a/myriad-commons/proto/mesos/executor.proto b/myriad-commons/proto/mesos/executor.proto
new file mode 100644
index 0000000..1b5fa5d
--- /dev/null
+++ b/myriad-commons/proto/mesos/executor.proto
@@ -0,0 +1,212 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+import "mesos/mesos.proto";
+
+package mesos.executor;
+
+option java_package = "org.apache.mesos.executor";
+option java_outer_classname = "Protos";
+
+
+/**
+ * Executor event API.
+ *
+ * An event is described using the standard protocol buffer "union"
+ * trick, see https://developers.google.com/protocol-buffers/docs/techniques#union.
+ */
+message Event {
+  // Possible event types, followed by message definitions if
+  // applicable.
+  enum Type {
+    // This must be the first enum value in this list, to
+    // ensure that if 'type' is not set, the default value
+    // is UNKNOWN. This enables enum values to be added
+    // in a backwards-compatible way. See: MESOS-4997.
+    UNKNOWN = 0;
+
+    SUBSCRIBED = 1;   // See 'Subscribed' below.
+    LAUNCH = 2;       // See 'Launch' below.
+    LAUNCH_GROUP = 8; // See 'LaunchGroup' below.
+    KILL = 3;         // See 'Kill' below.
+    ACKNOWLEDGED = 4; // See 'Acknowledged' below.
+    MESSAGE = 5;      // See 'Message' below.
+    ERROR = 6;        // See 'Error' below.
+
+    // Received when the agent asks the executor to shutdown/kill itself.
+    // The executor is then required to kill all its active tasks, send
+    // `TASK_KILLED` status updates and gracefully exit. The executor
+    // should terminate within a `MESOS_EXECUTOR_SHUTDOWN_GRACE_PERIOD`
+    // (an environment variable set by the agent upon executor startup);
+    // it can be configured via `ExecutorInfo.shutdown_grace_period`. If
+    // the executor fails to do so, the agent will forcefully destroy the
+    // container where the executor is running. The agent would then send
+    // `TASK_LOST` updates for any remaining active tasks of this executor.
+    //
+    // NOTE: The executor must not assume that it will always be allotted
+    // the full grace period, as the agent may decide to allot a shorter
+    // period and failures / forcible terminations may occur.
+    //
+    // TODO(alexr): Consider adding a duration field into the `Shutdown`
+    // message so that the agent can communicate when a shorter period
+    // has been allotted.
+    SHUTDOWN = 7;
+  }
+
+  // First event received when the executor subscribes.
+  // The 'id' field in the 'framework_info' will be set.
+   message Subscribed {
+    required ExecutorInfo executor_info = 1;
+    required FrameworkInfo framework_info = 2;
+    required SlaveInfo slave_info = 3;
+
+    // Uniquely identifies the container of an executor run.
+    optional ContainerID container_id = 4;
+  }
+
+  // Received when the framework attempts to launch a task. Once
+  // the task is successfuly launched, the executor must respond with
+  // a TASK_RUNNING update (See TaskState in mesos.proto).
+  message Launch {
+    required TaskInfo task = 1;
+  }
+
+  // Received when the framework attempts to launch a group of tasks atomically.
+  // Similar to `Launch` above the executor must send TASK_RUNNING updates for
+  // tasks that are successfully launched.
+  message LaunchGroup {
+    required TaskGroupInfo task_group = 1;
+  }
+
+  // Received when the scheduler wants to kill a specific task. Once
+  // the task is terminated, the executor should send a TASK_KILLED
+  // (or TASK_FAILED) update. The terminal update is necessary so
+  // Mesos can release the resources associated with the task.
+  message Kill {
+    required TaskID task_id = 1;
+
+    // If set, overrides any previously specified kill policy for this task.
+    // This includes 'TaskInfo.kill_policy' and 'Executor.kill.kill_policy'.
+    optional KillPolicy kill_policy = 2;
+  }
+
+  // Received when the slave acknowledges the receipt of status
+  // update. Schedulers are responsible for explicitly acknowledging
+  // the receipt of status updates that have 'update.status().uuid()'
+  // field set. Unacknowledged updates can be retried by the executor.
+  // They should also be sent by the executor whenever it
+  // re-subscribes.
+  message Acknowledged {
+    required TaskID task_id = 1;
+    required bytes uuid = 2;
+  }
+
+  // Received when a custom message generated by the scheduler is
+  // forwarded by the slave. Note that this message is not
+  // interpreted by Mesos and is only forwarded (without reliability
+  // guarantees) to the executor. It is up to the scheduler to retry
+  // if the message is dropped for any reason.
+  message Message {
+    required bytes data = 1;
+  }
+
+  // Received in case the executor sends invalid calls (e.g.,
+  // required values not set).
+  // TODO(arojas): Remove this once the old executor driver is no
+  // longer supported. With HTTP API all errors will be signaled via
+  // HTTP response codes.
+  message Error {
+    required string message = 1;
+  }
+
+  // Type of the event, indicates which optional field below should be
+  // present if that type has a nested message definition.
+  // Enum fields should be optional, see: MESOS-4997.
+  optional Type type = 1;
+
+  optional Subscribed subscribed = 2;
+  optional Acknowledged acknowledged = 3;
+  optional Launch launch = 4;
+  optional LaunchGroup launch_group = 8;
+  optional Kill kill = 5;
+  optional Message message = 6;
+  optional Error error = 7;
+}
+
+
+/**
+ * Executor call API.
+ *
+ * Like Event, a Call is described using the standard protocol buffer
+ * "union" trick (see above).
+ */
+message Call {
+  // Possible call types, followed by message definitions if
+  // applicable.
+  enum Type {
+    // See comments above on `Event::Type` for more details on this enum value.
+    UNKNOWN = 0;
+
+    SUBSCRIBE = 1;    // See 'Subscribe' below.
+    UPDATE = 2;       // See 'Update' below.
+    MESSAGE = 3;      // See 'Message' below.
+  }
+
+  // Request to subscribe with the slave. If subscribing after a disconnection,
+  // it must include a list of all the tasks and updates which haven't been
+  // acknowledged by the scheduler.
+  message Subscribe {
+    repeated TaskInfo unacknowledged_tasks = 1;
+    repeated Update unacknowledged_updates = 2;
+  }
+
+  // Notifies the scheduler that a task has transitioned from one
+  // state to another. Status updates should be used by executors
+  // to reliably communicate the status of the tasks that they
+  // manage. It is crucial that a terminal update (see TaskState
+  // in mesos.proto) is sent to the scheduler as soon as the task
+  // terminates, in order for Mesos to release the resources allocated
+  // to the task. It is the responsibility of the scheduler to
+  // explicitly acknowledge the receipt of a status update. See
+  // 'Acknowledged' in the 'Events' section above for the semantics.
+  message Update {
+    required TaskStatus status = 1;
+  }
+
+  // Sends arbitrary binary data to the scheduler. Note that Mesos
+  // neither interprets this data nor makes any guarantees about the
+  // delivery of this message to the scheduler.
+  // See 'Message' in the 'Events' section.
+  message Message {
+    required bytes data = 2;
+  }
+
+  // Identifies the executor which generated this call.
+  required ExecutorID executor_id = 1;
+  required FrameworkID framework_id = 2;
+
+  // Type of the call, indicates which optional field below should be
+  // present if that type has a nested message definition.
+  // In case type is SUBSCRIBED, no message needs to be set.
+  // See comments on `Event::Type` above on the reasoning behind this field being optional.
+  optional Type type = 3;
+
+  optional Subscribe subscribe = 4;
+  optional Update update = 5;
+  optional Message message = 6;
+}


Mime
View raw message