gobblin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a...@apache.org
Subject [1/2] incubator-gobblin git commit: Add `CLUSTER` to the launcher_type enum in the JobHistory DB
Date Thu, 27 Jul 2017 23:49:36 GMT
Repository: incubator-gobblin
Updated Branches:
  refs/heads/master 8d297de3e -> 0975312c7


Add `CLUSTER` to the launcher_type enum in the JobHistory DB

Modified DatabaseJobHistoryStoreV101 to return values of LauncherTypeEnum.CLUSTER as "YARN"
and created DatabaseJobHistoryStoreV102 which keeps them as "CLUSTER".

Have DatabaseJobHistoryStoreV102 implement VersionedDatabaseJobHistoryStore

Add missing import


Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/6e6883ad
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/6e6883ad
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/6e6883ad

Branch: refs/heads/master
Commit: 6e6883adf3240468f3feccb34e9f644d729d7548
Parents: 16dad89
Author: Joel Baranick <joel.baranick@ensighten.com>
Authored: Mon May 1 16:07:17 2017 -0700
Committer: Joel Baranick <joel.baranick@ensighten.com>
Committed: Thu Jun 22 14:32:49 2017 -0700

----------------------------------------------------------------------
 .../database/DatabaseJobHistoryStoreV101.java   | 12 ++++-
 .../database/DatabaseJobHistoryStoreV102.java   | 46 ++++++++++++++++++++
 .../V1_0_2__LauncherType_enum_expansion.sql     | 18 ++++++++
 3 files changed, 75 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/6e6883ad/gobblin-metastore/src/main/java/gobblin/metastore/database/DatabaseJobHistoryStoreV101.java
----------------------------------------------------------------------
diff --git a/gobblin-metastore/src/main/java/gobblin/metastore/database/DatabaseJobHistoryStoreV101.java
b/gobblin-metastore/src/main/java/gobblin/metastore/database/DatabaseJobHistoryStoreV101.java
index 6909afc..beeee3f 100644
--- a/gobblin-metastore/src/main/java/gobblin/metastore/database/DatabaseJobHistoryStoreV101.java
+++ b/gobblin-metastore/src/main/java/gobblin/metastore/database/DatabaseJobHistoryStoreV101.java
@@ -250,6 +250,16 @@ public class DatabaseJobHistoryStoreV101 implements VersionedDatabaseJobHistoryS
     return this.dataSource.getConnection();
   }
 
+  protected String getLauncherType(JobExecutionInfo info) {
+    if (info.hasLauncherType()) {
+      if (info.getLauncherType() == LauncherTypeEnum.CLUSTER) {
+        return LauncherTypeEnum.YARN.name();
+      }
+      return info.getLauncherType().name();
+    }
+    return null;
+  }
+
   private void upsertJobExecutionInfo(Connection connection, JobExecutionInfo info)
       throws SQLException {
     Preconditions.checkArgument(info.hasJobName());
@@ -267,7 +277,7 @@ public class DatabaseJobHistoryStoreV101 implements VersionedDatabaseJobHistoryS
       upsertStatement.setString(++index, info.hasState() ? info.getState().name() : null);
       upsertStatement.setInt(++index, info.hasLaunchedTasks() ? info.getLaunchedTasks() :
-1);
       upsertStatement.setInt(++index, info.hasCompletedTasks() ? info.getCompletedTasks()
: -1);
-      upsertStatement.setString(++index, info.hasLauncherType() ? info.getLauncherType().name()
: null);
+      upsertStatement.setString(++index, getLauncherType(info));
       upsertStatement.setString(++index, info.hasTrackingUrl() ? info.getTrackingUrl() :
null);
       upsertStatement.executeUpdate();
     }

http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/6e6883ad/gobblin-metastore/src/main/java/gobblin/metastore/database/DatabaseJobHistoryStoreV102.java
----------------------------------------------------------------------
diff --git a/gobblin-metastore/src/main/java/gobblin/metastore/database/DatabaseJobHistoryStoreV102.java
b/gobblin-metastore/src/main/java/gobblin/metastore/database/DatabaseJobHistoryStoreV102.java
new file mode 100644
index 0000000..f59bf78
--- /dev/null
+++ b/gobblin-metastore/src/main/java/gobblin/metastore/database/DatabaseJobHistoryStoreV102.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package gobblin.metastore.database;
+
+import gobblin.metastore.JobHistoryStore;
+
+import java.io.IOException;
+
+import gobblin.rest.JobExecutionInfo;
+
+
+/**
+ * An implementation of {@link JobHistoryStore} backed by MySQL.
+ *
+ * <p>
+ *     The DDLs for the MySQL job history store can be found under metastore/src/main/resources.
+ * </p>
+ *
+ * @author Joel Baranick
+ */
+@SupportedDatabaseVersion(isDefault = false, version = "1.0.2")
+public class DatabaseJobHistoryStoreV102 extends DatabaseJobHistoryStoreV101 implements VersionedDatabaseJobHistoryStore
{
+
+  @Override
+  protected String getLauncherType(JobExecutionInfo info) {
+    if (info.hasLauncherType()) {
+      return info.getLauncherType().name();
+    }
+    return null;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/6e6883ad/gobblin-metastore/src/main/resources/db/migration/V1_0_2__LauncherType_enum_expansion.sql
----------------------------------------------------------------------
diff --git a/gobblin-metastore/src/main/resources/db/migration/V1_0_2__LauncherType_enum_expansion.sql
b/gobblin-metastore/src/main/resources/db/migration/V1_0_2__LauncherType_enum_expansion.sql
new file mode 100644
index 0000000..af23b03
--- /dev/null
+++ b/gobblin-metastore/src/main/resources/db/migration/V1_0_2__LauncherType_enum_expansion.sql
@@ -0,0 +1,18 @@
+--
+--  Licensed to the Apache Software Foundation (ASF) under one or more
+--  contributor license agreements.  See the NOTICE file distributed with
+--  this work for additional information regarding copyright ownership.
+--  The ASF licenses this file to You under the Apache License, Version 2.0
+--  (the "License"); you may not use this file except in compliance with
+--  the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+--  Unless required by applicable law or agreed to in writing, software
+--  distributed under the License is distributed on an "AS IS" BASIS,
+--  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+--  See the License for the specific language governing permissions and
+--  limitations under the License.
+--
+
+ALTER TABLE `gobblin_job_executions` CHANGE `launcher_type` `launcher_type` ENUM('LOCAL',
'MAPREDUCE', 'CLUSTER', 'YARN');
\ No newline at end of file


Mime
View raw message