gobblin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a...@apache.org
Subject incubator-gobblin git commit: Making gobblin-admin a module
Date Sat, 29 Jul 2017 07:57:36 GMT
Repository: incubator-gobblin
Updated Branches:
  refs/heads/master a889431e0 -> 15ac4679b


Making gobblin-admin a module

Making gobblin-admin a module

Revert gobblin-admin move

Addressing review comments

Closes #1540 from chavdar/admin_deps


Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/15ac4679
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/15ac4679
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/15ac4679

Branch: refs/heads/master
Commit: 15ac4679b3d8c2048ab5eca68f0cc1bdfe02097c
Parents: a889431
Author: Chavdar Botev <cbotev@cbotev-ld.(none)>
Authored: Sat Jul 29 00:57:28 2017 -0700
Committer: Abhishek Tiwari <abhishektiwari.btech@gmail.com>
Committed: Sat Jul 29 00:57:28 2017 -0700

----------------------------------------------------------------------
 .../admin/DefaultAdminWebServerFactory.java     | 39 +++++++++++++++++
 .../configuration/ConfigurationKeys.java        |  7 +++-
 .../runtime/api/AdminWebServerFactory.java      | 44 ++++++++++++++++++++
 gobblin-aws/build.gradle                        |  1 -
 gobblin-cluster/build.gradle                    |  1 -
 gobblin-distribution/build.gradle               |  1 -
 .../gobblin-flavor-custom.gradle                |  3 ++
 gobblin-distribution/gobblin-flavor-full.gradle |  1 +
 gobblin-runtime/build.gradle                    |  1 -
 .../runtime/app/ServiceBasedAppLauncher.java    | 25 +++++++++--
 gobblin-yarn/build.gradle                       |  1 -
 .../gobblin/yarn/GobblinYarnAppLauncher.java    |  9 ++--
 12 files changed, 119 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/15ac4679/gobblin-admin/src/main/java/gobblin/admin/DefaultAdminWebServerFactory.java
----------------------------------------------------------------------
diff --git a/gobblin-admin/src/main/java/gobblin/admin/DefaultAdminWebServerFactory.java b/gobblin-admin/src/main/java/gobblin/admin/DefaultAdminWebServerFactory.java
new file mode 100644
index 0000000..87c2112
--- /dev/null
+++ b/gobblin-admin/src/main/java/gobblin/admin/DefaultAdminWebServerFactory.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package gobblin.admin;
+
+import java.net.URI;
+import java.util.Properties;
+
+import com.google.common.util.concurrent.Service;
+
+import gobblin.annotation.Alias;
+import gobblin.runtime.api.AdminWebServerFactory;
+
+/**
+ * A factory for the default implementation of the admin web server
+ */
+@Alias(value="default")
+public class DefaultAdminWebServerFactory implements AdminWebServerFactory {
+
+  /** {@inheritDoc} */
+  @Override
+  public Service createInstance(Properties config, URI executionInfoServerURI) {
+    return new AdminWebServer(config, executionInfoServerURI);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/15ac4679/gobblin-api/src/main/java/gobblin/configuration/ConfigurationKeys.java
----------------------------------------------------------------------
diff --git a/gobblin-api/src/main/java/gobblin/configuration/ConfigurationKeys.java b/gobblin-api/src/main/java/gobblin/configuration/ConfigurationKeys.java
index c20a7d3..e119d43 100644
--- a/gobblin-api/src/main/java/gobblin/configuration/ConfigurationKeys.java
+++ b/gobblin-api/src/main/java/gobblin/configuration/ConfigurationKeys.java
@@ -676,10 +676,13 @@ public class ConfigurationKeys {
   public static final String DEFAULT_REST_SERVER_PORT = "8080";
   public static final String REST_SERVER_ADVERTISED_URI_KEY = "rest.server.advertised.uri";
 
-  /**
+  /*
    * Admin server configuration properties.
    */
   public static final String ADMIN_SERVER_ENABLED_KEY = "admin.server.enabled";
+  /** The name of the class with the admin interface. The class must implement the
+   * AdminWebServerFactory interface .*/
+  public static final String ADMIN_SERVER_FACTORY_CLASS_KEY = "admin.server.factory.type";
   public static final String ADMIN_SERVER_HOST_KEY = "admin.server.host";
   public static final String DEFAULT_ADMIN_SERVER_HOST = "localhost";
   public static final String ADMIN_SERVER_PORT_KEY = "admin.server.port";
@@ -689,6 +692,8 @@ public class ConfigurationKeys {
   public static final String ADMIN_SERVER_REFRESH_INTERVAL_KEY = "admin.server.refresh_interval";
   public static final long DEFAULT_ADMIN_SERVER_REFRESH_INTERVAL = 30000;
 
+  public static final String DEFAULT_ADMIN_SERVER_FACTORY_CLASS = "gobblin.admin.DefaultAdminWebServerFactory";
+
   /**
    * Kafka job configurations.
    */

http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/15ac4679/gobblin-api/src/main/java/gobblin/runtime/api/AdminWebServerFactory.java
----------------------------------------------------------------------
diff --git a/gobblin-api/src/main/java/gobblin/runtime/api/AdminWebServerFactory.java b/gobblin-api/src/main/java/gobblin/runtime/api/AdminWebServerFactory.java
new file mode 100644
index 0000000..f9f998d
--- /dev/null
+++ b/gobblin-api/src/main/java/gobblin/runtime/api/AdminWebServerFactory.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package gobblin.runtime.api;
+
+import java.net.URI;
+import java.util.Properties;
+
+import com.google.common.util.concurrent.Service;
+
+import gobblin.annotation.Alpha;
+
+/**
+ * A factory interface for AdminWebServer.
+ *
+ * NOTE: This interface is provided for backwards compatibility and may change in the future.
+ * @author cbotev
+ *
+ */
+@Alpha
+public interface AdminWebServerFactory {
+
+  /**
+   * Creates a new AdminWebServer instance
+   * @param config                      the server config
+   * @param executionInfoServerURI      the URI to the job execution server
+   * @return the instance
+   */
+  Service createInstance(Properties config, URI executionInfoServerURI);
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/15ac4679/gobblin-aws/build.gradle
----------------------------------------------------------------------
diff --git a/gobblin-aws/build.gradle b/gobblin-aws/build.gradle
index a76f449..37d4389 100644
--- a/gobblin-aws/build.gradle
+++ b/gobblin-aws/build.gradle
@@ -18,7 +18,6 @@
 apply plugin: 'java'
 
 dependencies {
-  compile project(":gobblin-admin")
   compile project(":gobblin-api")
   compile project(":gobblin-cluster")
   compile project(":gobblin-core")

http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/15ac4679/gobblin-cluster/build.gradle
----------------------------------------------------------------------
diff --git a/gobblin-cluster/build.gradle b/gobblin-cluster/build.gradle
index b96ee1f..da892aa 100644
--- a/gobblin-cluster/build.gradle
+++ b/gobblin-cluster/build.gradle
@@ -18,7 +18,6 @@
 apply plugin: 'java'
 
 dependencies {
-  compile project(":gobblin-admin")
   compile project(":gobblin-api")
   compile project(":gobblin-core")
   compile project(":gobblin-metrics-libs:gobblin-metrics")

http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/15ac4679/gobblin-distribution/build.gradle
----------------------------------------------------------------------
diff --git a/gobblin-distribution/build.gradle b/gobblin-distribution/build.gradle
index 760edeb..fff1298 100644
--- a/gobblin-distribution/build.gradle
+++ b/gobblin-distribution/build.gradle
@@ -29,7 +29,6 @@ configurations {
 }
 
 dependencies {
-  compile project(':gobblin-admin')
   compile project(':gobblin-api')
   compile project(':gobblin-core')
   compile project(':gobblin-compaction')

http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/15ac4679/gobblin-distribution/gobblin-flavor-custom.gradle
----------------------------------------------------------------------
diff --git a/gobblin-distribution/gobblin-flavor-custom.gradle b/gobblin-distribution/gobblin-flavor-custom.gradle
index 4913c4f..1f935bf 100644
--- a/gobblin-distribution/gobblin-flavor-custom.gradle
+++ b/gobblin-distribution/gobblin-flavor-custom.gradle
@@ -19,6 +19,9 @@ dependencies {
   // Example jobs
   // compile project(':gobblin-example')
 
+  // Admin Web Server
+  // compile project(':gobblin-admin')
+
   // Gobblin Azkaban integration
   // compile project(':gobblin-modules:gobblin-azkaban')
 

http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/15ac4679/gobblin-distribution/gobblin-flavor-full.gradle
----------------------------------------------------------------------
diff --git a/gobblin-distribution/gobblin-flavor-full.gradle b/gobblin-distribution/gobblin-flavor-full.gradle
index 71b5d33..22ab209 100644
--- a/gobblin-distribution/gobblin-flavor-full.gradle
+++ b/gobblin-distribution/gobblin-flavor-full.gradle
@@ -17,6 +17,7 @@
 
 dependencies {
   compile project(':gobblin-example')
+  compile project(':gobblin-admin')
   compile project(':gobblin-modules:gobblin-azkaban')
   compile project(':gobblin-modules:gobblin-compliance')
   compile project(':gobblin-modules:gobblin-couchbase')

http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/15ac4679/gobblin-runtime/build.gradle
----------------------------------------------------------------------
diff --git a/gobblin-runtime/build.gradle b/gobblin-runtime/build.gradle
index 85b367a..ec99635 100644
--- a/gobblin-runtime/build.gradle
+++ b/gobblin-runtime/build.gradle
@@ -36,7 +36,6 @@ compileJava {
 }
 
 dependencies {
-  compile project(":gobblin-admin")
   compile project(":gobblin-api")
   compile project(":gobblin-core")
   compile project(":gobblin-hive-registration")

http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/15ac4679/gobblin-runtime/src/main/java/gobblin/runtime/app/ServiceBasedAppLauncher.java
----------------------------------------------------------------------
diff --git a/gobblin-runtime/src/main/java/gobblin/runtime/app/ServiceBasedAppLauncher.java
b/gobblin-runtime/src/main/java/gobblin/runtime/app/ServiceBasedAppLauncher.java
index 58010c5..6a07cf2 100644
--- a/gobblin-runtime/src/main/java/gobblin/runtime/app/ServiceBasedAppLauncher.java
+++ b/gobblin-runtime/src/main/java/gobblin/runtime/app/ServiceBasedAppLauncher.java
@@ -20,6 +20,7 @@ package gobblin.runtime.app;
 import java.io.IOException;
 import java.lang.reflect.Constructor;
 import java.lang.reflect.InvocationTargetException;
+import java.net.URI;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Properties;
@@ -34,14 +35,15 @@ import com.google.common.util.concurrent.Service;
 import com.google.common.util.concurrent.ServiceManager;
 
 import gobblin.annotation.Alpha;
-import gobblin.admin.AdminWebServer;
 import gobblin.configuration.ConfigurationKeys;
 import gobblin.configuration.State;
 import gobblin.metrics.GobblinMetrics;
 import gobblin.rest.JobExecutionInfoServer;
+import gobblin.runtime.api.AdminWebServerFactory;
 import gobblin.runtime.services.JMXReportingService;
 import gobblin.runtime.services.MetricsReportingService;
 import gobblin.util.ApplicationLauncherUtils;
+import gobblin.util.ClassAliasResolver;
 
 
 /**
@@ -52,7 +54,7 @@ import gobblin.util.ApplicationLauncherUtils;
  * <ul>
  *   <li>{@link MetricsReportingService} is optional and controlled by {@link ConfigurationKeys#METRICS_ENABLED_KEY}</li>
  *   <li>{@link JobExecutionInfoServer} is optional and controlled by {@link ConfigurationKeys#JOB_EXECINFO_SERVER_ENABLED_KEY}</li>
- *   <li>{@link AdminWebServer} is optional and controlled by {@link ConfigurationKeys#ADMIN_SERVER_ENABLED_KEY}</li>
+ *   <li>AdminWebServer is optional and controlled by {@link ConfigurationKeys#ADMIN_SERVER_ENABLED_KEY}</li>
  *   <li>{@link JMXReportingService} is mandatory</li>
  * </ul>
  *
@@ -224,13 +226,30 @@ public class ServiceBasedAppLauncher implements ApplicationLauncher
{
 
       if (adminUiServerEnabled) {
         LOG.info("Will launch the admin UI server");
-        addService(new AdminWebServer(properties, executionInfoServer.getAdvertisedServerUri()));
+        addService(createAdminServer(properties, executionInfoServer.getAdvertisedServerUri()));
       }
     } else if (adminUiServerEnabled) {
       LOG.warn("Not launching the admin UI because the job execution info server is not enabled");
     }
   }
 
+  public static Service createAdminServer(Properties properties,
+                                          URI executionInfoServerURI) {
+    String factoryClassName = properties.getProperty(ConfigurationKeys.ADMIN_SERVER_FACTORY_CLASS_KEY,
+                                                 ConfigurationKeys.DEFAULT_ADMIN_SERVER_FACTORY_CLASS);
+    ClassAliasResolver<AdminWebServerFactory> classResolver =
+        new ClassAliasResolver<>(AdminWebServerFactory.class);
+    try
+    {
+      AdminWebServerFactory factoryInstance = classResolver.resolveClass(factoryClassName).newInstance();
+      return factoryInstance.createInstance(properties, executionInfoServerURI);
+    }
+    catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) {
+      throw new RuntimeException("Unable to instantiate the AdminWebServer factory. " +
+           "Have you included the module in the gobblin distribution? :" + e, e);
+    }
+  }
+
   private void addMetricsService(Properties properties) {
     if (GobblinMetrics.isEnabled(properties)) {
       addService(new MetricsReportingService(properties, this.appId));

http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/15ac4679/gobblin-yarn/build.gradle
----------------------------------------------------------------------
diff --git a/gobblin-yarn/build.gradle b/gobblin-yarn/build.gradle
index 1ab9558..02959bf 100644
--- a/gobblin-yarn/build.gradle
+++ b/gobblin-yarn/build.gradle
@@ -22,7 +22,6 @@ plugins {
 apply plugin: 'java'
 
 dependencies {
-  compile project(":gobblin-admin")
   compile project(":gobblin-api")
   compile project(":gobblin-cluster")
   compile project(":gobblin-core")

http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/15ac4679/gobblin-yarn/src/main/java/gobblin/yarn/GobblinYarnAppLauncher.java
----------------------------------------------------------------------
diff --git a/gobblin-yarn/src/main/java/gobblin/yarn/GobblinYarnAppLauncher.java b/gobblin-yarn/src/main/java/gobblin/yarn/GobblinYarnAppLauncher.java
index 6603813..a5f119d 100644
--- a/gobblin-yarn/src/main/java/gobblin/yarn/GobblinYarnAppLauncher.java
+++ b/gobblin-yarn/src/main/java/gobblin/yarn/GobblinYarnAppLauncher.java
@@ -61,7 +61,6 @@ import org.apache.hadoop.yarn.client.api.YarnClientApplication;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.util.Records;
-
 import org.apache.helix.Criteria;
 import org.apache.helix.HelixManager;
 import org.apache.helix.HelixManagerFactory;
@@ -84,17 +83,16 @@ import com.google.common.io.Closer;
 import com.google.common.util.concurrent.ListeningExecutorService;
 import com.google.common.util.concurrent.Service;
 import com.google.common.util.concurrent.ServiceManager;
-
 import com.typesafe.config.Config;
 import com.typesafe.config.ConfigFactory;
 
-import gobblin.admin.AdminWebServer;
 import gobblin.cluster.GobblinClusterConfigurationKeys;
 import gobblin.cluster.GobblinClusterUtils;
 import gobblin.cluster.GobblinHelixConstants;
 import gobblin.cluster.HelixUtils;
 import gobblin.configuration.ConfigurationKeys;
 import gobblin.rest.JobExecutionInfoServer;
+import gobblin.runtime.app.ServiceBasedAppLauncher;
 import gobblin.util.ConfigUtils;
 import gobblin.util.EmailUtils;
 import gobblin.util.ExecutorsUtils;
@@ -290,7 +288,8 @@ public class GobblinYarnAppLauncher {
       services.add(executionInfoServer);
       if (config.getBoolean(ConfigurationKeys.ADMIN_SERVER_ENABLED_KEY)) {
         LOGGER.info("Starting the admin UI server since it is enabled");
-        services.add(new AdminWebServer(properties, executionInfoServer.getAdvertisedServerUri()));
+        services.add(ServiceBasedAppLauncher.createAdminServer(properties,
+                                                               executionInfoServer.getAdvertisedServerUri()));
       }
     } else if (config.getBoolean(ConfigurationKeys.ADMIN_SERVER_ENABLED_KEY)) {
       LOGGER.warn("NOT starting the admin UI because the job execution info server is NOT
enabled");
@@ -379,7 +378,7 @@ public class GobblinYarnAppLauncher {
 
   @Subscribe
   public void handleGetApplicationReportFailureEvent(
-      @SuppressWarnings("unused") GetApplicationReportFailureEvent getApplicationReportFailureEvent)
{
+      GetApplicationReportFailureEvent getApplicationReportFailureEvent) {
     int numConsecutiveFailures = this.getApplicationReportFailureCount.incrementAndGet();
     if (numConsecutiveFailures > this.maxGetApplicationReportFailures) {
       LOGGER.warn(String


Mime
View raw message