gobblin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a...@apache.org
Subject [8/9] incubator-gobblin git commit: Azkaban orchestrator findbugs fix, addition of default config fallback, updated config values
Date Wed, 30 Aug 2017 15:55:34 GMT
Azkaban orchestrator findbugs fix, addition of default config fallback, updated config values


Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/b0c96aca
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/b0c96aca
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/b0c96aca

Branch: refs/heads/master
Commit: b0c96acabf9056b8caa8e565f737be989f449056
Parents: ee3e548
Author: Abhishek Tiwari <abhishektiwari.btech@gmail.com>
Authored: Wed Aug 30 08:07:38 2017 -0700
Committer: Abhishek Tiwari <abhishektiwari.btech@gmail.com>
Committed: Wed Aug 30 08:07:38 2017 -0700

----------------------------------------------------------------------
 conf/service/log4j-cluster.properties           | 27 ------------------
 conf/service/log4j-service.properties           | 27 ++++++++++++++++++
 .../orchestration/AzkabanAjaxAPIClient.java     |  5 +++-
 .../modules/orchestration/AzkabanJobHelper.java | 30 ++++++++++++++++++--
 .../orchestration/AzkabanProjectConfig.java     |  4 +--
 .../AzkabanSpecExecutorInstance.java            |  4 ++-
 .../AzkabanSpecExecutorInstanceProducer.java    |  6 ++--
 .../orchestration/ServiceAzkabanConfigKeys.java |  1 +
 .../main/resources/default-service-azkaban.conf |  2 +-
 .../gobblin/service/ServiceConfigKeys.java      |  2 ++
 .../modules/core/GobblinServiceManager.java     |  1 +
 11 files changed, 70 insertions(+), 39 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/b0c96aca/conf/service/log4j-cluster.properties
----------------------------------------------------------------------
diff --git a/conf/service/log4j-cluster.properties b/conf/service/log4j-cluster.properties
deleted file mode 100755
index a7ffb68..0000000
--- a/conf/service/log4j-cluster.properties
+++ /dev/null
@@ -1,27 +0,0 @@
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-
-# log4j configuration used during build and unit tests
-
-log4j.rootLogger=info,stdout
-log4j.threshhold=ALL
-log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss z} %-5p [%t] %C %X{tableName}
- %m%n
-
-# Suppressed loggers
-log4j.logger.org.apache.helix.controller.GenericHelixController=ERROR
-log4j.logger.org.apache.helix.controller.stages=ERROR
-log4j.logger.org.apache.helix.controller.strategy.AutoRebalanceStrategy=ERROR
-log4j.logger.org.apache.helix.manager.zk=ERROR
-log4j.logger.org.apache.helix.monitoring.mbeans.ClusterStatusMonitor=ERROR
-log4j.logger.org.apache.helix.store.zk.AutoFallbackPropertyStore=ERROR
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/b0c96aca/conf/service/log4j-service.properties
----------------------------------------------------------------------
diff --git a/conf/service/log4j-service.properties b/conf/service/log4j-service.properties
new file mode 100755
index 0000000..a7ffb68
--- /dev/null
+++ b/conf/service/log4j-service.properties
@@ -0,0 +1,27 @@
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+# log4j configuration used during build and unit tests
+
+log4j.rootLogger=info,stdout
+log4j.threshhold=ALL
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss z} %-5p [%t] %C %X{tableName}
- %m%n
+
+# Suppressed loggers
+log4j.logger.org.apache.helix.controller.GenericHelixController=ERROR
+log4j.logger.org.apache.helix.controller.stages=ERROR
+log4j.logger.org.apache.helix.controller.strategy.AutoRebalanceStrategy=ERROR
+log4j.logger.org.apache.helix.manager.zk=ERROR
+log4j.logger.org.apache.helix.monitoring.mbeans.ClusterStatusMonitor=ERROR
+log4j.logger.org.apache.helix.store.zk.AutoFallbackPropertyStore=ERROR
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/b0c96aca/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanAjaxAPIClient.java
----------------------------------------------------------------------
diff --git a/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanAjaxAPIClient.java
b/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanAjaxAPIClient.java
index 90bf005..9c3cee4 100644
--- a/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanAjaxAPIClient.java
+++ b/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanAjaxAPIClient.java
@@ -308,7 +308,7 @@ public class AzkabanAjaxAPIClient {
       throws IOException {
     Map<String, String> params = Maps.newHashMap();
     params.put("ajax", "executeFlow");
-    params.put("projectName", azkabanProjectConfig.getAzkabanProjectName());
+    params.put("project", azkabanProjectConfig.getAzkabanProjectName());
     params.put("flow", azkabanProjectConfig.getAzkabanProjectFlowName());
 
     executePostRequest(preparePostRequest(azkabanProjectConfig.getAzkabanServerUrl() + "/executor",
sessionId, params));
@@ -454,6 +454,9 @@ public class AzkabanAjaxAPIClient {
    * @param delayMinutes If current time is within window, then additional delay for bootstrapping
if desired
    * @return Scheduled time string of the format hh,mm,a,z
    */
+  @edu.umd.cs.findbugs.annotations.SuppressWarnings(
+      value = "DMI_RANDOM_USED_ONLY_ONCE",
+      justification = "As expected for randomization")
   public static String getScheduledTimeInAzkabanFormat(int windowStartHour, int windowEndHour,
int delayMinutes) {
     // Validate
     if (windowStartHour < 0 || windowEndHour > 23 || windowStartHour >= windowEndHour)
{

http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/b0c96aca/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanJobHelper.java
----------------------------------------------------------------------
diff --git a/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanJobHelper.java
b/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanJobHelper.java
index a74a6ad..4fbe32b 100644
--- a/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanJobHelper.java
+++ b/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanJobHelper.java
@@ -263,7 +263,13 @@ public class AzkabanJobHelper {
     // Determine final zip file path
     String zipFilePath = String.format("%s/%s", directory, zipFilename);
     File zipFile = new File(zipFilePath);
-    zipFile.delete();
+    if (zipFile.exists()) {
+      if (zipFile.delete()) {
+        log.info("Zipfile existed and was deleted: " + zipFilePath);
+      } else {
+        log.warn("Zipfile exists but was not deleted: " + zipFilePath);
+      }
+    }
 
     // Create and add files to zip file
     addFilesToZip(zipFile, filesToAdd);
@@ -271,6 +277,9 @@ public class AzkabanJobHelper {
     return zipFilePath;
   }
 
+  @edu.umd.cs.findbugs.annotations.SuppressWarnings(
+      value = "OBL_UNSATISFIED_OBLIGATION",
+      justification = "Lombok construct of @Cleanup is handing this, but not detected by
FindBugs")
   private static void addFilesToZip(File zipFile, List<File> filesToAdd) throws IOException
{
     try {
       @Cleanup
@@ -300,7 +309,13 @@ public class AzkabanJobHelper {
     // Determine final config file path
     String jobFilePath = String.format("%s/%s.job", workDir, flowName);
     File jobFile = new File(jobFilePath);
-    jobFile.delete();
+    if (jobFile.exists()) {
+      if (jobFile.delete()) {
+        log.info("JobFile existed and was deleted: " + jobFilePath);
+      } else {
+        log.warn("JobFile exists but was not deleted: " + jobFilePath);
+      }
+    }
 
     StringBuilder propertyFileContent = new StringBuilder();
     for (Map.Entry entry : azkabanProjectConfig.getJobSpec().getConfigAsProperties().entrySet())
{
@@ -313,6 +328,9 @@ public class AzkabanJobHelper {
     return new File[] {jobFile};
   }
 
+  @edu.umd.cs.findbugs.annotations.SuppressWarnings(
+      value = "OBL_UNSATISFIED_OBLIGATION",
+      justification = "Lombok construct of @Cleanup is handing this, but not detected by
FindBugs")
   private static File downloadAzkabanJobJar(String workDir, String jobJarUrl)
       throws IOException {
     // Determine final jar file path
@@ -320,7 +338,13 @@ public class AzkabanJobHelper {
     String jobJarName = jobJarUrlParts[jobJarUrlParts.length-1];
     String jobJarFilePath = String.format("%s/%s", workDir, jobJarName);
     File jobJarFile = new File(jobJarFilePath);
-    jobJarFile.delete();
+    if (jobJarFile.exists()) {
+      if (jobJarFile.delete()) {
+      log.info("JobJarFilePath existed and was deleted: " + jobJarFilePath);
+    } else {
+        log.warn("JobJarFilePath exists but was not deleted: " + jobJarFilePath);
+      }
+    }
 
     // Create work directory if not already exists
     FileUtils.forceMkdir(new File(workDir));

http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/b0c96aca/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanProjectConfig.java
----------------------------------------------------------------------
diff --git a/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanProjectConfig.java
b/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanProjectConfig.java
index b99683d..583988b 100644
--- a/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanProjectConfig.java
+++ b/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanProjectConfig.java
@@ -39,8 +39,6 @@ import com.typesafe.config.ConfigFactory;
  * Class to hold Azkaban project specific configs
  */
 public class AzkabanProjectConfig {
-  private static final String DEFAULT_AZKABAN_PROJECT_CONFIG_FILE = "default-service-azkaban.conf";
-
   private final String azkabanServerUrl;
 
   private final String azkabanProjectName;
@@ -60,7 +58,7 @@ public class AzkabanProjectConfig {
   public AzkabanProjectConfig(JobSpec jobSpec) {
     // Extract config objects
     this.jobSpec = jobSpec;
-    Config defaultConfig = ConfigFactory.load(DEFAULT_AZKABAN_PROJECT_CONFIG_FILE);
+    Config defaultConfig = ConfigFactory.load(ServiceAzkabanConfigKeys.DEFAULT_AZKABAN_PROJECT_CONFIG_FILE);
     Config config  = jobSpec.getConfig().withFallback(defaultConfig);
 
     // Azkaban Infrastructure

http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/b0c96aca/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanSpecExecutorInstance.java
----------------------------------------------------------------------
diff --git a/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanSpecExecutorInstance.java
b/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanSpecExecutorInstance.java
index 65209c3..dcc89cc 100644
--- a/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanSpecExecutorInstance.java
+++ b/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanSpecExecutorInstance.java
@@ -35,6 +35,7 @@ import com.google.common.base.Splitter;
 import com.google.common.collect.Maps;
 import com.google.common.util.concurrent.AbstractIdleService;
 import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
 
 
 public class AzkabanSpecExecutorInstance extends AbstractIdleService implements SpecExecutorInstance
{
@@ -48,7 +49,8 @@ public class AzkabanSpecExecutorInstance extends AbstractIdleService implements
   protected final Map<String, String> _capabilities;
 
   public AzkabanSpecExecutorInstance(Config config, Optional<Logger> log) {
-    _config = config;
+    Config defaultConfig = ConfigFactory.load(ServiceAzkabanConfigKeys.DEFAULT_AZKABAN_PROJECT_CONFIG_FILE);
+    _config = config.withFallback(defaultConfig);
     _log = log.isPresent() ? log.get() : LoggerFactory.getLogger(getClass());
     try {
       _specExecutorInstanceUri = new URI(ConfigUtils.getString(config, ConfigurationKeys.SPECEXECUTOR_INSTANCE_URI_KEY,

http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/b0c96aca/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanSpecExecutorInstanceProducer.java
----------------------------------------------------------------------
diff --git a/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanSpecExecutorInstanceProducer.java
b/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanSpecExecutorInstanceProducer.java
index f73bc6c..47df250 100644
--- a/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanSpecExecutorInstanceProducer.java
+++ b/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanSpecExecutorInstanceProducer.java
@@ -47,9 +47,9 @@ public class AzkabanSpecExecutorInstanceProducer extends AzkabanSpecExecutorInst
 
     try {
       // Initialize Azkaban client / producer and cache credentials
-      String azkabanUsername = config.getString(ServiceAzkabanConfigKeys.AZKABAN_USERNAME_KEY);
-      String azkabanPassword = getAzkabanPassword(config);
-      String azkabanServerUrl = config.getString(ServiceAzkabanConfigKeys.AZKABAN_SERVER_URL_KEY);
+      String azkabanUsername = _config.getString(ServiceAzkabanConfigKeys.AZKABAN_USERNAME_KEY);
+      String azkabanPassword = getAzkabanPassword(_config);
+      String azkabanServerUrl = _config.getString(ServiceAzkabanConfigKeys.AZKABAN_SERVER_URL_KEY);
 
       _sessionId = AzkabanAjaxAPIClient.authenticateAndGetSessionId(azkabanUsername, azkabanPassword,
azkabanServerUrl);
     } catch (IOException | EncoderException e) {

http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/b0c96aca/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/ServiceAzkabanConfigKeys.java
----------------------------------------------------------------------
diff --git a/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/ServiceAzkabanConfigKeys.java
b/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/ServiceAzkabanConfigKeys.java
index b712a5a..4c24944 100644
--- a/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/ServiceAzkabanConfigKeys.java
+++ b/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/ServiceAzkabanConfigKeys.java
@@ -37,5 +37,6 @@ public class ServiceAzkabanConfigKeys {
 
   // Azkaban System Environment
   public static final String AZKABAN_PASSWORD_SYSTEM_KEY = "GOBBLIN_SERVICE_AZKABAN_PASSWORD";
+  public static final String DEFAULT_AZKABAN_PROJECT_CONFIG_FILE = "default-service-azkaban.conf";
 }
 

http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/b0c96aca/gobblin-modules/gobblin-azkaban/src/main/resources/default-service-azkaban.conf
----------------------------------------------------------------------
diff --git a/gobblin-modules/gobblin-azkaban/src/main/resources/default-service-azkaban.conf
b/gobblin-modules/gobblin-azkaban/src/main/resources/default-service-azkaban.conf
index 6d31984..caf6ebe 100644
--- a/gobblin-modules/gobblin-azkaban/src/main/resources/default-service-azkaban.conf
+++ b/gobblin-modules/gobblin-azkaban/src/main/resources/default-service-azkaban.conf
@@ -37,5 +37,5 @@ gobblin.service.azkaban.project.zip.failIfJarNotFound=false
 gobblin.service.azkaban.project.zip.additionalFilesUrl=""
 
 gobblin.service.azkaban.project.job.jar.mavenUrlTemplate="https://repo.maven.apache.org/maven2/com/linkedin/gobblin/<module-name>/<module-version>/<module-name>-<module-version>.jar"
-gobblin.service.azkaban.project.job.jar.mavenUrlTemplateModules="gobblin-admin.jar,gobblin-api.jar,gobblin-compaction.jar,gobblin-config-management.jar,gobblin-core.jar,gobblin-core-base.jar,gobblin-distribution.jar,gobblin-example.jar,gobblin-hive-registration.jar,gobblin-metrics-libs.jar,gobblin-metastore.jar,gobblin-modules.jar,gobblin-rest-service.jar,gobblin-runtime.jar,gobblin-runtime-hadoop.jar,gobblin-utility.jar,gobblin-salesforce.jar,gobblin-test-harness.jar,gobblin-tunnel.jar,gobblin-data-management.jar,gobblin-config-management.jar,gobblin-audit.jar,gobblin-yarn.jar,gobblin-cluster.jar,gobblin-aws.jar,gobblin-service.jar,gobblin-test-utils.jar"
+gobblin.service.azkaban.project.job.jar.mavenUrlTemplateModules="gobblin-admin,gobblin-api,gobblin-compaction,gobblin-config-management,gobblin-core,gobblin-core-base,gobblin-distribution,gobblin-example,gobblin-hive-registration,gobblin-metrics-libs,gobblin-metastore,gobblin-modules,gobblin-rest-service,gobblin-runtime,gobblin-runtime-hadoop,gobblin-utility,gobblin-salesforce,gobblin-test-harness,gobblin-tunnel,gobblin-data-management,gobblin-config-management,gobblin-audit,gobblin-yarn,gobblin-cluster,gobblin-aws,gobblin-service,gobblin-test-utils"
 gobblin.service.azkaban.project.job.jar.mavenGobblinVersion="0.11.0"
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/b0c96aca/gobblin-service/src/main/java/org/apache/gobblin/service/ServiceConfigKeys.java
----------------------------------------------------------------------
diff --git a/gobblin-service/src/main/java/org/apache/gobblin/service/ServiceConfigKeys.java
b/gobblin-service/src/main/java/org/apache/gobblin/service/ServiceConfigKeys.java
index a6f0199..8ea19c4 100644
--- a/gobblin-service/src/main/java/org/apache/gobblin/service/ServiceConfigKeys.java
+++ b/gobblin-service/src/main/java/org/apache/gobblin/service/ServiceConfigKeys.java
@@ -75,4 +75,6 @@ public class ServiceConfigKeys {
   // Template Catalog Keys
   public static final String TEMPLATE_CATALOGS_FULLY_QUALIFIED_PATH_KEY = GOBBLIN_SERVICE_PREFIX
+ "templateCatalogs.fullyQualifiedPath";
 
+  // Logging
+  public static final String GOBBLIN_SERVICE_LOG4J_CONFIGURATION_FILE = "log4j-service.properties";
 }

http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/b0c96aca/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GobblinServiceManager.java
----------------------------------------------------------------------
diff --git a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GobblinServiceManager.java
b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GobblinServiceManager.java
index 4707361..c2591e1 100644
--- a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GobblinServiceManager.java
+++ b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GobblinServiceManager.java
@@ -37,6 +37,7 @@ import org.apache.commons.cli.HelpFormatter;
 import org.apache.commons.cli.Options;
 import org.apache.commons.cli.ParseException;
 import org.apache.commons.lang3.reflect.ConstructorUtils;
+import org.apache.gobblin.util.logs.Log4jConfigurationHelper;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;


Mime
View raw message