airavata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From lah...@apache.org
Subject airavata git commit: First stage of kafka appender
Date Mon, 29 Aug 2016 04:39:28 GMT
Repository: airavata
Updated Branches:
  refs/heads/lahiru/AIRAVATA-2065 415e9b701 -> 46b8502eb


First stage of kafka appender


Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/46b8502e
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/46b8502e
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/46b8502e

Branch: refs/heads/lahiru/AIRAVATA-2065
Commit: 46b8502ebe3d2c76000db8d79329e58434810ef3
Parents: 415e9b7
Author: Lahiru Ginnaliya Gamathige <lahiru@apache.org>
Authored: Sun Aug 28 21:39:11 2016 -0700
Committer: Lahiru Ginnaliya Gamathige <lahiru@apache.org>
Committed: Sun Aug 28 21:39:11 2016 -0700

----------------------------------------------------------------------
 distribution/pom.xml                            |  11 +-
 modules/commons/pom.xml                         |  12 +-
 .../airavata/common/logging/Exception.java      |  66 +++++++++
 .../airavata/common/logging/GsonTest.java       |  20 +++
 .../airavata/common/logging/LogEntry.java       | 132 ++++++++++++++++++
 .../airavata/common/logging/ServerId.java       |  68 +++++++++
 .../common/logging/kafka/KafkaAppender.java     | 116 ++++++++++++++++
 .../airavata/common/utils/AiravataZKUtils.java  |  13 +-
 .../common/utils/ApplicationSettings.java       |  34 +++--
 .../airavata/common/utils/AwsMetadata.java      | 137 +++++++++++++++++++
 .../airavata/common/utils/BuildConstant.java    |  28 ++++
 .../apache/airavata/common/utils/JSONUtil.java  |   1 -
 .../airavata/common/utils/ServerSettings.java   |  36 +++++
 .../main/resources/airavata-server.properties   |   8 +-
 .../gfac/bes/provider/impl/BESProvider.java     |   7 +-
 .../impl/JSDLGeneratorTestWithMyProxyAuth.java  |   2 +-
 .../apache/airavata/gfac/core/GFacUtils.java    |   2 +-
 .../gfac/impl/task/BESJobSubmissionTask.java    |   7 +-
 .../services/impl/BigRed2TestWithSSHAuth.java   |   2 +-
 .../impl/GSISSHProviderTestWithMyProxyAuth.java |   2 +-
 .../impl/SSHProviderTestWithSSHAuth.java        |   2 +-
 modules/registry/registry-core/pom.xml          |  27 ----
 .../org/apache/airavata/server/ServerMain.java  |  23 +++-
 .../utils/PropertyReader.java                   |   3 +-
 .../core/AbstractThriftDeserializer.java        |   3 +-
 pom.xml                                         |  32 +++++
 26 files changed, 728 insertions(+), 66 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/distribution/pom.xml
----------------------------------------------------------------------
diff --git a/distribution/pom.xml b/distribution/pom.xml
index d3124eb..e3d1b33 100644
--- a/distribution/pom.xml
+++ b/distribution/pom.xml
@@ -559,7 +559,16 @@
             <artifactId>curator-framework</artifactId>
             <version>${curator.version}</version>
         </dependency>
-
+        <dependency>
+            <groupId>org.apache.kafka</groupId>
+            <artifactId>kafka-clients</artifactId>
+            <version>${kafka-clients.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>ch.qos.logback</groupId>
+            <artifactId>logback-classic</artifactId>
+            <version>${logback.version}</version>
+        </dependency>
         <!-- ======================== Sample =================== -->
         <dependency>
             <groupId>org.apache.airavata</groupId>

http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/commons/pom.xml
----------------------------------------------------------------------
diff --git a/modules/commons/pom.xml b/modules/commons/pom.xml
index a2136c6..f4231ce 100644
--- a/modules/commons/pom.xml
+++ b/modules/commons/pom.xml
@@ -115,7 +115,6 @@
         <dependency>
             <groupId>org.slf4j</groupId>
             <artifactId>slf4j-log4j12</artifactId>
-            <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.apache.curator</groupId>
@@ -127,12 +126,21 @@
             <artifactId>libthrift</artifactId>
             <version>${thrift.version}</version>
         </dependency>
-
         <dependency>
             <groupId>com.google.code.gson</groupId>
             <artifactId>gson</artifactId>
             <version>${google.gson.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.kafka</groupId>
+            <artifactId>kafka-clients</artifactId>
+            <version>${kafka-clients.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>ch.qos.logback</groupId>
+            <artifactId>logback-classic</artifactId>
+            <version>${logback.version}</version>
+        </dependency>
     </dependencies>
 
     <properties>

http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/commons/src/main/java/org/apache/airavata/common/logging/Exception.java
----------------------------------------------------------------------
diff --git a/modules/commons/src/main/java/org/apache/airavata/common/logging/Exception.java b/modules/commons/src/main/java/org/apache/airavata/common/logging/Exception.java
new file mode 100644
index 0000000..cea0c95
--- /dev/null
+++ b/modules/commons/src/main/java/org/apache/airavata/common/logging/Exception.java
@@ -0,0 +1,66 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.airavata.common.logging;
+
+
+public class Exception {
+
+    private String message;
+
+    private String[] stackTrace;
+
+    private String className;
+
+    public Exception(String message, String[] stackTrace) {
+        this.message = message;
+        this.stackTrace = stackTrace;
+    }
+
+    public Exception(String message, String[] stackTrace, String className) {
+        this.message = message;
+        this.stackTrace = stackTrace;
+        this.className = className;
+    }
+
+    public String getMessage() {
+        return message;
+    }
+
+    public void setMessage(String message) {
+        this.message = message;
+    }
+
+    public String[] getStackTrace() {
+        return stackTrace;
+    }
+
+    public void setStackTrace(String[] stackTrace) {
+        this.stackTrace = stackTrace;
+    }
+
+    public String getClassName() {
+        return className;
+    }
+
+    public void setClassName(String className) {
+        this.className = className;
+    }
+}

http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/commons/src/main/java/org/apache/airavata/common/logging/GsonTest.java
----------------------------------------------------------------------
diff --git a/modules/commons/src/main/java/org/apache/airavata/common/logging/GsonTest.java b/modules/commons/src/main/java/org/apache/airavata/common/logging/GsonTest.java
new file mode 100644
index 0000000..b4e9cb2
--- /dev/null
+++ b/modules/commons/src/main/java/org/apache/airavata/common/logging/GsonTest.java
@@ -0,0 +1,20 @@
+///
+// Copyright (c) 2016. Highfive Technologies, Inc.
+///
+package org.apache.airavata.common.logging;
+
+import com.google.gson.Gson;
+
+import java.util.HashMap;
+
+public class GsonTest {
+    public static void main(String[] args) {
+        final String[] strings = {"roles"};
+        final Exception exception = new Exception("message", strings, "test");
+        final LogEntry logEntry = new LogEntry(new ServerId("serverId", "serverId", "serverId", strings), "serverId", "serverId", "serverId", "serverId", new HashMap<String, String>(), "thread");
+        final LogEntry logEntry1 = new LogEntry(new ServerId("serverId", "serverId", "serverId", strings), "serverId", "serverId", "serverId", "serverId", new HashMap<String, String>(), "thread", exception);
+
+        System.out.println(new Gson().toJson(logEntry));
+        System.out.println(new Gson().toJson(logEntry1));
+    }
+}

http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/commons/src/main/java/org/apache/airavata/common/logging/LogEntry.java
----------------------------------------------------------------------
diff --git a/modules/commons/src/main/java/org/apache/airavata/common/logging/LogEntry.java b/modules/commons/src/main/java/org/apache/airavata/common/logging/LogEntry.java
new file mode 100644
index 0000000..72fc4a0
--- /dev/null
+++ b/modules/commons/src/main/java/org/apache/airavata/common/logging/LogEntry.java
@@ -0,0 +1,132 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.airavata.common.logging;
+
+
+import java.lang.*;
+import java.util.Map;
+
+public class LogEntry {
+
+    private ServerId serverId;
+
+    private String message;
+
+    private String timestamp;
+
+    private String level;
+
+    private String loggerName;
+
+    private Map<String, String> mdc;
+
+    private String threadName;
+
+    private Exception exception;
+
+    public LogEntry(ServerId serverId, String message, String timestamp, String level, String loggerName, Map<String,
+            String> mdc, String threadName, Exception exception) {
+        this.serverId = serverId;
+        this.message = message;
+        this.timestamp = timestamp;
+        this.level = level;
+        this.loggerName = loggerName;
+        this.mdc = mdc;
+        this.threadName = threadName;
+        this.exception = exception;
+    }
+
+    public LogEntry(ServerId serverId, String message, String timestamp, String level, String loggerName, Map<String,
+            String> mdc, String threadName) {
+        this.serverId = serverId;
+        this.message = message;
+        this.timestamp = timestamp;
+        this.level = level;
+        this.loggerName = loggerName;
+        this.mdc = mdc;
+        this.threadName = threadName;
+    }
+
+
+    public ServerId getServerId() {
+        return serverId;
+    }
+
+    public void setServerId(ServerId serverId) {
+        this.serverId = serverId;
+    }
+
+    public String getMessage() {
+        return message;
+    }
+
+    public void setMessage(String message) {
+        this.message = message;
+    }
+
+    public String getTimestamp() {
+        return timestamp;
+    }
+
+    public void setTimestamp(String timestamp) {
+        this.timestamp = timestamp;
+    }
+
+    public String getLevel() {
+        return level;
+    }
+
+    public void setLevel(String level) {
+        this.level = level;
+    }
+
+    public String getLoggerName() {
+        return loggerName;
+    }
+
+    public void setLoggerName(String loggerName) {
+        this.loggerName = loggerName;
+    }
+
+    public Map<String, String> getMdc() {
+        return mdc;
+    }
+
+    public void setMdc(Map<String, String> mdc) {
+        this.mdc = mdc;
+    }
+
+    public String getThreadName() {
+        return threadName;
+    }
+
+    public void setThreadName(String threadName) {
+        this.threadName = threadName;
+    }
+
+    public Exception getException() {
+        return exception;
+    }
+
+    public void setException(Exception exception) {
+        this.exception = exception;
+    }
+}

http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/commons/src/main/java/org/apache/airavata/common/logging/ServerId.java
----------------------------------------------------------------------
diff --git a/modules/commons/src/main/java/org/apache/airavata/common/logging/ServerId.java b/modules/commons/src/main/java/org/apache/airavata/common/logging/ServerId.java
new file mode 100644
index 0000000..9611302
--- /dev/null
+++ b/modules/commons/src/main/java/org/apache/airavata/common/logging/ServerId.java
@@ -0,0 +1,68 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.airavata.common.logging;
+
+public class ServerId {
+    private String serverId;
+    private String hostName;
+    private String version;
+
+    private String[] roles; // gfac, orchestrator, apiserver,
+
+    public ServerId(String serverId, String hostName, String version, String[] roles) {
+        this.serverId = serverId;
+        this.hostName = hostName;
+        this.version = version;
+        this.roles = roles;
+    }
+
+    public String getServerId() {
+        return serverId;
+    }
+
+    public void setServerId(String serverId) {
+        this.serverId = serverId;
+    }
+
+    public String getHostName() {
+        return hostName;
+    }
+
+    public void setHostName(String hostName) {
+        this.hostName = hostName;
+    }
+
+    public String getVersion() {
+        return version;
+    }
+
+    public void setVersion(String version) {
+        this.version = version;
+    }
+
+    public String[] getRoles() {
+        return roles;
+    }
+
+    public void setRoles(String[] roles) {
+        this.roles = roles;
+    }
+}

http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/commons/src/main/java/org/apache/airavata/common/logging/kafka/KafkaAppender.java
----------------------------------------------------------------------
diff --git a/modules/commons/src/main/java/org/apache/airavata/common/logging/kafka/KafkaAppender.java b/modules/commons/src/main/java/org/apache/airavata/common/logging/kafka/KafkaAppender.java
new file mode 100644
index 0000000..a4a6a7b
--- /dev/null
+++ b/modules/commons/src/main/java/org/apache/airavata/common/logging/kafka/KafkaAppender.java
@@ -0,0 +1,116 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.airavata.common.logging.kafka;
+
+
+import ch.qos.logback.classic.Level;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.classic.spi.IThrowableProxy;
+import ch.qos.logback.classic.spi.StackTraceElementProxy;
+import ch.qos.logback.core.UnsynchronizedAppenderBase;
+import com.google.gson.Gson;
+import org.apache.airavata.common.logging.Exception;
+import org.apache.airavata.common.logging.LogEntry;
+import org.apache.airavata.common.logging.ServerId;
+import org.apache.airavata.common.utils.AwsMetadata;
+import org.apache.airavata.common.utils.BuildConstant;
+import org.apache.airavata.common.utils.ServerSettings;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.Producer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.time.Instant;
+import java.util.Arrays;
+import java.util.Properties;
+
+public class KafkaAppender extends UnsynchronizedAppenderBase<ILoggingEvent> {
+    private final static Logger logger = LoggerFactory.getLogger(KafkaAppender.class);
+
+    private final Producer<String, String> producer;
+    private final String kafkaTopic;
+
+    private  ServerId serverId = null;
+
+    public KafkaAppender(String kafkaTopicPrefix, String kafkaHost) {
+        Properties props = new Properties();
+        props.put("bootstrap.servers", kafkaHost);
+        props.put("acks", "0");
+        props.put("retries", 0);
+        props.put("batch.size", 16384);
+        props.put("linger.ms", 10000); // Send the batch every 5 seconds
+        props.put("buffer.memory", 33554432);
+        props.put("producer.type", "async");
+        props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
+        props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
+        this.kafkaTopic = getKafkaTopicPrefix(kafkaTopicPrefix);
+
+        logger.info("Starting kafka producer: bootstrap-server:{}, topic : {}", kafkaHost, kafkaTopic);
+        this.producer = new KafkaProducer<>(props);
+        if(ServerSettings.isRunningOnAws()) {
+            final AwsMetadata awsMetadata = new AwsMetadata();
+            serverId = new ServerId(awsMetadata.getId(), awsMetadata.getHostname(),
+                    BuildConstant.getGitDescribeVersion(), ServerSettings.getServerRoles());
+        } else {
+            serverId = new ServerId(ServerSettings.getIp(), ServerSettings.getIp(),
+                    BuildConstant.getGitDescribeVersion(), ServerSettings.getServerRoles());
+        }
+    }
+
+    @Override
+    protected void append(ILoggingEvent event) {
+        //todo do more elegant streaming approach to publish logs
+        if (!event.getLevel().equals(Level.ALL) &&         // OFF AND ALL are not loggable levels
+                !event.getLevel().equals(Level.OFF)) {
+            final IThrowableProxy throwableProxy = event.getThrowableProxy();
+            final LogEntry entry = throwableProxy != null ?
+                    new LogEntry(serverId, event.getMessage(), Instant.ofEpochMilli(event.getTimeStamp()).toString(),
+                            event.getLevel().toString(), event.getLoggerName(), event.getMDCPropertyMap(),
+                            event.getThreadName() != null ? event.getThreadName() : null,
+                            new Exception(throwableProxy.getMessage(), toStringArray(throwableProxy.getStackTraceElementProxyArray())
+                            , throwableProxy.getClassName()))
+                    : new LogEntry(serverId, event.getMessage(), Instant.ofEpochMilli(event.getTimeStamp()).toString(),
+                    event.getLevel().toString(), event.getLoggerName(), event.getMDCPropertyMap(),
+                    event.getThreadName() != null ? event.getThreadName() : null);
+
+            producer.send(new ProducerRecord<>(kafkaTopic, new Gson().toJson(entry)));
+        }
+    }
+
+
+    private String[] toStringArray(StackTraceElementProxy[] stackTraceElement) {
+        return Arrays.stream(stackTraceElement).map(StackTraceElementProxy::getSTEAsString).toArray(String[]::new);
+    }
+
+    private String getKafkaTopicPrefix(String kafkaTopicPrefix) {
+        final StringBuilder stringBuffer = new StringBuilder("");
+        for (String role : ServerSettings.getServerRoles()) {
+            stringBuffer.append("_");
+            stringBuffer.append(role);
+        }
+        return kafkaTopicPrefix + stringBuffer.toString();
+    }
+
+    public void close() {
+        producer.close();
+    }
+}

http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/commons/src/main/java/org/apache/airavata/common/utils/AiravataZKUtils.java
----------------------------------------------------------------------
diff --git a/modules/commons/src/main/java/org/apache/airavata/common/utils/AiravataZKUtils.java b/modules/commons/src/main/java/org/apache/airavata/common/utils/AiravataZKUtils.java
index 5faf985..75f91fd 100644
--- a/modules/commons/src/main/java/org/apache/airavata/common/utils/AiravataZKUtils.java
+++ b/modules/commons/src/main/java/org/apache/airavata/common/utils/AiravataZKUtils.java
@@ -119,13 +119,14 @@ public class AiravataZKUtils implements Watcher {
     public static void startEmbeddedZK(ServerCnxnFactory cnxnFactory) {
         if (ServerSettings.isEmbeddedZK()) {
             ServerConfig serverConfig = new ServerConfig();
-            URL resource = AiravataZKUtils.class.getClassLoader().getResource("zoo.cfg");
-            if (resource == null) {
-                logger.error("There is no zoo.cfg file in the classpath... Failed to start Zookeeper Server");
-                System.exit(1);
-            }
+            URL resource = ApplicationSettings.loadFile("zoo.cfg");
             try {
+                if (resource == null) {
+                    logger.error("There is no zoo.cfg file in the classpath... Failed to start Zookeeper Server");
+                    System.exit(1);
+                }
                 serverConfig.parse(resource.getPath());
+
             } catch (QuorumPeerConfig.ConfigException e) {
                 logger.error("Error while starting embedded Zookeeper", e);
                 System.exit(2);
@@ -193,7 +194,5 @@ public class AiravataZKUtils implements Watcher {
         } else {
             return bytesToLong(curatorClient.getData().storingStatIn(exists).forPath(cancelDeliveryTagPath));
         }
-
-
     }
 }

http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/commons/src/main/java/org/apache/airavata/common/utils/ApplicationSettings.java
----------------------------------------------------------------------
diff --git a/modules/commons/src/main/java/org/apache/airavata/common/utils/ApplicationSettings.java b/modules/commons/src/main/java/org/apache/airavata/common/utils/ApplicationSettings.java
index 9ce0786..dc7944f 100644
--- a/modules/commons/src/main/java/org/apache/airavata/common/utils/ApplicationSettings.java
+++ b/modules/commons/src/main/java/org/apache/airavata/common/utils/ApplicationSettings.java
@@ -25,9 +25,11 @@ import org.apache.airavata.common.exception.ApplicationSettingsException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.MalformedURLException;
 import java.net.URL;
 import java.util.*;
 import java.util.regex.Matcher;
@@ -35,10 +37,12 @@ import java.util.regex.Pattern;
 
 public class ApplicationSettings {
     public static final String SERVER_PROPERTIES="airavata-server.properties";
-    
-	public static String ADDITIONAL_SETTINGS_FILES = "external.settings";
+    public static final String AIRAVATA_CONFIG_DIR = "airavata.config.dir";
+
+    public static String ADDITIONAL_SETTINGS_FILES = "external.settings";
 
 	protected Properties properties = new Properties();
+
     private Exception propertyLoadException;
 
 
@@ -63,7 +67,6 @@ public class ApplicationSettings {
 	private void loadProperties() {
 		URL url = getPropertyFileURL();
         try {
-        	
             properties.load(url.openStream());
             logger.info("Settings loaded from "+url.toString());
             URL[] externalSettingsFileURLs = getExternalSettingsFileURLs();
@@ -77,7 +80,7 @@ public class ApplicationSettings {
 	}
 
 	protected URL getPropertyFileURL() {
-		return ApplicationSettings.class.getClassLoader().getResource(SERVER_PROPERTIES);
+		return ApplicationSettings.loadFile(SERVER_PROPERTIES);
 	}
 	
 	protected URL[] getExternalSettingsFileURLs(){
@@ -86,7 +89,7 @@ public class ApplicationSettings {
 			String externalSettingsFileNames = getSettingImpl(ADDITIONAL_SETTINGS_FILES);
 			String[] externalSettingFiles = externalSettingsFileNames.split(",");
 			for (String externalSettingFile : externalSettingFiles) {
-				URL externalSettingFileURL = ApplicationSettings.class.getClassLoader().getResource(externalSettingFile);
+				URL externalSettingFileURL = ApplicationSettings.loadFile(externalSettingFile);
 				if (externalSettingFileURL==null){
 					logger.warn("Could not file external settings file "+externalSettingFile);
 				}else{
@@ -313,11 +316,11 @@ public class ApplicationSettings {
      * Static methods which will be used by the users
      */
     
-    public static String getSetting(String key) throws ApplicationSettingsException{
+    public static String getSetting(String key) throws ApplicationSettingsException {
     	return getInstance().getSettingImpl(key);
     }
-    
-    public static String getSetting(String key, String defaultValue){
+
+    public static String getSetting(String key, String defaultValue) {
     	return getInstance().getSettingImpl(key,defaultValue);
 
     }
@@ -426,4 +429,19 @@ public class ApplicationSettings {
     public static ShutdownStrategy getShutdownStrategy() throws Exception{
     	return getInstance().getShutdownStrategyImpl();
     }
+
+    public static URL loadFile(String fileName) {
+        final URL resource = ApplicationSettings.class.getClassLoader().getResource(fileName);
+        if(resource == null) {
+            if(System.getProperty(AIRAVATA_CONFIG_DIR) != null) {
+                final String airavataConfigDir = System.getProperty(AIRAVATA_CONFIG_DIR);
+                try {
+                     return new File(airavataConfigDir + File.separator + fileName).toURI().toURL();
+                } catch (MalformedURLException e) {
+                    logger.error("Error parsing the file from airavata.config.dir", airavataConfigDir);
+                }
+            }
+        }
+        return resource;
+    }
 }

http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/commons/src/main/java/org/apache/airavata/common/utils/AwsMetadata.java
----------------------------------------------------------------------
diff --git a/modules/commons/src/main/java/org/apache/airavata/common/utils/AwsMetadata.java b/modules/commons/src/main/java/org/apache/airavata/common/utils/AwsMetadata.java
new file mode 100644
index 0000000..f9b4a65
--- /dev/null
+++ b/modules/commons/src/main/java/org/apache/airavata/common/utils/AwsMetadata.java
@@ -0,0 +1,137 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.airavata.common.utils;
+
+
+import com.google.common.base.Preconditions;
+import com.google.common.net.InetAddresses;
+import com.google.gson.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.net.InetAddress;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+public class AwsMetadata {
+    private static final Logger log = LoggerFactory.getLogger(AwsMetadata.class);
+    private static final String METADATA_URI_BASE = "http://169.254.169.254/";
+    private static final String REGION_SUFFIX = "/latest/dynamic/instance-identity/document";
+    private static final String ZONE_SUFFIX = "/latest/meta-data/placement/availability-zone";
+    private static final String PUBLIC_IPV4_SUFFIX = "/latest/meta-data/public-ipv4";
+    private static final String PRIVATE_IPV4_SUFFIX = "latest/meta-data/local-ipv4";
+    private static final String HOSTNAME_SUFFIX = "/latest/meta-data/hostname";
+    private static final String ID_SUFFIX = "/latest/meta-data/instance-id";
+
+    private final URI baseUri;
+
+    private String id;
+    private String region;
+    private String hostname;
+    private String zone;
+    private InetAddress publicIp;
+    private InetAddress privateIp;
+
+    public AwsMetadata() {
+        try {
+            baseUri = new URI(METADATA_URI_BASE);
+        } catch (URISyntaxException e) {
+            Preconditions.checkState(false, "Unexpected URI Syntax Exception: {}", e);
+            throw new RuntimeException(e);
+        }
+    }
+
+    public String getRegion() {
+        if (region == null) {
+            try {
+                String dynamicData = getMetadataAt(REGION_SUFFIX);
+                if (dynamicData != null) {
+                    final JsonObject asJsonObject = new JsonParser().parse(dynamicData).getAsJsonObject();
+                    region = asJsonObject.get("region").getAsString();
+                }
+            } catch (ClassCastException e) {
+                log.error("Unable to get region, expecting a JSON Object", e);
+            }
+        }
+        return region;
+    }
+
+    public String getZoneName() {
+        if (zone == null) {
+            zone = getMetadataAt(ZONE_SUFFIX);
+        }
+        return zone;
+    }
+
+    public String getId() {
+        if (id == null) {
+            id = getMetadataAt(ID_SUFFIX);
+        }
+
+        return id;
+    }
+
+    public String getHostname() {
+        if (hostname == null) {
+            hostname = getMetadataAt(HOSTNAME_SUFFIX);
+        }
+        return hostname;
+    }
+
+    public InetAddress getPublicIpAddress() {
+        if (publicIp == null) {
+            String ip = getMetadataAt(PUBLIC_IPV4_SUFFIX);
+            if (ip != null) {
+                publicIp = InetAddresses.forString(ip);
+            }
+        }
+        return publicIp;
+    }
+
+    public InetAddress getInternalIpAddress() {
+        if (privateIp == null) {
+            String ip = getMetadataAt(PRIVATE_IPV4_SUFFIX);
+            if (ip != null) {
+                privateIp = InetAddresses.forString(ip);
+            }
+        }
+        return privateIp;
+    }
+
+    private String getMetadataAt(String suffix) {
+        try {
+            URI resolved = baseUri.resolve(suffix);
+            StringBuilder builder = new StringBuilder();
+            String line = null;
+            try (BufferedReader reader = new BufferedReader(new InputStreamReader(resolved.toURL().openStream()))) {
+                while ((line = reader.readLine()) != null) {
+                    builder.append(line);
+                }
+                return builder.toString();
+            }
+        } catch (Exception e) {
+            // ignore for now to make sure local servers don't go verbose
+        }
+        return null;
+    }
+}

http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/commons/src/main/java/org/apache/airavata/common/utils/BuildConstant.java
----------------------------------------------------------------------
diff --git a/modules/commons/src/main/java/org/apache/airavata/common/utils/BuildConstant.java b/modules/commons/src/main/java/org/apache/airavata/common/utils/BuildConstant.java
new file mode 100644
index 0000000..5598974
--- /dev/null
+++ b/modules/commons/src/main/java/org/apache/airavata/common/utils/BuildConstant.java
@@ -0,0 +1,28 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.airavata.common.utils;
+
+public class BuildConstant {
+    public static final String GIT_DESCRIBE_VERSION = "airavata-0.16-128-g415e9b7";
+    public static String getGitDescribeVersion() {
+        return GIT_DESCRIBE_VERSION;
+    }
+}

http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/commons/src/main/java/org/apache/airavata/common/utils/JSONUtil.java
----------------------------------------------------------------------
diff --git a/modules/commons/src/main/java/org/apache/airavata/common/utils/JSONUtil.java b/modules/commons/src/main/java/org/apache/airavata/common/utils/JSONUtil.java
index 222e5a2..59d065d 100644
--- a/modules/commons/src/main/java/org/apache/airavata/common/utils/JSONUtil.java
+++ b/modules/commons/src/main/java/org/apache/airavata/common/utils/JSONUtil.java
@@ -28,7 +28,6 @@ import com.google.gson.JsonElement;
 import com.google.gson.JsonObject;
 import com.google.gson.JsonParser;
 import com.google.gson.JsonPrimitive;
-import com.google.gson.stream.JsonReader;
 
 import java.io.File;
 import java.io.FileReader;

http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/commons/src/main/java/org/apache/airavata/common/utils/ServerSettings.java
----------------------------------------------------------------------
diff --git a/modules/commons/src/main/java/org/apache/airavata/common/utils/ServerSettings.java b/modules/commons/src/main/java/org/apache/airavata/common/utils/ServerSettings.java
index 985daad..a14cb9d 100644
--- a/modules/commons/src/main/java/org/apache/airavata/common/utils/ServerSettings.java
+++ b/modules/commons/src/main/java/org/apache/airavata/common/utils/ServerSettings.java
@@ -27,6 +27,8 @@ import org.slf4j.LoggerFactory;
 
 import java.net.InetAddress;
 import java.net.UnknownHostException;
+import java.util.HashMap;
+import java.util.Map;
 
 public class ServerSettings extends ApplicationSettings {
 
@@ -107,6 +109,15 @@ public class ServerSettings extends ApplicationSettings {
     private static final String EMAIL_BASED_MONITOR_STORE_PROTOCOL = "email.based.monitor.store.protocol";
     private static final String ENABLE_EMAIL_BASED_MONITORING = "enable.email.based.monitoring";
 
+    private static final String IS_RUNNING_ON_AWS = "isRunningOnAws";
+    private static final String ENABLE_KAFKA_LOGGING = "enable.kafka.logging";
+    private static final String KAFKA_BROKER_LIST = "kafka.broker.list";
+    private static final String KAFKA_TOPIC_PREFIX = "kafka.topic.prefix";
+    private static final String SERVER_ROLES = "server.roles";
+
+    // todo until AIRAVATA-2066 is finished, keep server side list configurations here.
+    private static Map<String, String[]> listConfigurations = new HashMap<>();
+
     private static boolean stopAllThreads = false;
     private static boolean emailBaseNotificationEnable;
     private static String outputLocation;
@@ -400,4 +411,29 @@ public class ServerSettings extends ApplicationSettings {
     public static String getLocalDataLocation() {
         return System.getProperty("java.io.tmpdir");
     }
+
+    public static boolean isRunningOnAws() {
+        return Boolean.valueOf(getSetting(IS_RUNNING_ON_AWS, "false"));
+    }
+
+    public static String getKafkaBrokerList() {
+        return getSetting(KAFKA_BROKER_LIST, null);
+    }
+
+    public static String getKafkaTopicPrefix() {
+        return getSetting(KAFKA_TOPIC_PREFIX, "all");
+    }
+
+    public static boolean isEnabledKafkaLogging() {
+        return Boolean.valueOf(getSetting(ENABLE_KAFKA_LOGGING, "false"));
+    }
+
+    public static void setServerRoles(String[] roles) {
+        listConfigurations.put(SERVER_ROLES, roles);
+    }
+
+    public static String[] getServerRoles() {
+        return listConfigurations.get(SERVER_ROLES);
+    }
+
 }

http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/configuration/server/src/main/resources/airavata-server.properties
----------------------------------------------------------------------
diff --git a/modules/configuration/server/src/main/resources/airavata-server.properties b/modules/configuration/server/src/main/resources/airavata-server.properties
index 0dad713..d079a6f 100644
--- a/modules/configuration/server/src/main/resources/airavata-server.properties
+++ b/modules/configuration/server/src/main/resources/airavata-server.properties
@@ -283,4 +283,10 @@ authorization.policy=airavata-default-xacml-policy
 #### authorization cache related configuration ####
 authz.cache.enabled=true
 authz.cache.manager.class=org.apache.airavata.api.server.security.authzcache.DefaultAuthzCacheManager
-in.memory.cache.size=1000
\ No newline at end of file
+in.memory.cache.size=1000
+
+# Kafka Logging related configuration
+isRunningOnAws= false
+kafka.broker.list= 54.84.144.132:9092
+kafka.topic.prefix= local
+enable.kafka.logging= true

http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/provider/impl/BESProvider.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/provider/impl/BESProvider.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/provider/impl/BESProvider.java
index b287b8e..efd2c5f 100644
--- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/provider/impl/BESProvider.java
+++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/provider/impl/BESProvider.java
@@ -144,10 +144,9 @@ public class BESProvider extends AbstractProvider implements GFacProvider,
             JobDetails jobDetails = new JobDetails();
             FactoryClient factory = new FactoryClient(eprt, secProperties);
 
-            log.info(String.format("Activity Submitting to %s ... \n",
-                    factoryUrl));
+            log.info("Activity Submitting to {} ... \n", factoryUrl);
             CreateActivityResponseDocument response = factory.createActivity(cad);
-            log.info(String.format("Activity Submitted to %s \n", factoryUrl));
+            log.info("Activity Submitted to {} \n", factoryUrl);
 
             EndpointReferenceType activityEpr = response.getCreateActivityResponse().getActivityIdentifier();
 
@@ -353,7 +352,7 @@ public class BESProvider extends AbstractProvider implements GFacProvider,
 	}
 
 	protected String formatStatusMessage(String activityUrl, String status) {
-		return String.format("Activity %s is %s.\n", activityUrl, status);
+		return String.format("Activity {} is {}.\n", activityUrl, status);
 	}
 
 	protected String subStatusAsString(ActivityStatusType statusType) {

http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/gfac/gfac-bes/src/test/java/org/apache/airavata/core/gfac/services/impl/JSDLGeneratorTestWithMyProxyAuth.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/test/java/org/apache/airavata/core/gfac/services/impl/JSDLGeneratorTestWithMyProxyAuth.java b/modules/gfac/gfac-bes/src/test/java/org/apache/airavata/core/gfac/services/impl/JSDLGeneratorTestWithMyProxyAuth.java
index cf9b82a..90f67a1 100644
--- a/modules/gfac/gfac-bes/src/test/java/org/apache/airavata/core/gfac/services/impl/JSDLGeneratorTestWithMyProxyAuth.java
+++ b/modules/gfac/gfac-bes/src/test/java/org/apache/airavata/core/gfac/services/impl/JSDLGeneratorTestWithMyProxyAuth.java
@@ -106,7 +106,7 @@ import static org.junit.Assert.assertTrue;
 //	}
 //
 //	protected GFacConfiguration getGFACConfig() throws Exception{
-//        URL resource = BESProviderTestWithMyProxyAuth.class.getClassLoader().getResource(org.apache.airavata.common.utils.Constants.GFAC_CONFIG_XML);
+//        URL resource = ApplicationSettings.loadFile(org.apache.airavata.common.utils.Constants.GFAC_CONFIG_XML);
 //        System.out.println(resource.getFile());
 //        GFacConfiguration gFacConfiguration = GFacConfiguration.create(new File(resource.getPath()),null,null);
 //		return gFacConfiguration;

http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java
index 0ed836f..545e122 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java
@@ -801,7 +801,7 @@ public class GFacUtils {
     public static File createJobFile(TaskContext taskContext, JobDescriptor jobDescriptor, JobManagerConfiguration jobManagerConfiguration) throws GFacException {
         try {
             TransformerFactory factory = TransformerFactory.newInstance();
-            URL resource = GFacUtils.class.getClassLoader().getResource(jobManagerConfiguration.getJobDescriptionTemplateName());
+            URL resource = ApplicationSettings.loadFile(jobManagerConfiguration.getJobDescriptionTemplateName());
 
             if (resource == null) {
                 String error = "System configuration file '" + jobManagerConfiguration.getJobDescriptionTemplateName()

http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/BESJobSubmissionTask.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/BESJobSubmissionTask.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/BESJobSubmissionTask.java
index 6529395..1955225 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/BESJobSubmissionTask.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/BESJobSubmissionTask.java
@@ -151,10 +151,9 @@ public class BESJobSubmissionTask implements JobSubmissionTask {
             jobDetails.setProcessId(taskContext.getProcessId());
             FactoryClient factory = new FactoryClient(eprt, secProperties);
 
-            log.info(String.format("Activity Submitting to %s ... \n",
-                    factoryUrl));
+            log.info("Activity Submitting to {} ... \n", factoryUrl);
             CreateActivityResponseDocument response = factory.createActivity(cad);
-            log.info(String.format("Activity Submitted to %s \n", factoryUrl));
+            log.info("Activity Submitted to {} \n", factoryUrl);
 
             EndpointReferenceType activityEpr = response.getCreateActivityResponse().getActivityIdentifier();
 
@@ -356,7 +355,7 @@ public class BESJobSubmissionTask implements JobSubmissionTask {
     }
 
     protected String formatStatusMessage(String activityUrl, String status) {
-        return String.format("Activity %s is %s.\n", activityUrl, status);
+        return String.format("Activity {} is {}.\n", activityUrl, status);
     }
 
     protected void waitUntilDone(EndpointReferenceType factoryEpr, EndpointReferenceType activityEpr, ProcessContext processContext, DefaultClientConfiguration secProperties) throws Exception {

http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/gfac/gfac-impl/src/test/java/org/apache/airavata/core/gfac/services/impl/BigRed2TestWithSSHAuth.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/test/java/org/apache/airavata/core/gfac/services/impl/BigRed2TestWithSSHAuth.java b/modules/gfac/gfac-impl/src/test/java/org/apache/airavata/core/gfac/services/impl/BigRed2TestWithSSHAuth.java
index 4aecd57..fbad51c 100644
--- a/modules/gfac/gfac-impl/src/test/java/org/apache/airavata/core/gfac/services/impl/BigRed2TestWithSSHAuth.java
+++ b/modules/gfac/gfac-impl/src/test/java/org/apache/airavata/core/gfac/services/impl/BigRed2TestWithSSHAuth.java
@@ -91,7 +91,7 @@
 //            System.out.println("Use -Dssh.username=xxx -Dssh.password=yyy -Dssh.keypass=zzz " +
 //                    "-Dprivate.ssh.key -Dpublic.ssh.key -Dssh.working.directory ");
 //        }
-//        URL resource = BigRed2TestWithSSHAuth.class.getClassLoader().getResource(org.apache.airavata.common.utils.Constants.GFAC_CONFIG_XML);
+//        URL resource = ApplicationSettings.loadFile(org.apache.airavata.common.utils.Constants.GFAC_CONFIG_XML);
 //        assert resource != null;
 //        System.out.println(resource.getFile());
 //        GFacConfiguration gFacConfiguration = GFacConfiguration.create(new File(resource.getPath()), null);

http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/gfac/gfac-impl/src/test/java/org/apache/airavata/core/gfac/services/impl/GSISSHProviderTestWithMyProxyAuth.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/test/java/org/apache/airavata/core/gfac/services/impl/GSISSHProviderTestWithMyProxyAuth.java b/modules/gfac/gfac-impl/src/test/java/org/apache/airavata/core/gfac/services/impl/GSISSHProviderTestWithMyProxyAuth.java
index 4ca5684..9d352a4 100644
--- a/modules/gfac/gfac-impl/src/test/java/org/apache/airavata/core/gfac/services/impl/GSISSHProviderTestWithMyProxyAuth.java
+++ b/modules/gfac/gfac-impl/src/test/java/org/apache/airavata/core/gfac/services/impl/GSISSHProviderTestWithMyProxyAuth.java
@@ -90,7 +90,7 @@
 //                    "E.g :- mvn clean install -Dmyproxy.username=xxx -Dmyproxy.password=xxx -Dgsi.working.directory=/path<<<<<<<");
 //            throw new Exception("Need my proxy user name password to run tests.");
 //        }
-//        URL resource = GSISSHProviderTestWithMyProxyAuth.class.getClassLoader().getResource(org.apache.airavata.common.utils.Constants.GFAC_CONFIG_XML);
+//        URL resource = ApplicationSettings.loadFile(org.apache.airavata.common.utils.Constants.GFAC_CONFIG_XML);
 //        assert resource != null;
 //        System.out.println(resource.getFile());
 //        GFacConfiguration gFacConfiguration = GFacConfiguration.create(new File(resource.getPath()), null);

http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/gfac/gfac-impl/src/test/java/org/apache/airavata/core/gfac/services/impl/SSHProviderTestWithSSHAuth.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/test/java/org/apache/airavata/core/gfac/services/impl/SSHProviderTestWithSSHAuth.java b/modules/gfac/gfac-impl/src/test/java/org/apache/airavata/core/gfac/services/impl/SSHProviderTestWithSSHAuth.java
index 4aa0df1..bdbadda 100644
--- a/modules/gfac/gfac-impl/src/test/java/org/apache/airavata/core/gfac/services/impl/SSHProviderTestWithSSHAuth.java
+++ b/modules/gfac/gfac-impl/src/test/java/org/apache/airavata/core/gfac/services/impl/SSHProviderTestWithSSHAuth.java
@@ -54,7 +54,7 @@
 //    @Before
 //    public void setUp() throws Exception {
 //
-//    	URL resource = SSHProviderTestWithSSHAuth.class.getClassLoader().getResource(org.apache.airavata.common.utils.Constants.GFAC_CONFIG_XML);
+//    	URL resource = ApplicationSettings.loadFile(org.apache.airavata.common.utils.Constants.GFAC_CONFIG_XML);
 //        GFacConfiguration gFacConfiguration = GFacConfiguration.create(new File(resource.getPath()),null);
 ////        gFacConfiguration.s
 //        //have to set InFlwo Handlers and outFlowHandlers

http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/registry/registry-core/pom.xml
----------------------------------------------------------------------
diff --git a/modules/registry/registry-core/pom.xml b/modules/registry/registry-core/pom.xml
index 85986ae..60da666 100644
--- a/modules/registry/registry-core/pom.xml
+++ b/modules/registry/registry-core/pom.xml
@@ -107,33 +107,6 @@
 
     <build>
         <plugins>
-            <plugin>
-                <groupId>org.apache.openjpa</groupId>
-                <artifactId>openjpa-maven-plugin</artifactId>
-                <version>2.2.0</version>
-                <configuration>
-                    <includes>**/entities/*.class</includes>
-                    <excludes>**/entities/XML*.class</excludes>
-                    <addDefaultConstructor>true</addDefaultConstructor>
-                    <enforcePropertyRestrictions>true</enforcePropertyRestrictions>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>enhancer</id>
-                        <phase>process-classes</phase>
-                        <goals>
-                            <goal>enhance</goal>
-                        </goals>
-                    </execution>
-                </executions>
-                <dependencies>
-                    <dependency>
-                        <groupId>org.apache.openjpa</groupId>
-                        <artifactId>openjpa</artifactId>
-                        <version>2.2.0</version>
-                    </dependency>
-                </dependencies>
-            </plugin>
             <!--<plugin>-->
                 <!--<groupId>org.apache.maven.plugins</groupId>-->
                 <!--<artifactId>maven-antrun-plugin</artifactId>-->

http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/server/src/main/java/org/apache/airavata/server/ServerMain.java
----------------------------------------------------------------------
diff --git a/modules/server/src/main/java/org/apache/airavata/server/ServerMain.java b/modules/server/src/main/java/org/apache/airavata/server/ServerMain.java
index 99387de..bcde1e9 100644
--- a/modules/server/src/main/java/org/apache/airavata/server/ServerMain.java
+++ b/modules/server/src/main/java/org/apache/airavata/server/ServerMain.java
@@ -20,13 +20,16 @@
  */
 package org.apache.airavata.server;
 
+import ch.qos.logback.classic.LoggerContext;
 import org.apache.airavata.common.exception.ApplicationSettingsException;
+import org.apache.airavata.common.logging.kafka.KafkaAppender;
 import org.apache.airavata.common.utils.*;
 import org.apache.airavata.common.utils.ApplicationSettings.ShutdownStrategy;
 import org.apache.airavata.common.utils.IServer.ServerStatus;
 import org.apache.airavata.common.utils.StringUtil.CommandLineParameters;
 import org.apache.commons.cli.ParseException;
 import org.apache.zookeeper.server.ServerCnxnFactory;
+import org.slf4j.ILoggerFactory;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -155,14 +158,29 @@ public class ServerMain {
 //	}
 	
 	public static void main(String args[]) throws ParseException, IOException {
-        CommandLineParameters commandLineParameters = StringUtil.getCommandLineParser(args);
+		ServerSettings.mergeSettingsCommandLineArgs(args);
+		ServerSettings.setServerRoles(ApplicationSettings.getSetting(SERVERS_KEY, "all").split(","));
+
+		if (ServerSettings.isEnabledKafkaLogging()) {
+			final ILoggerFactory iLoggerFactory = LoggerFactory.getILoggerFactory();
+			if (iLoggerFactory instanceof LoggerContext) {
+				final KafkaAppender kafkaAppender = new KafkaAppender(ServerSettings.getKafkaBrokerList(),
+						ServerSettings.getKafkaTopicPrefix());
+				kafkaAppender.setContext((LoggerContext) iLoggerFactory);
+				kafkaAppender.setName("kafka-appender");
+				kafkaAppender.clearAllFilters();
+				kafkaAppender.start();
+				((LoggerContext) iLoggerFactory).getLogger(Logger.ROOT_LOGGER_NAME).addAppender(kafkaAppender);
+			}
+		}
+
+		CommandLineParameters commandLineParameters = StringUtil.getCommandLineParser(args);
         if (commandLineParameters.getArguments().contains(STOP_COMMAND_STR)){
             performServerStopRequest(commandLineParameters);
         }else{
             AiravataZKUtils.startEmbeddedZK(cnxnFactory);
             performServerStart(args);
 		}
-
     }
 
 
@@ -173,7 +191,6 @@ public class ServerMain {
 		for (String string : args) {
 			logger.info("Server Arguments: " + string);
 		}
-		ServerSettings.mergeSettingsCommandLineArgs(args);
 		String serverNames;
 		try {
 			serverNames = ApplicationSettings.getSetting(SERVERS_KEY);

http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/test-suite/multi-tenanted-airavata/src/main/java/org/apache/airavata/testsuite/multitenantedairavata/utils/PropertyReader.java
----------------------------------------------------------------------
diff --git a/modules/test-suite/multi-tenanted-airavata/src/main/java/org/apache/airavata/testsuite/multitenantedairavata/utils/PropertyReader.java b/modules/test-suite/multi-tenanted-airavata/src/main/java/org/apache/airavata/testsuite/multitenantedairavata/utils/PropertyReader.java
index f7b8e1f..9df1745 100644
--- a/modules/test-suite/multi-tenanted-airavata/src/main/java/org/apache/airavata/testsuite/multitenantedairavata/utils/PropertyReader.java
+++ b/modules/test-suite/multi-tenanted-airavata/src/main/java/org/apache/airavata/testsuite/multitenantedairavata/utils/PropertyReader.java
@@ -21,6 +21,7 @@
 
 package org.apache.airavata.testsuite.multitenantedairavata.utils;
 
+import org.apache.airavata.common.utils.ApplicationSettings;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -41,7 +42,7 @@ public class PropertyReader {
     }
 
     protected void loadProperties() throws IOException {
-        URL airavataURL = PropertyFileType.class.getClassLoader().getResource(TestFrameworkConstants.AIRAVATA_CLIENT_PROPERTIES);
+        URL airavataURL = ApplicationSettings.loadFile(TestFrameworkConstants.AIRAVATA_CLIENT_PROPERTIES);
         if (airavataURL != null){
             airavataClientProperties.load(airavataURL.openStream());
         }

http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/user-profile/user-profile-core/src/main/java/org/apache/airavata/userprofile/core/AbstractThriftDeserializer.java
----------------------------------------------------------------------
diff --git a/modules/user-profile/user-profile-core/src/main/java/org/apache/airavata/userprofile/core/AbstractThriftDeserializer.java b/modules/user-profile/user-profile-core/src/main/java/org/apache/airavata/userprofile/core/AbstractThriftDeserializer.java
index 83af52b..14a4b9c 100644
--- a/modules/user-profile/user-profile-core/src/main/java/org/apache/airavata/userprofile/core/AbstractThriftDeserializer.java
+++ b/modules/user-profile/user-profile-core/src/main/java/org/apache/airavata/userprofile/core/AbstractThriftDeserializer.java
@@ -91,8 +91,7 @@ public abstract class AbstractThriftDeserializer<E extends TFieldIdEnum, T exten
             // Validate that the instance contains all required fields.
             validate(instance);
         } catch (final TException e) {
-            log.error(String.format("Unable to deserialize JSON '%s' to type '%s'.",
-                    jp.getValueAsString(), instance.getClass().getName(), e));
+            log.error("Unable to deserialize JSON {} to type {}.", jp.getValueAsString(), instance.getClass().getName(), e);
             ctxt.mappingException(e.getMessage());
         }
 

http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 6594026..00bc168 100644
--- a/pom.xml
+++ b/pom.xml
@@ -101,6 +101,8 @@
         <snakeyaml.version>1.15</snakeyaml.version>
 		<maven.javadoc.failOnError>false</maven.javadoc.failOnError>
 		<maven.replacer.plugin.version>1.5.3</maven.replacer.plugin.version>
+		<kafka-clients.version>0.8.2.2</kafka-clients.version>
+		<logback.version>1.1.6</logback.version>
 	</properties>
 
 	<developers>
@@ -434,6 +436,16 @@
 				<artifactId>curator-framework</artifactId>
 				<version>${curator.version}</version>
 			</dependency>
+			<dependency>
+				<groupId>org.apache.kafka</groupId>
+				<artifactId>kafka-clients</artifactId>
+				<version>${kafka-clients.version}</version>
+			</dependency>
+			<dependency>
+				<groupId>ch.qos.logback</groupId>
+				<artifactId>logback-classic</artifactId>
+				<version>${logback.version}</version>
+			</dependency>
 		</dependencies>
 	</dependencyManagement>
 
@@ -542,6 +554,26 @@
 							</excludes>
 						</configuration>
 					</plugin>
+					<plugin>
+						<groupId>com.lukegb.mojo</groupId>
+						<artifactId>gitdescribe-maven-plugin</artifactId>
+						<version>3.0</version>
+						<executions>
+							<execution>
+								<goals>
+									<goal>gitdescribe</goal>
+								</goals>
+								<id>git-describe</id>
+								<phase>initialize</phase>
+								<configuration>
+									<extraArguments>
+										<param>--tags</param>
+									</extraArguments>
+									<descriptionProperty>git-describe</descriptionProperty>
+								</configuration>
+							</execution>
+						</executions>
+					</plugin>
 				</plugins>
 			</build>
 			<activation>


Mime
View raw message