falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From pall...@apache.org
Subject falcon git commit: FALCON-2147 Excess logging due to SlaMonitoring service and Backlog e…
Date Thu, 20 Oct 2016 08:13:43 GMT
Repository: falcon
Updated Branches:
  refs/heads/master ef57d29ad -> 4746e039a


FALCON-2147 Excess logging due to SlaMonitoring service and Backlog e…

…mitter service

Author: Pallavi Rao <pallavi.rao@inmobi.com>

Reviewers: @sandeepSamudrala

Closes #286 from pallavi-rao/2147 and squashes the following commits:

b3cc9d0 [Pallavi Rao] Comments addressed
7069e2f [Pallavi Rao] Removing excessive user logging in in BacklogMetricEmitter
cacdc1d [Pallavi Rao] Addressed comment
4ddd4dd [Pallavi Rao] FALCON-2147 Excess logging due to SlaMonitoring service and Backlog
emitter service


Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/4746e039
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/4746e039
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/4746e039

Branch: refs/heads/master
Commit: 4746e039a1522af375a4e877848a6274b2de1c66
Parents: ef57d29
Author: Pallavi Rao <pallavi.rao@inmobi.com>
Authored: Thu Oct 20 13:43:24 2016 +0530
Committer: Pallavi Rao <pallavi.rao@inmobi.com>
Committed: Thu Oct 20 13:43:24 2016 +0530

----------------------------------------------------------------------
 .../falcon/workflow/WorkflowEngineFactory.java   |  4 ++--
 .../workflow/engine/OozieWorkflowEngine.java     |  2 +-
 .../service/BacklogMetricEmitterService.java     | 19 +++++++++++++------
 .../falcon/service/EntitySLAAlertService.java    |  5 ++---
 .../service/EntitySLAMonitoringService.java      | 19 +++++++++----------
 5 files changed, 27 insertions(+), 22 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/falcon/blob/4746e039/common/src/main/java/org/apache/falcon/workflow/WorkflowEngineFactory.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/workflow/WorkflowEngineFactory.java b/common/src/main/java/org/apache/falcon/workflow/WorkflowEngineFactory.java
index c713712..e5d0da2 100644
--- a/common/src/main/java/org/apache/falcon/workflow/WorkflowEngineFactory.java
+++ b/common/src/main/java/org/apache/falcon/workflow/WorkflowEngineFactory.java
@@ -56,7 +56,7 @@ public final class WorkflowEngineFactory {
             LOG.debug("Returning native workflow engine for entity {}", entity.getName());
             return nativeWorkflowEngine;
         }
-        LOG.debug("Returning configured workflow engine for entity {}.", entity);
+        LOG.debug("Returning configured workflow engine for entity {}", (entity == null)?
null : entity.getName());
         return getWorkflowEngine();
     }
 
@@ -70,7 +70,7 @@ public final class WorkflowEngineFactory {
         throws FalconException {
         // If entity is null or not schedulable and the engine property is not specified,
return the configured WE.
         if (entity == null || !entity.getEntityType().isSchedulable()) {
-            LOG.debug("Returning configured workflow engine for entity {}.", entity);
+            LOG.debug("Returning configured workflow engine for entity {}", (entity == null)?
null : entity.getName());
             return getWorkflowEngine();
         }
 

http://git-wip-us.apache.org/repos/asf/falcon/blob/4746e039/oozie/src/main/java/org/apache/falcon/workflow/engine/OozieWorkflowEngine.java
----------------------------------------------------------------------
diff --git a/oozie/src/main/java/org/apache/falcon/workflow/engine/OozieWorkflowEngine.java
b/oozie/src/main/java/org/apache/falcon/workflow/engine/OozieWorkflowEngine.java
index 394600c..6964200 100644
--- a/oozie/src/main/java/org/apache/falcon/workflow/engine/OozieWorkflowEngine.java
+++ b/oozie/src/main/java/org/apache/falcon/workflow/engine/OozieWorkflowEngine.java
@@ -371,7 +371,7 @@ public class OozieWorkflowEngine extends AbstractWorkflowEngine {
                         //Load bundle as coord info is not returned in getBundleJobsInfo()
                         BundleJob bundle = getBundleInfo(clusterName, job.getId());
                         filteredJobs.add(bundle);
-                        LOG.debug("Found bundle {} with app path {} and status {}",
+                        LOG.trace("Found bundle {} with app path {} and status {}",
                                 job.getId(), job.getAppPath(), job.getStatus());
                     }
                 }

http://git-wip-us.apache.org/repos/asf/falcon/blob/4746e039/prism/src/main/java/org/apache/falcon/service/BacklogMetricEmitterService.java
----------------------------------------------------------------------
diff --git a/prism/src/main/java/org/apache/falcon/service/BacklogMetricEmitterService.java
b/prism/src/main/java/org/apache/falcon/service/BacklogMetricEmitterService.java
index 801ab36..d9ac386 100644
--- a/prism/src/main/java/org/apache/falcon/service/BacklogMetricEmitterService.java
+++ b/prism/src/main/java/org/apache/falcon/service/BacklogMetricEmitterService.java
@@ -17,6 +17,7 @@
  */
 package org.apache.falcon.service;
 
+import org.apache.commons.lang3.StringUtils;
 import org.apache.falcon.FalconException;
 import org.apache.falcon.LifeCycle;
 import org.apache.falcon.entity.EntityUtil;
@@ -204,7 +205,7 @@ public final class BacklogMetricEmitterService implements FalconService,
 
         @Override
         public void run() {
-            LOG.debug("BacklogMetricEmitter running for entities");
+            LOG.debug("Starting periodic check for backlog");
             executor = new ScheduledThreadPoolExecutor(10);
             List<Future> futures = new ArrayList<>();
             try {
@@ -316,11 +317,7 @@ public final class BacklogMetricEmitterService implements FalconService,
                                 Date nominalTime;
                                 try {
                                     nominalTime = DATE_FORMAT.get().parse(nominalTimeStr);
-                                    if (entity.getACL().getOwner() != null && !entity.getACL().getOwner().isEmpty())
{
-                                        CurrentUser.authenticate(entity.getACL().getOwner());
-                                    } else {
-                                        CurrentUser.authenticate(System.getProperty("user.name"));
-                                    }
+                                    authenticateUser(entity);
                                     if (wfEngine.isMissing(entity)) {
                                         LOG.info("Entity of name {} was deleted so removing
instance of "
                                                 + "nominaltime {} ", entity.getName(), nominalTimeStr);
@@ -353,4 +350,14 @@ public final class BacklogMetricEmitterService implements FalconService,
         }
     }
 
+    private static void authenticateUser(Entity entity){
+        if (!CurrentUser.isAuthenticated()) {
+            if (StringUtils.isNotBlank(entity.getACL().getOwner())) {
+                CurrentUser.authenticate(entity.getACL().getOwner());
+            } else {
+                CurrentUser.authenticate(System.getProperty("user.name"));
+            }
+        }
+    }
+
 }

http://git-wip-us.apache.org/repos/asf/falcon/blob/4746e039/prism/src/main/java/org/apache/falcon/service/EntitySLAAlertService.java
----------------------------------------------------------------------
diff --git a/prism/src/main/java/org/apache/falcon/service/EntitySLAAlertService.java b/prism/src/main/java/org/apache/falcon/service/EntitySLAAlertService.java
index 9b1a594..09c6695 100644
--- a/prism/src/main/java/org/apache/falcon/service/EntitySLAAlertService.java
+++ b/prism/src/main/java/org/apache/falcon/service/EntitySLAAlertService.java
@@ -76,6 +76,7 @@ public final class EntitySLAAlertService implements FalconService, EntitySLAList
                     continue;
                 }
                 EntitySLAListener listener = ReflectionUtils.getInstanceByClassName(listenerClassName);
+                LOG.info("Registering listener {}" , listenerClassName);
                 registerListener(listener);
             }
         }
@@ -111,8 +112,7 @@ public final class EntitySLAAlertService implements FalconService, EntitySLAList
         if (pendingInstanceBeanList == null || pendingInstanceBeanList.isEmpty()){
             return;
         }
-
-        LOG.debug("In processSLACandidates :" + pendingInstanceBeanList.size());
+        LOG.trace("In processSLACandidates :" + pendingInstanceBeanList.size());
         try{
             for (PendingInstanceBean pendingInstanceBean : pendingInstanceBeanList) {
 
@@ -161,7 +161,6 @@ public final class EntitySLAAlertService implements FalconService, EntitySLAList
     @Override
     public void highSLAMissed(String entityName, String clusterName, EntityType entityType
, Date nominalTime
                               ) throws FalconException {
-        LOG.debug("Listners called...");
         for (EntitySLAListener listener : listeners) {
             listener.highSLAMissed(entityName, clusterName, entityType, nominalTime);
             store.deleteEntityAlertInstance(entityName, clusterName, nominalTime, entityType.name());

http://git-wip-us.apache.org/repos/asf/falcon/blob/4746e039/prism/src/main/java/org/apache/falcon/service/EntitySLAMonitoringService.java
----------------------------------------------------------------------
diff --git a/prism/src/main/java/org/apache/falcon/service/EntitySLAMonitoringService.java
b/prism/src/main/java/org/apache/falcon/service/EntitySLAMonitoringService.java
index 1e20a2b..816846d 100644
--- a/prism/src/main/java/org/apache/falcon/service/EntitySLAMonitoringService.java
+++ b/prism/src/main/java/org/apache/falcon/service/EntitySLAMonitoringService.java
@@ -303,14 +303,14 @@ public final class EntitySLAMonitoringService implements ConfigurationChangeList
 
         freq = StartupProperties.get().getProperty("entity.sla.lookAheadWindow.millis", "900000");
         lookAheadWindowMillis = Integer.parseInt(freq);
-        LOG.debug("No old state exists at: {}, Initializing a clean state.", filePath.toString());
+        LOG.info("Initializing EntitySLAMonitoringService from ", filePath.toString());
         ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(1);
         executor.scheduleWithFixedDelay(new Monitor(), 0, statusCheckFrequencySeconds, TimeUnit.SECONDS);
     }
 
     public void makeFeedInstanceAvailable(String feedName, String clusterName, Date nominalTime)
         throws FalconException {
-        LOG.info("Removing {} feed's instance {} in cluster {} from pendingSLA", feedName,
+        LOG.debug("Removing {} feed's instance {} in cluster {} from pendingSLA", feedName,
                 clusterName, nominalTime);
         List<Date> instances = (MONITORING_JDBC_STATE_STORE.getNominalInstances(feedName,
clusterName,
                 EntityType.FEED.toString()));
@@ -369,7 +369,6 @@ public final class EntitySLAMonitoringService implements ConfigurationChangeList
         for(MonitoredEntityBean monitoredEntityBean : entityBeanList) {
             String entityName = monitoredEntityBean.getFeedName();
             Entity entity = EntityUtil.getEntity(entityType, entityName);
-            LOG.debug("entityName:"+ entityName+"entity:"+entity);
             Set<String> clusters =  EntityUtil.getClustersDefined(entity);
             List<org.apache.falcon.entity.v0.cluster.Cluster> cluster = new ArrayList();
             for(String string : clusters){
@@ -391,7 +390,7 @@ public final class EntitySLAMonitoringService implements ConfigurationChangeList
                     org.apache.falcon.entity.v0.cluster.Cluster currentCluster =
                             EntityUtil.getEntity(EntityType.CLUSTER, entityCluster.getName());
                     nextInstanceTime = EntityUtil.getNextStartTime(entity, currentCluster,
nextInstanceTime);
-                    LOG.info("nextInstanceTime:"+ nextInstanceTime + "entityName:"+entityName);
+                    LOG.trace("nextInstanceTime:"+ nextInstanceTime + "entityName:"+entityName);
                     Date endDate;
                     if (entityType.equals(EntityType.FEED.toString())){
                         endDate =  FeedHelper.getClusterValidity((Feed) entity, currentCluster.getName()).getEnd();
@@ -400,7 +399,8 @@ public final class EntitySLAMonitoringService implements ConfigurationChangeList
                                 currentCluster.getName()).getEnd();
                     }
                     while (nextInstanceTime.before(to) && nextInstanceTime.before(endDate))
{
-                        LOG.info("Adding instance={} for <entity,cluster>={}", nextInstanceTime,
key);
+                        LOG.trace("Adding pending instance={} for <entity,cluster>={};
entityType={}",
+                                nextInstanceTime, key, entityType);
                         instances.add(nextInstanceTime);
                         nextInstanceTime = new Date(nextInstanceTime.getTime() + ONE_MS);
                         nextInstanceTime = EntityUtil.getNextStartTime(entity, currentCluster,
nextInstanceTime);
@@ -420,7 +420,6 @@ public final class EntitySLAMonitoringService implements ConfigurationChangeList
      * Checks the availability of all the pendingInstances and removes the ones which have
become available.
      */
     private void checkPendingInstanceAvailability(String entityType) throws FalconException
{
-        LOG.debug("Size "+MONITORING_JDBC_STATE_STORE.getAllMonitoredEntity().size());
         if (MONITORING_JDBC_STATE_STORE.getAllPendingInstances() == null){
             LOG.info("Returning as size of pending instance is zero");
             return;
@@ -445,26 +444,26 @@ public final class EntitySLAMonitoringService implements ConfigurationChangeList
         authenticateUser(entity);
         try {
             if (entityType.equals(EntityType.PROCESS.toString())){
-                LOG.debug("Checking instance availability status for entity:{}, cluster:{},
"
+                LOG.trace("Checking instance availability status for entity:{}, cluster:{},
"
                         + "instanceTime:{}", entity.getName(), clusterName, nominalTime,
entityType);
                 AbstractWorkflowEngine wfEngine = WorkflowEngineFactory.getWorkflowEngine();
                 InstancesResult instancesResult = wfEngine.getStatus(entity, nominalTime,
nominalTime, null, null);
                 if (instancesResult.getStatus().equals(APIResult.Status.SUCCEEDED)){
-                    LOG.debug("Entity instance(Process:{}, cluster:{}, instanceTime:{}) is
available.",
+                    LOG.trace("Entity instance(Process:{}, cluster:{}, instanceTime:{}) is
available.",
                             entity.getName(), clusterName, nominalTime);
                     return true;
                 }
                 return false;
             }
             if (entityType.equals(EntityType.FEED.toString())){
-                LOG.debug("Checking instance availability status for feed:{}, cluster:{},
instanceTime:{}",
+                LOG.trace("Checking instance availability status for feed:{}, cluster:{},
instanceTime:{}",
                         entity.getName(), clusterName, nominalTime);
 
                 FeedInstanceStatus.AvailabilityStatus status = FeedHelper.getFeedInstanceStatus((Feed)
entity,
                         clusterName, nominalTime);
                 if (status.equals(FeedInstanceStatus.AvailabilityStatus.AVAILABLE)
                         || status.equals(FeedInstanceStatus.AvailabilityStatus.EMPTY)) {
-                    LOG.debug("Feed instance(feed:{}, cluster:{}, instanceTime:{}) is available.",
entity.getName(),
+                    LOG.trace("Feed instance(feed:{}, cluster:{}, instanceTime:{}) is available.",
entity.getName(),
                             clusterName, nominalTime);
                     return true;
                 }


Mime
View raw message