chukwa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ey...@apache.org
Subject svn commit: r764142 - in /hadoop/chukwa/trunk: ./ src/java/org/apache/hadoop/chukwa/datacollection/agent/ src/java/org/apache/hadoop/chukwa/datacollection/collector/ src/java/org/apache/hadoop/chukwa/datacollection/connector/http/ src/java/org/apache/h...
Date Sat, 11 Apr 2009 01:42:08 GMT
Author: eyang
Date: Sat Apr 11 01:42:07 2009
New Revision: 764142

URL: http://svn.apache.org/viewvc?rev=764142&view=rev
Log:
CHUKWA-98.  Added Daemon watcher to capture signal for pid file management. (Cheng Zhang via
Eric Yang)


Modified:
    hadoop/chukwa/trunk/CHANGES.txt
    hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
    hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/collector/CollectorStub.java
    hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/connector/http/HttpConnector.java
    hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/writer/SeqFileWriter.java
    hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/archive/ChukwaArchiveManager.java
    hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/DailyChukwaRecordRolling.java
    hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/DemuxManager.java
    hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/HourlyChukwaRecordRolling.java
    hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/PostProcessorManager.java
    hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/inputtools/jplugin/JPluginAgent.java
    hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/inputtools/plugin/metrics/Exec.java

Modified: hadoop/chukwa/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/CHANGES.txt?rev=764142&r1=764141&r2=764142&view=diff
==============================================================================
--- hadoop/chukwa/trunk/CHANGES.txt (original)
+++ hadoop/chukwa/trunk/CHANGES.txt Sat Apr 11 01:42:07 2009
@@ -32,6 +32,8 @@
 
   IMPROVEMENTS
 
+    CHUKWA-80.  Extracted rpm spec file from build.xml file.  Fix start up script, and config
script. (Eric Yang)
+
     CHUKWA-26.  * DemuxManager, ArchiveManager and PostProcessorManager are now a single
daemon process each.
                 * Each one working independently from others, as soon as something is available.
                 * Start-data-processor is now using those new daemons instead of pocessSink.sh
@@ -83,6 +85,10 @@
 
   BUG FIXES
 
+    CHUKWA-98.  Added Daemon watcher to capture signal for pid file management. (Cheng Zhang
via Eric Yang)
+
+    CHUKWA-120. Added the missing commons-cli library. (Jerome Boulon via Eric Yang)
+
     CHUKWA-61.  Added report widget for accounting information. (Eric Yang)
 
     CHUKWA-77.  Added Database Schema for Hadoop accounting information. (Eric Yang)

Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java?rev=764142&r1=764141&r2=764142&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
(original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
Sat Apr 11 01:42:07 2009
@@ -25,7 +25,7 @@
 import org.apache.hadoop.chukwa.datacollection.connector.*;
 import org.apache.hadoop.chukwa.datacollection.connector.http.HttpConnector;
 import org.apache.hadoop.chukwa.datacollection.test.ConsoleOutConnector;
-import org.apache.hadoop.chukwa.util.PidFile;
+import org.apache.hadoop.chukwa.util.*;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.log4j.Logger;
@@ -45,7 +45,6 @@
   static final AgentMetrics agentMetrics = new AgentMetrics("ChukwaAgent", "chukwaAgent");;
   static Logger log = Logger.getLogger(ChukwaAgent.class);
   static ChukwaAgent agent = null;
-  private static PidFile pFile = null;
 
   public static ChukwaAgent getAgent() {
     return agent;
@@ -107,8 +106,7 @@
    */
   public static void main(String[] args) throws AdaptorException {
 
-    pFile = new PidFile("Agent");
-    Runtime.getRuntime().addShutdownHook(pFile);
+    DaemonWatcher.createInstance("Agent");
 
     try {
       if (args.length > 0 && args[0].equals("-help")) {
@@ -123,7 +121,7 @@
         System.out
             .println("another agent is running (or port has been usurped). "
                 + "Bailing out now");
-        System.exit(-1);
+        DaemonWatcher.bailout(-1);
       }
 
       int uriArgNumber = 0;

Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/collector/CollectorStub.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/collector/CollectorStub.java?rev=764142&r1=764141&r2=764142&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/collector/CollectorStub.java
(original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/collector/CollectorStub.java
Sat Apr 11 01:42:07 2009
@@ -24,19 +24,17 @@
 import org.mortbay.jetty.servlet.*;
 import org.apache.hadoop.chukwa.datacollection.collector.servlet.ServletCollector;
 import org.apache.hadoop.chukwa.datacollection.writer.*;
-import org.apache.hadoop.chukwa.util.PidFile;
+import org.apache.hadoop.chukwa.util.DaemonWatcher;
 import org.apache.hadoop.chukwa.conf.ChukwaConfiguration;
 
 public class CollectorStub {
 
   static int THREADS = 80;
-  private static PidFile pFile = null;
   public static Server jettyServer = null;
 
   public static void main(String[] args) {
 
-    pFile = new PidFile("Collector");
-    Runtime.getRuntime().addShutdownHook(pFile);
+    DaemonWatcher.createInstance("Collector");
     try {
       if (args.length >= 1 && args[0].equalsIgnoreCase("-help")) {
         System.out.println("usage:  CollectorStub [portno] [pretend]");
@@ -95,7 +93,7 @@
 
     } catch (Exception e) {
       e.printStackTrace();
-      System.exit(0);
+      DaemonWatcher.bailout(-1);
     }
 
   }

Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/connector/http/HttpConnector.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/connector/http/HttpConnector.java?rev=764142&r1=764141&r2=764142&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/connector/http/HttpConnector.java
(original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/connector/http/HttpConnector.java
Sat Apr 11 01:42:07 2009
@@ -47,6 +47,7 @@
 import org.apache.hadoop.chukwa.datacollection.agent.ChukwaAgent;
 import org.apache.hadoop.chukwa.datacollection.connector.Connector;
 import org.apache.hadoop.chukwa.datacollection.sender.*;
+import org.apache.hadoop.chukwa.util.DaemonWatcher;
 import org.apache.log4j.Logger;
 
 public class HttpConnector implements Connector, Runnable {
@@ -171,11 +172,11 @@
           .info("received stop() command so exiting run() loop to shutdown connector");
     } catch (OutOfMemoryError e) {
       log.warn("Bailing out", e);
-      System.exit(-1);
+      DaemonWatcher.bailout(-1);
     } catch (InterruptedException e) {
       // do nothing, let thread die.
       log.warn("Bailing out", e);
-      System.exit(-1);
+      DaemonWatcher.bailout(-1);
     } catch (java.io.IOException e) {
       log.error("connector failed; shutting down agent");
       agent.shutdown(true);

Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/writer/SeqFileWriter.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/writer/SeqFileWriter.java?rev=764142&r1=764141&r2=764142&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/writer/SeqFileWriter.java
(original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/writer/SeqFileWriter.java
Sat Apr 11 01:42:07 2009
@@ -29,6 +29,7 @@
 import org.apache.hadoop.chukwa.ChukwaArchiveKey;
 import org.apache.hadoop.chukwa.Chunk;
 import org.apache.hadoop.chukwa.ChunkImpl;
+import org.apache.hadoop.chukwa.util.DaemonWatcher;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
@@ -239,7 +240,7 @@
         // so there's nothing we can do
         // Shutting down the collector for now
         // Watchdog will re-start it automatically
-        System.exit(-1);
+        DaemonWatcher.bailout(-1);
       }
     }
 
@@ -302,7 +303,7 @@
         if (writeChunkRetries < 0) {
           log
               .fatal("Too many IOException when trying to write a chunk, Collector is going
to exit!");
-          System.exit(-1);
+          DaemonWatcher.bailout(-1);
         }
         throw new WriterException(e);
       }
@@ -361,7 +362,7 @@
         if (writeChunkRetries < 0) {
           log
               .fatal("Too many IOException when trying to write a chunk, Collector is going
to exit!");
-          System.exit(-1);
+          DaemonWatcher.bailout(-1);
         }
         throw new WriterException(e);
       }

Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/archive/ChukwaArchiveManager.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/archive/ChukwaArchiveManager.java?rev=764142&r1=764141&r2=764142&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/archive/ChukwaArchiveManager.java
(original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/archive/ChukwaArchiveManager.java
Sat Apr 11 01:42:07 2009
@@ -25,7 +25,7 @@
 
 import org.apache.hadoop.chukwa.conf.ChukwaConfiguration;
 import org.apache.hadoop.chukwa.extraction.CHUKWA_CONSTANT;
-import org.apache.hadoop.chukwa.util.PidFile;
+import org.apache.hadoop.chukwa.util.DaemonWatcher;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -60,8 +60,7 @@
   }
 
   public static void main(String[] args) throws Exception {
-    PidFile pFile = new PidFile("ArchiveManager");
-    Runtime.getRuntime().addShutdownHook(pFile);
+    DaemonWatcher.createInstance("ArchiveManager");
     
     ChukwaArchiveManager manager = new ChukwaArchiveManager();
     manager.start();
@@ -118,7 +117,7 @@
         if (errorCount >= 4 ) {
           // it's better to exit, Watchdog will re-start it
           log.warn("Too many error - bail out!");
-          System.exit(-1);
+          DaemonWatcher.bailout(-1);
         }
         // /chukwa/archives/<YYYYMMDD>/dataSinkDirXXX
         //  to

Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/DailyChukwaRecordRolling.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/DailyChukwaRecordRolling.java?rev=764142&r1=764141&r2=764142&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/DailyChukwaRecordRolling.java
(original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/DailyChukwaRecordRolling.java
Sat Apr 11 01:42:07 2009
@@ -28,6 +28,7 @@
 import org.apache.hadoop.chukwa.conf.ChukwaConfiguration;
 import org.apache.hadoop.chukwa.extraction.engine.ChukwaRecord;
 import org.apache.hadoop.chukwa.extraction.engine.ChukwaRecordKey;
+import org.apache.hadoop.chukwa.util.DaemonWatcher;
 import org.apache.hadoop.chukwa.util.PidFile;
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.FileStatus;
@@ -202,8 +203,7 @@
    */
   public static void main(String[] args) throws Exception {
     
-    PidFile pFile = new PidFile("DailyChukwaRecordRolling");
-    Runtime.getRuntime().addShutdownHook(pFile);
+    DaemonWatcher.createInstance("DailyChukwaRecordRolling");
     
     conf = new ChukwaConfiguration();
     String fsName = conf.get("writer.hdfs.filesystem");

Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/DemuxManager.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/DemuxManager.java?rev=764142&r1=764141&r2=764142&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/DemuxManager.java
(original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/DemuxManager.java
Sat Apr 11 01:42:07 2009
@@ -26,7 +26,7 @@
 import org.apache.hadoop.chukwa.conf.ChukwaConfiguration;
 import org.apache.hadoop.chukwa.extraction.CHUKWA_CONSTANT;
 import org.apache.hadoop.chukwa.util.NagiosHelper;
-import org.apache.hadoop.chukwa.util.PidFile;
+import org.apache.hadoop.chukwa.util.DaemonWatcher;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -62,8 +62,7 @@
 
 
   public static void main(String[] args) throws Exception {
-    PidFile pFile = new PidFile("DemuxManager");
-    Runtime.getRuntime().addShutdownHook(pFile);
+    DaemonWatcher.createInstance("DemuxManager");
     
     DemuxManager manager = new DemuxManager();
     manager.start();
@@ -154,7 +153,7 @@
 
          if (globalErrorcounter > 5) {
            log.warn("==================\nToo many errors, Bail out!\n==================");
-           System.exit(-1);
+           DaemonWatcher.bailout(-1);
          }
          
          // Check for anomalies

Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/HourlyChukwaRecordRolling.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/HourlyChukwaRecordRolling.java?rev=764142&r1=764141&r2=764142&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/HourlyChukwaRecordRolling.java
(original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/HourlyChukwaRecordRolling.java
Sat Apr 11 01:42:07 2009
@@ -28,6 +28,7 @@
 import org.apache.hadoop.chukwa.conf.ChukwaConfiguration;
 import org.apache.hadoop.chukwa.extraction.engine.ChukwaRecord;
 import org.apache.hadoop.chukwa.extraction.engine.ChukwaRecordKey;
+import org.apache.hadoop.chukwa.util.DaemonWatcher;
 import org.apache.hadoop.chukwa.util.PidFile;
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.FileStatus;
@@ -152,8 +153,7 @@
    * @throws Exception
    */
   public static void main(String[] args) throws Exception {
-    PidFile pFile = new PidFile("HourlyChukwaRecordRolling");
-    Runtime.getRuntime().addShutdownHook(pFile);
+    DaemonWatcher.createInstance("HourlyChukwaRecordRolling");
     
     conf = new ChukwaConfiguration();
     String fsName = conf.get("writer.hdfs.filesystem");

Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/PostProcessorManager.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/PostProcessorManager.java?rev=764142&r1=764141&r2=764142&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/PostProcessorManager.java
(original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/PostProcessorManager.java
Sat Apr 11 01:42:07 2009
@@ -29,7 +29,7 @@
 import org.apache.hadoop.chukwa.conf.ChukwaConfiguration;
 import org.apache.hadoop.chukwa.extraction.CHUKWA_CONSTANT;
 import org.apache.hadoop.chukwa.extraction.database.DatabaseLoader;
-import org.apache.hadoop.chukwa.util.PidFile;
+import org.apache.hadoop.chukwa.util.DaemonWatcher;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -71,8 +71,7 @@
   
   public static void main(String[] args) throws Exception {
  
-    PidFile pFile = new PidFile("PostProcessorManager");
-    Runtime.getRuntime().addShutdownHook(pFile);
+    DaemonWatcher.createInstance("PostProcessorManager");
     
 
     
@@ -106,7 +105,7 @@
     String[] datasources = conf.getStrings("postProcessorManager.dbloader.ds");
     if (datasources == null || datasources.length == 0) {
       log.warn("Cannot read postProcessorManager.dbloader.ds from configuration, bail out!");
-      System.exit(-1);
+      DaemonWatcher.bailout(-1);
     }
     for(String ds: datasources) {
       dataSources.put(ds.trim(), "");
@@ -120,7 +119,7 @@
       if (errorCount >= 4 ) {
         // it's better to exit, Watchdog will re-strat it
         log.warn("Too many error - bail out!");
-        System.exit(-1);
+        DaemonWatcher.bailout(-1);
       }
       
       try {

Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/inputtools/jplugin/JPluginAgent.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/inputtools/jplugin/JPluginAgent.java?rev=764142&r1=764141&r2=764142&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/inputtools/jplugin/JPluginAgent.java
(original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/inputtools/jplugin/JPluginAgent.java
Sat Apr 11 01:42:07 2009
@@ -24,6 +24,7 @@
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.chukwa.util.DaemonWatcher;
 import org.apache.hadoop.chukwa.util.PidFile;
 
 public class JPluginAgent {
@@ -102,8 +103,7 @@
     }
 
     try {
-      PidFile pFile = new PidFile(plugin.getRecordType() + "-data-loader");
-      Runtime.getRuntime().addShutdownHook(pFile);
+      DaemonWatcher.createInstance(plugin.getRecordType() + "-data-loader");
     } catch (Exception e) {
       e.printStackTrace();
     }

Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/inputtools/plugin/metrics/Exec.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/inputtools/plugin/metrics/Exec.java?rev=764142&r1=764141&r2=764142&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/inputtools/plugin/metrics/Exec.java
(original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/inputtools/plugin/metrics/Exec.java
Sat Apr 11 01:42:07 2009
@@ -9,14 +9,13 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.chukwa.inputtools.plugin.ExecPlugin;
 import org.apache.hadoop.chukwa.inputtools.plugin.IPlugin;
-import org.apache.hadoop.chukwa.util.PidFile;
+import org.apache.hadoop.chukwa.util.DaemonWatcher;
 import org.json.JSONException;
 import org.json.JSONObject;
 
 public class Exec extends TimerTask {
   private static Log log = LogFactory.getLog(Exec.class);
   private String cmde = null;
-  private static PidFile pFile = null;
   private Timer timer = null;
   private IPlugin plugin = null;
 
@@ -51,8 +50,7 @@
   }
 
   public static void main(String[] args) {
-    pFile = new PidFile(System.getProperty("RECORD_TYPE") + "-data-loader");
-    Runtime.getRuntime().addShutdownHook(pFile);
+    DaemonWatcher.createInstance(System.getProperty("RECORD_TYPE") + "-data-loader");
     int period = 60;
     try {
       if (System.getProperty("PERIOD") != null) {



Mime
View raw message