hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From asrab...@apache.org
Subject svn commit: r750140 - in /hadoop/core/trunk/src/contrib/chukwa/src: java/org/apache/hadoop/chukwa/datacollection/agent/ test/org/apache/hadoop/chukwa/datacollection/adaptor/ test/org/apache/hadoop/chukwa/datacollection/agent/
Date Wed, 04 Mar 2009 20:27:24 GMT
Author: asrabkin
Date: Wed Mar  4 20:27:23 2009
New Revision: 750140

URL: http://svn.apache.org/viewvc?rev=750140&view=rev
Log:
patch for HADOOP-5057
Improves test coverage for chukwa agent startup, in particular checkpoint restore corner cases.
Patch by asrabkin.

Modified:
    hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
    hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/ChukwaTestAdaptor.java
    hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/agent/TestAgentConfig.java

Modified: hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java?rev=750140&r1=750139&r2=750140&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
(original)
+++ hadoop/core/trunk/src/contrib/chukwa/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
Wed Mar  4 20:27:23 2009
@@ -219,7 +219,6 @@
     log.info("Config - tags: [" + tags + "]");
 
     if (DO_CHECKPOINT_RESTORE) {
-      needNewCheckpoint = true;
       log.info("checkpoints are enabled, period is " + CHECKPOINT_INTERVAL_MS);
     }
     
@@ -251,6 +250,9 @@
       controlSock.start(); // this sets us up as a daemon
       log.info("control socket started on port " + controlSock.portno);
 
+      
+        //shouldn't start checkpointing until we're finishing launching
+      //adaptors on boot
       if (CHECKPOINT_INTERVAL_MS > 0 && checkpointDir!= null)  {
         checkpointer = new Timer();
         checkpointer.schedule(new CheckpointTask(), 0, CHECKPOINT_INTERVAL_MS);
@@ -315,6 +317,7 @@
         adaptorID = ++lastAdaptorNumber;
         adaptorsByNumber.put(adaptorID, adaptor);
         adaptorPositions.put(adaptor, new Offset(offset, adaptorID));
+        needNewCheckpoint = true;
         try
         {
           adaptor.start(adaptorID, dataType, params, offset, DataFactory
@@ -330,7 +333,7 @@
       }
     } else
       if(cmd.length() > 0)
-        log.warn("only 'add' command supported in config files");
+        log.warn("only 'add' command supported in config files; cmd was: "+ cmd);
       //no warning for blank line
     
     return -1;
@@ -361,7 +364,7 @@
       });
       
       if (checkpointNames == null) {
-        log.error("Unable to list directories in checkpoint dir");
+        log.error("Unable to list files in checkpoint dir");
         return false;
       }
       if (checkpointNames.length == 0)
@@ -576,7 +579,8 @@
     if (checkpointer != null) {
       checkpointer.cancel();
       try {
-        writeCheckpoint(); // write a last checkpoint here, before stopping
+        if(needNewCheckpoint)
+          writeCheckpoint(); // write a last checkpoint here, before stopping
       } catch (IOException e) {
       }
     }

Modified: hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/ChukwaTestAdaptor.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/ChukwaTestAdaptor.java?rev=750140&r1=750139&r2=750140&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/ChukwaTestAdaptor.java
(original)
+++ hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/ChukwaTestAdaptor.java
Wed Mar  4 20:27:23 2009
@@ -31,13 +31,13 @@
   @Override
   public String getCurrentStatus() throws AdaptorException {
     // TODO Auto-generated method stub
-    return null;
+    return type+ " "+ params + " "+ startOffset;
   }
 
   @Override
   public String getStreamName() {
     // TODO Auto-generated method stub
-    return null;
+    return "";
   }
 
   @Override

Modified: hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/agent/TestAgentConfig.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/agent/TestAgentConfig.java?rev=750140&r1=750139&r2=750140&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/agent/TestAgentConfig.java
(original)
+++ hadoop/core/trunk/src/contrib/chukwa/src/test/org/apache/hadoop/chukwa/datacollection/agent/TestAgentConfig.java
Wed Mar  4 20:27:23 2009
@@ -73,4 +73,60 @@
       fail(e.toString());
     }
   }
+  
+  
+  public void testNoCheckpoints() {
+    try {
+      String tmpdir = System.getProperty("test.build.data", "/tmp");
+      File NONCE_DIR = new File(tmpdir,"/test_chukwa_checkpoints");
+      if(NONCE_DIR.exists()) {
+        for(File f: NONCE_DIR.listFiles())
+          f.delete();
+        NONCE_DIR.delete();
+      }
+//      assertFalse(NONCE_DIR.exists());
+      Configuration conf = new Configuration();
+      conf.set("chukwaAgent.checkpoint.dir", NONCE_DIR.getAbsolutePath());
+      conf.setBoolean("chukwaAgent.checkpoint.enabled", true);
+      conf.setInt("chukwaAgent.agent.control.port", 0);
+      
+      System.out.println("\n\n===checkpoints enabled, dir does not exist:");
+      ChukwaAgent agent = new ChukwaAgent(conf);
+      assertEquals(0, agent.getAdaptorList().size());
+      agent.shutdown();
+      assertTrue(NONCE_DIR.exists());
+      for(File f: NONCE_DIR.listFiles())
+        f.delete();
+
+      System.out.println("\n\n===checkpoints enabled, dir exists but is empty:");
+      agent = new ChukwaAgent(conf);
+      assertEquals(0, agent.getAdaptorList().size());
+      agent.shutdown();
+      for(File f: NONCE_DIR.listFiles())
+        f.delete();
+      
+      System.out.println("\n\n===checkpoints enabled, dir exists with zero-length file:");
+      (new File(NONCE_DIR, "chukwa_checkpoint_0")).createNewFile();
+      agent = new ChukwaAgent(conf);
+      assertEquals(0, agent.getAdaptorList().size());
+      agent.processCommand("ADD org.apache.hadoop.chukwa.datacollection.adaptor.ChukwaTestAdaptor
testdata  0");
+      agent.shutdown();
+      assertTrue(new File(NONCE_DIR, "chukwa_checkpoint_1").exists());
+
+      System.out.println("\n\n===checkpoints enabled, dir exists with valid checkpoint");
+      agent = new ChukwaAgent(conf);
+      assertEquals(1, agent.getAdaptorList().size());
+      agent.shutdown();
+      //checkpoint # increments by one on boot and reload
+      assertTrue(new File(NONCE_DIR, "chukwa_checkpoint_2").exists());
+
+      
+    }
+    catch(Exception e) {
+      fail(e.toString());
+    }
+  }
+  
+  
+  
 }



Mime
View raw message