chukwa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From asrab...@apache.org
Subject svn commit: r796032 - in /hadoop/chukwa/trunk: CHANGES.txt src/java/org/apache/hadoop/chukwa/extraction/archive/SinkArchiver.java src/test/org/apache/hadoop/chukwa/datacollection/adaptor/TestDirTailingAdaptor.java
Date Mon, 20 Jul 2009 21:44:48 GMT
Author: asrabkin
Date: Mon Jul 20 21:44:47 2009
New Revision: 796032

URL: http://svn.apache.org/viewvc?rev=796032&view=rev
Log:
CHUKWA-354. Fix corner case in SinkArchiver.

Modified:
    hadoop/chukwa/trunk/CHANGES.txt
    hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/archive/SinkArchiver.java
    hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/TestDirTailingAdaptor.java

Modified: hadoop/chukwa/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/CHANGES.txt?rev=796032&r1=796031&r2=796032&view=diff
==============================================================================
--- hadoop/chukwa/trunk/CHANGES.txt (original)
+++ hadoop/chukwa/trunk/CHANGES.txt Mon Jul 20 21:44:47 2009
@@ -94,6 +94,8 @@
 
   BUG FIXES
 
+    CHUKWA-354. Fix corner case in SinkArchiver. (asrabkin)
+
     CHUKWA-334. Fix LocalHdfsMover exception handling. (Jerome Boulon via asrabkin)
 
     CHUKWA-335. Discard meta line in job history file. (Cheng Zhang via asrabkin)

Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/archive/SinkArchiver.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/archive/SinkArchiver.java?rev=796032&r1=796031&r2=796032&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/archive/SinkArchiver.java
(original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/archive/SinkArchiver.java
Mon Jul 20 21:44:47 2009
@@ -99,6 +99,9 @@
         fs.delete(pMRInputDir, true);
       }
       
+      if(!fs.exists(archive)) {
+        fs.mkdirs(archive);
+      }
       FileStatus[] files = fs.listStatus(pOutputDir);
       for(FileStatus f: files) {
         if(!f.getPath().getName().endsWith("_logs"))
@@ -132,7 +135,15 @@
         args);
     return res;
   }
-  
+  /**
+   * Merges the contents of src into dest.
+   * If src is a file, moves it to dest.
+   * 
+   * @param fs the filesystem in question
+   * @param src a file or directory to merge into dest
+   * @param dest a directory to merge into
+   * @throws IOException
+   */
   public void promoteAndMerge(FileSystem fs, Path src, Path dest) 
   throws IOException {
     FileStatus stat = fs.getFileStatus(src);

Modified: hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/TestDirTailingAdaptor.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/TestDirTailingAdaptor.java?rev=796032&r1=796031&r2=796032&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/TestDirTailingAdaptor.java
(original)
+++ hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/datacollection/adaptor/TestDirTailingAdaptor.java
Mon Jul 20 21:44:47 2009
@@ -35,23 +35,25 @@
     
     Configuration conf = new Configuration();
     baseDir = new File(System.getProperty("test.build.data", "/tmp"));
-    conf.set("chukwaAgent.checkpoint.dir", baseDir.getCanonicalPath());
-    conf.set("chukwaAgent.checkpoint.name", "checkpoint_test_"+System.currentTimeMillis());
+    File checkpointDir = new File(baseDir, "dirtailerTestCheckpoints");
+    createEmptyDir(checkpointDir);
+    
+    conf.set("chukwaAgent.checkpoint.dir", checkpointDir.getCanonicalPath());
+    conf.set("chukwaAgent.checkpoint.name", "checkpoint_");
     conf.setInt("chukwaAgent.control.port", 0);
     conf.setBoolean("chukwaAgent.checkpoint.enabled", false);
     
     agent = new ChukwaAgent(conf);
     File emptyDir = new File(baseDir, "emptyDir");
-    if(emptyDir.exists())
-      emptyDir.delete();
-    emptyDir.mkdir();
+    createEmptyDir(emptyDir);
+    
     assertEquals(0, agent.adaptorCount());
     agent.processAddCommand("add DirTailingAdaptor raw " + emptyDir + " filetailer.CharFileTailingAdaptorUTF8
0");
     assertEquals(1, agent.adaptorCount());
 
     File dirWithFile = new File(baseDir, "dir2");
     dirWithFile.delete();
-    assertFalse( "temp directory not empty",dirWithFile.exists());
+    assertFalse("temp directory not empty",dirWithFile.exists());
       
     dirWithFile.mkdir();
     File inDir = File.createTempFile("atemp", "file", dirWithFile);
@@ -70,12 +72,43 @@
     aNewFile.deleteOnExit();
     anOldFile.setLastModified(10);//just after epoch
     agent = new ChukwaAgent(conf); //restart agent.
+    
+    //should be four adaptors: the DirTailer on emptyDir, the DirTailer on the full dir,
+    //and FileTailers for File inDir and file newfile
     assertEquals(4, agent.adaptorCount());
     
     //make sure we started tailing the new, not the old, file.
     for(Map.Entry<String, String> adaptors : agent.getAdaptorList().entrySet()) {
+      System.out.println(adaptors.getKey() +": " + adaptors.getValue());
       assertFalse(adaptors.getValue().contains("oldXYZ"));
     }
+    nukeDirContents(checkpointDir);//nuke dir
+    checkpointDir.delete();
+    emptyDir.delete();
+    nukeDirContents(dirWithFile);
+    dirWithFile.delete();
+  }
+
+  //returns true if dir exists
+  private boolean nukeDirContents(File dir) {
+    if(dir.exists()) {
+      if(dir.isDirectory()) {
+        for(File f: dir.listFiles()) {
+          nukeDirContents(f);
+          f.delete();
+        }
+      } else
+        dir.delete();
+      
+      return true;
+    }
+    return false;
+  }
+  
+  private void createEmptyDir(File dir) {
+    if(!nukeDirContents(dir))
+      dir.mkdir();
+    assertTrue(dir.isDirectory() && dir.listFiles().length == 0);
   }
 
 }



Mime
View raw message