flume-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hshreedha...@apache.org
Subject flume git commit: FLUME-2525. Handle a zero byte .flumespool-main.meta file for the spooldir source.
Date Thu, 06 Nov 2014 19:32:52 GMT
Repository: flume
Updated Branches:
  refs/heads/flume-1.6 e9633c6c2 -> c2b953449


FLUME-2525. Handle a zero byte .flumespool-main.meta file for the spooldir source.

(Johny Rufus via Hari)


Project: http://git-wip-us.apache.org/repos/asf/flume/repo
Commit: http://git-wip-us.apache.org/repos/asf/flume/commit/c2b95344
Tree: http://git-wip-us.apache.org/repos/asf/flume/tree/c2b95344
Diff: http://git-wip-us.apache.org/repos/asf/flume/diff/c2b95344

Branch: refs/heads/flume-1.6
Commit: c2b953449d033acaa02e3ced1d9a95e9cdcb5e02
Parents: e9633c6
Author: Hari Shreedharan <hshreedharan@apache.org>
Authored: Thu Nov 6 11:31:35 2014 -0800
Committer: Hari Shreedharan <hshreedharan@apache.org>
Committed: Thu Nov 6 11:32:39 2014 -0800

----------------------------------------------------------------------
 .../avro/ReliableSpoolingFileEventReader.java   |  3 ++
 .../TestReliableSpoolingFileEventReader.java    | 29 ++++++++++++++++++++
 2 files changed, 32 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flume/blob/c2b95344/flume-ng-core/src/main/java/org/apache/flume/client/avro/ReliableSpoolingFileEventReader.java
----------------------------------------------------------------------
diff --git a/flume-ng-core/src/main/java/org/apache/flume/client/avro/ReliableSpoolingFileEventReader.java
b/flume-ng-core/src/main/java/org/apache/flume/client/avro/ReliableSpoolingFileEventReader.java
index 1833076..27e9c1e 100644
--- a/flume-ng-core/src/main/java/org/apache/flume/client/avro/ReliableSpoolingFileEventReader.java
+++ b/flume-ng-core/src/main/java/org/apache/flume/client/avro/ReliableSpoolingFileEventReader.java
@@ -199,6 +199,9 @@ public class ReliableSpoolingFileEventReader implements ReliableEventReader
{
     }
 
     this.metaFile = new File(trackerDirectory, metaFileName);
+    if(metaFile.exists() && metaFile.length() == 0) {
+      deleteMetaFile();
+    }
   }
 
   @VisibleForTesting

http://git-wip-us.apache.org/repos/asf/flume/blob/c2b95344/flume-ng-core/src/test/java/org/apache/flume/client/avro/TestReliableSpoolingFileEventReader.java
----------------------------------------------------------------------
diff --git a/flume-ng-core/src/test/java/org/apache/flume/client/avro/TestReliableSpoolingFileEventReader.java
b/flume-ng-core/src/test/java/org/apache/flume/client/avro/TestReliableSpoolingFileEventReader.java
index a6b2473..c6ff63e 100644
--- a/flume-ng-core/src/test/java/org/apache/flume/client/avro/TestReliableSpoolingFileEventReader.java
+++ b/flume-ng-core/src/test/java/org/apache/flume/client/avro/TestReliableSpoolingFileEventReader.java
@@ -405,6 +405,35 @@ public class TestReliableSpoolingFileEventReader {
   @Test public void testLargeNumberOfFilesRANDOM() throws IOException {    
     templateTestForLargeNumberOfFiles(ConsumeOrder.RANDOM, null, 1000);
   }
+
+  @Test
+  public void testZeroByteTrackerFile() throws IOException {
+    String trackerDirPath =
+            SpoolDirectorySourceConfigurationConstants.DEFAULT_TRACKER_DIR;
+    File trackerDir = new File(WORK_DIR, trackerDirPath);
+    if(!trackerDir.exists()) {
+      trackerDir.mkdir();
+    }
+    File trackerFile = new File(trackerDir, ReliableSpoolingFileEventReader.metaFileName);
+    if(trackerFile.exists()) {
+      trackerFile.delete();
+    }
+    trackerFile.createNewFile();
+
+    ReliableEventReader reader = new ReliableSpoolingFileEventReader.Builder()
+            .spoolDirectory(WORK_DIR).trackerDirPath(trackerDirPath).build();
+    final int expectedLines = 1;
+    int seenLines = 0;
+    List<Event> events = reader.readEvents(10);
+    int numEvents = events.size();
+    if (numEvents > 0) {
+      seenLines += numEvents;
+      reader.commit();
+    }
+    // This line will fail, if the zero-byte tracker file has not been handled
+    Assert.assertEquals(expectedLines, seenLines);
+  }
+
   private void templateTestForLargeNumberOfFiles(ConsumeOrder order, 
       Comparator<Long> comparator,
       int N) throws IOException {


Mime
View raw message