commons-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Simone Tripodi <simonetrip...@apache.org>
Subject Re: [io] Tailer returning partial lines returned when EOF before newline
Date Wed, 25 May 2011 06:13:03 GMT
Hi Frank,
What I suggest you is following the usual workflow, I mean opening an
Issue on ASF Jira - see Niall's link - and build a patch against the
/trunk and attach it to the filled Issue.
Otherwise it will be very hard that IO maintainers could notice and
apply a textual patch inside the ML.
HTH, have a nice day!
Simo

http://people.apache.org/~simonetripodi/
http://www.99soft.org/



On Tue, May 24, 2011 at 10:39 PM, frankgrimes97 <frankgrimes97@gmail.com> wrote:
> The following is a patch against 2.0.1 in SVN which seems to address the
> limitation:
>
> Index: src/main/java/org/apache/commons/io/input/Tailer.java
> ===================================================================
> --- src/main/java/org/apache/commons/io/input/Tailer.java (revision 1127267)
> +++ src/main/java/org/apache/commons/io/input/Tailer.java (working copy)
> @@ -335,12 +335,56 @@
>      * @throws java.io.IOException if an I/O error occurs.
>      */
>     private long readLines(RandomAccessFile reader) throws IOException {
> -        String line = reader.readLine();
> +        String line = readLine(reader);
>         while (line != null) {
>             listener.handle(line);
> -            line = reader.readLine();
> +            line = readLine(reader);
>         }
>         return reader.getFilePointer();
>     }
>
> +    /**
> +     * Copied from RandomAccessFile.readLine() but returns null and resets
> file pointer on EOF.
> +     * @param reader
> +     * @return
> +     * @throws IOException
> +     */
> +    private final String readLine(RandomAccessFile reader) throws
> IOException {
> +     long start = reader.getFilePointer();
> +     StringBuffer input = new StringBuffer();
> +     int c = -1;
> +     boolean eol = false;
> +     boolean eof = false;
> +
> +     while (!eol && !eof) {
> +        switch (c = reader.read()) {
> +        case -1:
> +        eof =  true;
> +        break;
> +        case '\n':
> +     eol = true;
> +     break;
> +        case '\r':
> +     eol = true;
> +     long cur = reader.getFilePointer();
> +     if ((reader.read()) != '\n') {
> +     reader.seek(cur);
> +     }
> +     break;
> +        default:
> +     input.append((char)c);
> +     break;
> +        }
> +     }
> +
> +     if ((c == -1) && (input.length() == 0)) {
> +        return null;
> +     }
> +     if (eof) {
> +     reader.seek(start);
> +     return null;
> +     }
> +
> +     return input.toString();
> +    }
>  }
> Index: src/test/java/org/apache/commons/io/input/TailerTest.java
> ===================================================================
> --- src/test/java/org/apache/commons/io/input/TailerTest.java (revision
> 1127267)
> +++ src/test/java/org/apache/commons/io/input/TailerTest.java (working copy)
> @@ -45,6 +45,38 @@
>     protected void tearDown() throws Exception {
>         FileUtils.deleteDirectory(getTestDirectory());
>     }
> +
> +    public void testTailerEof() throws Exception {
> +        // Create & start the Tailer
> +        long delay = 50;
> +        final File file = new File(getTestDirectory(), "tailer2-test.txt");
> +        createFile(file, 0);
> +        final TestTailerListener listener = new TestTailerListener();
> +        final Tailer tailer = new Tailer(file, listener, delay, false);
> +        final Thread thread = new Thread(tailer);
> +        thread.start();
> +
> +        // Write some lines to the file
> +        FileWriter writer = null;
> +        try {
> +         writeString(file, "Line");
> +
> +            Thread.sleep(delay * 2);
> +            List<String> lines = listener.getLines();
> +            assertEquals("1 line count", 0, lines.size());
> +
> +            writeString(file, " one\n");
> +            Thread.sleep(delay * 2);
> +            lines = listener.getLines();
> +
> +            assertEquals("1 line count", 1, lines.size());
> +            assertEquals("1 line 1", "Line one", lines.get(0));
> +
> +            listener.clear();
> +        } finally {
> +            IOUtils.closeQuietly(writer);
> +        }
> +    }
>
>     public void testTailer() throws Exception {
>
> @@ -142,6 +174,17 @@
>             IOUtils.closeQuietly(writer);
>         }
>     }
> +
> +    /** Append a string to a file */
> +    private void writeString(File file, String string) throws Exception {
> +        FileWriter writer = null;
> +        try {
> +            writer = new FileWriter(file, true);
> +            writer.write(string);
> +        } finally {
> +            IOUtils.closeQuietly(writer);
> +        }
> +    }
>
>     public void testStopWithNoFile() throws Exception {
>         final File file = new File(getTestDirectory(),"nosuchfile");
>
>
> On Tue, May 24, 2011 at 1:32 PM, frankgrimes97 <frankgrimes97@gmail.com>wrote:
>
>> Hi All,
>>
>> We are using org.apache.commons.io.input.Tailer to process log files for
>> insertion into a database.
>>
>> What we are seeing is that occasionally a line fails to process because it
>> is incomplete.
>> In reviewing the code, it appears that Tailer.readLines delegates to
>> java.io.RandomAccessFile.readLine which returns a partial line if EOF is
>> reached.
>>
>> Shouldn't Tailer be providing a guarantee of complete lines?
>> Should we create a bug report for this?
>>
>> FYI, we are using 1.6.0_15 on Linux.
>>
>> Thanks,
>>
>> Frank Grimes
>>
>

---------------------------------------------------------------------
To unsubscribe, e-mail: user-unsubscribe@commons.apache.org
For additional commands, e-mail: user-help@commons.apache.org


Mime
View raw message