harmony-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From telli...@apache.org
Subject svn commit: r954817 - in /harmony/enhanced/java/trunk/classlib/modules/luni/src: main/java/java/util/Scanner.java test/api/common/org/apache/harmony/luni/tests/java/util/ScannerParseLargeFileBenchmarkTest.java
Date Tue, 15 Jun 2010 11:05:28 GMT
Author: tellison
Date: Tue Jun 15 11:05:28 2010
New Revision: 954817

URL: http://svn.apache.org/viewvc?rev=954817&view=rev
Log:
Apply patch for HARMONY-6470 ([classlib][luni] Scanner will exhaust all heap memory when parse
a large file)

Includes a (relatively) long running test case.

Added:
    harmony/enhanced/java/trunk/classlib/modules/luni/src/test/api/common/org/apache/harmony/luni/tests/java/util/ScannerParseLargeFileBenchmarkTest.java
  (with props)
Modified:
    harmony/enhanced/java/trunk/classlib/modules/luni/src/main/java/java/util/Scanner.java

Modified: harmony/enhanced/java/trunk/classlib/modules/luni/src/main/java/java/util/Scanner.java
URL: http://svn.apache.org/viewvc/harmony/enhanced/java/trunk/classlib/modules/luni/src/main/java/java/util/Scanner.java?rev=954817&r1=954816&r2=954817&view=diff
==============================================================================
--- harmony/enhanced/java/trunk/classlib/modules/luni/src/main/java/java/util/Scanner.java
(original)
+++ harmony/enhanced/java/trunk/classlib/modules/luni/src/main/java/java/util/Scanner.java
Tue Jun 15 11:05:28 2010
@@ -2017,6 +2017,8 @@ public final class Scanner implements It
         return negative;
     }
 
+    private int tokenStart = 0;
+    private int tokenEnd = 0;
     /*
      * Find the prefixed delimiter and posefixed delimiter in the input resource
      * and set the start index and end index of Matcher region. If postfixed
@@ -2031,11 +2033,13 @@ public final class Scanner implements It
         matcher.usePattern(delimiter);
         matcher.region(findStartIndex, bufferLength);
 
-        tokenStartIndex = findPreDelimiter();
-        if (setHeadTokenRegion(tokenStartIndex)) {
+        findPreDelimiter();
+        if (setHeadTokenRegion(tokenStart)) {
             return true;
         }
-        tokenEndIndex = findPostDelimiter();
+        findPostDelimiter();
+        tokenStartIndex = tokenStart;
+        tokenEndIndex = tokenEnd;
         // If the second delimiter is not found
         if (-1 == tokenEndIndex) {
             // Just first Delimiter Exists
@@ -2053,8 +2057,8 @@ public final class Scanner implements It
     /*
      * Find prefix delimiter
      */
-    private int findPreDelimiter() {
-        int tokenStartIndex;
+    private void findPreDelimiter() {
+    	tokenStart = 0;
         boolean findComplete = false;
         while (!findComplete) {
             if (matcher.find()) {
@@ -2074,13 +2078,14 @@ public final class Scanner implements It
                     readMore();
                     resetMatcher();
                 } else {
-                    return -1;
+                    tokenStart = -1;
+                    return;
                 }
             }
         }
-        tokenStartIndex = matcher.end();
+
+        tokenStart = matcher.end();
         findStartIndex = matcher.end();
-        return tokenStartIndex;
     }
 
     /*
@@ -2113,8 +2118,8 @@ public final class Scanner implements It
     /*
      * Find postfix delimiter
      */
-    private int findPostDelimiter() {
-        int tokenEndIndex = 0;
+    private void findPostDelimiter() {
+        tokenEnd = 0;
         boolean findComplete = false;
         while (!findComplete) {
             if (matcher.find()) {
@@ -2128,13 +2133,16 @@ public final class Scanner implements It
                     readMore();
                     resetMatcher();
                 } else {
-                    return -1;
+                    tokenStart = findStartIndex;
+                    tokenEnd = -1;
+                    return;
                 }
             }
         }
-        tokenEndIndex = matcher.start();
+
+        tokenStart = findStartIndex;
+        tokenEnd = matcher.start();
         findStartIndex = matcher.start();
-        return tokenEndIndex;
     }
 
     /*
@@ -2143,13 +2151,12 @@ public final class Scanner implements It
      * true, otherwise set to false.
      */
     private void readMore() {
-        int oldPosition = buffer.position();
-        int oldBufferLength = bufferLength;
         // Increase capacity if empty space is not enough
         if (bufferLength >= buffer.capacity()) {
             expandBuffer();
         }
-
+        int oldPosition = buffer.position();
+        int oldBufferLength = bufferLength;
         // Read input resource
         int readCount = 0;
         try {
@@ -2183,14 +2190,66 @@ public final class Scanner implements It
 
     // Expand the size of internal buffer.
     private void expandBuffer() {
-        int oldPosition = buffer.position();
-        int oldCapacity = buffer.capacity();
+        //inital new indexes values
+        int nFindStartIndex = 0;
+        int nPreStartIndex = 0;
+        int nBufferLength = 0;
+        int nLimit = 0;
+        int nPostion = 0;       
+        int nCachehasNextIndex = 0;
+
+        //store old buffer set
+        int oldLength = bufferLength;
+        int oldPostion = buffer.position();
         int oldLimit = buffer.limit();
-        int newCapacity = oldCapacity * DIPLOID;
-        char[] newBuffer = new char[newCapacity];
-        System.arraycopy(buffer.array(), 0, newBuffer, 0, oldLimit);
-        buffer = CharBuffer.wrap(newBuffer, 0, newCapacity);
-        buffer.position(oldPosition);
-        buffer.limit(oldLimit);
+        int oldCapacity = buffer.capacity();
+        
+        //new start index for new buffer
+        int startIndex = 0;
+        
+        if (preStartIndex == -1) {
+            startIndex = findStartIndex;
+            nPreStartIndex = -1;
+            nFindStartIndex = 0;
+        } else {
+            startIndex = preStartIndex;
+            nPreStartIndex = 0;
+            nFindStartIndex = findStartIndex - startIndex;
+        }
+        
+        int nCapacity = oldCapacity;
+        char[] newBuffer = null;
+        // set nCapacity
+        if (startIndex < oldCapacity / 2) {
+            nCapacity = nCapacity << 1;
+            newBuffer = new char[nCapacity];
+        } else {
+            newBuffer = buffer.array();
+        }
+
+        //new buffer set
+        nBufferLength = oldLength - startIndex;
+        nLimit = oldLimit - startIndex;
+        if (oldPostion < startIndex) {
+            nPostion = 0;
+        } else {
+            nPostion = oldPostion - startIndex;
+        }
+        // new CachedhasNextIndex;
+        if (cachehasNextIndex != -1) {
+            nCachehasNextIndex = cachehasNextIndex - startIndex;
+        } else {
+            nCachehasNextIndex = -1;
+        }
+        System.arraycopy(buffer.array(), startIndex, newBuffer, 0, nLimit);
+
+        buffer = CharBuffer.wrap(newBuffer, 0, nCapacity);
+        buffer.position(nPostion);
+        buffer.limit(nLimit);
+
+        preStartIndex = nPreStartIndex;
+        findStartIndex = nFindStartIndex;
+        bufferLength = nBufferLength;
+        cachehasNextIndex = nCachehasNextIndex;
     }
 }

Added: harmony/enhanced/java/trunk/classlib/modules/luni/src/test/api/common/org/apache/harmony/luni/tests/java/util/ScannerParseLargeFileBenchmarkTest.java
URL: http://svn.apache.org/viewvc/harmony/enhanced/java/trunk/classlib/modules/luni/src/test/api/common/org/apache/harmony/luni/tests/java/util/ScannerParseLargeFileBenchmarkTest.java?rev=954817&view=auto
==============================================================================
--- harmony/enhanced/java/trunk/classlib/modules/luni/src/test/api/common/org/apache/harmony/luni/tests/java/util/ScannerParseLargeFileBenchmarkTest.java
(added)
+++ harmony/enhanced/java/trunk/classlib/modules/luni/src/test/api/common/org/apache/harmony/luni/tests/java/util/ScannerParseLargeFileBenchmarkTest.java
Tue Jun 15 11:05:28 2010
@@ -0,0 +1,77 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.harmony.luni.tests.java.util;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.util.Scanner;
+
+import org.junit.Test;
+
+public class ScannerParseLargeFileBenchmarkTest {
+
+    /**
+     * This test will check when parse a large file like more than 200M bytes if
+     * the Scanner will exhaust all heap memory
+     */
+    @Test
+    public void testParseLargeFile() throws Exception {
+        MyReader reader = new MyReader();
+        String delimiter = "\r?\n";
+        Scanner scanner = new Scanner(reader).useDelimiter(delimiter);
+
+        while (scanner.hasNext()) {
+            scanner.next();
+        }
+        scanner.close();
+        reader.close();
+    }
+
+    private static class MyReader extends Reader {
+        static final char[] CONTENT = "large file!\n".toCharArray();
+
+        static long fileLength = (8 << 21) * 12;
+
+        static boolean first = true;
+
+        static int position = 0;
+
+        private int count = 0;
+
+        @Override
+        public void close() throws IOException {
+        }
+
+        @Override
+        public int read(char[] buf, int offset, int length) {
+            if (count >= fileLength) {
+                return -1;
+            }
+            if (first == true) {
+                position = 0;
+                first = false;
+            }
+            for (int i = offset; i < length; i++) {
+                buf[i] = CONTENT[(i + position) % CONTENT.length];
+                count++;
+            }
+
+            position = (length + position) % CONTENT.length;
+
+            return length - offset;
+        }
+    }
+}

Propchange: harmony/enhanced/java/trunk/classlib/modules/luni/src/test/api/common/org/apache/harmony/luni/tests/java/util/ScannerParseLargeFileBenchmarkTest.java
------------------------------------------------------------------------------
    svn:eol-style = native



Mime
View raw message