db-derby-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kmars...@apache.org
Subject svn commit: r672818 - in /db/derby/code/trunk: ./ java/engine/org/apache/derby/iapi/types/ java/testing/org/apache/derbyTesting/functionTests/suites/ java/testing/org/apache/derbyTesting/functionTests/tests/memory/
Date Mon, 30 Jun 2008 16:45:38 GMT
Author: kmarsden
Date: Mon Jun 30 09:45:38 2008
New Revision: 672818

URL: http://svn.apache.org/viewvc?rev=672818&view=rev
Log:
DERBY-3732  SQL Length function materializes BLOB into memory


Added:
    db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memory/BlobMemTest.java
  (with props)
    db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memory/_Suite.java
  (with props)
Modified:
    db/derby/code/trunk/build.xml
    db/derby/code/trunk/java/engine/org/apache/derby/iapi/types/SQLBinary.java
    db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/suites/AllPackages.java

Modified: db/derby/code/trunk/build.xml
URL: http://svn.apache.org/viewvc/db/derby/code/trunk/build.xml?rev=672818&r1=672817&r2=672818&view=diff
==============================================================================
--- db/derby/code/trunk/build.xml (original)
+++ db/derby/code/trunk/build.xml Mon Jun 30 09:45:38 2008
@@ -2017,6 +2017,33 @@
       </junit>	
   </target>
 
+	  
+  <!-- low memory suite runs with 16MB heap  -->
+  <target name="junit-lowmem"  depends="checkVMLevel,junit-init" if="vmLevelIsAtLeast1.5">
+    <condition property="derby.junit.classpath" value="">
+      <not>
+        <isset property="derby.junit.classpath"/>
+      </not>
+    </condition>
+  	 <junit printsummary="on"
+           fork="yes" forkmode="perTest"
+           jvm="${derby.junit.jvm}" maxmemory="16m"
+           showoutput="yes"
+           dir="junit_${derby.junit.timestamp}"
+           tempdir="junit_${derby.junit.timestamp}"
+  	 	   errorproperty="tests.failed"
+  	       failureproperty="tests.failed">
+      <formatter type="xml"/>
+  	 	
+      <test name="org.apache.derbyTesting.functionTests.tests.memory._Suite"
+      	todir="junit_${derby.junit.timestamp}"/>
+        <classpath>
+       	    <pathelement path="${derby.junit.classpath}"/>
+            <pathelement location="${javatools.dir}/junit.jar"/>
+        </classpath>
+      </junit>	
+  </target>
+
   <!-- Runs Derby's JMX tests if the required VM level (1.5 or higher) is detected -->
   <target name="junit-jmx" depends="checkVMLevel,junit-init" if="vmLevelIsAtLeast1.5">
     <junit  printsummary="on"
@@ -2095,7 +2122,7 @@
       
   </target>
    
-  <target name="junit-all" depends="junit-core,junit-jdbc4,junit-jmx">
+  <target name="junit-all" depends="junit-core,junit-jdbc4,junit-jmx,junit-lowmem">
   </target>
  
   <target name="junitreport" depends="junit-all,junit-html"/>

Modified: db/derby/code/trunk/java/engine/org/apache/derby/iapi/types/SQLBinary.java
URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/engine/org/apache/derby/iapi/types/SQLBinary.java?rev=672818&r1=672817&r2=672818&view=diff
==============================================================================
--- db/derby/code/trunk/java/engine/org/apache/derby/iapi/types/SQLBinary.java (original)
+++ db/derby/code/trunk/java/engine/org/apache/derby/iapi/types/SQLBinary.java Mon Jun 30
09:45:38 2008
@@ -229,7 +229,7 @@
 		}
 		catch (IOException ioe)
 		{
-			throw StandardException.newException(SQLState.LANG_STREAMING_COLUMN_I_O_EXCEPTION, ioe,
getTypeName());
+			throwStreamingIOException(ioe);
 		}
 		return dataValue;
 	}
@@ -242,12 +242,49 @@
 	public final int	getLength() throws StandardException
 	{
 		if (stream != null) {
-
 			if (streamValueLength != -1)
 				return streamValueLength;
+			else if (stream instanceof Resetable){
+				try {
+					// If we have the stream length encoded.
+					// just read that.
+					streamValueLength = readBinaryLength((ObjectInput) stream);
+					if (streamValueLength != 0)
+						return streamValueLength;
+					// Otherwise we will have to read the whole stream.
+					for (;;) {
+						long skipsize = stream.skip(Integer.MAX_VALUE);
+						streamValueLength += skipsize;
+						if (stream.read() == -1)
+							break;
+						else
+							streamValueLength++;
+					}
+					return streamValueLength;
+				}
+				catch (IOException ioe) {
+					throwStreamingIOException(ioe);
+				}
+				finally {
+					try {
+						((Resetable) stream).resetStream();
+					} catch (IOException ioe) {
+						throwStreamingIOException(ioe);
+					}
+				}
+				
+			}
 		}
+		byte[] bytes = getBytes();
+		return (bytes == null) ? 0 : bytes.length;
+		
+	}
+
 
-		return (getBytes() == null) ? 0 : getBytes().length;
+	private void throwStreamingIOException(IOException ioe) throws StandardException {
+		throw StandardException.
+			newException(SQLState.LANG_STREAMING_COLUMN_I_O_EXCEPTION,
+						 ioe, getTypeName());
 	}
 
 	/*

Modified: db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/suites/AllPackages.java
URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/suites/AllPackages.java?rev=672818&r1=672817&r2=672818&view=diff
==============================================================================
--- db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/suites/AllPackages.java
(original)
+++ db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/suites/AllPackages.java
Mon Jun 30 09:45:38 2008
@@ -59,6 +59,7 @@
 	 suite.addTest(org.apache.derbyTesting.functionTests.tests.tools._Suite.suite());
         suite.addTest(org.apache.derbyTesting.functionTests.tests.engine._Suite.suite());
         suite.addTest(org.apache.derbyTesting.functionTests.tests.demo._Suite.suite());
+        suite.addTest(org.apache.derbyTesting.functionTests.tests.memory._Suite.suite());
 
         // Suites that are compiled using Java SE 6 target need to
         // be added this way, otherwise creating the suite

Added: db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memory/BlobMemTest.java
URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memory/BlobMemTest.java?rev=672818&view=auto
==============================================================================
--- db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memory/BlobMemTest.java
(added)
+++ db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memory/BlobMemTest.java
Mon Jun 30 09:45:38 2008
@@ -0,0 +1,198 @@
+/**
+ * Derby - Class org.apache.derbyTesting.functionTests.tests.memory.BlobMemTest
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.derbyTesting.functionTests.tests.memory;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+import java.sql.PreparedStatement;
+import java.util.Arrays;
+import java.util.Properties;
+
+import junit.framework.Test;
+import junit.framework.TestSuite;
+
+import org.apache.derbyTesting.functionTests.harness.JavaVersionHolder;
+import org.apache.derbyTesting.functionTests.tests.lang.SimpleTest;
+import org.apache.derbyTesting.functionTests.util.streams.LoopingAlphabetStream;
+import org.apache.derbyTesting.junit.BaseJDBCTestCase;
+import org.apache.derbyTesting.junit.DatabasePropertyTestSetup;
+import org.apache.derbyTesting.junit.JDBC;
+import org.apache.derbyTesting.junit.SystemPropertyTestSetup;
+import org.apache.derbyTesting.junit.TestConfiguration;
+
+public class BlobMemTest extends BaseJDBCTestCase {
+
+    private static final int LONG_BLOB_LENGTH = 18000000;
+    private static final String LONG_BLOB_LENGTH_STRING= "18000000";
+    private static final byte[] SHORT_BLOB_BYTES = new byte[] {0x01,0x02,0x03};
+
+    public BlobMemTest(String name) {
+        super(name);
+    }
+
+    public void tearDown() throws SQLException {
+        rollback();
+    }
+    
+    
+    
+    /**
+     * Insert a blob and test length.    
+     * 
+     * @param lengthless  if true use the lengthless setBinaryStream api
+     * 
+     * @throws SQLException
+     * @throws IOException 
+     * @throws InvocationTargetException 
+     * @throws IllegalAccessException 
+     * @throws IllegalArgumentException 
+     */
+    private void testBlobLength(boolean lengthless) throws SQLException, IOException, IllegalArgumentException,
IllegalAccessException, InvocationTargetException {
+        setAutoCommit(false);
+        Statement s = createStatement();
+        s.executeUpdate("CREATE TABLE BLOBTAB (K INT CONSTRAINT PK PRIMARY KEY, B BLOB("
+ LONG_BLOB_LENGTH + "))");
+        
+        PreparedStatement ps = prepareStatement("INSERT INTO BLOBTAB VALUES(?,?)");
+        // We allocate 16MB for the test so use something bigger than that.
+        ps.setInt(1,1);
+        LoopingAlphabetStream stream = new LoopingAlphabetStream(LONG_BLOB_LENGTH);
+        if (lengthless) {
+            Method m = null;
+            try {
+                Class c = ps.getClass();
+                m = c.getMethod("setBinaryStream",new Class[] {Integer.TYPE,
+                            InputStream.class});                
+            } catch (NoSuchMethodException e) {
+                // ignore method not found as method may not be present for 
+                // jdk's lower than 1.6.
+                println("Skipping lengthless insert because method is not available");
+                return;                
+            }
+            m.invoke(ps, new Object[] {new Integer(2),stream});
+        }
+        else
+            ps.setBinaryStream(2, stream,LONG_BLOB_LENGTH);
+        ps.executeUpdate();
+        // insert a zero length blob.
+        ps.setInt(1, 2);
+        ps.setBytes(2, new byte[] {});
+        ps.executeUpdate();
+        // insert a null blob.
+        ps.setInt(1, 3);
+        ps.setBytes(2,null);
+        ps.executeUpdate();
+        // insert a short blob
+        ps.setInt(1, 4);
+        ps.setBytes(2, SHORT_BLOB_BYTES);
+        ps.executeUpdate();
+        // Currently need to use optimizer override to force use of the index.
+        // Derby should use sort avoidance and do it automatically, but there
+        // appears to be a bug.
+        ResultSet rs = s.executeQuery("SELECT K, LENGTH(B), B FROM BLOBTAB" +
+                "-- DERBY-PROPERTIES constraint=pk\n ORDER BY K"); 
+        rs.next();
+        assertEquals(LONG_BLOB_LENGTH_STRING,rs.getString(2));
+        // make sure we can still access the blob after getting length.
+        // It should be ok because we reset the stream
+        InputStream rsstream = rs.getBinaryStream(3);
+        int len= 0;
+        byte[] buf = new byte[32672];
+        for (;;)  {
+                int size = rsstream.read(buf);
+                if (size == -1)
+                        break;
+                len += size;
+                int expectedValue = ((len -1) % 26) + 'a';
+                if (size != 0)
+                    assertEquals(expectedValue,buf[size -1]);      
+        }
+
+        assertEquals(LONG_BLOB_LENGTH,len);
+        // empty blob
+        rs.next();
+        assertEquals("0",rs.getString(2));
+        byte[] bytes = rs.getBytes(3);
+        assertEquals(0, bytes.length);
+        // null blob
+        rs.next();
+        assertEquals(null,rs.getString(2));
+        bytes = rs.getBytes(3);
+        assertEquals(null,bytes);
+        // short blob
+        rs.next();
+        assertEquals("3",rs.getString(2));
+        bytes = rs.getBytes(3);
+        assertTrue(Arrays.equals(SHORT_BLOB_BYTES, bytes));
+        rs.close();         
+        
+        // Select just length without selecting the blob.
+        rs = s.executeQuery("SELECT K, LENGTH(B)  FROM BLOBTAB " +
+                "ORDER BY K");
+        JDBC.assertFullResultSet(rs, new String [][] {{"1",LONG_BLOB_LENGTH_STRING},{"2","0"},
+                {"3",null},{"4","3"}});
+    }
+    
+    /**
+     * Test the length after inserting with the setBinaryStream api 
+     * that takes length.  In this case the length will be encoded at the
+     * begining of the stream and the call should be fairly low overhead.
+     * 
+     * @throws SQLException
+     * @throws IOException 
+     * @throws InvocationTargetException 
+     * @throws IllegalAccessException 
+     * @throws IllegalArgumentException 
+     */
+    public void testBlobLength() throws SQLException, IOException, IllegalArgumentException,
IllegalAccessException, InvocationTargetException {
+        testBlobLength(false);
+    }
+    
+    /**
+     * Test the length after inserting the blob value with the lengthless
+     * setBinaryStream api. In this case we will have to read the whole 
+     * stream to get the length.
+     * 
+     * @throws SQLException
+     * @throws IOException 
+     * @throws InvocationTargetException 
+     * @throws IllegalAccessException 
+     * @throws IllegalArgumentException 
+     */
+    public void testBlobLengthWithLengthlessInsert() throws SQLException, IOException, IllegalArgumentException,
IllegalAccessException, InvocationTargetException {        
+        testBlobLength(true);  
+    }
+       public static Test suite() {
+        Test suite =  TestConfiguration.defaultSuite(BlobMemTest.class);
+        Properties p = new Properties();
+        // use small pageCacheSize so we don't run out of memory on the insert.
+        p.setProperty("derby.storage.pageCacheSize", "100");
+        return new SystemPropertyTestSetup(suite,p);
+    }
+
+      
+
+}

Propchange: db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memory/BlobMemTest.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memory/_Suite.java
URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memory/_Suite.java?rev=672818&view=auto
==============================================================================
--- db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memory/_Suite.java
(added)
+++ db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memory/_Suite.java
Mon Jun 30 09:45:38 2008
@@ -0,0 +1,41 @@
+/**
+ * Derby - Class org.apache.derbyTesting.functionTests.tests.memory._Suite
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.derbyTesting.functionTests.tests.memory;
+
+import junit.framework.Test;
+import junit.framework.TestSuite;
+
+import org.apache.derbyTesting.junit.BaseJDBCTestCase;
+
+public class _Suite extends BaseJDBCTestCase {
+
+    public _Suite(String name) {
+        super(name);
+      
+    }
+
+    public static Test suite() throws Exception{
+        TestSuite suite = new TestSuite("Memory Suite");
+        suite.addTest(BlobMemTest.suite());
+                return suite;
+    }
+}

Propchange: db/derby/code/trunk/java/testing/org/apache/derbyTesting/functionTests/tests/memory/_Suite.java
------------------------------------------------------------------------------
    svn:eol-style = native



Mime
View raw message