db-derby-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Knut Anders Hatlen (JIRA)" <j...@apache.org>
Subject [jira] [Commented] (DERBY-6884) SYSCS_IMPORT_TABLE_LOBS_FROM_EXTFILE can't import more than Integer.MAX_VALUE bytes of blob data
Date Mon, 02 May 2016 13:42:13 GMT

    [ https://issues.apache.org/jira/browse/DERBY-6884?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15266628#comment-15266628
] 

Knut Anders Hatlen commented on DERBY-6884:
-------------------------------------------

JustChangeOffset.diff looks good to me. Except that jardriftcheck fails when building the
jar files, because of the new class in derbyTesting.jar. +1 to commit when that's fixed.

A couple of nits:

{noformat}
--- java/engine/org/apache/derby/impl/load/ImportLobFile.java	(revision 1741376)
+++ java/engine/org/apache/derby/impl/load/ImportLobFile.java	(working copy)
@@ -128,7 +128,7 @@
      * @param length  length of the the data.
      * @exception  IOException  on any I/O error.     
      */
-    public String getString(int offset, int length) throws IOException {
+    public String getString(long offset, int length) throws IOException {
         lobInputStream.seek(offset);
         lobLimitIn.clearLimit();
         lobLimitIn.setLimit((int) length);
{noformat}

While you're at it, maybe also remove the redundant cast of length to int in the above call
to setLimit(), so that readers don't have to spend cycles figuring out what the purpose of
the cast is?

You might also want to clean up the indentation in the test case. It uses a mix of tabs and
spaces, and it doesn't always seem to agree with itself if tabs are 4 or 8 characters wide.

{noformat}
+        PreparedStatement ps = getConnection().prepareStatement(
+                     "insert into DERBY_6884_TESTCLOB values(? , ?)" );
{noformat}

BaseJDBCTestCase has a helper method for preparing statments, so the above could have been
replaced with the slightly simpler

{code}
        PreparedStatement ps = prepareStatement(
                     "insert into DERBY_6884_TESTCLOB values(? , ?)" );
{code}

Then you don't have to close the prepared statement manually in the test case.

> SYSCS_IMPORT_TABLE_LOBS_FROM_EXTFILE can't import more than Integer.MAX_VALUE bytes of
blob data
> ------------------------------------------------------------------------------------------------
>
>                 Key: DERBY-6884
>                 URL: https://issues.apache.org/jira/browse/DERBY-6884
>             Project: Derby
>          Issue Type: Bug
>          Components: SQL
>    Affects Versions: 10.11.1.1
>            Reporter: Edward Howe
>         Attachments: DerbyIssue.java, JustChangeOffset.diff, firstTryAtTest.diff, testForLargeDataSuite.diff,
trivial.diff
>
>
> Using SYSCS_EXPORT_TABLE_LOBS_TO_EXTFILE to export a table containing a blob column,
SYSCS_IMPORT_TABLE_LOBS_FROM_EXTFILE  will fail with a NumberFormatException if the offset
for a blob record is > Integer.MAX_VALUE.  This is because ImportReadData.initExternalLobFile()
is parsing the offset as an Integer.
> The stack trace and a program to reproduce are below.
> java.lang.NumberFormatException: For input string: "2147483770"
> 	at java.lang.NumberFormatException.forInputString(NumberFormatException.java:65) ~[na:1.8.0_45]
> 	at java.lang.Integer.parseInt(Integer.java:583) ~[na:1.8.0_45]
> 	at java.lang.Integer.parseInt(Integer.java:615) ~[na:1.8.0_45]
> 	at org.apache.derby.impl.load.ImportReadData.initExternalLobFile(Unknown Source) ~[derby-10.11.1.1.jar:na]
> 	at org.apache.derby.impl.load.ImportReadData.getBlobColumnFromExtFile(Unknown Source)
~[derby-10.11.1.1.jar:na]
> 	at org.apache.derby.impl.load.ImportAbstract.getBlob(Unknown Source) ~[derby-10.11.1.1.jar:na]
> 	at org.apache.derby.impl.load.Import.getBlob(Unknown Source) ~[derby-10.11.1.1.jar:na]
> 	at org.apache.derby.iapi.types.SQLBlob.setValueFromResultSet(Unknown Source) ~[derby-10.11.1.1.jar:na]
> 	at org.apache.derby.impl.sql.execute.VTIResultSet.populateFromResultSet(Unknown Source)
~[derby-10.11.1.1.jar:na]
> 	at org.apache.derby.impl.sql.execute.VTIResultSet.getNextRowCore(Unknown Source) ~[derby-10.11.1.1.jar:na]
> 	at org.apache.derby.impl.sql.execute.ProjectRestrictResultSet.getNextRowCore(Unknown
Source) ~[derby-10.11.1.1.jar:na]
> 	at org.apache.derby.impl.sql.execute.NormalizeResultSet.getNextRowCore(Unknown Source)
~[derby-10.11.1.1.jar:na]
> 	at org.apache.derby.impl.sql.execute.NoPutResultSetImpl.getNextRowFromRowSource(Unknown
Source) ~[derby-10.11.1.1.jar:na]
> 	at org.apache.derby.impl.store.access.heap.HeapController.load(Unknown Source) ~[derby-10.11.1.1.jar:na]
> 	at org.apache.derby.impl.store.access.heap.Heap.load(Unknown Source) ~[derby-10.11.1.1.jar:na]
> 	at org.apache.derby.impl.store.access.RAMTransaction.loadConglomerate(Unknown Source)
~[derby-10.11.1.1.jar:na]
> 	at org.apache.derby.impl.store.access.RAMTransaction.recreateAndLoadConglomerate(Unknown
Source) ~[derby-10.11.1.1.jar:na]
> 	at org.apache.derby.impl.sql.execute.InsertResultSet.bulkInsertCore(Unknown Source)
~[derby-10.11.1.1.jar:na]
> 	at org.apache.derby.impl.sql.execute.InsertResultSet.open(Unknown Source) ~[derby-10.11.1.1.jar:na]
> 	at org.apache.derby.impl.sql.GenericPreparedStatement.executeStmt(Unknown Source) ~[derby-10.11.1.1.jar:na]
> 	at org.apache.derby.impl.sql.GenericPreparedStatement.execute(Unknown Source) ~[derby-10.11.1.1.jar:na]
> 	... 36 common frames omitted
> ==================================
> package blob;
> import java.io.BufferedInputStream;
> import java.io.File;
> import java.io.FileInputStream;
> import java.io.IOException;
> import java.sql.*;
> public final class DerbyIssue {
>     // derby url
>     public static final String DBURL = "jdbc:derby:testdb;create=true";
>     // any random binary file such as a large image or document
>     public static final String BLOB_DATA_FILE = "...";
>     public static final String EXPORT_TABLE_FILE = "table-data";
>     public static final String EXPORT_BLOB_FILE = "blob-data";
>     public static void main(String... args) throws Exception {
>         final DerbyIssue test = new DerbyIssue();
>         test.run();
>     }
>     public void run() throws Exception {
>         Class.forName("org.apache.derby.jdbc.ClientDriver").getConstructor().newInstance();
>         try(final Connection con = DriverManager.getConnection(DBURL)) {
>             try (final Statement stmt = con.createStatement()) {
>                 stmt.execute("CREATE TABLE TESTBLOB(id BIGINT, content BLOB)");
>             }
>             System.out.printf("inserting test data%n");
>             try (final PreparedStatement pstmt = con.prepareStatement("INSERT INTO TESTBLOB
(id, content) VALUES (?, ?)")) {
>                 long id = 1;
>                 long byteCount = 0;
>                 final File content = new File(BLOB_DATA_FILE);
>                 while (byteCount < Integer.MAX_VALUE) {
>                     insertBlob(pstmt, id, content);
>                     id++;
>                     byteCount += content.length();
>                     if (id % 100 == 0) {
>                         System.out.printf("%d%n", byteCount);
>                     }
>                 }
>                 insertBlob(pstmt, id, content);
>                 byteCount += content.length();
>                 System.out.printf("%d bytes written to testblob table%n", byteCount);
>             }
>             final File exportFile = new File(EXPORT_TABLE_FILE);
>             final File blobFile = new File(EXPORT_BLOB_FILE);
>             try (final CallableStatement stmt = con.prepareCall(
>                     "CALL SYSCS_UTIL.SYSCS_EXPORT_TABLE_LOBS_TO_EXTFILE (null, ?, ?,
null, null, null, ?)")) {
>                 stmt.setString(1, "TESTBLOB");
>                 stmt.setString(2, exportFile.toString());
>                 stmt.setString(3, blobFile.toString());
>                 stmt.execute();
>             }
>             System.out.printf("testblob table exported%n");
>             try (final Statement stmt = con.createStatement()) {
>                 stmt.execute("TRUNCATE TABLE TESTBLOB");
>             }
>             System.out.printf("testblob table truncated%n");
>             try (final CallableStatement stmt = con.prepareCall(
>                     "CALL SYSCS_UTIL.SYSCS_IMPORT_TABLE_LOBS_FROM_EXTFILE (null, ?, ?,
null, null, null, 0)")) {
>                 stmt.setString(1, "TESTBLOB");
>                 stmt.setString(2, exportFile.toString());
>                 stmt.execute();
>             }
>             System.out.printf("testblob data imported%n");
>         }
>     }
>     private void insertBlob(PreparedStatement pstmt, long id, File content) throws IOException,
SQLException {
>         try(BufferedInputStream contentStream = new BufferedInputStream(new FileInputStream(content)))
{
>             pstmt.setLong(1, id);
>             pstmt.setBinaryStream(2, contentStream);
>             pstmt.executeUpdate();
>         }
>     }
> }



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

Mime
View raw message