Return-Path: X-Original-To: apmail-db-derby-dev-archive@www.apache.org Delivered-To: apmail-db-derby-dev-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 47F321902B for ; Sat, 30 Apr 2016 20:40:13 +0000 (UTC) Received: (qmail 47031 invoked by uid 500); 30 Apr 2016 20:40:13 -0000 Delivered-To: apmail-db-derby-dev-archive@db.apache.org Received: (qmail 47003 invoked by uid 500); 30 Apr 2016 20:40:13 -0000 Mailing-List: contact derby-dev-help@db.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: Delivered-To: mailing list derby-dev@db.apache.org Received: (qmail 46993 invoked by uid 99); 30 Apr 2016 20:40:13 -0000 Received: from arcas.apache.org (HELO arcas) (140.211.11.28) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 30 Apr 2016 20:40:13 +0000 Received: from arcas.apache.org (localhost [127.0.0.1]) by arcas (Postfix) with ESMTP id F306C2C044E for ; Sat, 30 Apr 2016 20:40:12 +0000 (UTC) Date: Sat, 30 Apr 2016 20:40:12 +0000 (UTC) From: "Bryan Pendleton (JIRA)" To: derby-dev@db.apache.org Message-ID: In-Reply-To: References: Subject: [jira] [Commented] (DERBY-6884) SYSCS_IMPORT_TABLE_LOBS_FROM_EXTFILE can't import more than Integer.MAX_VALUE bytes of blob data MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 7bit X-JIRA-FingerPrint: 30527f35849b9dde25b450d4833f0394 [ https://issues.apache.org/jira/browse/DERBY-6884?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15265479#comment-15265479 ] Bryan Pendleton commented on DERBY-6884: ---------------------------------------- I believe I've got the test able to reproduce both the CLOB and BLOB issues. However, it still takes a very long time to run. And, I don't have enough disk space on my (virtual) machine to run the test anymore, so I'll need to add some more disk space. It's clear that these tests don't belong in the regular test suite, so I'll look into placing these tests into the "largeData" suite suggested by Mike. > SYSCS_IMPORT_TABLE_LOBS_FROM_EXTFILE can't import more than Integer.MAX_VALUE bytes of blob data > ------------------------------------------------------------------------------------------------ > > Key: DERBY-6884 > URL: https://issues.apache.org/jira/browse/DERBY-6884 > Project: Derby > Issue Type: Bug > Components: SQL > Affects Versions: 10.11.1.1 > Reporter: Edward Howe > Attachments: DerbyIssue.java, firstTryAtTest.diff, trivial.diff > > > Using SYSCS_EXPORT_TABLE_LOBS_TO_EXTFILE to export a table containing a blob column, SYSCS_IMPORT_TABLE_LOBS_FROM_EXTFILE will fail with a NumberFormatException if the offset for a blob record is > Integer.MAX_VALUE. This is because ImportReadData.initExternalLobFile() is parsing the offset as an Integer. > The stack trace and a program to reproduce are below. > java.lang.NumberFormatException: For input string: "2147483770" > at java.lang.NumberFormatException.forInputString(NumberFormatException.java:65) ~[na:1.8.0_45] > at java.lang.Integer.parseInt(Integer.java:583) ~[na:1.8.0_45] > at java.lang.Integer.parseInt(Integer.java:615) ~[na:1.8.0_45] > at org.apache.derby.impl.load.ImportReadData.initExternalLobFile(Unknown Source) ~[derby-10.11.1.1.jar:na] > at org.apache.derby.impl.load.ImportReadData.getBlobColumnFromExtFile(Unknown Source) ~[derby-10.11.1.1.jar:na] > at org.apache.derby.impl.load.ImportAbstract.getBlob(Unknown Source) ~[derby-10.11.1.1.jar:na] > at org.apache.derby.impl.load.Import.getBlob(Unknown Source) ~[derby-10.11.1.1.jar:na] > at org.apache.derby.iapi.types.SQLBlob.setValueFromResultSet(Unknown Source) ~[derby-10.11.1.1.jar:na] > at org.apache.derby.impl.sql.execute.VTIResultSet.populateFromResultSet(Unknown Source) ~[derby-10.11.1.1.jar:na] > at org.apache.derby.impl.sql.execute.VTIResultSet.getNextRowCore(Unknown Source) ~[derby-10.11.1.1.jar:na] > at org.apache.derby.impl.sql.execute.ProjectRestrictResultSet.getNextRowCore(Unknown Source) ~[derby-10.11.1.1.jar:na] > at org.apache.derby.impl.sql.execute.NormalizeResultSet.getNextRowCore(Unknown Source) ~[derby-10.11.1.1.jar:na] > at org.apache.derby.impl.sql.execute.NoPutResultSetImpl.getNextRowFromRowSource(Unknown Source) ~[derby-10.11.1.1.jar:na] > at org.apache.derby.impl.store.access.heap.HeapController.load(Unknown Source) ~[derby-10.11.1.1.jar:na] > at org.apache.derby.impl.store.access.heap.Heap.load(Unknown Source) ~[derby-10.11.1.1.jar:na] > at org.apache.derby.impl.store.access.RAMTransaction.loadConglomerate(Unknown Source) ~[derby-10.11.1.1.jar:na] > at org.apache.derby.impl.store.access.RAMTransaction.recreateAndLoadConglomerate(Unknown Source) ~[derby-10.11.1.1.jar:na] > at org.apache.derby.impl.sql.execute.InsertResultSet.bulkInsertCore(Unknown Source) ~[derby-10.11.1.1.jar:na] > at org.apache.derby.impl.sql.execute.InsertResultSet.open(Unknown Source) ~[derby-10.11.1.1.jar:na] > at org.apache.derby.impl.sql.GenericPreparedStatement.executeStmt(Unknown Source) ~[derby-10.11.1.1.jar:na] > at org.apache.derby.impl.sql.GenericPreparedStatement.execute(Unknown Source) ~[derby-10.11.1.1.jar:na] > ... 36 common frames omitted > ================================== > package blob; > import java.io.BufferedInputStream; > import java.io.File; > import java.io.FileInputStream; > import java.io.IOException; > import java.sql.*; > public final class DerbyIssue { > // derby url > public static final String DBURL = "jdbc:derby:testdb;create=true"; > // any random binary file such as a large image or document > public static final String BLOB_DATA_FILE = "..."; > public static final String EXPORT_TABLE_FILE = "table-data"; > public static final String EXPORT_BLOB_FILE = "blob-data"; > public static void main(String... args) throws Exception { > final DerbyIssue test = new DerbyIssue(); > test.run(); > } > public void run() throws Exception { > Class.forName("org.apache.derby.jdbc.ClientDriver").getConstructor().newInstance(); > try(final Connection con = DriverManager.getConnection(DBURL)) { > try (final Statement stmt = con.createStatement()) { > stmt.execute("CREATE TABLE TESTBLOB(id BIGINT, content BLOB)"); > } > System.out.printf("inserting test data%n"); > try (final PreparedStatement pstmt = con.prepareStatement("INSERT INTO TESTBLOB (id, content) VALUES (?, ?)")) { > long id = 1; > long byteCount = 0; > final File content = new File(BLOB_DATA_FILE); > while (byteCount < Integer.MAX_VALUE) { > insertBlob(pstmt, id, content); > id++; > byteCount += content.length(); > if (id % 100 == 0) { > System.out.printf("%d%n", byteCount); > } > } > insertBlob(pstmt, id, content); > byteCount += content.length(); > System.out.printf("%d bytes written to testblob table%n", byteCount); > } > final File exportFile = new File(EXPORT_TABLE_FILE); > final File blobFile = new File(EXPORT_BLOB_FILE); > try (final CallableStatement stmt = con.prepareCall( > "CALL SYSCS_UTIL.SYSCS_EXPORT_TABLE_LOBS_TO_EXTFILE (null, ?, ?, null, null, null, ?)")) { > stmt.setString(1, "TESTBLOB"); > stmt.setString(2, exportFile.toString()); > stmt.setString(3, blobFile.toString()); > stmt.execute(); > } > System.out.printf("testblob table exported%n"); > try (final Statement stmt = con.createStatement()) { > stmt.execute("TRUNCATE TABLE TESTBLOB"); > } > System.out.printf("testblob table truncated%n"); > try (final CallableStatement stmt = con.prepareCall( > "CALL SYSCS_UTIL.SYSCS_IMPORT_TABLE_LOBS_FROM_EXTFILE (null, ?, ?, null, null, null, 0)")) { > stmt.setString(1, "TESTBLOB"); > stmt.setString(2, exportFile.toString()); > stmt.execute(); > } > System.out.printf("testblob data imported%n"); > } > } > private void insertBlob(PreparedStatement pstmt, long id, File content) throws IOException, SQLException { > try(BufferedInputStream contentStream = new BufferedInputStream(new FileInputStream(content))) { > pstmt.setLong(1, id); > pstmt.setBinaryStream(2, contentStream); > pstmt.executeUpdate(); > } > } > } -- This message was sent by Atlassian JIRA (v6.3.4#6332)