db-derby-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sure...@apache.org
Subject svn commit: r520197 - in /db/derby/code/trunk/java/engine/org/apache/derby/impl/load: ImportAbstract.java ImportClob.java ImportLobFile.java ImportReadData.java
Date Tue, 20 Mar 2007 00:40:23 GMT
Author: suresht
Date: Mon Mar 19 17:40:22 2007
New Revision: 520197

URL: http://svn.apache.org/viewvc?view=rev&rev=520197
Log:
DERBY-378 (partial)
This patch adds some code required to support import/exoprt of lob data.
1) Addded  code to read clob data using getCharacterStream()
instead of getString() while importing clob data from an extern file. 
(Note: Clobs are read using getString() until DERBY-2465 is fixed).
2) Made some code changes to make each lob column has it it's own file handle to
   the lob file to read the data, otherwise streams can get corrupted when 
   there are more than one clob/blob type column in the table.



Added:
    db/derby/code/trunk/java/engine/org/apache/derby/impl/load/ImportClob.java   (with props)
Modified:
    db/derby/code/trunk/java/engine/org/apache/derby/impl/load/ImportAbstract.java
    db/derby/code/trunk/java/engine/org/apache/derby/impl/load/ImportLobFile.java
    db/derby/code/trunk/java/engine/org/apache/derby/impl/load/ImportReadData.java

Modified: db/derby/code/trunk/java/engine/org/apache/derby/impl/load/ImportAbstract.java
URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/engine/org/apache/derby/impl/load/ImportAbstract.java?view=diff&rev=520197&r1=520196&r2=520197
==============================================================================
--- db/derby/code/trunk/java/engine/org/apache/derby/impl/load/ImportAbstract.java (original)
+++ db/derby/code/trunk/java/engine/org/apache/derby/impl/load/ImportAbstract.java Mon Mar
19 17:40:22 2007
@@ -145,12 +145,13 @@
 
     if (columnIndex <= numberOfColumns) {
 		String val = nextRow[columnIndex-1];
-		if (isColumnInExtFile(columnIndex)) {
+        if (isColumnInExtFile(columnIndex)) {
             // a clob column data is stored in an external 
             // file, the reference to it is in the main file. 
             // read the data from the external file using the 
             // reference from the main file. 
-			val = importReadData.getClobColumnFromExtFile(val);
+			val = importReadData.getClobColumnFromExtFileAsString(val, 
+                                                                  columnIndex);
         }
 		wasNull = (val == null);
 		return val;
@@ -160,6 +161,33 @@
     }
   }
 
+
+    /**
+     * Returns <code> java.sql.Clob </code> type object that 
+     * contains the columnn data from the import file. 
+     * @param columnIndex number of the column. starts at 1.
+     * @exception SQLException if any occurs during create of the clob object.
+     */
+	public java.sql.Clob getClob(int columnIndex) throws SQLException {
+
+        java.sql.Clob clob = null;
+		if (lobsInExtFile) 
+        {
+            // lob data is in another file, read from the external file.
+            clob = importReadData.getClobColumnFromExtFile(
+                                         nextRow[columnIndex-1], columnIndex);
+        } else {
+            // data is in the main export file.
+            String data =  nextRow[columnIndex-1];
+            if (data != null) {
+                clob = new ImportClob(data);                
+            }
+        }
+        
+        wasNull = (clob == null);
+        return clob;
+	}
+
 	
     /**
      * Returns <code> java.sql.Blob </code> type object that 
@@ -174,7 +202,7 @@
         {
             // lob data is in another file, read from the external file.
             blob = importReadData.getBlobColumnFromExtFile(
-                                         nextRow[columnIndex-1]);
+                                         nextRow[columnIndex-1], columnIndex);
         } else {
             // data is in the main export file, stored in hex format.
             String hexData = nextRow[columnIndex-1];
@@ -211,6 +239,8 @@
         wasNull = (blob == null);
         return blob;
 	}
+
+
 
 
     /**

Added: db/derby/code/trunk/java/engine/org/apache/derby/impl/load/ImportClob.java
URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/engine/org/apache/derby/impl/load/ImportClob.java?view=auto&rev=520197
==============================================================================
--- db/derby/code/trunk/java/engine/org/apache/derby/impl/load/ImportClob.java (added)
+++ db/derby/code/trunk/java/engine/org/apache/derby/impl/load/ImportClob.java Mon Mar 19
17:40:22 2007
@@ -0,0 +1,227 @@
+/*
+
+   Derby - Class org.apache.derby.impl.load.ImportClob
+
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
+ */
+
+package org.apache.derby.impl.load;
+import org.apache.derby.iapi.services.io.LimitReader;
+import java.sql.Clob;
+import java.sql.SQLException;
+import java.io.Reader;
+import java.io.IOException;
+
+/**
+ * This class implements  <code > java.sql.CLOB interface </code>.  
+ * Objects created using the <code> ImportClob </code> class  are 
+ * intended to be be used to create a clob object of the data  stored 
+ * in an import file.  Only the routines that are needed  to read the 
+ * clob data for the clob columns by the  inserts done through the VTI  
+ * have real implementations,  Other routines are dummy ones to satisfy
+ * <code> java.sql.Clob </code>  interface.
+ */
+
+class ImportClob implements java.sql.Clob {
+
+    private ImportLobFile lobFile;
+	private long position; // postion in the import file in bytes.
+    private long length;  // length in bytes
+	private long clobLength; // length of clob in chars. 
+    private String clobData = null;
+
+
+    /**
+     * Create a import Clob object, that reads <code> length </code> amount of

+     * data  from an external file, starting at <code> position </code>. 
+     * @param lobFile  lob file resource object, using which data is read.
+     * @param position  byte offset in the file, of this clob column data. 
+     * @param length   length of this clob object data in bytes. 
+     */
+    public ImportClob(ImportLobFile lobFile, long position, long length) 
+        throws IOException 
+	{
+		this.lobFile = lobFile;
+		this.position = position;
+        this.length = length;
+		this.clobLength = lobFile.getClobDataLength(position, length);
+	}
+
+
+    /**
+     * Create a Clob object, whose value is given as string.  
+     * @param data  String that contains the clob data. 
+     */
+    public ImportClob(String data) 
+    {
+       clobData = data;
+       clobLength = data.length();
+    }
+
+
+  /**
+   * Returns the number of characters in this <code>CLOB</code>  object.
+   * @return length of the <code>CLOB</code> in characters
+   * @exception SQLException on any error.
+   */
+	public long length() throws SQLException {
+		return clobLength;
+	}
+
+
+    /**
+     * Returns  <code>CLOB</code> value designated by this
+     * <code>Clob</code> object as a <code> Reader </code>.
+     *
+     * @return a Reader containing the <code>CLOB</code> data.
+     * @exception SQLException if any error occurs while setting up 
+     *                         this clob data in the import file as Reader. 
+     * @see java.sql.Clob 
+     */
+    public java.io.Reader getCharacterStream() throws SQLException {
+        try {
+            Reader ir;
+            if(clobData != null) {
+                // the data is string is already in the user spefied code set.
+                ir = new java.io.StringReader(clobData);
+                // wrap the Reader with a LimitReader class,
+                // so that only the data of the clob length is read.
+                LimitReader lr = new  LimitReader(ir);
+                lr.setLimit((int) clobLength);
+                return lr;
+            } else {
+                return lobFile.getCharacterStream(position, length);
+            }
+		} catch (Exception e) {
+			throw LoadError.unexpectedError(e);
+		}
+    }
+
+
+        
+    /** following rotines does not have implmentation because they
+     * are not used by the VTI that is used to import the data. 
+     */
+
+    /**
+     * This routine is not used by the VTI to read the data, so no 
+     * implementation is provided, an exception is thrown if it is
+     * called.  
+     *
+     * @see java.sql.Clob 
+     */
+    public String getSubString(long pos, int length) throws SQLException {
+        throw LoadError.unexpectedError(
+                         new Exception("Method not implemented"));
+    }
+
+
+    /**
+     * This routine is not used by the VTI to read the data, so no 
+     * implementation is provided, an exception is thrown if it is
+     * called.  
+     * @see java.sql.Clob 
+     */
+    public java.io.InputStream getAsciiStream() throws SQLException {
+        throw LoadError.unexpectedError(
+                         new Exception("Method not implemented")); 
+    }
+
+
+    /** 
+     * This routine is not used by the VTI to read the data, so no 
+     * implementation is provided, an exception is thrown if it is
+     * called.  
+     * @see java.sql.Clob 
+     */
+    public long position(String searchstr, long start) throws SQLException {
+        throw LoadError.unexpectedError(
+                         new Exception("Method not implemented")); 
+    }
+
+    /** 
+     * This routine is not used by the VTI to read the data, so no 
+     * implementation is provided, an exception is thrown if it is
+     * called.  
+     * @see java.sql.Clob 
+     */
+    public long position(Clob searchstr, long start) throws SQLException {
+        throw LoadError.unexpectedError(
+                         new Exception("Method not implemented")); 
+    }
+
+    /**
+     * This routine is not used by the VTI to read the data, so no 
+     * implementation is provided, an exception is thrown if it is
+     * called.  
+     * @see java.sql.Clob 
+     */
+    public int setString(long pos, String str) throws SQLException {
+        throw LoadError.unexpectedError(
+                         new Exception("Method not implemented"));  
+    }
+
+    /**
+     * This routine is not used by the VTI to read the data, so no 
+     * implementation is provided, an exception is thrown if it is
+     * called.  
+     * @see java.sql.Clob 
+     */
+    public int setString(long pos, String str, int offset, int len) 
+        throws SQLException
+    {
+        throw LoadError.unexpectedError(
+                         new Exception("Method not implemented")); 
+    }
+
+    /**
+     * This routine is not used by the VTI to read the data, so no 
+     * implementation is provided, an exception is thrown if it is
+     * called.  
+     * @see java.sql.Clob 
+     */
+    public java.io.OutputStream setAsciiStream(long pos) throws SQLException
+    {
+        throw LoadError.unexpectedError(
+                         new Exception("Method not implemented")); 
+    }
+
+    /**
+     * This routine is not used by the VTI to read the data, so no 
+     * implementation is provided, an exception is thrown if it is
+     * called.  
+     * @see java.sql.Clob 
+     */
+    public java.io.Writer setCharacterStream(long pos) throws SQLException
+    {
+        throw LoadError.unexpectedError(
+                         new Exception("Method not implemented")); 
+     }
+
+    /**
+     * This routine is not used by the VTI to read the data, so no 
+     * implementation is provided, an exception is thrown if it is
+     * called.  
+     * @see java.sql.Clob 
+     */
+    public void truncate(long len) throws SQLException 
+    {
+        throw LoadError.unexpectedError(
+                         new Exception("Method not implemented"));  
+    }
+}
+

Propchange: db/derby/code/trunk/java/engine/org/apache/derby/impl/load/ImportClob.java
------------------------------------------------------------------------------
    svn:eol-style = native

Modified: db/derby/code/trunk/java/engine/org/apache/derby/impl/load/ImportLobFile.java
URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/engine/org/apache/derby/impl/load/ImportLobFile.java?view=diff&rev=520197&r1=520196&r2=520197
==============================================================================
--- db/derby/code/trunk/java/engine/org/apache/derby/impl/load/ImportLobFile.java (original)
+++ db/derby/code/trunk/java/engine/org/apache/derby/impl/load/ImportLobFile.java Mon Mar
19 17:40:22 2007
@@ -91,13 +91,7 @@
         // of data is read from the file, for example to read one 
         // column data from the file. 
         lobLimitIn = new  LimitInputStream(lobInputStream);
-
-        // setup a reader on top of the stream, so that calls 
-        // to read the clob data from the file can read the 
-        // with approapriate  data code set. 
-        lobReader = dataCodeset == null ?
-    		new InputStreamReader(lobLimitIn) : 
-            new InputStreamReader(lobLimitIn, dataCodeset);    
+   
     }
 
 
@@ -128,6 +122,13 @@
         lobInputStream.seek(offset);
         lobLimitIn.clearLimit();
         lobLimitIn.setLimit((int) length);
+        
+        // wrap a reader on top of the stream, so that calls 
+        // to read the clob data from the file can read the 
+        // with approapriate  data code set. 
+        lobReader = dataCodeset == null ?
+    		new InputStreamReader(lobLimitIn) : 
+            new InputStreamReader(lobLimitIn, dataCodeset);    
 
         // read data from the file, and return it as string. 
         StringBuffer sb = new StringBuffer();
@@ -140,6 +141,60 @@
 		return sb.toString();
     }
 
+
+    /* 
+     * Returns a stream that points to the clob data from file at the 
+     * given <code> offset </code>.
+     * @param offset  byte offset of the column data in the file. 
+     * @param length  length of the the data in bytes.
+     * @exception  IOException  on any I/O error.     
+     */
+    public java.io.Reader getCharacterStream(long offset, long length) 
+        throws IOException 
+    {
+        lobInputStream.seek(offset);
+        lobLimitIn.clearLimit();
+        lobLimitIn.setLimit((int) length);
+
+        // wrap a reader on top of the stream, so that calls 
+        // to read the clob data from the file can read the 
+        // with approapriate  data code set. 
+        lobReader = dataCodeset == null ?
+    		new InputStreamReader(lobLimitIn) : 
+            new InputStreamReader(lobLimitIn, dataCodeset);    
+
+        return lobReader;
+    }
+
+    /*
+     * Returns the clob data length in characters at the give location. 
+     * @param offset  byte offset of the column data in the file. 
+     * @param length  length of the the data in bytes.
+     * @exception  IOException  on any I/O error.     
+     */
+    public long getClobDataLength(long offset, long length) throws IOException {
+        lobInputStream.seek(offset);
+        lobLimitIn.clearLimit();
+        lobLimitIn.setLimit((int) length);
+        
+        // wrap a reader on top of the stream, so that calls 
+        // to read the clob data from the file can read the 
+        // with approapriate  data code set. 
+        lobReader = dataCodeset == null ?
+            new InputStreamReader(lobLimitIn) : 
+            new InputStreamReader(lobLimitIn, dataCodeset);   
+
+        // find the length in characters 
+        char[] buf= new char[1024];
+        long lengthInChars = 0;
+        int noChars = lobReader.read(buf , 0 , 1024);
+        while (noChars != -1) {
+            lengthInChars += noChars;
+            noChars = lobReader.read(buf , 0 , 1024);
+        }
+        
+        return lengthInChars;
+    }
 
     /* 
      * close all the resources realate to the lob file.

Modified: db/derby/code/trunk/java/engine/org/apache/derby/impl/load/ImportReadData.java
URL: http://svn.apache.org/viewvc/db/derby/code/trunk/java/engine/org/apache/derby/impl/load/ImportReadData.java?view=diff&rev=520197&r1=520196&r2=520197
==============================================================================
--- db/derby/code/trunk/java/engine/org/apache/derby/impl/load/ImportReadData.java (original)
+++ db/derby/code/trunk/java/engine/org/apache/derby/impl/load/ImportReadData.java Mon Mar
19 17:40:22 2007
@@ -105,7 +105,7 @@
 
 
   // variables realted to reading lob data from files.
-  private ImportLobFile lobFile; // lob file object 
+  private ImportLobFile[] lobFileHandles; // lob file handle object 
   private String lobFileName; // current file name
   private int lobOffset; // offset of the current large object
   private int lobLength; //length of the current large object
@@ -153,6 +153,9 @@
     //read the first row to find how many columns make a row and then save that
     //column information for further use
     loadMetaData();
+
+    lobFileHandles = new ImportLobFile[numberOfColumns];
+
   }
 
   //just a getter returning number of columns for a row in the data file
@@ -281,9 +284,14 @@
     }
 
     // close external lob file resources.
-    if (lobFile !=null) {
-        lobFile.close();
+    if (lobFileHandles != null) {
+        for (int i = 0 ; i < numberOfColumns ; i++) 
+        {
+            if(lobFileHandles[i] != null) 
+                lobFileHandles[i].close();
+        }
     }
+
   }
 
   //actually looks at the data file to find how many columns make up a row
@@ -934,22 +942,51 @@
      * to external file is stored in the main import file.
      */
 
+    
     /**
      * Returns a clob columnn data stored at the specified location.
      * @param lobLocationStr location of the clob data.
+     * @param colIndex number of the column. starts at 1.      
      * @exception  SQLException  on any errors. 
      */
-    String getClobColumnFromExtFile(String lobLocationStr) 
+    String getClobColumnFromExtFileAsString(String lobLocationStr, int colIndex) 
         throws SQLException 
     {
 		try {
-            initExternalLobFile(lobLocationStr);
+            initExternalLobFile(lobLocationStr, colIndex);
             if (lobLength == -1 ){
                 // lob length -1 indicates columnn value is a NULL, 
                 // just return null. 
                 return null;
             } else {
-                return lobFile.getString(lobOffset,lobLength);
+                return lobFileHandles[colIndex-1].getString(lobOffset,lobLength);
+            }
+            
+		}catch(Exception ex) {
+			throw LoadError.unexpectedError(ex);
+		}
+	}
+
+
+    /**
+     * Returns a clob columnn data stored at the specified location as
+     * a java.sql.Clob object. 
+     * @param lobLocationStr location of the clob data.
+     * @param colIndex number of the column. starts at 1. 
+     * @exception  SQLException  on any errors. 
+     */
+    java.sql.Clob getClobColumnFromExtFile(String lobLocationStr, int colIndex) 
+        throws SQLException 
+    {
+		try {
+            initExternalLobFile(lobLocationStr, colIndex);
+            if (lobLength == -1 ){
+                // lob length -1 indicates columnn value is a NULL, 
+                // just return null. 
+                return null;
+            } else {
+                return new ImportClob(lobFileHandles[colIndex -1],
+                                      lobOffset,lobLength);
             }
             
 		}catch(Exception ex) {
@@ -961,19 +998,21 @@
      * Returns a blob columnn data stored at the specified location as
      * a java.sql.Blob object. 
      * @param lobLocationStr location of the clob data.
+     * @param colIndex number of the column. starts at 1.                   
      * @exception  SQLException  on any errors. 
      */
-    java.sql.Blob getBlobColumnFromExtFile(String lobLocationStr)
+    java.sql.Blob getBlobColumnFromExtFile(String lobLocationStr, int colIndex)
         throws SQLException
     {
-        initExternalLobFile(lobLocationStr);
+        initExternalLobFile(lobLocationStr, colIndex);
         if (lobLength == -1) {
             // lob length -1 indicates columnn value is a NULL, 
             // just return null. 
             return null;
         }
         else {
-            return new ImportBlob(lobFile, lobOffset, lobLength);
+            return new ImportBlob(lobFileHandles[colIndex -1], 
+                                  lobOffset, lobLength);
         }
     }
 
@@ -983,9 +1022,10 @@
      * the file on first  invocaton. 
      *
      * @param lobLocationStr location of the clob data.
+     * @param colIndex number of the column. starts at 1.
      * @exception  SQLException  on any errors. 
      */
-    private void initExternalLobFile(String lobLocationStr) 
+    private void initExternalLobFile(String lobLocationStr, int colIndex) 
         throws SQLException 
     {
 		// extract file name, offset, and the length from the 
@@ -1004,20 +1044,20 @@
                                      offsetIndex+1, 
                                      lengthIndex));
         lobFileName = lobLocationStr.substring(0 , offsetIndex);
-
-        if (lobFile == null) {
+        if (lobFileHandles[colIndex-1] == null) {
             // open external file where the lobs are stored.
             try {
-                lobFile = new ImportLobFile(lobFileName, 
-                                            controlFileReader.getDataCodeset());
+                // each lob column in the table has it's own file handle. 
+                // separate file handles are must, lob stream objects
+                // can not be reused until the whole row is inserted.
+                lobFileHandles[colIndex-1] = new ImportLobFile(lobFileName, 
+                                         controlFileReader.getDataCodeset());
             }catch(Exception ex) {
                 throw LoadError.unexpectedError(ex);
             }
         }
     }
 }
-
-
 
 
 



Mime
View raw message