Mailing-List: contact derby-commits-help@db.apache.org; run by ezmlm
Precedence: bulk
Reply-To: "Derby Development" <derby-dev@db.apache.org>
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r647312 - in
 /db/derby/code/branches/10.4/java/engine/org/apache/derby/impl/store/raw/data:
 FileContainer.java InputStreamContainer.java RAFContainer.java
 RAFContainer4.java
Date: Fri, 11 Apr 2008 21:12:47 -0000
To: derby-commits@db.apache.org
From: kahatlen@apache.org
Message-Id: <20080411211248.7EBB21A9832@eris.apache.org>

Author: kahatlen
Date: Fri Apr 11 14:12:42 2008
New Revision: 647312

URL: http://svn.apache.org/viewvc?rev=647312&view=rev
Log:
DERBY-3347: ERROR XSDB3: Container information cannot change once written

(merged from trunk, revision 647139)

On JVMs that support the NIO API, use NIO consistently when accessing
the data files. Mixing old style I/O and NIO has caused problems on
some platforms since thread-safety is only guaranteed if all access
happens through the FileChannel interface.

Modified:
    db/derby/code/branches/10.4/java/engine/org/apache/derby/impl/store/raw/data/FileContainer.java
    db/derby/code/branches/10.4/java/engine/org/apache/derby/impl/store/raw/data/InputStreamContainer.java
    db/derby/code/branches/10.4/java/engine/org/apache/derby/impl/store/raw/data/RAFContainer.java
    db/derby/code/branches/10.4/java/engine/org/apache/derby/impl/store/raw/data/RAFContainer4.java

Modified: db/derby/code/branches/10.4/java/engine/org/apache/derby/impl/store/raw/data/FileContainer.java
URL: http://svn.apache.org/viewvc/db/derby/code/branches/10.4/java/engine/org/apache/derby/impl/store/raw/data/FileContainer.java?rev=647312&r1=647311&r2=647312&view=diff
==============================================================================
--- db/derby/code/branches/10.4/java/engine/org/apache/derby/impl/store/raw/data/FileContainer.java (original)
+++ db/derby/code/branches/10.4/java/engine/org/apache/derby/impl/store/raw/data/FileContainer.java Fri Apr 11 14:12:42 2008
@@ -26,19 +26,11 @@
 import org.apache.derby.iapi.reference.Limits;
 import org.apache.derby.iapi.reference.SQLState;
 
-import org.apache.derby.impl.store.raw.data.BaseContainer;
-import org.apache.derby.impl.store.raw.data.BaseContainerHandle;
-import org.apache.derby.impl.store.raw.data.BasePage;
-import org.apache.derby.impl.store.raw.data.PageVersion;
-
 import org.apache.derby.iapi.services.cache.Cacheable;
 import org.apache.derby.iapi.services.cache.CacheManager;
 import org.apache.derby.iapi.services.context.ContextService;
-import org.apache.derby.iapi.services.daemon.DaemonService;
-import org.apache.derby.iapi.services.daemon.Serviceable;
 import org.apache.derby.iapi.services.monitor.Monitor;
 import org.apache.derby.iapi.services.sanity.SanityManager;
-import org.apache.derby.iapi.services.io.FormatIdUtil;
 import org.apache.derby.iapi.services.io.FormatIdOutputStream;
 import org.apache.derby.iapi.services.io.StoredFormatIds;
 import org.apache.derby.iapi.services.io.TypedFormat;
@@ -46,11 +38,8 @@
 import org.apache.derby.iapi.error.StandardException;
 import org.apache.derby.iapi.store.raw.ContainerHandle;
 import org.apache.derby.iapi.store.raw.ContainerKey;
-import org.apache.derby.iapi.store.raw.LockingPolicy;
-import org.apache.derby.iapi.store.raw.Loggable;
 import org.apache.derby.iapi.store.raw.Page;
 import org.apache.derby.iapi.store.raw.PageKey;
-import org.apache.derby.iapi.store.raw.PageTimeStamp;
 import org.apache.derby.iapi.store.raw.RecordHandle;
 import org.apache.derby.iapi.store.raw.RawStoreFactory;
 import org.apache.derby.iapi.store.raw.Transaction;
@@ -69,11 +58,11 @@
 
 import java.io.IOException;
 import java.io.DataInput;
-import java.io.DataOutput;
 
 import java.util.Properties;
 import java.util.zip.CRC32;
 
+import org.apache.derby.io.StorageRandomAccessFile;
 
 /**
 	FileContainer is an abstract base class for containers
@@ -692,39 +681,33 @@
     }
 
 	/**
-		Read the container's header.  Assumes the input stream (fileData)
-		is positioned at the beginning of the file.
+        Read the container's header.
 
-		Subclass that implements openContainer is expected to manufacture a DataInput 
-		stream which is used here to read the header.
+        When this method is called, the embryonic page that is passed in must
+        have been read directly from the file or the input stream, even if the
+        alloc page may still be in cache.  This is because a stubbify operation
+        only writes the stub to disk, it does not get rid of any stale page
+        from the page cache.  So if it so happens that the stubbified container
+        object is aged out of the container cache but the first alloc page
+        hasn't, then when any stale page of this container wants to be written
+        out, the container needs to be reopened, which is when this routine is
+        called.  We must not get the alloc page in cache because it may be
+        stale page and it may still say the container has not been dropped.
 
 		<BR> MT - single thread required - Enforced by caller.
 
+        @param epage the embryonic page to read the header from
 		@exception StandardException Derby Standard error policy
 		@exception IOException error in reading the header from file
 	*/
-	protected void readHeader(DataInput fileData) 
+	protected void readHeader(byte[] epage)
 		 throws IOException, StandardException
 	{
-		// Always read the header from the input stread even if the alloc page may
-		// still be in cache.  This is because a stubbify operation only writes
-		// the stub to disk, it did not get rid of any stale page from the page
-		// cache.  So if it so happen that the stubbified container object is
-		// aged out of the container cache but the first alloc page hasn't,
-		// then when any stale page of this container wants to be written out,
-		// the container needs to be reopened, which is when this routine is
-		// called.  We must not get the alloc page in cache because it may be
-		// stale page and it may still say the container has not been dropped.
-
-		byte[] epage = getEmbryonicPage(fileData);
-
 		// read persistent container header into containerInfo
 		AllocPage.ReadContainerInfo(containerInfo, epage);
 
 		// initialize header from information stored in containerInfo
 		readHeaderFromArray(containerInfo);
-
-		epage = null;
 	}
 
 	// initialize header information so this container object can be safely
@@ -869,8 +852,7 @@
 	}
 
 	/**
-		Write the container header directly to output stream (fileData).
-		Assumes the output stream is positioned at the beginning of the file.
+		Write the container header directly to file.
 
 		Subclasses that can writes the container header is expected to
 		manufacture a DataOutput stream which is used here.
@@ -880,7 +862,8 @@
 		@exception StandardException Derby Standard error policy
 		@exception IOException error in writing the header to file
 	 */
-	protected void writeHeader(DataOutput fileData, boolean create, byte[] epage)
+	protected void writeHeader(StorageRandomAccessFile file,
+                               boolean create, byte[] epage)
 		 throws IOException, StandardException
 	{
 		// write out the current containerInfo in the borrowed space to byte
@@ -903,7 +886,7 @@
 		dataFactory.writeInProgress();
 		try
 		{
-			fileData.write(epage);
+            writeAtOffset(file, epage, FIRST_ALLOC_PAGE_OFFSET);
 		}
 		finally
 		{
@@ -911,6 +894,23 @@
 		}
 	}
 
+    /**
+     * Write a sequence of bytes at the given offset in a file. This method
+     * is not thread safe, so the caller must make sure that no other thread
+     * is performing operations that may change current position in the file.
+     *
+     * @param file the file to write to
+     * @param bytes the bytes to write
+     * @param offset the offset to start writing at
+     * @throws IOException if an I/O error occurs while writing
+     */
+    void writeAtOffset(StorageRandomAccessFile file, byte[] bytes, long offset)
+            throws IOException
+    {
+        file.seek(offset);
+        file.write(bytes);
+    }
+
 	/**
 		Get an embryonic page from the dataInput stream.
 
@@ -933,6 +933,26 @@
 		}
 		return epage;
 	}
+
+    /**
+     * Read an embryonic page (that is, a section of the first alloc page that
+     * is so large that we know all the borrowed space is included in it) from
+     * the specified offset in a {@code StorageRandomAccessFile}. This method
+     * is not thread safe, so the caller must make sure that no other thread
+     * is performing operations that may change current position in the file.
+     *
+     * @param file the file to read from
+     * @param offset where to start reading (normally
+     * {@code FileContainer.FIRST_ALLOC_PAGE_OFFSET})
+     * @return a byte array containing the embryonic page
+     * @throws IOException if an I/O error occurs while reading
+     */
+    byte[] getEmbryonicPage(StorageRandomAccessFile file, long offset)
+            throws IOException
+    {
+        file.seek(offset);
+        return getEmbryonicPage(file);
+    }
 
 	/**
 		Write containerInfo into a byte array

Modified: db/derby/code/branches/10.4/java/engine/org/apache/derby/impl/store/raw/data/InputStreamContainer.java
URL: http://svn.apache.org/viewvc/db/derby/code/branches/10.4/java/engine/org/apache/derby/impl/store/raw/data/InputStreamContainer.java?rev=647312&r1=647311&r2=647312&view=diff
==============================================================================
--- db/derby/code/branches/10.4/java/engine/org/apache/derby/impl/store/raw/data/InputStreamContainer.java (original)
+++ db/derby/code/branches/10.4/java/engine/org/apache/derby/impl/store/raw/data/InputStreamContainer.java Fri Apr 11 14:12:42 2008
@@ -89,7 +89,7 @@
 			// of the first allocation page. And it is because we
 			// just opened the stream and the first allocation page
 			// is located at the beginning of the file.
-			readHeader(dis);
+			readHeader(getEmbryonicPage(dis));
 
 			return true;
 

Modified: db/derby/code/branches/10.4/java/engine/org/apache/derby/impl/store/raw/data/RAFContainer.java
URL: http://svn.apache.org/viewvc/db/derby/code/branches/10.4/java/engine/org/apache/derby/impl/store/raw/data/RAFContainer.java?rev=647312&r1=647311&r2=647312&view=diff
==============================================================================
--- db/derby/code/branches/10.4/java/engine/org/apache/derby/impl/store/raw/data/RAFContainer.java (original)
+++ db/derby/code/branches/10.4/java/engine/org/apache/derby/impl/store/raw/data/RAFContainer.java Fri Apr 11 14:12:42 2008
@@ -679,14 +679,11 @@
 		}
 		else
 		{
-			file.seek(FIRST_ALLOC_PAGE_OFFSET);
-			epage = getEmbryonicPage(file);
+			epage = getEmbryonicPage(file, FIRST_ALLOC_PAGE_OFFSET);
 		}
 
 		// need to check for frozen state
 
-
-		file.seek(FIRST_ALLOC_PAGE_OFFSET);
 		writeHeader(file, create, epage);
 
 		// leave the end of the file at a page boundry. This
@@ -1393,8 +1390,8 @@
              try {
 
                  fileData = file.getRandomAccessFile(canUpdate ? "rw" : "r");
-                 fileData.seek(FIRST_ALLOC_PAGE_OFFSET);
-                 readHeader(fileData);
+                 readHeader(getEmbryonicPage(fileData,
+                                             FIRST_ALLOC_PAGE_OFFSET));
 
                  if (SanityManager.DEBUG)
                  {
@@ -1435,7 +1432,8 @@
                          fileData = 
                              stub.getRandomAccessFile(canUpdate ? "rw" : "r");
 
-                         readHeader(fileData);
+                         readHeader(getEmbryonicPage(fileData,
+                                                     FIRST_ALLOC_PAGE_OFFSET));
                      }
                      catch (IOException ioe2)
                      {

Modified: db/derby/code/branches/10.4/java/engine/org/apache/derby/impl/store/raw/data/RAFContainer4.java
URL: http://svn.apache.org/viewvc/db/derby/code/branches/10.4/java/engine/org/apache/derby/impl/store/raw/data/RAFContainer4.java?rev=647312&r1=647311&r2=647312&view=diff
==============================================================================
--- db/derby/code/branches/10.4/java/engine/org/apache/derby/impl/store/raw/data/RAFContainer4.java (original)
+++ db/derby/code/branches/10.4/java/engine/org/apache/derby/impl/store/raw/data/RAFContainer4.java Fri Apr 11 14:12:42 2008
@@ -24,9 +24,7 @@
 
 import org.apache.derby.iapi.error.StandardException;
 import org.apache.derby.iapi.services.sanity.SanityManager;
-import org.apache.derby.iapi.services.io.FormatIdUtil;
 
-import org.apache.derby.impl.store.raw.data.BaseDataFileFactory;
 import org.apache.derby.iapi.store.raw.ContainerKey;
 
 import java.io.EOFException;
@@ -35,6 +33,7 @@
 import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
 import java.nio.channels.ClosedChannelException;
+import org.apache.derby.io.StorageRandomAccessFile;
 
 /**
  * RAFContainer4 overrides a few methods in RAFContainer in an attempt to use
@@ -84,6 +83,51 @@
         super(factory);
     }
 
+    /**
+     * Return the {@code FileChannel} for the specified
+     * {@code StorageRandomAccessFile} if it is a {@code RandomAccessFile}.
+     * Otherwise, return {@code null}.
+     *
+     * @param file the file to get the channel for
+     * @return a {@code FileChannel} if {@code file} is an instance of
+     * {@code RandomAccessFile}, {@code null} otherwise
+     */
+    private FileChannel getChannel(StorageRandomAccessFile file) {
+        if (file instanceof RandomAccessFile) {
+            /** XXX - this cast isn't testing friendly.
+             * A testing class that implements StorageRandomAccessFile but isn't
+             * a RandomAccessFile will be "worked around" by this class. An
+             * example of such a class is
+             * functionTests/util/corruptio/CorruptRandomAccessFile.java.
+             * An interface rework may be necessary.
+             */
+            return ((RandomAccessFile) file).getChannel();
+        }
+        return null;
+    }
+
+    /**
+     * <p>
+     * Return the file channel for the current value of the {@code fileData}
+     * field. If {@code fileData} doesn't support file channels, return
+     * {@code null}.
+     * </p>
+     *
+     * <p>
+     * Callers of this method must synchronize on the container object since
+     * two shared fields ({@code fileData} and {@code ourChannel}) are
+     * accessed.
+     * </p>
+     *
+     * @return a {@code FileChannel} object, if supported, or {@code null}
+     */
+    private FileChannel getChannel() {
+        if (ourChannel == null) {
+            ourChannel = getChannel(fileData);
+        }
+        return ourChannel;
+    }
+
     /*
      * Wrapping methods that retrieve the FileChannel from RAFContainer's
      * fileData after calling the real methods in RAFContainer.
@@ -95,21 +139,11 @@
             SanityManager.ASSERT(iosInProgress == 0,
                     "Container opened while IO operations are in progress. "
                     + "This should not happen.");
+            SanityManager.ASSERT(fileData == null, "fileData isn't null");
+            SanityManager.ASSERT(ourChannel == null, "ourChannel isn't null");
         }
 
-        boolean result = super.openContainer(newIdentity);
-        if (result == true && super.fileData != null &&
-                super.fileData instanceof java.io.RandomAccessFile) {
-            /** XXX - this cast isn't testing friendly.
-             * A testing class that implements StorageRandomAccessFile but isn't
-             * a RandomAccessFile will be "worked around" by this class. An
-             * example of such a class is
-             * functionTests/util/corruptio/CorruptRandomAccessFile.java.
-             * An interface rework may be necessary.
-             */
-            ourChannel = ((RandomAccessFile)super.fileData).getChannel();
-        }
-        return result;
+        return super.openContainer(newIdentity);
     }
 
     synchronized void createContainer(ContainerKey newIdentity)
@@ -119,14 +153,10 @@
             SanityManager.ASSERT(iosInProgress == 0,
                     "Container created while IO operations are in progress. "
                     + "This should not happen.");
+            SanityManager.ASSERT(fileData == null, "fileData isn't null");
+            SanityManager.ASSERT(ourChannel == null, "ourChannel isn't null");
         }
         super.createContainer(newIdentity);
-
-        if (super.fileData != null &&
-                super.fileData instanceof java.io.RandomAccessFile) {
-            // XXX - see "XXX" comment above.
-            ourChannel = ((RandomAccessFile) super.fileData).getChannel();
-        }
     }
 
 
@@ -188,18 +218,10 @@
     {
         FileChannel ioChannel;
         synchronized (this) {
-            ioChannel = ourChannel;
             if (SanityManager.DEBUG) {
                 SanityManager.ASSERT(!getCommittedDropState());
-                // If ioChannel == null and fileData supports getChannel()
-                // we have a problem. See this.openContainer(ContainerKey 
-                // newIdentity).
-                SanityManager.ASSERT(! ((ioChannel == null) &&
-                        super.fileData instanceof java.io.RandomAccessFile),
-                        "RAFContainer4: New style readPage attempted" +
-                        " with uninitialized ioChannel");
-
             }
+            ioChannel = getChannel();
         }
 
         if(ioChannel != null) {
@@ -272,21 +294,14 @@
          throws IOException, StandardException
     {
         FileChannel ioChannel;
-        synchronized(this) {
+        synchronized (this) {
             // committed and dropped, do nothing.
             // This file container may only be a stub
             if (getCommittedDropState())
                 return;
-            ioChannel = ourChannel;
-            if (SanityManager.DEBUG) {
-                // If ioChannel == null and fileData supports getChannel()
-                // we have a problem
-                SanityManager.ASSERT(! ((ioChannel == null) &&
-                        super.fileData instanceof java.io.RandomAccessFile),
-                        "RAFContainer4: New style writePage attempted " +
-                        "with uninitialized ioChannel");
-            }
+            ioChannel = getChannel();
         }
+
         if(ioChannel != null) {
             ///////////////////////////////////////////////////
             //
@@ -398,6 +413,49 @@
         }
     }
 
+    /**
+     * Write a sequence of bytes at the given offset in a file.
+     *
+     * @param file the file to write to
+     * @param bytes the bytes to write
+     * @param offset the offset to start writing at
+     * @throws IOException if an I/O error occurs while writing
+     */
+    void writeAtOffset(StorageRandomAccessFile file, byte[] bytes, long offset)
+            throws IOException
+    {
+        FileChannel ioChannel = getChannel(file);
+        if (ioChannel != null) {
+            writeFull(ByteBuffer.wrap(bytes), ioChannel, offset);
+        } else {
+            super.writeAtOffset(file, bytes, offset);
+        }
+    }
+
+    /**
+     * Read an embryonic page (that is, a section of the first alloc page that
+     * is so large that we know all the borrowed space is included in it) from
+     * the specified offset in a {@code StorageRandomAccessFile}.
+     *
+     * @param file the file to read from
+     * @param offset where to start reading (normally
+     * {@code FileContainer.FIRST_ALLOC_PAGE_OFFSET})
+     * @return a byte array containing the embryonic page
+     * @throws IOException if an I/O error occurs while reading
+     */
+    byte[] getEmbryonicPage(StorageRandomAccessFile file, long offset)
+            throws IOException
+    {
+        FileChannel ioChannel = getChannel(file);
+        if (ioChannel != null) {
+            ByteBuffer buffer =
+                    ByteBuffer.allocate(AllocPage.MAX_BORROWED_SPACE);
+            readFull(buffer, ioChannel, offset);
+            return buffer.array();
+        } else {
+            return super.getEmbryonicPage(file, offset);
+        }
+    }
 
     /**
      * Attempts to fill buf completely from start until it's full.