jackrabbit-oak-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From resc...@apache.org
Subject svn commit: r1594207 - in /jackrabbit/oak/trunk/oak-core/src: main/java/org/apache/jackrabbit/oak/plugins/document/rdb/ test/java/org/apache/jackrabbit/oak/plugins/document/blob/ test/java/org/apache/jackrabbit/oak/plugins/document/rdb/
Date Tue, 13 May 2014 12:11:17 GMT
Author: reschke
Date: Tue May 13 12:11:16 2014
New Revision: 1594207

URL: http://svn.apache.org/r1594207
Log:
OAK-1812

- catch attempts to write blobs that are too long
- add tests
- increase default blob size in DB2

Added:
    jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/rdb/
    jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBBlobStoreFriend.java
  (with props)
Modified:
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBBlobStore.java
    jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/blob/RDBBlobStoreTest.java

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBBlobStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBBlobStore.java?rev=1594207&r1=1594206&r2=1594207&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBBlobStore.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBBlobStore.java
Tue May 13 12:11:16 2014
@@ -66,6 +66,9 @@ public class RDBBlobStore extends Cachin
 
     private static final Logger LOG = LoggerFactory.getLogger(RDBBlobStore.class);
 
+    // blob size we need to support
+    private static final int MINBLOB = 2 * 1024 * 1024;
+
     private Exception callStack;
 
     private DataSource ds;
@@ -97,12 +100,12 @@ public class RDBBlobStore extends Cachin
                                 + " (ID varchar(1000) not null primary key, LEVEL int, LASTMOD
bigint)");
                     } else {
                         // the code below likely will need to be extended for
-                        // new
-                        // database types
+                        // new database types
                         if ("PostgreSQL".equals(dbtype)) {
                             stmt.execute("create table " + tableName + " (ID varchar(1000)
not null primary key, DATA bytea)");
                         } else if ("DB2".equals(dbtype) || (dbtype != null && dbtype.startsWith("DB2/")))
{
-                            stmt.execute("create table " + tableName + " (ID varchar(1000)
not null primary key, DATA blob)");
+                            stmt.execute("create table " + tableName + " (ID varchar(1000)
not null primary key, DATA blob("
+                                    + MINBLOB + "))");
                         } else {
                             stmt.execute("create table " + tableName + " (ID varchar(1000)
not null primary key, DATA blob)");
                         }
@@ -158,8 +161,11 @@ public class RDBBlobStore extends Cachin
                     } finally {
                         prep.close();
                     }
-                } catch (SQLException e) {
-                    // already exists - ok
+                } catch (SQLException ex) {
+                    // TODO: this code used to ignore exceptions here, assuming that it might
be a case where the blob is already in the database (maybe this requires inspecting the exception
code)
+                    String message = "insert document failed for id " + id + " with length
" + data.length + " (check max size of datastore_data.data)";
+                    LOG.error(message, ex);
+                    throw new RuntimeException(message, ex);
                 }
                 try {
                     prep = con.prepareStatement("insert into datastore_meta(id, level, lastMod)
values(?, ?, ?)");
@@ -181,6 +187,31 @@ public class RDBBlobStore extends Cachin
         }
     }
 
+    // needed in test
+    protected byte[] readBlockFromBackend(byte[] digest) throws Exception {
+        String id = StringUtils.convertBytesToHex(digest);
+        Connection con = ds.getConnection();
+        byte[] data;
+
+        try {
+            PreparedStatement prep = con.prepareStatement("select data from datastore_data
where id = ?");
+            try {
+                prep.setString(1, id);
+                ResultSet rs = prep.executeQuery();
+                if (!rs.next()) {
+                    throw new IOException("Datastore block " + id + " not found");
+                }
+                data = rs.getBytes(1);
+            } finally {
+                prep.close();
+            }
+        } finally {
+            con.commit();
+            con.close();
+        }
+        return data;
+    }
+
     @Override
     protected byte[] readBlockFromBackend(BlockId blockId) throws Exception {
 

Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/blob/RDBBlobStoreTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/blob/RDBBlobStoreTest.java?rev=1594207&r1=1594206&r2=1594207&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/blob/RDBBlobStoreTest.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/blob/RDBBlobStoreTest.java
Tue May 13 12:11:16 2014
@@ -16,14 +16,25 @@
  */
 package org.apache.jackrabbit.oak.plugins.document.blob;
 
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Arrays;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Random;
 
 import org.apache.jackrabbit.oak.plugins.document.rdb.RDBBlobStore;
+import org.apache.jackrabbit.oak.plugins.document.rdb.RDBBlobStoreFriend;
 import org.apache.jackrabbit.oak.plugins.document.rdb.RDBDataSourceFactory;
 import org.apache.jackrabbit.oak.spi.blob.AbstractBlobStoreTest;
 import org.junit.After;
 import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import com.google.common.collect.Lists;
 
@@ -40,6 +51,8 @@ public class RDBBlobStoreTest extends Ab
 
     private static final String PASSWD = System.getProperty("rdb.jdbc-passwd", "");
 
+    private static final Logger LOG = LoggerFactory.getLogger(RDBBlobStoreTest.class);
+
     @Before
     @Override
     public void setUp() throws Exception {
@@ -68,4 +81,46 @@ public class RDBBlobStoreTest extends Ab
         }
         blobStore.deleteChunks(ids, 0);
     }
+
+    @Test
+    public void testBigBlob() throws Exception {
+        int min = 0;
+        int max = 8 * 1024 * 1024;
+        int test = 0;
+
+        while (max - min >= 2) {
+            test = (max + min) / 2;
+            System.err.println(test);
+            byte[] data = new byte[test];
+            Random r = new Random(0);
+            r.nextBytes(data);
+            byte[] digest = getDigest(data);
+            try {
+                RDBBlobStoreFriend.storeBlock(blobStore, getDigest(data), 0, data);
+                byte[] data2 = RDBBlobStoreFriend.readBlockFromBackend(blobStore, digest);
+                if (!Arrays.equals(data, data2)) {
+                    throw new Exception("data mismatch for length " + data.length);
+                }
+                min = test;
+            } catch (Exception ex) {
+                max = test;
+            }
+        }
+
+        LOG.info("max id length for " + URL + " was " + test);
+
+        int expected = Math.max(blobStore.getBlockSize(), 2 * 1024 * 1024);
+        assertTrue("expected supported block size is " + expected + ", but measured: " +
test, test >= expected);
+    }
+
+    private byte[] getDigest(byte[] bytes) throws IOException {
+        MessageDigest messageDigest;
+        try {
+            messageDigest = MessageDigest.getInstance("SHA-256");
+        } catch (NoSuchAlgorithmException e) {
+            throw new IOException(e);
+        }
+        messageDigest.update(bytes, 0, bytes.length);
+        return messageDigest.digest();
+    }
 }

Added: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBBlobStoreFriend.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBBlobStoreFriend.java?rev=1594207&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBBlobStoreFriend.java
(added)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBBlobStoreFriend.java
Tue May 13 12:11:16 2014
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.document.rdb;
+
+import java.io.IOException;
+
+public class RDBBlobStoreFriend {
+
+    public static void storeBlock(RDBBlobStore ds, byte[] digest, int level, byte[] data)
throws IOException {
+        ds.storeBlock(digest, level, data);
+    }
+
+    public static byte[] readBlockFromBackend(RDBBlobStore ds, byte[] digest) throws Exception
{
+        return ds.readBlockFromBackend(digest);
+    }
+}

Propchange: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBBlobStoreFriend.java
------------------------------------------------------------------------------
    svn:eol-style = native



Mime
View raw message