Return-Path: Delivered-To: apmail-jackrabbit-commits-archive@www.apache.org Received: (qmail 5084 invoked from network); 3 Mar 2008 09:12:41 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.2) by minotaur.apache.org with SMTP; 3 Mar 2008 09:12:41 -0000 Received: (qmail 81169 invoked by uid 500); 3 Mar 2008 09:12:37 -0000 Delivered-To: apmail-jackrabbit-commits-archive@jackrabbit.apache.org Received: (qmail 81140 invoked by uid 500); 3 Mar 2008 09:12:37 -0000 Mailing-List: contact commits-help@jackrabbit.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@jackrabbit.apache.org Delivered-To: mailing list commits@jackrabbit.apache.org Received: (qmail 81131 invoked by uid 99); 3 Mar 2008 09:12:37 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 03 Mar 2008 01:12:37 -0800 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.3] (HELO eris.apache.org) (140.211.11.3) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 03 Mar 2008 09:11:57 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 978101A9832; Mon, 3 Mar 2008 01:12:16 -0800 (PST) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r632993 - in /jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/persistence: AbstractPersistenceManager.java PersistenceManager.java bundle/AbstractBundlePersistenceManager.java bundle/BundleDbPersistenceManager.java Date: Mon, 03 Mar 2008 09:12:15 -0000 To: commits@jackrabbit.apache.org From: jukka@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20080303091216.978101A9832@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: jukka Date: Mon Mar 3 01:12:14 2008 New Revision: 632993 URL: http://svn.apache.org/viewvc?rev=632993&view=rev Log: JCR-1428: Add API for selective bundle consistency check (Jackrabbit-specific) - Patch by Alexander Klimetschek Modified: jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/persistence/AbstractPersistenceManager.java jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/persistence/PersistenceManager.java jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/persistence/bundle/AbstractBundlePersistenceManager.java jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/persistence/bundle/BundleDbPersistenceManager.java Modified: jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/persistence/AbstractPersistenceManager.java URL: http://svn.apache.org/viewvc/jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/persistence/AbstractPersistenceManager.java?rev=632993&r1=632992&r2=632993&view=diff ============================================================================== --- jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/persistence/AbstractPersistenceManager.java (original) +++ jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/persistence/AbstractPersistenceManager.java Mon Mar 3 01:12:14 2008 @@ -98,6 +98,14 @@ } /** + * This implementation does nothing. + * + * {@inheritDoc} + */ + public void checkConsistency(String[] uuids, boolean recursive, boolean fix) { + } + + /** * Store a node state. Subclass responsibility. * * @param state node state to store Modified: jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/persistence/PersistenceManager.java URL: http://svn.apache.org/viewvc/jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/persistence/PersistenceManager.java?rev=632993&r1=632992&r2=632993&view=diff ============================================================================== --- jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/persistence/PersistenceManager.java (original) +++ jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/persistence/PersistenceManager.java Mon Mar 3 01:12:14 2008 @@ -179,4 +179,23 @@ */ void store(ChangeLog changeLog) throws ItemStateException; + /** + * Perform a consistency check of the data. An example are non-existent + * nodes referenced in a child node entry. The existence of this feature and + * the scope of the implementation can vary in different PersistenceManager + * implementations. + * + * @param uuids + * list of UUIDs of nodes to be checked. if null, all nodes will + * be checked + * @param recursive + * if true, the tree(s) below the given node(s) will be traversed + * and checked as well + * @param fix + * if true, any problems found that can be repaired will be + * repaired. if false, no data will be modified, instead all + * inconsistencies will only get logged + */ + void checkConsistency(String[] uuids, boolean recursive, boolean fix); + } Modified: jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/persistence/bundle/AbstractBundlePersistenceManager.java URL: http://svn.apache.org/viewvc/jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/persistence/bundle/AbstractBundlePersistenceManager.java?rev=632993&r1=632992&r2=632993&view=diff ============================================================================== --- jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/persistence/bundle/AbstractBundlePersistenceManager.java (original) +++ jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/persistence/bundle/AbstractBundlePersistenceManager.java Mon Mar 3 01:12:14 2008 @@ -711,4 +711,12 @@ } } + /** + * This implementation does nothing. + * + * {@inheritDoc} + */ + public void checkConsistency(String[] uuids, boolean recursive, boolean fix) { + } + } Modified: jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/persistence/bundle/BundleDbPersistenceManager.java URL: http://svn.apache.org/viewvc/jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/persistence/bundle/BundleDbPersistenceManager.java?rev=632993&r1=632992&r2=632993&view=diff ============================================================================== --- jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/persistence/bundle/BundleDbPersistenceManager.java (original) +++ jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/persistence/bundle/BundleDbPersistenceManager.java Mon Mar 3 01:12:14 2008 @@ -16,33 +16,6 @@ */ package org.apache.jackrabbit.core.persistence.bundle; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.apache.commons.io.IOUtils; -import org.apache.jackrabbit.util.Text; -import org.apache.jackrabbit.core.state.ChangeLog; -import org.apache.jackrabbit.core.state.ItemStateException; -import org.apache.jackrabbit.core.state.NoSuchItemStateException; -import org.apache.jackrabbit.core.state.NodeReferencesId; -import org.apache.jackrabbit.core.state.NodeReferences; -import org.apache.jackrabbit.core.persistence.PMContext; -import org.apache.jackrabbit.core.persistence.bundle.util.ConnectionRecoveryManager; -import org.apache.jackrabbit.core.persistence.bundle.util.DbNameIndex; -import org.apache.jackrabbit.core.persistence.bundle.util.NodePropBundle; -import org.apache.jackrabbit.core.persistence.bundle.util.BundleBinding; -import org.apache.jackrabbit.core.persistence.bundle.util.ErrorHandling; -import org.apache.jackrabbit.core.persistence.bundle.util.StringIndex; -import org.apache.jackrabbit.core.persistence.util.Serializer; -import org.apache.jackrabbit.core.persistence.util.BLOBStore; -import org.apache.jackrabbit.core.persistence.util.FileSystemBLOBStore; -import org.apache.jackrabbit.core.fs.FileSystemResource; -import org.apache.jackrabbit.core.fs.FileSystem; -import org.apache.jackrabbit.core.fs.local.LocalFileSystem; -import org.apache.jackrabbit.core.NodeId; -import org.apache.jackrabbit.core.NodeIdIterator; -import org.apache.jackrabbit.core.PropertyId; -import org.apache.jackrabbit.uuid.UUID; - import java.io.BufferedReader; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -68,6 +41,33 @@ import javax.jcr.RepositoryException; +import org.apache.commons.io.IOUtils; +import org.apache.jackrabbit.core.NodeId; +import org.apache.jackrabbit.core.NodeIdIterator; +import org.apache.jackrabbit.core.PropertyId; +import org.apache.jackrabbit.core.fs.FileSystem; +import org.apache.jackrabbit.core.fs.FileSystemResource; +import org.apache.jackrabbit.core.fs.local.LocalFileSystem; +import org.apache.jackrabbit.core.persistence.PMContext; +import org.apache.jackrabbit.core.persistence.bundle.util.BundleBinding; +import org.apache.jackrabbit.core.persistence.bundle.util.ConnectionRecoveryManager; +import org.apache.jackrabbit.core.persistence.bundle.util.DbNameIndex; +import org.apache.jackrabbit.core.persistence.bundle.util.ErrorHandling; +import org.apache.jackrabbit.core.persistence.bundle.util.NodePropBundle; +import org.apache.jackrabbit.core.persistence.bundle.util.StringIndex; +import org.apache.jackrabbit.core.persistence.util.BLOBStore; +import org.apache.jackrabbit.core.persistence.util.FileSystemBLOBStore; +import org.apache.jackrabbit.core.persistence.util.Serializer; +import org.apache.jackrabbit.core.state.ChangeLog; +import org.apache.jackrabbit.core.state.ItemStateException; +import org.apache.jackrabbit.core.state.NoSuchItemStateException; +import org.apache.jackrabbit.core.state.NodeReferences; +import org.apache.jackrabbit.core.state.NodeReferencesId; +import org.apache.jackrabbit.util.Text; +import org.apache.jackrabbit.uuid.UUID; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * This is a generic persistence manager that stores the {@link NodePropBundle}s * in a database. @@ -570,7 +570,8 @@ initialized = true; if (consistencyCheck) { - checkConsistency(); + // check all bundles + checkConsistency(null, true, consistencyFix); } } @@ -671,105 +672,205 @@ } /** - * Performs a consistency check. + * Checks a single bundle for inconsistencies, ie. inexistent child nodes + * and inexistent parents. + * + * @param id + * node id for the bundle to check + * @param bundle + * the bundle to check + * @param fix + * if true, repair things that can be repaired + * @param modifications + * if fix == true, collect the repaired + * {@linkplain NodePropBundle bundles} here + */ + protected void checkBundleConsistency(NodeId id, NodePropBundle bundle, boolean fix, Collection modifications) { + //log.info(name + ": checking bundle '" + id + "'"); + + // look at the node's children + Collection missingChildren = new ArrayList(); + Iterator iter = bundle.getChildNodeEntries().iterator(); + while (iter.hasNext()) { + NodePropBundle.ChildNodeEntry entry = (NodePropBundle.ChildNodeEntry) iter.next(); + + // skip check for system nodes (root, system root, version storage, nodetypes) + if (entry.getId().toString().endsWith("babecafebabe")) { + continue; + } + if (id.toString().endsWith("babecafebabe")) { + continue; + } + + try { + // analyze child node bundles + NodePropBundle child = loadBundle(entry.getId(), true); + if (child == null) { + log.error( + "NodeState '" + id + "' references inexistent child" + + " '" + entry.getName() + "' with id " + + "'" + entry.getId() + "'"); + missingChildren.add(entry); + } else { + NodeId cp = child.getParentId(); + if (cp == null) { + log.error("ChildNode has invalid parent uuid: "); + } else if (!cp.equals(id)) { + log.error("ChildNode has invalid parent uuid: '" + cp + "' (instead of '" + id + "')"); + } + } + } catch (ItemStateException e) { + // problem already logged (loadBundle called with logDetailedErrors=true) + } + } + // remove child node entry (if fixing is enabled) + if (fix && !missingChildren.isEmpty()) { + Iterator iterator = missingChildren.iterator(); + while (iterator.hasNext()) { + bundle.getChildNodeEntries().remove(iterator.next()); + } + modifications.add(bundle); + } + + // check parent reference + NodeId parentId = bundle.getParentId(); + try { + // skip root nodes (that point to itself) + if (parentId != null && !id.toString().endsWith("babecafebabe")) { + if (!existsBundle(parentId)) { + log.error("NodeState '" + id + "' references inexistent parent uuid '" + parentId + "'"); + } + } + } catch (ItemStateException e) { + log.error("Error reading node '" + parentId + "' (parent of '" + id + "'): " + e); + } + } + + /** + * {@inheritDoc} */ - private void checkConsistency() { - int count = 0; - int total = 0; + public void checkConsistency(String[] uuids, boolean recursive, boolean fix) { log.info("{}: checking workspace consistency...", name); + int count = 0; + int total = 0; Collection modifications = new ArrayList(); - ResultSet rs = null; - DataInputStream din = null; - try { - String sql; - if (getStorageModel() == SM_BINARY_KEYS) { - sql = "select NODE_ID, BUNDLE_DATA from " + schemaObjectPrefix + "BUNDLE"; - } else { - sql = "select NODE_ID_HI, NODE_ID_LO, BUNDLE_DATA from " + schemaObjectPrefix + "BUNDLE"; - } - Statement stmt = connectionManager.executeStmt(sql, new Object[0]); - rs = stmt.getResultSet(); - while (rs.next()) { - NodeId id; - Blob blob; + + if (uuids == null) { + // get all node bundles in the database with a single sql statement, + // which is (probably) faster than loading each bundle and traversing the tree + ResultSet rs = null; + DataInputStream din = null; + try { + String sql; if (getStorageModel() == SM_BINARY_KEYS) { - id = new NodeId(new UUID(rs.getBytes(1))); - blob = rs.getBlob(2); + sql = "select NODE_ID, BUNDLE_DATA from " + schemaObjectPrefix + "BUNDLE"; } else { - id = new NodeId(new UUID(rs.getLong(1), rs.getLong(2))); - blob = rs.getBlob(3); + sql = "select NODE_ID_HI, NODE_ID_LO, BUNDLE_DATA from " + schemaObjectPrefix + "BUNDLE"; } - din = new DataInputStream(blob.getBinaryStream()); - try { - NodePropBundle bundle = binding.readBundle(din, id); - Collection missingChildren = new ArrayList(); - Iterator iter = bundle.getChildNodeEntries().iterator(); - while (iter.hasNext()) { - NodePropBundle.ChildNodeEntry entry = (NodePropBundle.ChildNodeEntry) iter.next(); - if (entry.getId().toString().endsWith("babecafebabe")) { - continue; - } - if (id.toString().endsWith("babecafebabe")) { - continue; - } - try { - NodePropBundle child = loadBundle(entry.getId()); - if (child == null) { - log.error("NodeState " + id.getUUID() - + " references inexistent child " + entry.getName() - + " with id " + entry.getId().getUUID()); - missingChildren.add(entry); - } else { - NodeId cp = child.getParentId(); - if (cp == null) { - log.error("ChildNode has invalid parent uuid: null"); - } else if (!cp.equals(id)) { - log.error("ChildNode has invalid parent uuid: " + cp + " (instead of " + id.getUUID() + ")"); - } - } - } catch (ItemStateException e) { - log.error("Error while loading child node: " + e); - } + Statement stmt = connectionManager.executeStmt(sql, new Object[0]); + rs = stmt.getResultSet(); + + // iterate over all nodebundles in the db + while (rs.next()) { + NodeId id; + Blob blob; + if (getStorageModel() == SM_BINARY_KEYS) { + id = new NodeId(new UUID(rs.getBytes(1))); + blob = rs.getBlob(2); + } else { + id = new NodeId(new UUID(rs.getLong(1), rs.getLong(2))); + blob = rs.getBlob(3); } - if (consistencyFix && !missingChildren.isEmpty()) { - Iterator iterator = missingChildren.iterator(); - while (iterator.hasNext()) { - bundle.getChildNodeEntries().remove(iterator.next()); + din = new DataInputStream(blob.getBinaryStream()); + try { + // parse and check bundle + // check bundle will log any problems itself + if (binding.checkBundle(din)) { + // reset stream for readBundle() + din = new DataInputStream(blob.getBinaryStream()); + NodePropBundle bundle = binding.readBundle(din, id); + checkBundleConsistency(id, bundle, fix, modifications); + } else { + log.error("invalid bundle '" + id + "', see previous BundleBinding error log entry"); } - modifications.add(bundle); + } catch (Exception e) { + log.error("Error in bundle " + id + ": " + e); + } + count++; + if (count % 1000 == 0) { + log.info(name + ": checked " + count + " bundles..."); + } + } + } catch (Exception e) { + log.error("Error loading bundle", e); + } finally { + IOUtils.closeQuietly(din); + closeResultSet(rs); + total = count; + } + } else { + // check only given uuids, handle recursive flag + + // 1) convert uuid array to modifiable list + // 2) for each uuid do + // a) load node bundle + // b) check bundle, store any bundle-to-be-modified in collection + // c) if recursive, add child uuids to list of uuids + + List uuidList = new ArrayList(uuids.length); + // convert uuid string array to list of UUID objects + for (int i = 0; i < uuids.length; i++) { + try { + uuidList.add(new UUID(uuids[i])); + } catch (IllegalArgumentException e) { + log.error("Invalid uuid for consistency check, skipping: '" + uuids[i] + "': " + e); + } + } + + // iterate over UUIDs (including ones that are newly added inside the loop!) + for (int i = 0; i < uuidList.size(); i++) { + final UUID uuid = (UUID) uuidList.get(i); + try { + // load the node from the database + NodeId id = new NodeId(uuid); + NodePropBundle bundle = loadBundle(id, true); + + if (bundle == null) { + log.error("No bundle found for uuid '" + uuid + "'"); + continue; } - NodeId parentId = bundle.getParentId(); - if (parentId != null) { - if (!exists(parentId)) { - log.error("NodeState " + id + " references inexistent parent id " + parentId); + checkBundleConsistency(id, bundle, fix, modifications); + + if (recursive) { + Iterator iter = bundle.getChildNodeEntries().iterator(); + while (iter.hasNext()) { + NodePropBundle.ChildNodeEntry entry = (NodePropBundle.ChildNodeEntry) iter.next(); + uuidList.add(entry.getId().getUUID()); } } - } catch (IOException e) { - log.error("Error in bundle " + id + ": " + e); - din = new DataInputStream(blob.getBinaryStream()); - binding.checkBundle(din); - } - count++; - if (count % 1000 == 0) { - log.info(name + ": checked " + count + "/" + total + " bundles..."); + + count++; + if (count % 1000 == 0) { + log.info(name + ": checked " + count + "/" + uuidList.size() + " bundles..."); + } + } catch (ItemStateException e) { + // problem already logged (loadBundle called with logDetailedErrors=true) } } - } catch (Exception e) { - log.error("Error in bundle", e); - } finally { - IOUtils.closeQuietly(din); - closeResultSet(rs); + + total = uuidList.size(); } + // repair collected broken bundles if (consistencyFix && !modifications.isEmpty()) { log.info(name + ": Fixing " + modifications.size() + " inconsistent bundle(s)..."); Iterator iterator = modifications.iterator(); while (iterator.hasNext()) { NodePropBundle bundle = (NodePropBundle) iterator.next(); try { - log.info(name + ": Fixing bundle " + bundle.getId()); + log.info(name + ": Fixing bundle '" + bundle.getId() + "'"); bundle.markOld(); // use UPDATE instead of INSERT storeBundle(bundle); } catch (ItemStateException e) { @@ -781,7 +882,6 @@ log.info(name + ": checked " + count + "/" + total + " bundles."); } - /** * Makes sure that schemaObjectPrefix does only consist of * characters that are allowed in names on the target database. Illegal @@ -964,8 +1064,25 @@ */ protected synchronized NodePropBundle loadBundle(NodeId id) throws ItemStateException { + return loadBundle(id, false); + } + + /** + * Loads a bundle from the underlying system and optionally performs + * a check on the bundle first. + * + * @param id the node id of the bundle + * @param checkBeforeLoading check the bundle before loading it and log + * detailed informations about it (slower) + * @return the loaded bundle or null if the bundle does not + * exist. + * @throws ItemStateException if an error while loading occurs. + */ + protected synchronized NodePropBundle loadBundle(NodeId id, boolean checkBeforeLoading) + throws ItemStateException { ResultSet rs = null; InputStream in = null; + byte[] bytes = null; try { Statement stmt = connectionManager.executeStmt(bundleSelectSQL, getKey(id.getUUID())); rs = stmt.getResultSet(); @@ -975,13 +1092,24 @@ Blob b = rs.getBlob(1); // JCR-1039: pre-fetch/buffer blob data long length = b.length(); - byte[] bytes = new byte[(int) length]; + bytes = new byte[(int) length]; in = b.getBinaryStream(); int read, pos = 0; while ((read = in.read(bytes, pos, bytes.length - pos)) > 0) { pos += read; } DataInputStream din = new DataInputStream(new ByteArrayInputStream(bytes)); + + if (checkBeforeLoading) { + if (binding.checkBundle(din)) { + // reset stream for readBundle() + din = new DataInputStream(new ByteArrayInputStream(bytes)); + } else { + // gets wrapped as proper ItemStateException below + throw new Exception("invalid bundle, see previous BundleBinding error log entry"); + } + } + NodePropBundle bundle = binding.readBundle(din, id); bundle.setSize(length); return bundle;