incubator-sling-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bdelacre...@apache.org
Subject svn commit: r979260 - in /sling/trunk/contrib/extensions/bgservlets/src: main/java/org/apache/sling/bgservlets/impl/ main/java/org/apache/sling/bgservlets/impl/nodestream/ test/java/org/apache/sling/bgservlets/impl/nodestream/
Date Mon, 26 Jul 2010 12:21:43 GMT
Author: bdelacretaz
Date: Mon Jul 26 12:21:43 2010
New Revision: 979260

URL: http://svn.apache.org/viewvc?rev=979260&view=rev
Log:
SLING-505 - use hierarchical paths to store streams, to avoid large number of child nodes

Added:
    sling/trunk/contrib/extensions/bgservlets/src/main/java/org/apache/sling/bgservlets/impl/nodestream/NodeStreamPath.java
  (with props)
    sling/trunk/contrib/extensions/bgservlets/src/test/java/org/apache/sling/bgservlets/impl/nodestream/NodeStreamPathTest.java
  (with props)
Modified:
    sling/trunk/contrib/extensions/bgservlets/src/main/java/org/apache/sling/bgservlets/impl/BackgroundTestServlet.java
    sling/trunk/contrib/extensions/bgservlets/src/main/java/org/apache/sling/bgservlets/impl/nodestream/NodeInputStream.java
    sling/trunk/contrib/extensions/bgservlets/src/main/java/org/apache/sling/bgservlets/impl/nodestream/NodeOutputStream.java
    sling/trunk/contrib/extensions/bgservlets/src/test/java/org/apache/sling/bgservlets/impl/nodestream/NodeStreamTest.java

Modified: sling/trunk/contrib/extensions/bgservlets/src/main/java/org/apache/sling/bgservlets/impl/BackgroundTestServlet.java
URL: http://svn.apache.org/viewvc/sling/trunk/contrib/extensions/bgservlets/src/main/java/org/apache/sling/bgservlets/impl/BackgroundTestServlet.java?rev=979260&r1=979259&r2=979260&view=diff
==============================================================================
--- sling/trunk/contrib/extensions/bgservlets/src/main/java/org/apache/sling/bgservlets/impl/BackgroundTestServlet.java
(original)
+++ sling/trunk/contrib/extensions/bgservlets/src/main/java/org/apache/sling/bgservlets/impl/BackgroundTestServlet.java
Mon Jul 26 12:21:43 2010
@@ -53,7 +53,7 @@ public class BackgroundTestServlet exten
         final PrintWriter w = response.getWriter();
 
         final int cycles = getIntParam(request, "cycles", 10);
-        final int interval = getIntParam(request, "interval", 1);
+        final int interval = getIntParam(request, "interval", 1000);
         final int flushEvery = getIntParam(request, "flushEvery", 2);
 
         w.println("Start at " + new Date());
@@ -65,7 +65,7 @@ public class BackgroundTestServlet exten
                 }
                 w.printf("Cycle %d of %d\n", i, cycles);
                 try {
-                    Thread.sleep(interval * 1000);
+                    Thread.sleep(interval);
                 } catch (InterruptedException iex) {
                     throw new ServletException("InterruptedException", iex);
                 }

Modified: sling/trunk/contrib/extensions/bgservlets/src/main/java/org/apache/sling/bgservlets/impl/nodestream/NodeInputStream.java
URL: http://svn.apache.org/viewvc/sling/trunk/contrib/extensions/bgservlets/src/main/java/org/apache/sling/bgservlets/impl/nodestream/NodeInputStream.java?rev=979260&r1=979259&r2=979260&view=diff
==============================================================================
--- sling/trunk/contrib/extensions/bgservlets/src/main/java/org/apache/sling/bgservlets/impl/nodestream/NodeInputStream.java
(original)
+++ sling/trunk/contrib/extensions/bgservlets/src/main/java/org/apache/sling/bgservlets/impl/nodestream/NodeInputStream.java
Mon Jul 26 12:21:43 2010
@@ -39,31 +39,30 @@ public class NodeInputStream extends Inp
     /** The Node under which we read our data */
     private final Node node;
     
-    /** Counter used to build the name of Property from 
-     *  which we currently read */
-    private int counter;
+    /** Computes path for stream storage */
+    private final NodeStreamPath streamPath;
     
     /** Current stream that we are reading */
     private InputStream currentStream;
     
     public NodeInputStream(Node n) throws IOException {
         node = n;
+        streamPath = new NodeStreamPath();
         selectNextStream();
     }
     
     /** Select next property to read from and open its stream */
     private void selectNextStream() throws IOException {
-        counter++;
-        // TODO use hierarchy to allow for arbitrary number of flush calls
-        final String name = NodeOutputStream.STREAM_PROPERTY_NAME_PREFIX + counter;
+        streamPath.selectNextPath();
+        final String propertyPath = streamPath.getNodePath() + "/" + NodeStreamPath.PROPERTY_NAME;
         try {
-            if(node.hasProperty(name)) {
-                final Property p = node.getProperty(name); 
+            if(node.hasProperty(propertyPath)) {
+                final Property p = node.getProperty(propertyPath); 
                 currentStream = p.getStream();
                 log.debug("Switched to the InputStream of Property {}", p.getPath());
             } else {
                 currentStream = null;
-                log.debug("Property {} not found, end of stream", node.getPath() + "/" +
name);
+                log.debug("Property {} not found, end of stream", node.getPath() + "/" +
propertyPath);
             }
         } catch(RepositoryException re) {
             throw new IOException("RepositoryException in selectNextProperty()", re);

Modified: sling/trunk/contrib/extensions/bgservlets/src/main/java/org/apache/sling/bgservlets/impl/nodestream/NodeOutputStream.java
URL: http://svn.apache.org/viewvc/sling/trunk/contrib/extensions/bgservlets/src/main/java/org/apache/sling/bgservlets/impl/nodestream/NodeOutputStream.java?rev=979260&r1=979259&r2=979260&view=diff
==============================================================================
--- sling/trunk/contrib/extensions/bgservlets/src/main/java/org/apache/sling/bgservlets/impl/nodestream/NodeOutputStream.java
(original)
+++ sling/trunk/contrib/extensions/bgservlets/src/main/java/org/apache/sling/bgservlets/impl/nodestream/NodeOutputStream.java
Mon Jul 26 12:21:43 2010
@@ -26,6 +26,7 @@ import java.io.OutputStream;
 import javax.jcr.Node;
 import javax.jcr.RepositoryException;
 
+import org.apache.sling.bgservlets.impl.DeepNodeCreator;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -50,12 +51,14 @@ public class NodeOutputStream extends Ou
     /** Prefix for Property names used to store our streams */
     public static final String STREAM_PROPERTY_NAME_PREFIX = "_NODE_STREAM_";
     
+    /** Node type for our stream nodes */
+    public static final String STREAM_NODE_TYPE = "nt:unstructured";
+    
     /** The Node under which we write our data */
     private final Node node;
     
-    /** Counter used to build the name of Property to
-     *  which we currently write */
-    private int counter;
+    /** Computes path for stream storage */
+    private final NodeStreamPath streamPath;
     
     /** Buffer to hold data before writing it to a Property */
     private final ByteArrayOutputStream buffer = new ByteArrayOutputStream(BUFFER_SIZE);
@@ -65,6 +68,7 @@ public class NodeOutputStream extends Ou
     
     public NodeOutputStream(Node n) {
         node = n;
+        streamPath = new NodeStreamPath();
     }
     
     /** Calls flush to persist our stream, before closing */
@@ -79,15 +83,22 @@ public class NodeOutputStream extends Ou
      */
     @Override
     public void flush() throws IOException {
-        counter++;
-        // TODO use hierarchy to allow for arbitrary number of flush calls
-        final String name = NodeOutputStream.STREAM_PROPERTY_NAME_PREFIX + counter;
+        
+        streamPath.selectNextPath();
+        
         try {
             if(!node.getSession().isLive()) {
                 log.warn("Session closed, unable to flush stream");
             } else {
-                node.setProperty(name, new ByteArrayInputStream(buffer.toByteArray()));
-                log.debug("Saved {} bytes to Property {}", buffer.size(), node.getProperty(name).getPath());
+                // Create node that will store the stream
+                final String streamNodePath = node.getPath() + "/" + streamPath.getNodePath();
+                final Node streamNode = new DeepNodeCreator().deepCreateNode(streamNodePath,

+                        node.getSession(), STREAM_NODE_TYPE);
+                
+                streamNode.setProperty(NodeStreamPath.PROPERTY_NAME, 
+                        new ByteArrayInputStream(buffer.toByteArray()));
+                log.debug("Saved {} bytes to Property {}", buffer.size(), 
+                        streamNode.getProperty(NodeStreamPath.PROPERTY_NAME).getPath());
                 node.save();
                 buffer.reset();
             }

Added: sling/trunk/contrib/extensions/bgservlets/src/main/java/org/apache/sling/bgservlets/impl/nodestream/NodeStreamPath.java
URL: http://svn.apache.org/viewvc/sling/trunk/contrib/extensions/bgservlets/src/main/java/org/apache/sling/bgservlets/impl/nodestream/NodeStreamPath.java?rev=979260&view=auto
==============================================================================
--- sling/trunk/contrib/extensions/bgservlets/src/main/java/org/apache/sling/bgservlets/impl/nodestream/NodeStreamPath.java
(added)
+++ sling/trunk/contrib/extensions/bgservlets/src/main/java/org/apache/sling/bgservlets/impl/nodestream/NodeStreamPath.java
Mon Jul 26 12:21:43 2010
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.sling.bgservlets.impl.nodestream;
+
+
+/** Builds sequential hierachical paths to store node 
+ *  streams: increment a counter and build path using
+ *  CHARS_PER_LEVEL characters of the String value of
+ *  the counter per path level.    
+ */
+class NodeStreamPath {
+    private int counter;
+    private String path;
+    private final int CHARS_PER_LEVEL = 2;
+    
+    /** Property name to use for streams */
+    static final String PROPERTY_NAME = "stream";
+    
+    /** Select the next path to use, must be
+     *  called before using getPath().
+     *  Not thread-safe.  */
+    void selectNextPath() {
+        counter++;
+        final StringBuilder sb = new StringBuilder();
+        final String str = String.valueOf(counter);
+        for(int i = 0; i < str.length(); i++) {
+            if(i> 0 && i % CHARS_PER_LEVEL == 0) {
+                sb.append('/');
+            }
+            sb.append(str.charAt(i));
+        }
+        path = sb.toString();
+    }
+    
+    /** Return the last path computed by selectNextPath() */
+    String getNodePath() {
+        return path;
+    }
+}

Propchange: sling/trunk/contrib/extensions/bgservlets/src/main/java/org/apache/sling/bgservlets/impl/nodestream/NodeStreamPath.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: sling/trunk/contrib/extensions/bgservlets/src/main/java/org/apache/sling/bgservlets/impl/nodestream/NodeStreamPath.java
------------------------------------------------------------------------------
    svn:keywords = Author Date Id Revision Rev URL

Added: sling/trunk/contrib/extensions/bgservlets/src/test/java/org/apache/sling/bgservlets/impl/nodestream/NodeStreamPathTest.java
URL: http://svn.apache.org/viewvc/sling/trunk/contrib/extensions/bgservlets/src/test/java/org/apache/sling/bgservlets/impl/nodestream/NodeStreamPathTest.java?rev=979260&view=auto
==============================================================================
--- sling/trunk/contrib/extensions/bgservlets/src/test/java/org/apache/sling/bgservlets/impl/nodestream/NodeStreamPathTest.java
(added)
+++ sling/trunk/contrib/extensions/bgservlets/src/test/java/org/apache/sling/bgservlets/impl/nodestream/NodeStreamPathTest.java
Mon Jul 26 12:21:43 2010
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.sling.bgservlets.impl.nodestream;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+import org.junit.Test;
+
+public class NodeStreamPathTest {
+    
+    @Test
+    public void testNullOnFirstCall() {
+        final NodeStreamPath nsp = new NodeStreamPath();
+        assertNull(nsp.getNodePath());
+    }
+    
+    @Test
+    public void testPaths() {
+        final String [] data = {
+            "1", "1",
+            "9", "9",
+            "99", "99",
+            "100", "10/0",
+            "199", "19/9",
+            "200", "20/0",
+            "1234", "12/34",
+        };
+        
+        for(int i=0; i < data.length; i += 2) {
+            final NodeStreamPath nsp = new NodeStreamPath();
+            for(int j = 0; j < Integer.parseInt(data[i]); j++) {
+                nsp.selectNextPath();
+            }
+            final String exp = data[i+1];
+            assertEquals("At index " + i + ", expected " + exp, exp, nsp.getNodePath());
+        }
+    }
+}

Propchange: sling/trunk/contrib/extensions/bgservlets/src/test/java/org/apache/sling/bgservlets/impl/nodestream/NodeStreamPathTest.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: sling/trunk/contrib/extensions/bgservlets/src/test/java/org/apache/sling/bgservlets/impl/nodestream/NodeStreamPathTest.java
------------------------------------------------------------------------------
    svn:keywords = Author Date Id Revision Rev URL

Modified: sling/trunk/contrib/extensions/bgservlets/src/test/java/org/apache/sling/bgservlets/impl/nodestream/NodeStreamTest.java
URL: http://svn.apache.org/viewvc/sling/trunk/contrib/extensions/bgservlets/src/test/java/org/apache/sling/bgservlets/impl/nodestream/NodeStreamTest.java?rev=979260&r1=979259&r2=979260&view=diff
==============================================================================
--- sling/trunk/contrib/extensions/bgservlets/src/test/java/org/apache/sling/bgservlets/impl/nodestream/NodeStreamTest.java
(original)
+++ sling/trunk/contrib/extensions/bgservlets/src/test/java/org/apache/sling/bgservlets/impl/nodestream/NodeStreamTest.java
Mon Jul 26 12:21:43 2010
@@ -23,6 +23,8 @@ import java.io.ByteArrayOutputStream;
 import java.io.InputStream;
 
 import javax.jcr.Node;
+import javax.jcr.NodeIterator;
+import javax.jcr.RepositoryException;
 
 import org.apache.sling.commons.testing.jcr.RepositoryTestBase;
 
@@ -97,14 +99,28 @@ public class NodeStreamTest extends Repo
         nos.close();
         
         assertFalse("Expecting no pending changes in testNode session", testNode.getSession().hasPendingChanges());
-        final long propCount = testNode.getProperties().getSize();
-        final long expect = 10;
-        assertTrue("Expecting > " + expect + " properties on test node", propCount >
expect);
+        
+        // Stream must be stored in a hierarchy under testNode, to
+        // avoid limitations if flush() is called many times
+        final int childCount = getChildCount(testNode);
+        final int expect = 10;
+        assertTrue("Expecting > " + expect + " child nodes under testNode, got " + childCount,
childCount > expect);
 
         final NodeInputStream nis = new NodeInputStream(testNode);
         assertStream(new ByteArrayInputStream(BIG_DATA), nis);
     }
     
+    private int getChildCount(Node n) throws RepositoryException {
+        int result = 0;
+        final NodeIterator it = n.getNodes();
+        while(it.hasNext()) {
+            result++;
+            final Node kid = it.nextNode();
+            result += getChildCount(kid);
+        }
+        return result;
+    }
+    
     public void testWriteWithOffset() throws Exception {
         final Node testNode = getTestRootNode().addNode(NAME_PREFIX + counter++);
         testNode.getSession().save();



Mime
View raw message