Mailing-List: contact connectors-commits-help@incubator.apache.org;
 run by ezmlm
Precedence: bulk
Reply-To: connectors-dev@incubator.apache.org
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r1179737 - in
 /incubator/lcf/branches/CONNECTORS-256/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki:
 ByteBuffer.java PageBuffer.java
Date: Thu, 06 Oct 2011 18:14:13 -0000
To: connectors-commits@incubator.apache.org
From: kwright@apache.org
Message-Id: <20111006181413.B84B323888E4@eris.apache.org>

Author: kwright
Date: Thu Oct  6 18:14:13 2011
New Revision: 1179737

URL: http://svn.apache.org/viewvc?rev=1179737&view=rev
Log:
Add class for pushing bytes through XML to the indexing interface.

Added:
    incubator/lcf/branches/CONNECTORS-256/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/ByteBuffer.java   (with props)
Modified:
    incubator/lcf/branches/CONNECTORS-256/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/PageBuffer.java

Added: incubator/lcf/branches/CONNECTORS-256/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/ByteBuffer.java
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-256/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/ByteBuffer.java?rev=1179737&view=auto
==============================================================================
--- incubator/lcf/branches/CONNECTORS-256/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/ByteBuffer.java (added)
+++ incubator/lcf/branches/CONNECTORS-256/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/ByteBuffer.java Thu Oct  6 18:14:13 2011
@@ -0,0 +1,132 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.crawler.connectors.wiki;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import java.util.*;
+import java.io.*;
+
+/** Thread-safe class that functions as a limited-size buffer of pageIDs */
+public class ByteBuffer extends InputStream
+{
+  protected static int MAX_SIZE = 65536;
+  
+  protected byte[] buffer = new byte[MAX_SIZE];
+  protected int offset = 0;
+  protected int size = 0;
+  
+  protected boolean complete = false;
+  
+  /** Constructor */
+  public ByteBuffer()
+  {
+  }
+  
+  /** Add bytes to the buffer, and block if the buffer is full */
+  public synchronized void add(byte[] bytes, int boffset, int bsize)
+    throws ManifoldCFException
+  {
+    while (bsize > 0)
+    {
+      try
+      {
+        while (size == MAX_SIZE)
+        {
+          wait();
+        }
+      }
+      catch (InterruptedException e)
+      {
+        throw new ManifoldCFException(e.getMessage(),e,ManifoldCFException.INTERRUPTED);
+      }
+      int pos = offset + size;
+      if (pos >= MAX_SIZE)
+        pos -= MAX_SIZE;
+      buffer[pos] = bytes[boffset++];
+      bsize--;
+      size++;
+      // Notify threads that are waiting on there being stuff in the queue
+      if (size == 1)
+        notifyAll();
+    }
+  }
+  
+  /** Signal that the operation is complete, and that no more pageID's
+  * will be added.
+  */
+  public synchronized void signalDone()
+  {
+    complete = true;
+    // Notify threads that are waiting for stuff to appear, because it won't
+    notifyAll();
+  }
+  
+  /** Pull a byte off the buffer, and wait if there's more to come.
+  * Returns -1 if the operation is complete.
+  */
+  public synchronized int read()
+    throws IOException
+  {
+    try
+    {
+      while (size == 0 && !complete)
+        wait();
+    }
+    catch (InterruptedException e)
+    {
+      throw new InterruptedIOException(e.getMessage());
+    }
+    if (size == 0)
+      return -1;
+    boolean isBufferFull = (size == MAX_SIZE);
+    int rval = ((int)buffer[offset++]) & 0xff;
+    size--;
+    if (offset == MAX_SIZE)
+      offset = 0;
+    // Notify those threads waiting on buffer being not completely full to wake
+    if (isBufferFull)
+      notifyAll();
+    return rval;
+  }
+  
+  /** Pull multiple bytes off the buffer.
+  */
+  public synchronized int read(byte[] bytes, int boffset, int bcount)
+    throws IOException
+  {
+    int amt = 0;
+    while (bcount > 0)
+    {
+      int x = read();
+      if (x == -1)
+        return amt;
+      bytes[boffset++] = (byte)x;
+      bcount--;
+    }
+    return amt;
+  }
+  
+  /** Close */
+  public void close()
+    throws IOException
+  {
+    // Does nothing
+  }
+  
+}

Propchange: incubator/lcf/branches/CONNECTORS-256/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/ByteBuffer.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: incubator/lcf/branches/CONNECTORS-256/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/ByteBuffer.java
------------------------------------------------------------------------------
    svn:keywords = Id

Modified: incubator/lcf/branches/CONNECTORS-256/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/PageBuffer.java
URL: http://svn.apache.org/viewvc/incubator/lcf/branches/CONNECTORS-256/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/PageBuffer.java?rev=1179737&r1=1179736&r2=1179737&view=diff
==============================================================================
--- incubator/lcf/branches/CONNECTORS-256/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/PageBuffer.java (original)
+++ incubator/lcf/branches/CONNECTORS-256/connectors/wiki/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/wiki/PageBuffer.java Thu Oct  6 18:14:13 2011
@@ -85,7 +85,8 @@ public class PageBuffer
     boolean isBufferFull = (buffer.size() == MAX_SIZE);
     String rval = buffer.remove(buffer.size()-1);
     // Notify those threads waiting on buffer being not completely full to wake
-    notifyAll();
+    if (isBufferFull)
+      notifyAll();
     return rval;
   }