manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1546493 - in /manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler: bins/BinManager.java interfaces/IBinManager.java jobs/JobManager.java system/ReprioritizationTracker.java
Date Fri, 29 Nov 2013 01:59:42 GMT
Author: kwright
Date: Fri Nov 29 01:59:41 2013
New Revision: 1546493

URL: http://svn.apache.org/r1546493
Log:
Add ability to get multiple bin values at once.

Modified:
    manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/bins/BinManager.java
    manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IBinManager.java
    manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
    manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ReprioritizationTracker.java

Modified: manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/bins/BinManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/bins/BinManager.java?rev=1546493&r1=1546492&r2=1546493&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/bins/BinManager.java
(original)
+++ manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/bins/BinManager.java
Fri Nov 29 01:59:41 2013
@@ -22,6 +22,7 @@ import org.apache.manifoldcf.core.interf
 import org.apache.manifoldcf.crawler.interfaces.*;
 import org.apache.manifoldcf.crawler.interfaces.CacheKeyFactory;
 import org.apache.manifoldcf.crawler.system.ManifoldCF;
+import org.apache.manifoldcf.crawler.system.Logging;
 import java.util.*;
 
 /** This class manages the docbins table.
@@ -101,17 +102,18 @@ public class BinManager extends org.apac
     performDelete("", null, null);
   }
 
-  /** Get a bin value (and set next one).  If the record does not yet exist, create it with
a starting value.
-  * We expect this to happen within a transaction!! 
+  /** Get N bin values (and set next one).  If the record does not yet exist, create it with
a starting value.
+  * We expect this to happen within a transaction!!.
   *@param binName is the name of the bin (256 char max)
   *@param newBinValue is the value to use if there is no such bin yet.  This is the value
that will be
   * returned; what will be stored will be that value + 1.
-  *@return the counter value.
+  *@param count is the number of values desired.
+  *@return the counter values.
   */
-  @Override
-  public double getIncrementBinValue(String binName, double newBinValue)
+  public double[] getIncrementBinValues(String binName, double newBinValue, int count)
     throws ManifoldCFException
   {
+    double[] returnValues = new double[count];
     // SELECT FOR UPDATE/MODIFY is the most common path
     ArrayList params = new ArrayList();
     String query = buildConjunctionClause(params,new ClauseDescription[]{
@@ -124,18 +126,78 @@ public class BinManager extends org.apac
       double rval = value.doubleValue();
       if (rval < newBinValue)
         rval = newBinValue;
+      // rval is the starting value; compute the entire array based on it.
+      for (int i = 0; i < count; i++)
+      {
+        returnValues[i] = rval;
+        rval += 1.0;
+      }
       HashMap map = new HashMap();
-      map.put(binCounterField,new Double(rval+1.0));
+      map.put(binCounterField,new Double(rval));
       performUpdate(map," WHERE "+query,params,null);
-      return rval;
     }
     else
     {
+      for (int i = 0; i < count; i++)
+      {
+        returnValues[i] = newBinValue;
+        newBinValue += 1.0;
+      }
       HashMap map = new HashMap();
       map.put(binNameField,binName);
-      map.put(binCounterField,new Double(newBinValue+1.0));
+      map.put(binCounterField,new Double(newBinValue));
       performInsert(map,null);
-      return newBinValue;
+    }
+    return returnValues;
+  }
+
+  /** Get N bin values (and set next one).  If the record does not yet exist, create it with
a starting value.
+  * This method invokes its own retry-able transaction.
+  *@param binName is the name of the bin (256 char max)
+  *@param newBinValue is the value to use if there is no such bin yet.  This is the value
that will be
+  * returned; what will be stored will be that value + 1.
+  *@param count is the number of values desired.
+  *@return the counter values.
+  */
+  @Override
+  public double[] getIncrementBinValuesInTransaction(String binName, double newBinValue,
int count)
+    throws ManifoldCFException
+  {
+    while (true)
+    {
+      long sleepAmt = 0L;
+      beginTransaction();
+      try
+      {
+        return getIncrementBinValues(binName, newBinValue, count);
+      }
+      catch (Error e)
+      {
+        signalRollback();
+        throw e;
+      }
+      catch (RuntimeException e)
+      {
+        signalRollback();
+        throw e;
+      }
+      catch (ManifoldCFException e)
+      {
+        signalRollback();
+        if (e.getErrorCode() == e.DATABASE_TRANSACTION_ABORT)
+        {
+          if (Logging.perf.isDebugEnabled())
+            Logging.perf.debug("Aborted transaction obtaining docpriorities: "+e.getMessage());
+          sleepAmt = getSleepAmt();
+          continue;
+        }
+        throw e;
+      }
+      finally
+      {
+        endTransaction();
+        sleepFor(sleepAmt);
+      }
     }
   }
 

Modified: manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IBinManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IBinManager.java?rev=1546493&r1=1546492&r2=1546493&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IBinManager.java
(original)
+++ manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IBinManager.java
Fri Nov 29 01:59:41 2013
@@ -41,14 +41,26 @@ public interface IBinManager
   public void reset()
     throws ManifoldCFException;
 
-  /** Get a bin value (and set next one).  If the record does not yet exist, create it with
a starting value.
+  /** Get N bin values (and set next one).  If the record does not yet exist, create it with
a starting value.
   * We expect this to happen within a transaction!! 
   *@param binName is the name of the bin (256 char max)
   *@param newBinValue is the value to use if there is no such bin yet.  This is the value
that will be
   * returned; what will be stored will be that value + 1.
-  *@return the counter value.
+  *@param count is the number of values desired.
+  *@return the counter values.
   */
-  public double getIncrementBinValue(String binName, double newBinValue)
+  public double[] getIncrementBinValues(String binName, double newBinValue, int count)
+    throws ManifoldCFException;
+
+  /** Get N bin values (and set next one).  If the record does not yet exist, create it with
a starting value.
+  * This method invokes its own retry-able transaction.
+  *@param binName is the name of the bin (256 char max)
+  *@param newBinValue is the value to use if there is no such bin yet.  This is the value
that will be
+  * returned; what will be stored will be that value + 1.
+  *@param count is the number of values desired.
+  *@return the counter values.
+  */
+  public double[] getIncrementBinValuesInTransaction(String binName, double newBinValue,
int count)
     throws ManifoldCFException;
 
 }

Modified: manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java?rev=1546493&r1=1546492&r2=1546493&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
(original)
+++ manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
Fri Nov 29 01:59:41 2013
@@ -4536,8 +4536,6 @@ public class JobManager implements IJobM
               
           IResultSet set = database.performQuery(sb.toString(),list,null,null);
 
-          boolean priorityUsed;
-
           if (set.getRowCount() > 0)
           {
             // Found a row, and it is now locked.

Modified: manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ReprioritizationTracker.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ReprioritizationTracker.java?rev=1546493&r1=1546492&r2=1546493&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ReprioritizationTracker.java
(original)
+++ manifoldcf/branches/CONNECTORS-781/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/system/ReprioritizationTracker.java
Fri Nov 29 01:59:41 2013
@@ -255,7 +255,7 @@ public class ReprioritizationTracker
   public double getIncrementBinValue(String binName, double weightedMinimumDepth)
     throws ManifoldCFException
   {
-    return binManager.getIncrementBinValue(binName, weightedMinimumDepth);
+    return binManager.getIncrementBinValues(binName, weightedMinimumDepth,1)[0];
   }
   
   // Protected methods



Mime
View raw message