manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1636390 - in /manifoldcf/branches/release-1.7-branch: ./ framework/core/src/main/java/org/apache/manifoldcf/core/database/ framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/ framework/pull-agent/src/main/java/org/apache/ma...
Date Mon, 03 Nov 2014 16:57:28 GMT
Author: kwright
Date: Mon Nov  3 16:57:28 2014
New Revision: 1636390

URL: http://svn.apache.org/r1636390
Log:
Pull up fixes for CONNECTORS-1090 from dev_1x branch

Modified:
    manifoldcf/branches/release-1.7-branch/   (props changed)
    manifoldcf/branches/release-1.7-branch/CHANGES.txt
    manifoldcf/branches/release-1.7-branch/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java
    manifoldcf/branches/release-1.7-branch/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/JoinClause.java
    manifoldcf/branches/release-1.7-branch/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java

Propchange: manifoldcf/branches/release-1.7-branch/
------------------------------------------------------------------------------
  Merged /manifoldcf/branches/dev_1x:r1635810,1636147,1636182
  Merged /manifoldcf/trunk:r1635809,1636146,1636180

Modified: manifoldcf/branches/release-1.7-branch/CHANGES.txt
URL: http://svn.apache.org/viewvc/manifoldcf/branches/release-1.7-branch/CHANGES.txt?rev=1636390&r1=1636389&r2=1636390&view=diff
==============================================================================
--- manifoldcf/branches/release-1.7-branch/CHANGES.txt (original)
+++ manifoldcf/branches/release-1.7-branch/CHANGES.txt Mon Nov  3 16:57:28 2014
@@ -2,8 +2,9 @@ ManifoldCF Change Log
 $Id$
 
 
-CONNECTORS-1043: Fix list item URL.
-(Lalit Jangra, Karl Wright)
+CONNECTORS-1090: More PostgreSQL performance improvements.
+(Karl Wright)
+
 
 ======================= Release 1.7.1 =====================
 

Modified: manifoldcf/branches/release-1.7-branch/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/release-1.7-branch/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java?rev=1636390&r1=1636389&r2=1636390&view=diff
==============================================================================
--- manifoldcf/branches/release-1.7-branch/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java
(original)
+++ manifoldcf/branches/release-1.7-branch/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java
Mon Nov  3 16:57:28 2014
@@ -611,7 +611,10 @@ public class DBInterfacePostgreSQL exten
       return theException;
     Throwable e = theException.getCause();
     if (!(e instanceof java.sql.SQLException))
+    {
+      //e.printStackTrace();
       return theException;
+    }
     if (Logging.db.isDebugEnabled())
       Logging.db.debug("Exception "+theException.getMessage()+" is possibly a transaction
abort signal");
     java.sql.SQLException sqlException = (java.sql.SQLException)e;
@@ -628,8 +631,14 @@ public class DBInterfacePostgreSQL exten
     // one could make.)
     if (sqlState != null && sqlState.equals("23505"))
       return new ManifoldCFException(message,e,ManifoldCFException.DATABASE_TRANSACTION_ABORT);
+    // New Postgresql behavior (9.3): sometimes we don't get an exception thrown, but the
transaction is dead nonetheless.
+    if (sqlState != null && sqlState.equals("25P02"))
+      return new ManifoldCFException(message,e,ManifoldCFException.DATABASE_TRANSACTION_ABORT);
+      
     if (Logging.db.isDebugEnabled())
       Logging.db.debug("Exception "+theException.getMessage()+" is NOT a transaction abort
signal");
+    //e.printStackTrace();
+    //System.err.println("sqlstate = "+sqlState);
     return theException;
   }
 

Modified: manifoldcf/branches/release-1.7-branch/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/JoinClause.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/release-1.7-branch/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/JoinClause.java?rev=1636390&r1=1636389&r2=1636390&view=diff
==============================================================================
--- manifoldcf/branches/release-1.7-branch/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/JoinClause.java
(original)
+++ manifoldcf/branches/release-1.7-branch/framework/core/src/main/java/org/apache/manifoldcf/core/interfaces/JoinClause.java
Mon Nov  3 16:57:28 2014
@@ -24,14 +24,21 @@ public class JoinClause implements Claus
 {
   public static final String _rcsid = "@(#)$Id$";
 
-  protected String columnName;
-  protected String joinColumnName;
+  protected final String columnName;
+  protected final String joinColumnName;
+  protected final String operation;
   
   /** Construct */
   public JoinClause(String columnName, String joinColumnName)
   {
+    this(columnName,joinColumnName,"=");
+  }
+  
+  public JoinClause(String columnName, String joinColumnName, String operation)
+  {
     this.columnName = columnName;
     this.joinColumnName = joinColumnName;
+    this.operation = "=";
   }
   
   /** Get the column name */
@@ -43,7 +50,7 @@ public class JoinClause implements Claus
   /** Get the operation string */
   public String getOperation()
   {
-    return "=";
+    return operation;
   }
   
   /** Get the individual values */

Modified: manifoldcf/branches/release-1.7-branch/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/release-1.7-branch/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java?rev=1636390&r1=1636389&r2=1636390&view=diff
==============================================================================
--- manifoldcf/branches/release-1.7-branch/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
(original)
+++ manifoldcf/branches/release-1.7-branch/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobManager.java
Mon Nov  3 16:57:28 2014
@@ -1522,18 +1522,17 @@ public class JobManager implements IJobM
           
           sb.append("NOT EXISTS(SELECT 'x' FROM ").append(jobQueue.getTableName()).append("
t2 WHERE ")
             .append(database.buildConjunctionClause(list,new ClauseDescription[]{
-              new JoinClause("t2."+jobQueue.docHashField,"t0."+jobQueue.docHashField)})).append("
AND ")
-            .append("t2.").append(jobQueue.statusField).append(" IN (?,?,?,?,?,?) AND ")
-            .append("t2.").append(jobQueue.jobIDField).append("!=t0.").append(jobQueue.jobIDField)
+              new JoinClause("t2."+jobQueue.docHashField,"t0."+jobQueue.docHashField),
+              new MultiClause("t2."+jobQueue.statusField,new String[]{
+                jobQueue.statusToString(jobQueue.STATUS_ACTIVE),
+                jobQueue.statusToString(jobQueue.STATUS_ACTIVEPURGATORY),
+                jobQueue.statusToString(jobQueue.STATUS_ACTIVENEEDRESCAN),
+                jobQueue.statusToString(jobQueue.STATUS_ACTIVENEEDRESCANPURGATORY),
+                jobQueue.statusToString(jobQueue.STATUS_BEINGDELETED),
+                jobQueue.statusToString(jobQueue.STATUS_BEINGCLEANED)}),
+              new JoinClause("t2."+jobQueue.jobIDField,"t0."+jobQueue.jobIDField,"!=")}))
             .append(") ");
             
-          list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVE));
-          list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVEPURGATORY));
-          list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVENEEDRESCAN));
-          list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVENEEDRESCANPURGATORY));
-          list.add(jobQueue.statusToString(jobQueue.STATUS_BEINGDELETED));
-          list.add(jobQueue.statusToString(jobQueue.STATUS_BEINGCLEANED));
-
           sb.append(database.constructOffsetLimitClause(0,maxCount));
           
           // The checktime is null field check is for backwards compatibility
@@ -1765,18 +1764,17 @@ public class JobManager implements IJobM
             
           sb.append("NOT EXISTS(SELECT 'x' FROM ").append(jobQueue.getTableName()).append("
t2 WHERE ")
             .append(database.buildConjunctionClause(list,new ClauseDescription[]{
-              new JoinClause("t2."+jobQueue.docHashField,"t0."+jobQueue.docHashField)})).append("
AND ")
-            .append("t2.").append(jobQueue.statusField).append(" IN (?,?,?,?,?,?) AND ")
-            .append("t2.").append(jobQueue.jobIDField).append("!=t0.").append(jobQueue.jobIDField)
+              new JoinClause("t2."+jobQueue.docHashField,"t0."+jobQueue.docHashField),
+              new MultiClause("t2."+jobQueue.statusField,new String[]{
+                jobQueue.statusToString(jobQueue.STATUS_ACTIVE),
+                jobQueue.statusToString(jobQueue.STATUS_ACTIVEPURGATORY),
+                jobQueue.statusToString(jobQueue.STATUS_ACTIVENEEDRESCAN),
+                jobQueue.statusToString(jobQueue.STATUS_ACTIVENEEDRESCANPURGATORY),
+                jobQueue.statusToString(jobQueue.STATUS_BEINGDELETED),
+                jobQueue.statusToString(jobQueue.STATUS_BEINGCLEANED)}),
+              new JoinClause("t2."+jobQueue.jobIDField,"t0."+jobQueue.jobIDField,"!=")}))
             .append(") ");
             
-          list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVE));
-          list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVEPURGATORY));
-          list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVENEEDRESCAN));
-          list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVENEEDRESCANPURGATORY));
-          list.add(jobQueue.statusToString(jobQueue.STATUS_BEINGDELETED));
-          list.add(jobQueue.statusToString(jobQueue.STATUS_BEINGCLEANED));
-          
           sb.append(database.constructOffsetLimitClause(0,maxCount));
           
           // The checktime is null field check is for backwards compatibility
@@ -2304,16 +2302,16 @@ public class JobManager implements IJobM
     
     sb.append("NOT EXISTS(SELECT 'x' FROM ").append(jobQueue.getTableName()).append(" t2
WHERE ")
       .append(database.buildConjunctionClause(list,new ClauseDescription[]{
-        new JoinClause("t2."+jobQueue.docHashField,"t0."+jobQueue.docHashField)})).append("
AND ")
-      .append("t2.").append(jobQueue.statusField).append(" IN (?,?,?,?,?,?)").append(" AND
")
-      .append("t2.").append(jobQueue.jobIDField).append("!=t0.").append(jobQueue.jobIDField).append(")
");
-
-    list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVE));
-    list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVEPURGATORY));
-    list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVENEEDRESCAN));
-    list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVENEEDRESCANPURGATORY));
-    list.add(jobQueue.statusToString(jobQueue.STATUS_BEINGDELETED));
-    list.add(jobQueue.statusToString(jobQueue.STATUS_BEINGCLEANED));
+        new JoinClause("t2."+jobQueue.docHashField,"t0."+jobQueue.docHashField),
+        new MultiClause("t2."+jobQueue.statusField,new String[]{
+          jobQueue.statusToString(jobQueue.STATUS_ACTIVE),
+          jobQueue.statusToString(jobQueue.STATUS_ACTIVEPURGATORY),
+          jobQueue.statusToString(jobQueue.STATUS_ACTIVENEEDRESCAN),
+          jobQueue.statusToString(jobQueue.STATUS_ACTIVENEEDRESCANPURGATORY),
+          jobQueue.statusToString(jobQueue.STATUS_BEINGDELETED),
+          jobQueue.statusToString(jobQueue.STATUS_BEINGCLEANED)}),
+        new JoinClause("t2."+jobQueue.jobIDField,"t0."+jobQueue.jobIDField,"!=")}))
+      .append(") ");
 
     sb.append(database.constructOffsetLimitClause(0,n));
 
@@ -2836,7 +2834,7 @@ public class JobManager implements IJobM
       .append(" t0 ").append(jobQueue.getGetNextDocumentsIndexHint()).append(" WHERE ");
     
     sb.append(database.buildConjunctionClause(list,new ClauseDescription[]{
-      //new UnitaryClause("t0."+jobQueue.docPriorityField,">=",new Long(0L)),
+      new UnitaryClause("t0."+jobQueue.docPriorityField,"<",JobQueue.nullDocPriority),
 // Note: This is technically correct, but I need to confirm that it works OK for MySQL and
HSQLDB
       new MultiClause("t0."+jobQueue.statusField,new Object[]{
         jobQueue.statusToString(JobQueue.STATUS_PENDING),
         jobQueue.statusToString(JobQueue.STATUS_PENDINGPURGATORY)}),
@@ -2854,18 +2852,17 @@ public class JobManager implements IJobM
     
     sb.append("NOT EXISTS(SELECT 'x' FROM ").append(jobQueue.getTableName()).append(" t2
WHERE ")
       .append(database.buildConjunctionClause(list,new ClauseDescription[]{
-        new JoinClause("t2."+jobQueue.docHashField,"t0."+jobQueue.docHashField)})).append("
AND ")
-      .append("t2.").append(jobQueue.statusField).append(" IN (?,?,?,?,?,?) AND ")
-      .append("t2.").append(jobQueue.jobIDField).append("!=t0.").append(jobQueue.jobIDField)
+        new JoinClause("t2."+jobQueue.docHashField,"t0."+jobQueue.docHashField),
+        new MultiClause("t2."+jobQueue.statusField,new String[]{
+          jobQueue.statusToString(jobQueue.STATUS_ACTIVE),
+          jobQueue.statusToString(jobQueue.STATUS_ACTIVEPURGATORY),
+          jobQueue.statusToString(jobQueue.STATUS_ACTIVENEEDRESCAN),
+          jobQueue.statusToString(jobQueue.STATUS_ACTIVENEEDRESCANPURGATORY),
+          jobQueue.statusToString(jobQueue.STATUS_BEINGDELETED),
+          jobQueue.statusToString(jobQueue.STATUS_BEINGCLEANED)}),
+        new JoinClause("t2."+jobQueue.jobIDField,"t0."+jobQueue.jobIDField,"!=")}))
       .append(") AND ");
 
-    list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVE));
-    list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVEPURGATORY));
-    list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVENEEDRESCAN));
-    list.add(jobQueue.statusToString(jobQueue.STATUS_ACTIVENEEDRESCANPURGATORY));
-    list.add(jobQueue.statusToString(jobQueue.STATUS_BEINGDELETED));
-    list.add(jobQueue.statusToString(jobQueue.STATUS_BEINGCLEANED));
-        
     // Prerequisite event clause: AND NOT EXISTS(SELECT 'x' FROM prereqevents t3,events t4
WHERE t3.ownerid=t0.id AND t3.name=t4.name)
     sb.append("NOT EXISTS(SELECT 'x' FROM ").append(jobQueue.prereqEventManager.getTableName()).append("
t3,").append(eventManager.getTableName()).append(" t4 WHERE t0.")
       .append(jobQueue.idField).append("=t3.").append(jobQueue.prereqEventManager.ownerField).append("
AND t3.")



Mime
View raw message