manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1618145 - in /manifoldcf/branches/release-1.7-branch: ./ CHANGES.txt framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
Date Fri, 15 Aug 2014 10:24:29 GMT
Author: kwright
Date: Fri Aug 15 10:24:29 2014
New Revision: 1618145

URL: http://svn.apache.org/r1618145
Log:
Pull up fix for CONNECTORS-1010 from trunk

Modified:
    manifoldcf/branches/release-1.7-branch/   (props changed)
    manifoldcf/branches/release-1.7-branch/CHANGES.txt
    manifoldcf/branches/release-1.7-branch/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java

Propchange: manifoldcf/branches/release-1.7-branch/
------------------------------------------------------------------------------
  Merged /manifoldcf/trunk:r1618144

Modified: manifoldcf/branches/release-1.7-branch/CHANGES.txt
URL: http://svn.apache.org/viewvc/manifoldcf/branches/release-1.7-branch/CHANGES.txt?rev=1618145&r1=1618144&r2=1618145&view=diff
==============================================================================
--- manifoldcf/branches/release-1.7-branch/CHANGES.txt (original)
+++ manifoldcf/branches/release-1.7-branch/CHANGES.txt Fri Aug 15 10:24:29 2014
@@ -3,6 +3,10 @@ $Id$
 
 ======================= Release 1.7 =====================
 
+CONNECTORS-1010: Use uri hash instead of full URI as lock key
+to prevent "file name too long" errors on windows.
+(Erlend Garåsen, Shigeki Kobayashi, Karl Wright)
+
 CONNECTORS-1007: Fix CMIS connector so that tests pass.
 (Karl Wright)
 

Modified: manifoldcf/branches/release-1.7-branch/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/release-1.7-branch/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java?rev=1618145&r1=1618144&r2=1618145&view=diff
==============================================================================
--- manifoldcf/branches/release-1.7-branch/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
(original)
+++ manifoldcf/branches/release-1.7-branch/framework/agents/src/main/java/org/apache/manifoldcf/agents/incrementalingest/IncrementalIngester.java
Fri Aug 15 10:24:29 2014
@@ -1036,13 +1036,15 @@ public class IncrementalIngester extends
       }
       String[] lockArray = new String[validURIcount];
       String[] validURIArray = new String[validURIcount];
+      String[] validURIHashArray = new String[validURIcount];
       validURIcount = 0;
       for (int i = 0; i < uris.length; i++)
       {
         if (uris[i] != null && uris[i].getURI() != null)
         {
           validURIArray[validURIcount] = uris[i].getURI();
-          lockArray[validURIcount] = outputConnectionName+":"+validURIArray[validURIcount];
+          validURIHashArray[validURIcount] = uris[i].getURIHash();
+          lockArray[validURIcount] = outputConnectionName+":"+validURIHashArray[validURIcount];
           validURIcount++;
         }
       }
@@ -1079,9 +1081,11 @@ public class IncrementalIngester extends
           Set<String> docURIValues = new HashSet<String>();
           for (String docDBString : validURIArray)
           {
-            String docDBHashString = ManifoldCF.hash(docDBString);
             docURIValues.add(docDBString);
-            docURIHashValues.add(docDBHashString);
+          }
+          for (String docDBString : validURIHashArray)
+          {
+            docURIHashValues.add(docDBString);
           }
 
           // Now, perform n queries, each of them no larger the maxInClause in length.
@@ -1248,13 +1252,15 @@ public class IncrementalIngester extends
       }
       String[] lockArray = new String[validURIcount];
       String[] validURIArray = new String[validURIcount];
+      String[] validURIHashArray = new String[validURIcount];
       validURIcount = 0;
       for (int i = 0; i < uris.length; i++)
       {
         if (uris[i] != null && uris[i].getURI() != null)
         {
           validURIArray[validURIcount] = uris[i].getURI();
-          lockArray[validURIcount] = outputConnectionName+":"+validURIArray[validURIcount];
+          validURIHashArray[validURIcount] = uris[i].getURIHash();
+          lockArray[validURIcount] = outputConnectionName+":"+validURIHashArray[validURIcount];
           validURIcount++;
         }
       }
@@ -1291,9 +1297,11 @@ public class IncrementalIngester extends
           Set<String> docURIValues = new HashSet<String>();
           for (String docDBString : validURIArray)
           {
-            String docDBHashString = ManifoldCF.hash(docDBString);
             docURIValues.add(docDBString);
-            docURIHashValues.add(docDBHashString);
+          }
+          for (String docDBString : validURIHashArray)
+          {
+            docURIHashValues.add(docDBString);
           }
 
           // Now, perform n queries, each of them no larger the maxInClause in length.
@@ -2237,7 +2245,7 @@ public class IncrementalIngester extends
       new MultiClause(docKeyField,list),
       new UnitaryClause(outputConnNameField,outputConnectionName)});
       
-    IResultSet set = performQuery("SELECT "+docKeyField+","+docURIField+","+lastOutputVersionField+"
FROM "+getTableName()+" WHERE "+
+    IResultSet set = performQuery("SELECT "+docKeyField+","+docURIField+","+uriHashField+","+lastOutputVersionField+"
FROM "+getTableName()+" WHERE "+
       query,newList,null,null);
 
     // Go through list and put into buckets.
@@ -2245,14 +2253,17 @@ public class IncrementalIngester extends
     {
       IResultRow row = set.getRow(i);
       String docHash = row.getValue(docKeyField).toString();
-      Integer position = (Integer)map.get(docHash);
+      Integer position = map.get(docHash);
       if (position != null)
       {
         String lastURI = (String)row.getValue(docURIField);
         if (lastURI != null && lastURI.length() == 0)
           lastURI = null;
+        String lastURIHash = (String)row.getValue(uriHashField);
+        if (lastURIHash != null && lastURIHash.length() == 0)
+          lastURIHash = null;
         String lastOutputVersion = (String)row.getValue(lastOutputVersionField);
-        rval[position.intValue()] = new DeleteInfo(lastURI,lastOutputVersion);
+        rval[position.intValue()] = new DeleteInfo(lastURI,lastURIHash,lastOutputVersion);
       }
     }
   }
@@ -2283,7 +2294,7 @@ public class IncrementalIngester extends
       new UnitaryClause(outputConnNameField,outputConnectionName),
       ((componentHash==null)?new NullCheckClause(componentHashField,true):new UnitaryClause(componentHashField,componentHash))});
       
-    IResultSet set = performQuery("SELECT "+docKeyField+","+docURIField+","+lastOutputVersionField+"
FROM "+getTableName()+" WHERE "+
+    IResultSet set = performQuery("SELECT "+docKeyField+","+docURIField+","+uriHashField+","+lastOutputVersionField+"
FROM "+getTableName()+" WHERE "+
       query,newList,null,null);
 
     // Go through list and put into buckets.
@@ -2297,8 +2308,11 @@ public class IncrementalIngester extends
         String lastURI = (String)row.getValue(docURIField);
         if (lastURI != null && lastURI.length() == 0)
           lastURI = null;
+        String lastURIHash = (String)row.getValue(uriHashField);
+        if (lastURIHash != null && lastURIHash.length() == 0)
+          lastURIHash = null;
         String lastOutputVersion = (String)row.getValue(lastOutputVersionField);
-        rval[position.intValue()] = new DeleteInfo(lastURI,lastOutputVersion);
+        rval[position.intValue()] = new DeleteInfo(lastURI,lastURIHash,lastOutputVersion);
       }
     }
   }
@@ -2351,11 +2365,13 @@ public class IncrementalIngester extends
   protected static class DeleteInfo
   {
     protected String uriValue;
+    protected String uriHashValue;
     protected String outputVersion;
 
-    public DeleteInfo(String uriValue, String outputVersion)
+    public DeleteInfo(String uriValue, String uriHashValue, String outputVersion)
     {
       this.uriValue = uriValue;
+      this.uriHashValue = uriHashValue;
       this.outputVersion = outputVersion;
     }
 
@@ -2364,6 +2380,11 @@ public class IncrementalIngester extends
       return uriValue;
     }
 
+    public String getURIHash()
+    {
+      return uriHashValue;
+    }
+
     public String getOutputVersion()
     {
       return outputVersion;
@@ -3325,7 +3346,7 @@ public class IncrementalIngester extends
       // But, since we need to insure that any given URI is only worked on by one thread
at a time, use critical sections
       // to block the rare case that multiple threads try to work on the same URI.
       
-      String[] lockArray = computeLockArray(documentURI,oldURI,outputConnectionName);
+      String[] lockArray = computeLockArray(documentURIHash,oldURIHash,outputConnectionName);
       lockManager.enterLocks(null,null,lockArray);
       try
       {



Mime
View raw message