roller-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From snoopd...@apache.org
Subject svn commit: r350046 - in /incubator/roller/trunk: metadata/database/ src/org/roller/business/ src/org/roller/business/hibernate/ src/org/roller/pojos/ src/org/roller/presentation/servlets/ src/org/roller/presentation/website/actions/ src/org/roller/uti...
Date Wed, 30 Nov 2005 20:48:16 GMT
Author: snoopdave
Date: Wed Nov 30 12:47:55 2005
New Revision: 350046

URL: http://svn.apache.org/viewcvs?rev=350046&view=rev
Log:
Done with unified blacklist, may still need way to migrate old ignore words to new blacklist

Modified:
    incubator/roller/trunk/metadata/database/200-to-210-migration-raw.sql
    incubator/roller/trunk/metadata/database/createdb-raw.sql
    incubator/roller/trunk/src/org/roller/business/PropertiesManagerImpl.java
    incubator/roller/trunk/src/org/roller/business/RefererManagerImpl.java
    incubator/roller/trunk/src/org/roller/business/UserManagerImpl.java
    incubator/roller/trunk/src/org/roller/business/hibernate/HibernateRefererManagerImpl.java
    incubator/roller/trunk/src/org/roller/pojos/WebsiteData.java
    incubator/roller/trunk/src/org/roller/presentation/servlets/CommentServlet.java
    incubator/roller/trunk/src/org/roller/presentation/servlets/TrackbackServlet.java
    incubator/roller/trunk/src/org/roller/presentation/website/actions/WebsiteFormAction.java
    incubator/roller/trunk/src/org/roller/util/Blacklist.java
    incubator/roller/trunk/src/org/roller/util/SpamChecker.java
    incubator/roller/trunk/tests/org/roller/business/RefererManagerTest.java
    incubator/roller/trunk/web/WEB-INF/classes/ApplicationResources.properties
    incubator/roller/trunk/web/WEB-INF/classes/roller.properties
    incubator/roller/trunk/web/WEB-INF/classes/rollerRuntimeConfigDefs.xml
    incubator/roller/trunk/web/website/edit-website.jsp

Modified: incubator/roller/trunk/metadata/database/200-to-210-migration-raw.sql
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/metadata/database/200-to-210-migration-raw.sql?rev=350046&r1=350045&r2=350046&view=diff
==============================================================================
--- incubator/roller/trunk/metadata/database/200-to-210-migration-raw.sql (original)
+++ incubator/roller/trunk/metadata/database/200-to-210-migration-raw.sql Wed Nov 30 12:47:55 2005
@@ -1,17 +1,29 @@
 
+-- Add to roller_comment table: approved and pending fields
+
 alter table roller_comment add column approved @BOOLEAN_SQL_TYPE_FALSE@;
 @ALTER_TABLE_ROLLER_COMMENT_APPROVED_NOT_NULL@;
+
 alter table roller_comment add column pending @BOOLEAN_SQL_TYPE_TRUE@;
 @ALTER_TABLE_ROLLER_COMMENT_PENDING_NOT_NULL@;
+
 update roller_comment set approved=1, pending=0, posttime=posttime;
 
+
+-- Add to website table: commentmod, blacklist, defaultallowcomments and defaultcommentdays 
+
 alter table website add column commentmod @BOOLEAN_SQL_TYPE_FALSE@;
 @ALTER_TABLE_WEBSITE_COMMENTMOD_NOT_NULL@;
+
 alter table website add column defaultallowcomments @BOOLEAN_SQL_TYPE_TRUE@;
 @ALTER_TABLE_WEBSITE_DEFAULTALLOWCOMMENTS_NOT_NULL@;
+
 alter table website add column defaultcommentdays integer;
 @ALTER_TABLE_WEBSITE_DEFAULTCOMMENTDAYS_NOT_NULL@;
-update website set commentmod=0, defaultallowcomments=1, defaultcommentdays=0, datecreated=datecreated;
+
+alter table website add column blacklist @TEXT_SQL_TYPE@;
+
+update website set commentmod=0, defaultallowcomments=1, defaultcommentdays=0, blacklist='' datecreated=datecreated;
 
 
 

Modified: incubator/roller/trunk/metadata/database/createdb-raw.sql
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/metadata/database/createdb-raw.sql?rev=350046&r1=350045&r2=350046&view=diff
==============================================================================
--- incubator/roller/trunk/metadata/database/createdb-raw.sql (original)
+++ incubator/roller/trunk/metadata/database/createdb-raw.sql Wed Nov 30 12:47:55 2005
@@ -107,7 +107,11 @@
     defaultplugins    varchar(255),
     pinnedtomain      @BOOLEAN_SQL_TYPE_FALSE@ not null,
     isenabled         @BOOLEAN_SQL_TYPE_TRUE@ not null,
-    datecreated     timestamp not null
+    datecreated      timestamp not null,
+    blacklist               @TEXT_SQL_TYPE@,
+    defaultallowcomments    @BOOLEAN_SQL_TYPE_TRUE@ not null,
+    defaultallowcommentdays integer default 7 not null,
+    commentmod              @BOOLEAN_SQL_TYPE_FALSE@ not null    
 );
 create index ws_userid_idx    on website(userid);
 create index ws_isenabled_idx on website(isenabled);
@@ -215,9 +219,13 @@
     posttime timestamp   not null,
     spam    @BOOLEAN_SQL_TYPE_FALSE@ not null,
     notify  @BOOLEAN_SQL_TYPE_FALSE@ not null,
-    remotehost varchar(128)
+    remotehost varchar(128),
+    pending  @BOOLEAN_SQL_TYPE_TRUE@ not null,
+    approved  @BOOLEAN_SQL_TYPE_FALSE@ not null
 );
 create index co_entryid_idx on roller_comment( entryid );
+create index co_pending_idx on roller_comment( pending );
+create index co_approved_idx on roller_comment( approved );
 
 -- Ping Feature Tables
 -- name: short descriptive name of the ping target

Modified: incubator/roller/trunk/src/org/roller/business/PropertiesManagerImpl.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/roller/business/PropertiesManagerImpl.java?rev=350046&r1=350045&r2=350046&view=diff
==============================================================================
--- incubator/roller/trunk/src/org/roller/business/PropertiesManagerImpl.java (original)
+++ incubator/roller/trunk/src/org/roller/business/PropertiesManagerImpl.java Wed Nov 30 12:47:55 2005
@@ -187,8 +187,8 @@
             props.put("debug.memory.enabled",
                 new RollerPropertyData("debug.memory.enabled", rollerConfig.getMemDebug().toString()));
             */
-            props.put("spam.referers.ignorewords",
-                new RollerPropertyData("spam.referers.ignorewords", rollerConfig.getRefererSpamWords()));
+            props.put("spam.blacklist",
+                new RollerPropertyData("spam.blacklist", rollerConfig.getRefererSpamWords()));
         }
         else
         {

Modified: incubator/roller/trunk/src/org/roller/business/RefererManagerImpl.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/roller/business/RefererManagerImpl.java?rev=350046&r1=350045&r2=350046&view=diff
==============================================================================
--- incubator/roller/trunk/src/org/roller/business/RefererManagerImpl.java (original)
+++ incubator/roller/trunk/src/org/roller/business/RefererManagerImpl.java Wed Nov 30 12:47:55 2005
@@ -218,7 +218,7 @@
                     }
                     else
                     {
-                        // If referer URL contains spamWords or ignoreWords then don't log it.
+                        // If referer URL is blacklisted, don't log it
                         boolean isRefererSpam = checkForSpam(refererUrl, website);
                         if (isRefererSpam) return true;
                     }
@@ -356,7 +356,7 @@
     
     /**
      * Check the Referer URL against the Site-wide RefererSpamWords list
-     * and against the user's own IgnoreWords list.  If the Referer contains
+     * and against the user's own blacklist list.  If the Referer contains
      * any of the words from either list consider it Spam.
      * 
      * @param refererUrl
@@ -365,7 +365,7 @@
      */
     private boolean checkForSpam(String refererUrl, WebsiteData website) throws RollerException
     {
-        String spamwords = RollerRuntimeConfig.getProperty("spam.referers.ignorewords");
+        String spamwords = RollerRuntimeConfig.getProperty("spam.blacklist");
         if (spamwords == null) {
         		// Oracle returns nulls instead of empty string so next line would throw NPE.
         		spamwords = "";
@@ -373,12 +373,12 @@
         LinkedList spamWords = new LinkedList(Arrays.asList(
                 StringUtils.split(StringUtils.deleteWhitespace(spamwords), ",")));
     
-        if ( website.getIgnoreWords() != null )
+        if ( website.getBlacklist() != null )
         {
             spamWords.addAll( 
                 Arrays.asList(StringUtils.split(
                     StringUtils.deleteWhitespace(
-                        website.getIgnoreWords()),",")));
+                        website.getBlacklist()),",")));
         }
         for( Iterator i = spamWords.iterator(); i.hasNext(); )
         {

Modified: incubator/roller/trunk/src/org/roller/business/UserManagerImpl.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/roller/business/UserManagerImpl.java?rev=350046&r1=350045&r2=350046&view=diff
==============================================================================
--- incubator/roller/trunk/src/org/roller/business/UserManagerImpl.java (original)
+++ incubator/roller/trunk/src/org/roller/business/UserManagerImpl.java Wed Nov 30 12:47:55 2005
@@ -249,7 +249,7 @@
             null,                // bloggerCategory
             null,                // defaultCategory
             "editor-text.jsp",   // editorPage
-            "",                  // ignoreWords
+            "",                  // blacklist
             Boolean.TRUE,        // allowComments  
             Boolean.FALSE,       // emailComments
             "",                  // emailFromAddress

Modified: incubator/roller/trunk/src/org/roller/business/hibernate/HibernateRefererManagerImpl.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/roller/business/hibernate/HibernateRefererManagerImpl.java?rev=350046&r1=350045&r2=350046&view=diff
==============================================================================
--- incubator/roller/trunk/src/org/roller/business/hibernate/HibernateRefererManagerImpl.java (original)
+++ incubator/roller/trunk/src/org/roller/business/hibernate/HibernateRefererManagerImpl.java Wed Nov 30 12:47:55 2005
@@ -61,13 +61,13 @@
             Session session = ((HibernateStrategy)mStrategy).getSession();
             Criteria criteria = session.createCriteria(RefererData.class);
             
-            String spamwords = RollerRuntimeConfig.getProperty("spam.referers.ignorewords");
+            String spamwords = RollerRuntimeConfig.getProperty("spam.blacklist");
             
-            String[] ignoreWords = StringUtils.split(
+            String[] blacklist = StringUtils.split(
                     StringUtils.deleteWhitespace(spamwords),",");
             Junction or = Expression.disjunction();
-            for (int i=0; i<ignoreWords.length; i++) {
-                String ignoreWord = ignoreWords[i].trim();
+            for (int i=0; i<blacklist.length; i++) {
+                String ignoreWord = blacklist[i].trim();
                 or.add(Expression.ilike("refererUrl","%"+ignoreWord+"%"));
             }
             criteria.add(Expression.conjunction()
@@ -90,19 +90,19 @@
      */
     public void applyRefererFilters(WebsiteData website) throws RollerException {
         if (null == website) throw new RollerException("website is null");
-        if (null == website.getIgnoreWords()) return;
+        if (null == website.getBlacklist()) return;
         
         try {
             Session session = ((HibernateStrategy)mStrategy).getSession();
             Criteria criteria = session.createCriteria(RefererData.class);
             
-            String[] ignoreWords = StringUtils.split(
-                    StringUtils.deleteWhitespace(website.getIgnoreWords()),",");
-            if (ignoreWords.length == 0) return;
+            String[] blacklist = StringUtils.split(
+                    StringUtils.deleteWhitespace(website.getBlacklist()),",");
+            if (blacklist.length == 0) return;
             
             Junction or = Expression.disjunction();
-            for (int i=0; i<ignoreWords.length; i++) {
-                String ignoreWord = ignoreWords[i].trim();
+            for (int i=0; i<blacklist.length; i++) {
+                String ignoreWord = blacklist[i].trim();
                 or.add(Expression.ilike("refererUrl","%"+ignoreWord+"%"));
             }
             criteria.add(Expression.conjunction()

Modified: incubator/roller/trunk/src/org/roller/pojos/WebsiteData.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/roller/pojos/WebsiteData.java?rev=350046&r1=350045&r2=350046&view=diff
==============================================================================
--- incubator/roller/trunk/src/org/roller/pojos/WebsiteData.java (original)
+++ incubator/roller/trunk/src/org/roller/pojos/WebsiteData.java Wed Nov 30 12:47:55 2005
@@ -7,7 +7,9 @@
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.StringTokenizer;
 import java.util.TimeZone;
+import java.util.regex.Pattern;
 import org.apache.commons.lang.StringUtils;
 import org.roller.RollerException;
 import org.roller.model.Roller;
@@ -48,7 +50,7 @@
     private String  weblogDayPageId = null;
     private Boolean enableBloggerApi = Boolean.TRUE;
     private String  editorPage = null;
-    private String  ignoreWords = null;
+    private String  blacklist = null;
     private Boolean allowComments = Boolean.TRUE;
     private Boolean emailComments = Boolean.FALSE;
     private String  emailFromAddress = null;
@@ -85,7 +87,7 @@
                        final WeblogCategoryData bloggerCategory,
                        final WeblogCategoryData defaultCategory,
                        final String   editorPage,
-                       final String   ignoreWords,
+                       final String   blacklist,
                        final Boolean  allowComments,
                        final Boolean  emailComments,
                        final String   emailFromAddress,
@@ -108,7 +110,7 @@
         this.bloggerCategory = bloggerCategory;
         this.defaultCategory = defaultCategory;
         this.editorPage = editorPage;
-        this.ignoreWords = ignoreWords;
+        this.blacklist = blacklist;
         this.allowComments = allowComments;
         this.emailComments = emailComments;
         this.emailFromAddress = emailFromAddress;
@@ -554,17 +556,17 @@
     /**
      * @roller.wrapPojoMethod type="simple"
      * @ejb:persistent-field
-     * @hibernate.property column="ignorewords" non-null="true" unique="false"
+     * @hibernate.property column="blacklist" non-null="true" unique="false"
      */
-    public String getIgnoreWords()
+    public String getBlacklist()
     {
-        return this.ignoreWords;
+        return this.blacklist;
     }
 
     /** @ejb:persistent-field */
-    public void setIgnoreWords(String ignoreWords)
+    public void setBlacklist(String blacklist)
     {
-        this.ignoreWords = ignoreWords;
+        this.blacklist = blacklist;
     }
 
     /**
@@ -801,7 +803,7 @@
                    "bloggerCategory=" + bloggerCategory + " " +
                    "defaultCategory=" + defaultCategory + " " +
                    "editorPage=" + editorPage + " " +
-                   "ignoreWords=" + ignoreWords + " " +
+                   "blacklist=" + blacklist + " " +
                    "allowComments=" + allowComments + " " +
                    "emailAddress=" + emailAddress + " " + 
                    "emailComments=" + emailComments + " " + 
@@ -831,7 +833,7 @@
             lEquals = PojoUtil.equals(lEquals, this.getBloggerCategory(), lTest.getBloggerCategory());
             lEquals = PojoUtil.equals(lEquals, this.getDefaultCategory(), lTest.getDefaultCategory());
             lEquals = PojoUtil.equals(lEquals, this.getEditorPage(), lTest.getEditorPage());
-            lEquals = PojoUtil.equals(lEquals, this.getIgnoreWords(), lTest.getIgnoreWords());
+            lEquals = PojoUtil.equals(lEquals, this.getBlacklist(), lTest.getBlacklist());
             lEquals = PojoUtil.equals(lEquals, this.getAllowComments(), lTest.getAllowComments());           
             lEquals = PojoUtil.equals(lEquals, this.getEmailComments(), lTest.getEmailComments());
             lEquals = PojoUtil.equals(lEquals, this.getEmailAddress(), lTest.getEmailAddress());            
@@ -848,59 +850,6 @@
         }
     }
         
-    /*public boolean equals(Object pOther)
-    {
-        if (pOther instanceof WebsiteData)
-        {
-            WebsiteData lTest = (WebsiteData) pOther;
-            boolean lEquals = true;
-
-            lEquals = PojoUtil.equals(lEquals, this.id, lTest.id);
-
-            lEquals = PojoUtil.equals(lEquals, this.name, lTest.name);
-
-            lEquals = PojoUtil.equals(lEquals, this.description, lTest.description);
-
-            lEquals = PojoUtil.equals(lEquals, this.mUser, lTest.mUser);
-
-            lEquals = PojoUtil.equals(lEquals, this.defaultPageId, lTest.defaultPageId);
-
-            lEquals = PojoUtil.equals(lEquals, this.weblogDayPageId, lTest.weblogDayPageId);
-
-            lEquals = PojoUtil.equals(lEquals, this.enableBloggerApi, lTest.enableBloggerApi);
-
-            lEquals = PojoUtil.equals(lEquals, this.bloggerCategory.getId(), lTest.bloggerCategory.getId());
-
-            lEquals = PojoUtil.equals(lEquals, this.defaultCategory.getId(), lTest.defaultCategory.getId());
-
-            lEquals = PojoUtil.equals(lEquals, this.editorPage, lTest.editorPage);
-
-            lEquals = PojoUtil.equals(lEquals, this.ignoreWords, lTest.ignoreWords);
-
-            lEquals = PojoUtil.equals(lEquals, this.allowComments, lTest.allowComments);
-            
-            lEquals = PojoUtil.equals(lEquals, this.emailComments, lTest.emailComments);
-            
-            lEquals = PojoUtil.equals(lEquals, this.emailAddress, lTest.emailAddress);
-            
-            lEquals = PojoUtil.equals(lEquals, this.emailFromAddress, lTest.emailFromAddress);
-
-            lEquals = PojoUtil.equals(lEquals, this.editorTheme, lTest.editorTheme);
-
-            lEquals = PojoUtil.equals(lEquals, this.locale, lTest.locale);
-
-            lEquals = PojoUtil.equals(lEquals, this.timeZone, lTest.timeZone);
-
-            lEquals = PojoUtil.equals(lEquals, this.mDefaultPlugins, lTest.mDefaultPlugins);
-            
-            return lEquals;
-        }
-        else
-        {
-            return false;
-        }
-    }*/
-
     public int hashCode()
     {
         int result = 17;
@@ -914,7 +863,7 @@
         //result = PojoUtil.addHashCode(result, this.bloggerCategory);
         //result = PojoUtil.addHashCode(result, this.defaultCategory);
         result = PojoUtil.addHashCode(result, this.editorPage);
-        result = PojoUtil.addHashCode(result, this.ignoreWords);
+        result = PojoUtil.addHashCode(result, this.blacklist);
         result = PojoUtil.addHashCode(result, this.allowComments);
         result = PojoUtil.addHashCode(result, this.emailComments);
         result = PojoUtil.addHashCode(result, this.emailAddress);
@@ -945,7 +894,7 @@
         this.bloggerCategory = other.getBloggerCategory();
         this.defaultCategory = other.getDefaultCategory();
         this.editorPage = other.getEditorPage();
-        this.ignoreWords = other.getIgnoreWords();
+        this.blacklist = other.getBlacklist();
         this.allowComments = other.getAllowComments();
         this.emailComments = other.getEmailComments();
         this.emailAddress = other.getEmailAddress();
@@ -1104,7 +1053,6 @@
     {
         // no-op
     }
-
-
+    
 }
 

Modified: incubator/roller/trunk/src/org/roller/presentation/servlets/CommentServlet.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/roller/presentation/servlets/CommentServlet.java?rev=350046&r1=350045&r2=350046&view=diff
==============================================================================
--- incubator/roller/trunk/src/org/roller/presentation/servlets/CommentServlet.java (original)
+++ incubator/roller/trunk/src/org/roller/presentation/servlets/CommentServlet.java Wed Nov 30 12:47:55 2005
@@ -88,7 +88,9 @@
      * Here we handle incoming comment postings.  We will collect the data,
      * validate it, and save it.
      */
-    public void doPost(HttpServletRequest request, HttpServletResponse response)
+    public void doPost(
+            HttpServletRequest request, 
+            HttpServletResponse response)
         throws IOException, ServletException {
         
         boolean preview = false;
@@ -146,7 +148,13 @@
             
             if (preview) {
                 message = "This is a comment preview only";
-                error = bundle.getString("commentServlet.previewMarkedAsSpam");
+                
+                // If comment contains blacklisted text, warn commenter
+                SpamChecker checker = new SpamChecker();
+                if (checker.checkComment(comment)) {
+                   error = bundle.getString("commentServlet.previewMarkedAsSpam");
+                   mLogger.debug("Comment marked as spam"); 
+                }
                 request.setAttribute("previewComments", "dummy");
                 mLogger.debug("Comment is a preview");
                 
@@ -159,6 +167,7 @@
                     // If comment contains blacklisted text, mark as spam
                     SpamChecker checker = new SpamChecker();
                     if (checker.checkComment(comment)) {
+                       comment.setSpam(Boolean.TRUE);
                        error = bundle.getString("commentServlet.commentMarkedAsSpam");
                        mLogger.debug("Comment marked as spam"); 
                     }
@@ -177,8 +186,7 @@
                     RollerFactory.getRoller().commit();
                     reindexEntry(entry);
                     
-                    // Refresh user's entries in page cache
-                    //PageCacheFilter.removeFromCache(request, website);
+                    // Clear all caches associated with comment
                     CacheManager.invalidate(comment);
                     
                     // Send email notifications
@@ -202,9 +210,8 @@
         if (message != null)
             session.setAttribute(RollerSession.STATUS_MESSAGE, message);
         
-        if(error == null && message == null && !preview) {
-            entry_permalink = request.getContextPath()+entry_permalink;
-            
+        if(error == null && !preview) {
+            entry_permalink = request.getContextPath()+entry_permalink;            
             mLogger.debug("comment complete, redirecting to "+entry_permalink);
             response.sendRedirect(entry_permalink);
         } else {

Modified: incubator/roller/trunk/src/org/roller/presentation/servlets/TrackbackServlet.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/roller/presentation/servlets/TrackbackServlet.java?rev=350046&r1=350045&r2=350046&view=diff
==============================================================================
--- incubator/roller/trunk/src/org/roller/presentation/servlets/TrackbackServlet.java (original)
+++ incubator/roller/trunk/src/org/roller/presentation/servlets/TrackbackServlet.java Wed Nov 30 12:47:55 2005
@@ -149,6 +149,7 @@
                     // If comment contains blacklisted text, mark as spam
                     SpamChecker checker = new SpamChecker();
                     if (checker.checkTrackback(comment)) {
+                       comment.setSpam(Boolean.TRUE);
                        logger.debug("Trackback marked as spam"); 
                     }
                         
@@ -165,8 +166,7 @@
                     comment.save();
                     RollerFactory.getRoller().commit();
 
-                    // Refresh user's entries in page cache
-                    // PageCacheFilter.removeFromCache(req, entry.getWebsite());
+                    // Clear all caches associated with comment
                     CacheManager.invalidate(comment);
 
                     // Send email notifications

Modified: incubator/roller/trunk/src/org/roller/presentation/website/actions/WebsiteFormAction.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/roller/presentation/website/actions/WebsiteFormAction.java?rev=350046&r1=350045&r2=350046&view=diff
==============================================================================
--- incubator/roller/trunk/src/org/roller/presentation/website/actions/WebsiteFormAction.java (original)
+++ incubator/roller/trunk/src/org/roller/presentation/website/actions/WebsiteFormAction.java Wed Nov 30 12:47:55 2005
@@ -42,106 +42,96 @@
 import org.roller.presentation.RollerSession;
 import org.roller.presentation.cache.CacheManager;
 import org.roller.presentation.website.formbeans.WebsiteFormEx;
+import org.roller.util.SpamChecker;
 
 
 /////////////////////////////////////////////////////////////////////////////
 /**
  * Website Settings action.
- * 
- * @struts.action name="websiteFormEx" path="/editor/website" 
+ *
+ * @struts.action name="websiteFormEx" path="/editor/website"
  * 		scope="session" parameter="method"
- * 
- * @struts.action-forward name="editWebsite.page" path=".edit-website" 
- * @struts.action-forward name="removeWebsite.page" path=".WebsiteRemove" 
+ *
+ * @struts.action-forward name="editWebsite.page" path=".edit-website"
+ * @struts.action-forward name="removeWebsite.page" path=".WebsiteRemove"
  */
-public final class WebsiteFormAction extends DispatchAction
-{
-    private static Log mLogger = 
-        LogFactory.getFactory().getInstance(WebsiteFormAction.class);
-
+public final class WebsiteFormAction extends DispatchAction {
+    private static Log mLogger =
+            LogFactory.getFactory().getInstance(WebsiteFormAction.class);
+    
     public ActionForward add(
-        ActionMapping       mapping,
-        ActionForm          actionForm,
-        HttpServletRequest  request,
-        HttpServletResponse response)
-        throws Exception
-    {
+            ActionMapping       mapping,
+            ActionForm          actionForm,
+            HttpServletRequest  request,
+            HttpServletResponse response)
+            throws Exception {
         return update( mapping, actionForm, request, response );
     }
-
+    
     //-----------------------------------------------------------------------
     public ActionForward edit(
-        ActionMapping       mapping,
-        ActionForm          actionForm,
-        HttpServletRequest  request,
-        HttpServletResponse response)
-        throws IOException, ServletException
-    {
+            ActionMapping       mapping,
+            ActionForm          actionForm,
+            HttpServletRequest  request,
+            HttpServletResponse response)
+            throws IOException, ServletException {
         ActionForward forward = mapping.findForward("editWebsite.page");
-        try
-        {
+        try {
             RollerRequest rreq = RollerRequest.getRollerRequest(request);
             WebsiteData website = rreq.getWebsite();
             RollerSession rses = RollerSession.getRollerSession(request);
-            if (rses.isUserAuthorizedToAdmin(website))
-            {
+            if (rses.isUserAuthorizedToAdmin(website)) {
                 Roller roller = RollerFactory.getRoller();
                 UserManager umgr = roller.getUserManager();
                 WeblogManager wmgr = roller.getWeblogManager();
                 UserData ud = rses.getAuthenticatedUser();
                 request.setAttribute("user",ud);
-
+                
                 WebsiteFormEx wf = (WebsiteFormEx)actionForm;
                 wf.copyFrom(website, request.getLocale());
-
+                
                 List cd = wmgr.getWeblogCategories(website, true);
                 request.setAttribute("categories",cd);
-
+                
                 List bcd = wmgr.getWeblogCategories(website, true);
                 request.setAttribute("bloggerCategories",bcd);
-
+                
                 List pages = umgr.getPages(website);
                 request.setAttribute("pages",pages);
-
+                
                 ServletContext ctx = request.getSession().getServletContext();
-                String editorPages = 
+                String editorPages =
                         RollerRuntimeConfig.getProperty("users.editor.pages");
                 
                 List epages = Arrays.asList(StringUtils.split(
                         StringUtils.deleteWhitespace(editorPages), ","));
                 request.setAttribute("editorPagesList", epages);
                 
-                    
+                
                 WebsitePageModel pageModel = new WebsitePageModel(
-                  "websiteSettings.title", request, response, mapping, website);
-                request.setAttribute("model",pageModel);   
-            }
-            else
-            {
+                        "websiteSettings.title", request, response, mapping, website);
+                request.setAttribute("model",pageModel);
+            } else {
                 forward = mapping.findForward("access-denied");
             }
-        }
-        catch (Exception e)
-        {
+        } catch (Exception e) {
             throw new ServletException(e);
         }
         return forward;
     }
-
-
+    
+    
     //-----------------------------------------------------------------------
     public ActionForward update(
-        ActionMapping       mapping,
-        ActionForm          actionForm,
-        HttpServletRequest  request,
-        HttpServletResponse response)
-        throws Exception
-    {
+            ActionMapping       mapping,
+            ActionForm          actionForm,
+            HttpServletRequest  request,
+            HttpServletResponse response)
+            throws Exception {
         ActionErrors errors = new ActionErrors();
-        ActionMessages messages = new ActionMessages();        
+        ActionMessages messages = new ActionMessages();
         ActionForward forward = mapping.findForward("editWebsite");
-        try
-        {
+        try {
             WebsiteFormEx form = (WebsiteFormEx)actionForm;
             WeblogManager wmgr = RollerFactory.getRoller().getWeblogManager();
             UserManager umgr = RollerFactory.getRoller().getUserManager();
@@ -152,184 +142,162 @@
             RollerRequest.getRollerRequest(request).setWebsite(wd);
             
             RollerSession rollerSession = RollerSession.getRollerSession(request);
-            if ( rollerSession.isUserAuthorizedToAdmin(wd))
-            {
-
-                /* we don't need this check any longer -- Allen G
-                if(!form.getDefaultPageId().equals(form.getWeblogDayPageId()))
-                {
-                */               
-                    wd.save(); // should throw if save not permitted
-
+            if ( rollerSession.isUserAuthorizedToAdmin(wd)) {
+                checkBlacklist(errors, messages, form.getBlacklist());
+                if (errors.isEmpty()) {
                     // ensure getEnabled can't be changed
                     form.setEnabled(wd.getEnabled());
                     form.copyTo(wd, request.getLocale());
-                                        
+                    
                     wd.save();
-                    RollerFactory.getRoller().getRefererManager().applyRefererFilters(wd);                    
+                    RollerFactory.getRoller().getRefererManager().applyRefererFilters(wd);
                     RollerFactory.getRoller().commit();
-
-                   messages.add(null, 
-                     new ActionMessage("websiteSettings.savedChanges"));
-
+                    
+                    messages.add(null,
+                        new ActionMessage("websiteSettings.savedChanges"));
+                    
                     request.getSession().setAttribute(
-                        RollerRequest.WEBSITEID_KEY, form.getId());
-
-                    // clear the page cache for this user
-                    //PageCacheFilter.removeFromCache(request, wd);
+                            RollerRequest.WEBSITEID_KEY, form.getId());
+                    
+                    // Clear cache entries associated with website
                     CacheManager.invalidate(wd);
-
-                    // set the Editor Page list
-                    ServletContext ctx = request.getSession().getServletContext();
-                    String editorPages = 
+            
+                    actionForm.reset(mapping,request);
+                }
+                
+                // set the Editor Page list
+                ServletContext ctx = request.getSession().getServletContext();
+                String editorPages =
                         RollerRuntimeConfig.getProperty("users.editor.pages");
                 
-                    List epages = Arrays.asList(StringUtils.split(
+                List epages = Arrays.asList(StringUtils.split(
                         org.apache.commons.lang.StringUtils.deleteWhitespace(editorPages), ","));
-                    request.setAttribute("editorPagesList", epages);   
-                    
-                    WebsitePageModel pageModel = 
-                        new WebsitePageModel("websiteSettings.title", 
-                                request, response, mapping, wd);
-                    request.setAttribute("model",pageModel);                
-                /*
-                }
-                else
-                {
-                    errors.add(null, 
-                        new ActionError("websiteSettings.error.sameTemplate"));                   
-                    saveErrors(request, errors);
-                }
-                */
-            }
-            else
-            {
+                request.setAttribute("editorPagesList", epages);
+                
+                WebsitePageModel pageModel =
+                        new WebsitePageModel("websiteSettings.title",
+                        request, response, mapping, wd);
+                request.setAttribute("model",pageModel);
+                
+            } else {
                 forward = mapping.findForward("access-denied");
             }
-                    
-            actionForm.reset(mapping,request);            
-        }
-        catch (RollerPermissionsException e)
-        {
+            
+        } catch (RollerPermissionsException e) {
             errors.add(null, new ActionError("error.permissions.deniedSave"));
             forward = mapping.findForward("access-denied");
-        }
-        catch (RollerException re)
-        {
+        } catch (RollerException re) {
             mLogger.error("Unexpected exception",re.getRootCause());
             throw new ServletException(re);
-        }
-        catch (Exception e)
-        {
+        } catch (Exception e) {
             mLogger.error("Unexpected exception",e);
             throw new ServletException(e);
         }
         if (errors.size() > 0) saveErrors(request, errors);
-        if (messages.size() > 0) saveMessages(request, messages);  
+        if (messages.size() > 0) saveMessages(request, messages);
         return forward;
     }
     
+    /** Count string and regex ignore words, check regex for Syntax errors */
+    private void checkBlacklist(
+            ActionErrors errors, ActionMessages messages, String blacklist) {
+        List regexRules = new ArrayList();
+        List stringRules = new ArrayList();
+        try {
+            // just for testing/counting, this does not persist rules in any way
+            SpamChecker.populateSpamRules(blacklist, stringRules, regexRules, null);
+            messages.add(null, new ActionMessage(
+                "websiteSettings.acceptedBlacklist",
+                new Integer(stringRules.size()), new Integer(regexRules.size())));
+        } catch (Throwable e) {
+            errors.add(null, new ActionMessage(
+                "websiteSettings.error.processingBlacklist", e.getMessage()));
+        }
+    }
+    
     //-----------------------------------------------------------------------
     /** Send user to remove confirmation page */
     public ActionForward removeOk(
-        ActionMapping       mapping,
-        ActionForm          actionForm,
-        HttpServletRequest  request,
-        HttpServletResponse response)
-        throws Exception
-    {
+            ActionMapping       mapping,
+            ActionForm          actionForm,
+            HttpServletRequest  request,
+            HttpServletResponse response)
+            throws Exception {
         WebsiteFormEx form = (WebsiteFormEx)actionForm;
         UserManager umgr = RollerFactory.getRoller().getUserManager();
         WebsiteData website = umgr.retrieveWebsite(form.getId());
         ActionForward forward = mapping.findForward("removeWebsite.page");
         request.setAttribute("model", new WebsitePageModel(
-            "websiteRemove.title", request, response, mapping, website));
-        try
-        {
-            RollerSession rses = RollerSession.getRollerSession(request);            
-            if (rses.isUserAuthorizedToAdmin(website))
-            {
-                form.copyFrom(website, request.getLocale());               
+                "websiteRemove.title", request, response, mapping, website));
+        try {
+            RollerSession rses = RollerSession.getRollerSession(request);
+            if (rses.isUserAuthorizedToAdmin(website)) {
+                form.copyFrom(website, request.getLocale());
                 request.setAttribute("website", website);
-            }
-            else
-            {
+            } else {
                 forward = mapping.findForward("access-denied");
             }
-        }
-        catch (Exception e)
-        {
+        } catch (Exception e) {
             mLogger.error("ERROR in action",e);
             throw new ServletException(e);
         }
         return forward;
     }
-
+    
     //-----------------------------------------------------------------------
     public ActionForward remove(
-        ActionMapping       mapping,
-        ActionForm          actionForm,
-        HttpServletRequest  request,
-        HttpServletResponse response)
-        throws Exception
-    {
+            ActionMapping       mapping,
+            ActionForm          actionForm,
+            HttpServletRequest  request,
+            HttpServletResponse response)
+            throws Exception {
         ActionForward forward = mapping.findForward("yourWebsites");
-        try
-        {
+        try {
             UserManager umgr = RollerFactory.getRoller().getUserManager();
             WebsiteFormEx form = (WebsiteFormEx)actionForm;
             WebsiteData website = umgr.retrieveWebsite(form.getId());
             
-            RollerSession rses = RollerSession.getRollerSession(request);          
-            if ( rses.isUserAuthorizedToAdmin(website) )
-            {
+            RollerSession rses = RollerSession.getRollerSession(request);
+            if ( rses.isUserAuthorizedToAdmin(website) ) {
                 website.remove();
                 RollerFactory.getRoller().commit();
-
+                
                 //PageCacheFilter.removeFromCache(request, website);
                 CacheManager.invalidate(website);
                 
                 actionForm.reset(mapping, request);
-            }
-            else
-            {
+            } else {
                 forward = mapping.findForward("access-denied");
             }
-        }
-        catch (RollerException e)
-        {
+        } catch (RollerException e) {
             ActionErrors errors = new ActionErrors();
             errors.add(null, new ActionError(
                     "error.internationalized",e.getRootCauseMessage()));
-            saveErrors(request, errors);       
-        }
-        catch (Exception e)
-        {
+            saveErrors(request, errors);
+        } catch (Exception e) {
             mLogger.error("ERROR in action",e);
             throw new ServletException(e);
         }
         return forward;
     }
-
-    public class WebsitePageModel extends BasePageModel 
-    {
+    
+    public class WebsitePageModel extends BasePageModel {
         private List permissions = new ArrayList();
         private boolean groupBloggingEnabled = false;
         public WebsitePageModel(
                 String titleKey,
                 HttpServletRequest request,
                 HttpServletResponse response,
-                ActionMapping mapping, 
-                WebsiteData website) throws RollerException
-        {
+                ActionMapping mapping,
+                WebsiteData website) throws RollerException {
             super(titleKey, request, response, mapping);
             this.website = website;
             Roller roller = RollerFactory.getRoller();
             RollerSession rollerSession = RollerSession.getRollerSession(request);
             UserData user = rollerSession.getAuthenticatedUser();
             permissions = roller.getUserManager().getAllPermissions(website);
-            groupBloggingEnabled = 
-                RollerConfig.getBooleanProperty("groupblogging.enabled");
+            groupBloggingEnabled =
+                    RollerConfig.getBooleanProperty("groupblogging.enabled");
         }
         public boolean isGroupBloggingEnabled() {
             return groupBloggingEnabled;
@@ -337,9 +305,8 @@
         public void setGroupBloggingEnabled(boolean groupBloggingEnabled) {
             this.groupBloggingEnabled = groupBloggingEnabled;
         }
-
-        public boolean getHasPagePlugins()
-        {
+        
+        public boolean getHasPagePlugins() {
             boolean ret = false;
             try {
                 Roller roller = RollerFactory.getRoller();
@@ -350,20 +317,18 @@
             }
             return ret;
         }
-
-        public List getPagePlugins() 
-        {
+        
+        public List getPagePlugins() {
             List list = new ArrayList();
             try {
-                if (getHasPagePlugins()) 
-                {
+                if (getHasPagePlugins()) {
                     Roller roller = RollerFactory.getRoller();
                     PagePluginManager ppmgr = roller.getPagePluginManager();
                     Map plugins = ppmgr.createAndInitPagePlugins(
-                        getWebsite(),
-                        RollerContext.getRollerContext(request).getServletContext(),
-                        RollerContext.getRollerContext(request).getAbsoluteContextUrl(),
-                        new VelocityContext());
+                            getWebsite(),
+                            RollerContext.getRollerContext(request).getServletContext(),
+                            RollerContext.getRollerContext(request).getAbsoluteContextUrl(),
+                            new VelocityContext());
                     Iterator it = plugins.values().iterator();
                     while (it.hasNext()) list.add(it.next());
                 }

Modified: incubator/roller/trunk/src/org/roller/util/Blacklist.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/roller/util/Blacklist.java?rev=350046&r1=350045&r2=350046&view=diff
==============================================================================
--- incubator/roller/trunk/src/org/roller/util/Blacklist.java (original)
+++ incubator/roller/trunk/src/org/roller/util/Blacklist.java Wed Nov 30 12:47:55 2005
@@ -297,8 +297,8 @@
         List stringRules = blacklistStr;
         if (moreStringRules != null && moreStringRules.size() > 0) {
             stringRules = new ArrayList();
-            stringRules.addAll(blacklistStr);
             stringRules.addAll(moreStringRules);
+            stringRules.addAll(blacklistStr);
         }
         if (testStringRules(str, stringRules)) return true;
         
@@ -306,19 +306,19 @@
         List regexRules = blacklistRegex;
         if (moreRegexRules != null && moreRegexRules.size() > 0) {
             regexRules = new ArrayList();
-            regexRules.addAll(blacklistRegex);
             regexRules.addAll(moreRegexRules);
+            regexRules.addAll(blacklistRegex);
         }
         return testRegExRules(str, regexRules);
     }      
 
     /** 
-     * Test string only against rules provided by caller.
+     * Test string only against rules provided by caller, NOT against built-in blacklist.
      * @param str             String to be checked against rules
      * @param moreStringRules String rules to consider
      * @param moreRegexRules  Regex rules to consider 
      */
-    public static boolean matches(
+    public static boolean matchesRulesOnly(
         String str, List stringRules, List regexRules) {
         if (testStringRules(str, stringRules)) return true;
         return testRegExRules(str, regexRules);  
@@ -338,13 +338,11 @@
                 if (matcher.find()) {
                     mLogger.debug(matcher.group() 
                          + " matched by " + testPattern.pattern());
-                    hit = true;
-                    break;
+                    return true;
                 }
             } else {
                 if (testPattern.matcher(str).find()) {
-                    hit = true;
-                    break;
+                    return true;
                 }
             }
         }
@@ -363,8 +361,7 @@
                 if (mLogger.isDebugEnabled()) {
                     mLogger.debug("matched:" + test + ":");
                 }
-                hit = true;
-                break;
+                return true;
             }
         }
         return hit;

Modified: incubator/roller/trunk/src/org/roller/util/SpamChecker.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/src/org/roller/util/SpamChecker.java?rev=350046&r1=350045&r2=350046&view=diff
==============================================================================
--- incubator/roller/trunk/src/org/roller/util/SpamChecker.java (original)
+++ incubator/roller/trunk/src/org/roller/util/SpamChecker.java Wed Nov 30 12:47:55 2005
@@ -10,6 +10,7 @@
 import org.roller.config.RollerRuntimeConfig;
 import org.roller.pojos.CommentData;
 import org.roller.pojos.RefererData;
+import org.roller.pojos.WebsiteData;
 
 /**
  * Checks comment, trackbacks and referrers for spam.
@@ -35,67 +36,56 @@
         }
         return false;
     }
-    
-    /** Test comment against built in blacklist + ignoreWords */
+
+    /** Test comment against built blacklist only */
+    public boolean checkReferrer(RefererData referrer) {
+        boolean ret = false;        
+        List stringRules = new ArrayList();
+        List regexRules = new ArrayList();
+        WebsiteData website = referrer.getWebsite();
+        populateSpamRules(
+            website.getBlacklist(), stringRules, regexRules, 
+            RollerRuntimeConfig.getProperty("spam.blacklist")); 
+        // the blacklist.matches() (doesn't use the built-in blacklist)
+        if (Blacklist.matchesRulesOnly(referrer.getRefererUrl(), stringRules, regexRules)) {
+            ret = true;
+        }
+        return ret;
+    }
+
+    /** Test comment against built in blacklist + blacklist */
     private boolean testComment(CommentData c) {
         boolean ret = false;
-        
-        String weblogWords = c.getWeblogEntry().getWebsite().getIgnoreWords();
-        weblogWords = (weblogWords == null) ? "" : weblogWords;
-        String siteWords = 
-                RollerRuntimeConfig.getProperty("spam.referers.ignorewords");
-        siteWords = (siteWords == null) ? "" : siteWords;
-        
         List stringRules = new ArrayList();
         List regexRules = new ArrayList();
-        StringTokenizer toker = new StringTokenizer(siteWords + weblogWords,"\n");
-        while (toker.hasMoreTokens()) {
-            String token = toker.nextToken();
-            if (token.startsWith("#")) continue;
-            if (token.startsWith("(")) {
-                regexRules.add(Pattern.compile(token));
-            } else {
-                stringRules.add(token);
-            }
-        }
+        WebsiteData website = c.getWeblogEntry().getWebsite();
+        populateSpamRules(
+            website.getBlacklist(), stringRules, regexRules, 
+            RollerRuntimeConfig.getProperty("spam.blacklist"));
         if (   blacklist.isBlacklisted(c.getUrl(),     stringRules, regexRules)
             || blacklist.isBlacklisted(c.getEmail(),   stringRules, regexRules)
+            || blacklist.isBlacklisted(c.getName(),    stringRules, regexRules)
             || blacklist.isBlacklisted(c.getContent(), stringRules, regexRules)) {
-            c.setSpam(Boolean.TRUE);
             ret = true;
         }
         return ret;
     }
         
-    /** Test comment against built ignoreWords only */
-    public boolean testReferrer(RefererData referrer) {
-        boolean ret = false;
-        
-        String weblogWords = referrer.getWebsite().getIgnoreWords();
+    public static void populateSpamRules(
+        String blacklist, List stringRules, List regexRules, String addendum) {
+        String weblogWords = blacklist;
         weblogWords = (weblogWords == null) ? "" : weblogWords;
-        String siteWords = 
-                RollerRuntimeConfig.getProperty("spam.referers.ignorewords");
-        siteWords = (siteWords == null) ? "" : siteWords;
-        
-        List stringRules = new ArrayList();
-        List regexRules = new ArrayList();
-        StringTokenizer toker = 
-            new StringTokenizer(siteWords + weblogWords,"\n");
+        String siteWords = (addendum != null) ? addendum : "";
+        StringTokenizer toker = new StringTokenizer(siteWords + weblogWords,"\n");
         while (toker.hasMoreTokens()) {
-            String token = toker.nextToken();
+            String token = toker.nextToken().trim();
             if (token.startsWith("#")) continue;
             if (token.startsWith("(")) {
-                regexRules.add(token);
+                regexRules.add(Pattern.compile(token));
             } else {
                 stringRules.add(token);
             }
-        }   
-        // the blacklist.matches() doesn't use the built-in blacklist
-        if (Blacklist.matches(
-                referrer.getRefererUrl(), stringRules, regexRules)) {
-            ret = true;
-        }
-        return ret;
+        }        
     }
 }
 

Modified: incubator/roller/trunk/tests/org/roller/business/RefererManagerTest.java
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/tests/org/roller/business/RefererManagerTest.java?rev=350046&r1=350045&r2=350046&view=diff
==============================================================================
--- incubator/roller/trunk/tests/org/roller/business/RefererManagerTest.java (original)
+++ incubator/roller/trunk/tests/org/roller/business/RefererManagerTest.java Wed Nov 30 12:47:55 2005
@@ -1 +1 @@
-package org.roller.business; 

import java.sql.Timestamp;
import java.util.Calendar;
import java.util.Date;
import java.util.List;

import junit.framework.Test;
import junit.framework.TestSuite;
 
import org.roller.RollerException;
import org.roller.model.PropertiesManager;
import org.roller.model.RefererManager;
import org.roller.model.Roller;
import org.roller.pojos.RefererData;
import org.roller.pojos.RollerPropertyData;
import org.roller.pojos.UserData;
import org.roller.pojos.WeblogEntryData;
import org.roller.pojos.WebsiteData;
import org.roller.util.DateUtil;
import org.roller.RollerTestBase;

/**
 * Test Roller Referer Management.
 */
public class RefererManagerTest extends RollerTestBase 
{
    RefererManager rmgr;
    //List refs;
    int count = 20;        
    String testDay;
    String origSpamWords;

    //------------------------------------------------------------------------
	public RefererManagerTest(String name) 
    {
		super(name);
	}
    
    //------------------------------------------------------------------------
	public static void main(String args[]) 
    {
		junit.textui.TestRunner.run(RefererManagerTest.class);
	}
    
    //------------------------------------------------------------------------
    public static Test suite() 
    {
        return new TestSuite(RefererManagerTest.class);
    }

    protected void setUp() throws Exception
    {
        super.setUp();
        
        // add "spamtest" to refererSpamWords
        Roller mRoller = getRoller();
        PropertiesManager pmgr = mRoller.getPropertiesManager();
        RollerPropertyData spamprop = pmgr.getProperty("spam.referers.ignorewords");
        this.origSpamWords = spamprop.getValue();
        spamprop.setValue(spamprop.getValue() + ", spamtest");
        pmgr.store(spamprop);
        
        // Process count unique referers
        rmgr = getRoller().getRefererManager();
        Calendar lCalendar = Calendar.getInstance();
        lCalendar.setTime(new Date());
        for (int i = 0; i < count; i++)
        {
            lCalendar.add(Calendar.DATE, -1);
            Timestamp day = new Timestamp(lCalendar.getTime().getTime());

            getRoller().begin(UserData.SYSTEM_USER);
            MockRequest mock = new MockRequest(
                                DateUtil.format8chars(day),
                                "http://test"+i,
                                "http://test"+i,
                                null,
                                mWebsite
            );
            rmgr.processRequest(mock);
            getRoller().commit();
            
            testDay = mock.getDateString();
        }
    }

    public void tearDown() throws Exception
    {      
        List refs = rmgr.getReferers(mWebsite);
        // Remove all referers processes
        for (int i = 0; i < refs.size(); i++)
        {
            rmgr.removeReferer(((RefererData)refs.get(i)).getId());    
        }
        
        // Make sure all were removed
        refs = rmgr.getReferers(mWebsite);            
        assertEquals(0,refs.size());

        // reset refererSpamWords to original value
        Roller mRoller = getRoller();
        PropertiesManager pmgr = mRoller.getPropertiesManager();
        RollerPropertyData spamprop = pmgr.getProperty("spam.referers.ignorewords");
        spamprop.setValue(this.origSpamWords);
        pmgr.store(spamprop);
        
        super.tearDown();
    }
    
    //------------------------------------------------------------------------
    public void testGetReferersToDate() throws Exception
    {
        List referers = rmgr.getReferersToDate(mWebsite, testDay);
        assertEquals("Should be one Referer.", referers.size(), 1);
    }
    
    //------------------------------------------------------------------------
    public void testRefererProcessing() throws RollerException
    {
        List refs = rmgr.getReferers(mWebsite);            
        assertEquals("number of referers should equal count", count, refs.size());
        
        int hits = rmgr.getDayHits(mWebsite);
        assertEquals("There should be one fewer hits than referers", count, hits);         
    }
    
    public void testSelfRefererDenial() throws RollerException
    {
        // test against "self referrals"
        getRoller().begin(UserData.SYSTEM_USER);
        // create "direct" referer
        boolean isSpam = rmgr.processRequest(
            new MockRequest(
                "20020101",
                "direct",
                "http://test.com",
                null, mWebsite
            )
        );
        getRoller().commit();
        assertFalse("is not spam", isSpam);
        int newRefCount = rmgr.getReferers(mWebsite).size();
        
        // now create self-referer
        getRoller().begin(UserData.SYSTEM_USER);
        
        isSpam = rmgr.processRequest(
            new MockRequest(
                "20020202",
                "http://test.com/page/" + mWebsite.getHandle(),
                "http://test.com",
                null, mWebsite
            )
        );
        getRoller().commit();
        assertFalse("is not spam", isSpam);
        
        // number of referrers should not have changed
        List refs = rmgr.getReferers(mWebsite);            
        assertEquals("self referal not ignored", newRefCount, refs.size()); 
        
        // now create self-referer from editor page
        isSpam = rmgr.processRequest(
            new MockRequest(
                "20020202",
                "http://test.com/weblog.do",
                "http://test.com",
                null, mWebsite
            )
        );
        getRoller().commit();
        assertFalse("is not spam", isSpam);
        
        // number of referrers should not have changed
        refs = rmgr.getReferers(mWebsite);            
        assertEquals("editor referal not ignored", newRefCount, refs.size()); 
    }
    
    /**
     * Test to see if Referer Spam detection works.
     */
    public void testSpamBlocking()
    {
        boolean isSpam = rmgr.processRequest(
            new MockRequest(
                "20040101",
                "http://www.spamtest.com",
                "http://test.com",
                null, mWebsite
            )
        );
        //assertTrue("failed to detect referer spam", isSpam);
    }
    
    public void testApplyRefererFilters() throws Exception
    {
        List refs = rmgr.getReferers(mWebsite);
        assertEquals(count, refs.size());
        String origWords = null;
        
        getRoller().begin(UserData.SYSTEM_USER);
        Roller mRoller = getRoller();
        PropertiesManager pmgr = mRoller.getPropertiesManager();
        RollerPropertyData spamprop = pmgr.getProperty("spam.referers.ignorewords");
        origWords = spamprop.getValue();
        spamprop.setValue(spamprop.getValue() + ", test");
        pmgr.store(spamprop);
        getRoller().commit();
        
        getRoller().begin(UserData.SYSTEM_USER);
        getRoller().getRefererManager().applyRefererFilters();
        getRoller().commit();
        
        refs = rmgr.getReferers(mWebsite);
        assertEquals(0, refs.size());

        getRoller().begin(UserData.SYSTEM_USER);
        spamprop = pmgr.getProperty("spam.referers.ignorewords");
        spamprop.setValue(origWords);
        pmgr.store(spamprop);
        getRoller().commit();
    }
    
    public void testApplyRefererFiltersWebsite() throws Exception
    {
        List refs = rmgr.getReferers(mWebsite);
        assertEquals(count, refs.size());
        String origWords = null;
        
        getRoller().begin(UserData.SYSTEM_USER);
        mWebsite = getRoller().getUserManager().retrieveWebsite(mWebsite.getId());
        origWords = mWebsite.getIgnoreWords();
        mWebsite.setIgnoreWords("test");
        mWebsite.save();
        getRoller().commit();
        
        getRoller().begin(UserData.SYSTEM_USER);
        getRoller().getRefererManager().applyRefererFilters();
        getRoller().commit();
        
        refs = rmgr.getReferers(mWebsite);
        assertEquals(0, refs.size());
    }
}

class MockRequest implements org.roller.model.ParsedRequest
{  
    private String mDateStr = null;
    private String mRefUrl = null;
    private String mReqUrl = null;
    private WeblogEntryData mEntry = null;
    private WebsiteData mWebsite = null;
    private boolean mIsDateSpecified = false;
    
    public MockRequest( 
        String dateStr, String refUrl, String reqUrl, 
        WeblogEntryData entry, WebsiteData website) 
    {
        mDateStr = dateStr;
        mRefUrl = refUrl;
        mReqUrl = reqUrl;
        mEntry = entry;
        mWebsite = website;
        
        if (mDateStr != null) mIsDateSpecified = true;
    }
    
    /**
     * @see org.roller.pojos.ParsedRequest#getDateString()
     */
    public String getDateString()
    {
        return mDateStr;
    }

    /**
     * @see org.roller.pojos.ParsedRequest#getRefererURL()
     */
    public String getRefererURL()
    {
        return mRefUrl;
    }

    /**
     * @see org.roller.pojos.ParsedRequest#getRequestURL()
     */
    public String getRequestURL()
    {
        return mReqUrl;
    }

    /**
     * @see org.roller.pojos.ParsedRequest#getWeblogEntry()
     */
    public WeblogEntryData getWeblogEntry()
    {
        return mEntry;
    }

    /**
     * @see org.roller.pojos.ParsedRequest#getWebsite()
     */
    public WebsiteData getWebsite()
    {
        return mWebsite;
    }

    /**
     * Returns the isDateSpecified.
     * @return boolean
     */
    public boolean isDateSpecified()
    {
        return mIsDateSpecified;
    }

    /**
     * Sets the isDateSpecified.
     * @param isDateSpecified The isDateSpecified to set
     */
    public void setDateSpecified(boolean isDateSpecified)
    {
        mIsDateSpecified = isDateSpecified;
    }

    /** 
     * @see org.roller.pojos.ParsedRequest#isEnableLinkback()
     */
    public boolean isEnableLinkback()
    {
        return false;
    }
}
\ No newline at end of file
+package org.roller.business; 

import java.sql.Timestamp;
import java.util.Calendar;
import java.util.Date;
import java.util.List;

import junit.framework.Test;
import junit.framework.TestSuite;
 
import org.roller.RollerException;
import org.roller.model.PropertiesManager;
import org.roller.model.RefererManager;
import org.roller.model.Roller;
import org.roller.pojos.RefererData;
import org.roller.pojos.RollerPropertyData;
import org.roller.pojos.UserData;
import org.roller.pojos.WeblogEntryData;
import org.roller.pojos.WebsiteData;
import org.roller.util.DateUtil;
import org.roller.RollerTestBase;

/**
 * Test Roller Referer Management.
 */
public class RefererManagerTest extends RollerTestBase 
{
    RefererManager rmgr;
    //List refs;
    int count = 20;        
    String testDay;
    String origSpamWords;

    //------------------------------------------------------------------------
	public RefererManagerTest(String name) 
    {
		super(name);
	}
    
    //------------------------------------------------------------------------
	public static void main(String args[]) 
    {
		junit.textui.TestRunner.run(RefererManagerTest.class);
	}
    
    //------------------------------------------------------------------------
    public static Test suite() 
    {
        return new TestSuite(RefererManagerTest.class);
    }

    protected void setUp() throws Exception
    {
        super.setUp();
        
        // add "spamtest" to refererSpamWords
        Roller mRoller = getRoller();
        PropertiesManager pmgr = mRoller.getPropertiesManager();
        RollerPropertyData spamprop = pmgr.getProperty("spam.referers.ignorewords");
        this.origSpamWords = spamprop.getValue();
        spamprop.setValue(spamprop.getValue() + ", spamtest");
        pmgr.store(spamprop);
        
        // Process count unique referers
        rmgr = getRoller().getRefererManager();
        Calendar lCalendar = Calendar.getInstance();
        lCalendar.setTime(new Date());
        for (int i = 0; i < count; i++)
        {
            lCalendar.add(Calendar.DATE, -1);
            Timestamp day = new Timestamp(lCalendar.getTime().getTime());

            getRoller().begin(UserData.SYSTEM_USER);
            MockRequest mock = new MockRequest(
                                DateUtil.format8chars(day),
                                "http://test"+i,
                                "http://test"+i,
                                null,
                                mWebsite
            );
            rmgr.processRequest(mock);
            getRoller().commit();
            
            testDay = mock.getDateString();
        }
    }

    public void tearDown() throws Exception
    {      
        List refs = rmgr.getReferers(mWebsite);
        // Remove all referers processes
        for (int i = 0; i < refs.size(); i++)
        {
            rmgr.removeReferer(((RefererData)refs.get(i)).getId());    
        }
        
        // Make sure all were removed
        refs = rmgr.getReferers(mWebsite);            
        assertEquals(0,refs.size());

        // reset refererSpamWords to original value
        Roller mRoller = getRoller();
        PropertiesManager pmgr = mRoller.getPropertiesManager();
        RollerPropertyData spamprop = pmgr.getProperty("spam.referers.ignorewords");
        spamprop.setValue(this.origSpamWords);
        pmgr.store(spamprop);
        
        super.tearDown();
    }
    
    //------------------------------------------------------------------------
    public void testGetReferersToDate() throws Exception
    {
        List referers = rmgr.getReferersToDate(mWebsite, testDay);
        assertEquals("Should be one Referer.", referers.size(), 1);
    }
    
    //------------------------------------------------------------------------
    public void testRefererProcessing() throws RollerException
    {
        List refs = rmgr.getReferers(mWebsite);            
        assertEquals("number of referers should equal count", count, refs.size());
        
        int hits = rmgr.getDayHits(mWebsite);
        assertEquals("There should be one fewer hits than referers", count, hits);         
    }
    
    public void testSelfRefererDenial() throws RollerException
    {
        // test against "self referrals"
        getRoller().begin(UserData.SYSTEM_USER);
        // create "direct" referer
        boolean isSpam = rmgr.processRequest(
            new MockRequest(
                "20020101",
                "direct",
                "http://test.com",
                null, mWebsite
            )
        );
        getRoller().commit();
        assertFalse("is not spam", isSpam);
        int newRefCount = rmgr.getReferers(mWebsite).size();
        
        // now create self-referer
        getRoller().begin(UserData.SYSTEM_USER);
        
        isSpam = rmgr.processRequest(
            new MockRequest(
                "20020202",
                "http://test.com/page/" + mWebsite.getHandle(),
                "http://test.com",
                null, mWebsite
            )
        );
        getRoller().commit();
        assertFalse("is not spam", isSpam);
        
        // number of referrers should not have changed
        List refs = rmgr.getReferers(mWebsite);            
        assertEquals("self referal not ignored", newRefCount, refs.size()); 
        
        // now create self-referer from editor page
        isSpam = rmgr.processRequest(
            new MockRequest(
                "20020202",
                "http://test.com/weblog.do",
                "http://test.com",
                null, mWebsite
            )
        );
        getRoller().commit();
        assertFalse("is not spam", isSpam);
        
        // number of referrers should not have changed
        refs = rmgr.getReferers(mWebsite);            
        assertEquals("editor referal not ignored", newRefCount, refs.size()); 
    }
    
    /**
     * Test to see if Referer Spam detection works.
     */
    public void testSpamBlocking()
    {
        boolean isSpam = rmgr.processRequest(
            new MockRequest(
                "20040101",
                "http://www.spamtest.com",
                "http://test.com",
                null, mWebsite
            )
        );
        //assertTrue("failed to detect referer spam", isSpam);
    }
    
    public void testApplyRefererFilters() throws Exception
    {
        List refs = rmgr.getReferers(mWebsite);
        assertEquals(count, refs.size());
        String origWords = null;
        
        getRoller().begin(UserData.SYSTEM_USER);
        Roller mRoller = getRoller();
        PropertiesManager pmgr = mRoller.getPropertiesManager();
        RollerPropertyData spamprop = pmgr.getProperty("spam.referers.ignorewords");
        origWords = spamprop.getValue();
        spamprop.setValue(spamprop.getValue() + ", test");
        pmgr.store(spamprop);
        getRoller().commit();
        
        getRoller().begin(UserData.SYSTEM_USER);
        getRoller().getRefererManager().applyRefererFilters();
        getRoller().commit();
        
        refs = rmgr.getReferers(mWebsite);
        assertEquals(0, refs.size());

        getRoller().begin(UserData.SYSTEM_USER);
        spamprop = pmgr.getProperty("spam.referers.ignorewords");
        spamprop.setValue(origWords);
        pmgr.store(spamprop);
        getRoller().commit();
    }
    
    public void testApplyRefererFiltersWebsite() throws Exception
    {
        List refs = rmgr.getReferers(mWebsite);
        assertEquals(count, refs.size());
        String origWords = null;
        
        getRoller().begin(UserData.SYSTEM_USER);
        mWebsite = getRoller().getUserManager().retrieveWebsite(mWebsite.getId());
        origWords = mWebsite.getBlacklist();
        mWebsite.setBlacklist("test");
        mWebsite.save();
        getRoller().commit();
        
        getRoller().begin(UserData.SYSTEM_USER);
        getRoller().getRefererManager().applyRefererFilters();
        getRoller().commit();
        
        refs = rmgr.getReferers(mWebsite);
        assertEquals(0, refs.size());
    }
}

class MockRequest implements org.roller.model.ParsedRequest
{  
    private String mDateStr = null;
    private String mRefUrl = null;
    private String mReqUrl = null;
    private WeblogEntryData mEntry = null;
    private WebsiteData mWebsite = null;
    private boolean mIsDateSpecified = false;
    
    public MockRequest( 
        String dateStr, String refUrl, String reqUrl, 
        WeblogEntryData entry, WebsiteData website) 
    {
        mDateStr = dateStr;
        mRefUrl = refUrl;
        mReqUrl = reqUrl;
        mEntry = entry;
        mWebsite = website;
        
        if (mDateStr != null) mIsDateSpecified = true;
    }
    
    /**
     * @see org.roller.pojos.ParsedRequest#getDateString()
     */
    public String getDateString()
    {
        return mDateStr;
    }

    /**
     * @see org.roller.pojos.ParsedRequest#getRefererURL()
     */
    public String getRefererURL()
    {
        return mRefUrl;
    }

    /**
     * @see org.roller.pojos.ParsedRequest#getRequestURL()
     */
    public String getRequestURL()
    {
        return mReqUrl;
    }

    /**
     * @see org.roller.pojos.ParsedRequest#getWeblogEntry()
     */
    public WeblogEntryData getWeblogEntry()
    {
        return mEntry;
    }

    /**
     * @see org.roller.pojos.ParsedRequest#getWebsite()
     */
    public WebsiteData getWebsite()
    {
        return mWebsite;
    }

    /**
     * Returns the isDateSpecified.
     * @return boolean
     */
    public boolean isDateSpecified()
    {
        return mIsDateSpecified;
    }

    /**
     * Sets the isDateSpecified.
     * @param isDateSpecified The isDateSpecified to set
     */
    public void setDateSpecified(boolean isDateSpecified)
    {
        mIsDateSpecified = isDateSpecified;
    }

    /** 
     * @see org.roller.pojos.ParsedRequest#isEnableLinkback()
     */
    public boolean isEnableLinkback()
    {
        return false;
    }
}
\ No newline at end of file

Modified: incubator/roller/trunk/web/WEB-INF/classes/ApplicationResources.properties
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/web/WEB-INF/classes/ApplicationResources.properties?rev=350046&r1=350045&r2=350046&view=diff
==============================================================================
--- incubator/roller/trunk/web/WEB-INF/classes/ApplicationResources.properties (original)
+++ incubator/roller/trunk/web/WEB-INF/classes/ApplicationResources.properties Wed Nov 30 12:47:55 2005
@@ -236,7 +236,7 @@
 # -------------------------------------------------------------- CommentServlet
 
 commentServlet.commentMarkedAsSpam=Your comment was marked as spam and will not be displayed.
-commentServlet.previewMarkedAsSpam=If you submit this comment, it will be marked as spam ane not displayed.
+commentServlet.previewMarkedAsSpam=If you submit this comment, it will be marked as spam and not displayed.
 commentServlet.submittedToModerator=Your comment has been submitted to the moderator for approval.
 
 # --------------------------------------------------------------- Configuration
@@ -1429,8 +1429,8 @@
 websiteSettings.enableBloggerApi=Enable Blogger and MetaWeblog APIs for your weblog?
 websiteSettings.bloggerApiCategory=Category for posts received via Blogger API 
 websiteSettings.spamPrevention=Spam Prevention
-websiteSettings.ignoreUrls=Ignore referring URLs that<br />contain any of \
-these<br />(comma separated) words
+websiteSettings.ignoreUrls=List of words and regex expressions listed one per \
+line to be added to the blacklist used to check comments, trackbacks and referrers.
 websiteSettings.editorSettings=Editor
 websiteSettings.commentSettings=Comments
 
@@ -1439,6 +1439,8 @@
 websiteSettings.button.remove=Remove Weblog
 
 websiteSettings.savedChanges=Saved changes to weblog settings
+websiteSettings.acceptedBlacklist=Accepted {0} string and {1} regex blacklist rules
+websiteSettings.error.processingBlacklist=Error processing blacklist: {0}
 websiteSettings.error.sameTemplate=CHANGES REJECTED: cannot set default page \
 template and day template to same template
 

Modified: incubator/roller/trunk/web/WEB-INF/classes/roller.properties
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/web/WEB-INF/classes/roller.properties?rev=350046&r1=350045&r2=350046&view=diff
==============================================================================
--- incubator/roller/trunk/web/WEB-INF/classes/roller.properties (original)
+++ incubator/roller/trunk/web/WEB-INF/classes/roller.properties Wed Nov 30 12:47:55 2005
@@ -181,14 +181,14 @@
 comment.notification.separateOwnerMessage=false
 comment.notification.hideCommenterAddresses=false
 
-# enables blacklist + ignoreWords check on comment post (default: true)
+# enables site full blacklist check on comment posts (default: true)
 site.blacklist.enable.comments=true
 
-# enables blacklist + ignoreWords check at time of trackback post (default: true)
+# enables site full blacklist check at time of trackback post (default: true)
 site.blacklist.enable.trackbacks=true
 
-# enables ignoreWords check for each incoming referrer
-site.ignorewords.enable.referrers=true
+# enables partial blacklist check (not including blacklist.txt) for each incoming referrer
+site.blacklist.enable.referrers=true
 
 # Trackback protection. Set this only if you need to limit the URLs to
 # which users may send trackbacks. Regex expressions are allowed, for example:

Modified: incubator/roller/trunk/web/WEB-INF/classes/rollerRuntimeConfigDefs.xml
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/web/WEB-INF/classes/rollerRuntimeConfigDefs.xml?rev=350046&r1=350045&r2=350046&view=diff
==============================================================================
--- incubator/roller/trunk/web/WEB-INF/classes/rollerRuntimeConfigDefs.xml (original)
+++ incubator/roller/trunk/web/WEB-INF/classes/rollerRuntimeConfigDefs.xml Wed Nov 30 12:47:55 2005
@@ -169,7 +169,7 @@
    <!-- Spam Prevention Settings Group -->
    <display-group name="spamSettings" key="websiteSettings.spamPrevention" >
    
-      <property-def  name="spam.referers.ignorewords"  key="websiteSettings.ignoreUrls">
+      <property-def  name="spam.blacklist"  key="websiteSettings.ignoreUrls">
          <type>text</type>
          <default-value></default-value>
          <rows>7</rows>

Modified: incubator/roller/trunk/web/website/edit-website.jsp
URL: http://svn.apache.org/viewcvs/incubator/roller/trunk/web/website/edit-website.jsp?rev=350046&r1=350045&r2=350046&view=diff
==============================================================================
--- incubator/roller/trunk/web/website/edit-website.jsp (original)
+++ incubator/roller/trunk/web/website/edit-website.jsp Wed Nov 30 12:47:55 2005
@@ -192,7 +192,7 @@
 
     <tr>
         <td class="label"><fmt:message key="websiteSettings.ignoreUrls" /></td>
-        <td class="field"><html:textarea property="ignoreWords" rows="7" cols="40"/></td>
+        <td class="field"><html:textarea property="blacklist" rows="7" cols="40"/></td>
         <td class="description"><%-- <fmt:message key="websiteSettings.tip." /> --%></td>
     </tr>
 



Mime
View raw message