ace-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ange...@apache.org
Subject svn commit: r1537630 - in /ace/trunk: org.apache.ace.authentication.itest/src/org/apache/ace/it/authentication/ org.apache.ace.client.repository.itest/src/org/apache/ace/it/repositoryadmin/ org.apache.ace.gogo/ org.apache.ace.gogo/src/org/apache/ace/go...
Date Thu, 31 Oct 2013 20:43:15 GMT
Author: angelos
Date: Thu Oct 31 20:43:14 2013
New Revision: 1537630

URL: http://svn.apache.org/r1537630
Log:
ACE-378 We can now limit server logs on file-based log stores.

Added:
    ace/trunk/org.apache.ace.gogo/src/org/apache/ace/gogo/log/
    ace/trunk/org.apache.ace.gogo/src/org/apache/ace/gogo/log/LogCommands.java
    ace/trunk/run-server-allinone/conf/org.apache.ace.log.server.store.filebased.cfg
    ace/trunk/run-server/conf/org.apache.ace.log.server.store.filebased.cfg
Modified:
    ace/trunk/org.apache.ace.authentication.itest/src/org/apache/ace/it/authentication/LogAuthenticationTest.java
    ace/trunk/org.apache.ace.authentication.itest/src/org/apache/ace/it/authentication/ObrAuthenticationTest.java
    ace/trunk/org.apache.ace.client.repository.itest/src/org/apache/ace/it/repositoryadmin/BaseRepositoryAdminTest.java
    ace/trunk/org.apache.ace.gogo/bnd.bnd
    ace/trunk/org.apache.ace.gogo/src/org/apache/ace/gogo/Activator.java
    ace/trunk/org.apache.ace.log.itest/src/org/apache/ace/it/log/LogIntegrationTest.java
    ace/trunk/org.apache.ace.log.server.store.itest/src/org/apache/ace/log/server/store/tests/MongoLogStoreTest.java
    ace/trunk/org.apache.ace.log/src/org/apache/ace/log/server/store/LogStore.java
    ace/trunk/org.apache.ace.log/src/org/apache/ace/log/server/store/impl/Activator.java
    ace/trunk/org.apache.ace.log/src/org/apache/ace/log/server/store/impl/LogStoreImpl.java
    ace/trunk/org.apache.ace.log/src/org/apache/ace/log/server/store/mongo/Activator.java
    ace/trunk/org.apache.ace.log/src/org/apache/ace/log/server/store/mongo/MongoLogStore.java
    ace/trunk/org.apache.ace.log/test/org/apache/ace/log/server/servlet/LogServletTest.java
    ace/trunk/org.apache.ace.log/test/org/apache/ace/log/server/store/impl/ServerLogStoreTester.java

Modified: ace/trunk/org.apache.ace.authentication.itest/src/org/apache/ace/it/authentication/LogAuthenticationTest.java
URL: http://svn.apache.org/viewvc/ace/trunk/org.apache.ace.authentication.itest/src/org/apache/ace/it/authentication/LogAuthenticationTest.java?rev=1537630&r1=1537629&r2=1537630&view=diff
==============================================================================
--- ace/trunk/org.apache.ace.authentication.itest/src/org/apache/ace/it/authentication/LogAuthenticationTest.java
(original)
+++ ace/trunk/org.apache.ace.authentication.itest/src/org/apache/ace/it/authentication/LogAuthenticationTest.java
Thu Oct 31 20:43:14 2013
@@ -129,6 +129,8 @@ public class LogAuthenticationTest exten
         configure("org.apache.ace.scheduler",
             "org.apache.ace.configurator.useradmin.task.UpdateUserAdminTask", "100");
 
+        configure("org.apache.ace.log.server.store.filebased", "MaxEvents", "0");
+
         configure(DiscoveryConstants.DISCOVERY_PID,
             DiscoveryConstants.DISCOVERY_URL_KEY, baseURL);
         configure(IdentificationConstants.IDENTIFICATION_PID,

Modified: ace/trunk/org.apache.ace.authentication.itest/src/org/apache/ace/it/authentication/ObrAuthenticationTest.java
URL: http://svn.apache.org/viewvc/ace/trunk/org.apache.ace.authentication.itest/src/org/apache/ace/it/authentication/ObrAuthenticationTest.java?rev=1537630&r1=1537629&r2=1537630&view=diff
==============================================================================
--- ace/trunk/org.apache.ace.authentication.itest/src/org/apache/ace/it/authentication/ObrAuthenticationTest.java
(original)
+++ ace/trunk/org.apache.ace.authentication.itest/src/org/apache/ace/it/authentication/ObrAuthenticationTest.java
Thu Oct 31 20:43:14 2013
@@ -97,6 +97,8 @@ public class ObrAuthenticationTest exten
             RepositoryConstants.REPOSITORY_CUSTOMER, "apache",
             RepositoryConstants.REPOSITORY_MASTER, "true");
 
+        configure("org.apache.ace.log.server.store.filebased", "MaxEvents", "0");
+
         configure("org.apache.ace.configurator.useradmin.task.UpdateUserAdminTask",
             "repositoryName", "users",
             "repositoryCustomer", "apache");

Modified: ace/trunk/org.apache.ace.client.repository.itest/src/org/apache/ace/it/repositoryadmin/BaseRepositoryAdminTest.java
URL: http://svn.apache.org/viewvc/ace/trunk/org.apache.ace.client.repository.itest/src/org/apache/ace/it/repositoryadmin/BaseRepositoryAdminTest.java?rev=1537630&r1=1537629&r2=1537630&view=diff
==============================================================================
--- ace/trunk/org.apache.ace.client.repository.itest/src/org/apache/ace/it/repositoryadmin/BaseRepositoryAdminTest.java
(original)
+++ ace/trunk/org.apache.ace.client.repository.itest/src/org/apache/ace/it/repositoryadmin/BaseRepositoryAdminTest.java
Thu Oct 31 20:43:14 2013
@@ -179,6 +179,8 @@ public abstract class BaseRepositoryAdmi
 
         getService(SessionFactory.class).createSession("test-session-ID", null);
 
+        configure("org.apache.ace.log.server.store.filebased", "MaxEvents", "0");
+
         configureFactory("org.apache.ace.log.server.store.factory",
             "name", "auditlog", "authentication.enabled", "false");
     }

Modified: ace/trunk/org.apache.ace.gogo/bnd.bnd
URL: http://svn.apache.org/viewvc/ace/trunk/org.apache.ace.gogo/bnd.bnd?rev=1537630&r1=1537629&r2=1537630&view=diff
==============================================================================
--- ace/trunk/org.apache.ace.gogo/bnd.bnd (original)
+++ ace/trunk/org.apache.ace.gogo/bnd.bnd Thu Oct 31 20:43:14 2013
@@ -4,7 +4,8 @@
 	org.osgi.impl.bundle.repoindex.lib,\
 	org.apache.felix.dependencymanager,\
 	org.apache.felix.gogo.runtime,\
-	org.apache.ace.bnd.repository;version=latest
+	org.apache.ace.bnd.repository;version=latest,\
+	org.apache.ace.log.server.store.api;version=latest
 
 Bundle-Name: Apache ACE Gogo commands
 Bundle-Description: Provides Gogo commands for working with ACE	
@@ -75,5 +76,6 @@ Private-Package: org.apache.ace.bnd.repo
 	org.osgi.impl.bundle.bindex.*;-split-package:=merge-last,\
 	org.osgi.impl.bundle.obr.*;-split-package:=merge-last,\
 	org.xmlpull.v1;-split-package:=first,\
-	org.kxml2.io;-split-package:=first
+	org.kxml2.io;-split-package:=first,\
+	org.apache.ace.gogo.log
 	
\ No newline at end of file

Modified: ace/trunk/org.apache.ace.gogo/src/org/apache/ace/gogo/Activator.java
URL: http://svn.apache.org/viewvc/ace/trunk/org.apache.ace.gogo/src/org/apache/ace/gogo/Activator.java?rev=1537630&r1=1537629&r2=1537630&view=diff
==============================================================================
--- ace/trunk/org.apache.ace.gogo/src/org/apache/ace/gogo/Activator.java (original)
+++ ace/trunk/org.apache.ace.gogo/src/org/apache/ace/gogo/Activator.java Thu Oct 31 20:43:14
2013
@@ -22,10 +22,12 @@ import java.util.Properties;
 
 import org.apache.ace.gogo.execute.ExecuteCommands;
 import org.apache.ace.gogo.execute.ScriptExecutor;
+import org.apache.ace.gogo.log.LogCommands;
 import org.apache.ace.gogo.math.MathCommands;
 import org.apache.ace.gogo.misc.MiscCommands;
 import org.apache.ace.gogo.queue.QueueCommands;
 import org.apache.ace.gogo.repo.RepoCommands;
+import org.apache.ace.log.server.store.LogStore;
 import org.apache.felix.dm.DependencyActivatorBase;
 import org.apache.felix.dm.DependencyManager;
 import org.apache.felix.service.command.CommandProcessor;
@@ -63,6 +65,13 @@ public class Activator extends Dependenc
                 .setService(CommandProcessor.class)
                 .setRequired(true)));
 
+        manager.add(createComponent()
+            .setInterface(Object.class.getName(), createProps(LogCommands.SCOPE, LogCommands.FUNCTIONS))
+            .setImplementation(LogCommands.class)
+            .add(createServiceDependency()
+                .setService(LogStore.class)
+                .setRequired(true)));
+
         String script = System.getProperty("ace.gogo.script");
         if (script != null) {
             long delay = Long.getLong("ace.gogo.script.delay", 300L);

Added: ace/trunk/org.apache.ace.gogo/src/org/apache/ace/gogo/log/LogCommands.java
URL: http://svn.apache.org/viewvc/ace/trunk/org.apache.ace.gogo/src/org/apache/ace/gogo/log/LogCommands.java?rev=1537630&view=auto
==============================================================================
--- ace/trunk/org.apache.ace.gogo/src/org/apache/ace/gogo/log/LogCommands.java (added)
+++ ace/trunk/org.apache.ace.gogo/src/org/apache/ace/gogo/log/LogCommands.java Thu Oct 31
20:43:14 2013
@@ -0,0 +1,20 @@
+package org.apache.ace.gogo.log;
+
+import org.apache.ace.log.server.store.LogStore;
+import org.apache.felix.service.command.Descriptor;
+
+public class LogCommands {
+
+    public final static String SCOPE = "ace-log";
+    public final static String[] FUNCTIONS = new String[] { "cleanup" };
+
+    // Injected by Felix DM...
+    private volatile LogStore m_logStore;
+
+    @Descriptor("Apply the configured maximum to all existing logs")
+    public void cleanup() throws Exception {
+        m_logStore.clean();
+        System.out.println("All logfiles processed");
+    }
+
+}

Modified: ace/trunk/org.apache.ace.log.itest/src/org/apache/ace/it/log/LogIntegrationTest.java
URL: http://svn.apache.org/viewvc/ace/trunk/org.apache.ace.log.itest/src/org/apache/ace/it/log/LogIntegrationTest.java?rev=1537630&r1=1537629&r2=1537630&view=diff
==============================================================================
--- ace/trunk/org.apache.ace.log.itest/src/org/apache/ace/it/log/LogIntegrationTest.java (original)
+++ ace/trunk/org.apache.ace.log.itest/src/org/apache/ace/it/log/LogIntegrationTest.java Thu
Oct 31 20:43:14 2013
@@ -77,6 +77,8 @@ public class LogIntegrationTest extends 
         configure("org.apache.ace.deployment.servlet",
                 HttpConstants.ENDPOINT, DEPLOYMENT, "authentication.enabled", "false");
 
+        configure("org.apache.ace.log.server.store.filebased", "MaxEvents", "0");
+        
         configureFactory("org.apache.ace.log.server.servlet.factory",
                 "name", "auditlog",
                 HttpConstants.ENDPOINT, AUDITLOG, "authentication.enabled", "false");

Modified: ace/trunk/org.apache.ace.log.server.store.itest/src/org/apache/ace/log/server/store/tests/MongoLogStoreTest.java
URL: http://svn.apache.org/viewvc/ace/trunk/org.apache.ace.log.server.store.itest/src/org/apache/ace/log/server/store/tests/MongoLogStoreTest.java?rev=1537630&r1=1537629&r2=1537630&view=diff
==============================================================================
--- ace/trunk/org.apache.ace.log.server.store.itest/src/org/apache/ace/log/server/store/tests/MongoLogStoreTest.java
(original)
+++ ace/trunk/org.apache.ace.log.server.store.itest/src/org/apache/ace/log/server/store/tests/MongoLogStoreTest.java
Thu Oct 31 20:43:14 2013
@@ -165,6 +165,8 @@ public class MongoLogStoreTest extends I
 
     @Override
     protected void configureProvisionedServices() throws Exception {
+        configure("org.apache.ace.log.server.store.mongo", "MaxEvents", "0");
+
         configureFactory("org.amdatu.mongo", "dbName", "ace");
         configureFactory("org.apache.ace.log.server.store.factory", "name", "serverlog");
     }

Modified: ace/trunk/org.apache.ace.log/src/org/apache/ace/log/server/store/LogStore.java
URL: http://svn.apache.org/viewvc/ace/trunk/org.apache.ace.log/src/org/apache/ace/log/server/store/LogStore.java?rev=1537630&r1=1537629&r2=1537630&view=diff
==============================================================================
--- ace/trunk/org.apache.ace.log/src/org/apache/ace/log/server/store/LogStore.java (original)
+++ ace/trunk/org.apache.ace.log/src/org/apache/ace/log/server/store/LogStore.java Thu Oct
31 20:43:14 2013
@@ -90,4 +90,12 @@ public interface LogStore
      * @throws java.io.IOException in case of any error.
      */
     public List<Descriptor> getDescriptors() throws IOException;
+    
+    /**
+     * Cleanup the events in the store. This method will check each target and log in this
store and remove all
+     * events exceeding the maximum number of events that can be configured for this store.
+     * 
+     * @throws IOException in case of any error
+     */
+    public void clean() throws IOException;
 }
\ No newline at end of file

Modified: ace/trunk/org.apache.ace.log/src/org/apache/ace/log/server/store/impl/Activator.java
URL: http://svn.apache.org/viewvc/ace/trunk/org.apache.ace.log/src/org/apache/ace/log/server/store/impl/Activator.java?rev=1537630&r1=1537629&r2=1537630&view=diff
==============================================================================
--- ace/trunk/org.apache.ace.log/src/org/apache/ace/log/server/store/impl/Activator.java (original)
+++ ace/trunk/org.apache.ace.log/src/org/apache/ace/log/server/store/impl/Activator.java Thu
Oct 31 20:43:14 2013
@@ -36,6 +36,7 @@ import org.osgi.service.event.EventAdmin
 import org.osgi.service.log.LogService;
 
 public class Activator extends DependencyActivatorBase implements ManagedServiceFactory {
+    public static final String PID = "org.apache.ace.log.server.store.filebased";
 
     private static final String LOG_NAME = "name";
     private DependencyManager m_manager;
@@ -94,7 +95,8 @@ public class Activator extends Dependenc
             service = m_manager.createComponent()
                 .setInterface(LogStore.class.getName(), props)
                 .setImplementation(new LogStoreImpl(baseDir, name))
-                .add(createServiceDependency().setService(EventAdmin.class).setRequired(false));
+                .add(createServiceDependency().setService(EventAdmin.class).setRequired(false))
+                .add(createConfigurationDependency().setPid(PID));
             m_instances.put(pid, service);
             m_manager.add(service);
         } else {

Modified: ace/trunk/org.apache.ace.log/src/org/apache/ace/log/server/store/impl/LogStoreImpl.java
URL: http://svn.apache.org/viewvc/ace/trunk/org.apache.ace.log/src/org/apache/ace/log/server/store/impl/LogStoreImpl.java?rev=1537630&r1=1537629&r2=1537630&view=diff
==============================================================================
--- ace/trunk/org.apache.ace.log/src/org/apache/ace/log/server/store/impl/LogStoreImpl.java
(original)
+++ ace/trunk/org.apache.ace.log/src/org/apache/ace/log/server/store/impl/LogStoreImpl.java
Thu Oct 31 20:43:14 2013
@@ -29,27 +29,39 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Dictionary;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Hashtable;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
 
 import org.apache.ace.feedback.Descriptor;
 import org.apache.ace.feedback.Event;
 import org.apache.ace.log.server.store.LogStore;
 import org.apache.ace.range.Range;
 import org.apache.ace.range.SortedRangeSet;
+import org.osgi.service.cm.ConfigurationException;
+import org.osgi.service.cm.ManagedService;
 import org.osgi.service.event.EventAdmin;
 
 /**
  * A simple implementation of the LogStore interface.
  */
-public class LogStoreImpl implements LogStore {
+public class LogStoreImpl implements LogStore, ManagedService {
+
+    private static final String MAXIMUM_NUMBER_OF_EVENTS = "MaxEvents";
 
     private volatile EventAdmin m_eventAdmin; /* Injected by dependency manager */
 
     // the dir to store logs in - init is in the start method
     private final File m_dir;
     private final String m_name;
+    private int m_maxEvents = 0;
+
+    private final ConcurrentMap<String, Set<Long>> m_locks = new ConcurrentHashMap<String,
Set<Long>>();
+    private final Map<String, Long> m_fileToID = new HashMap<String, Long>();
 
     public LogStoreImpl(File baseDir, String name) {
         m_name = name;
@@ -57,8 +69,7 @@ public class LogStoreImpl implements Log
     }
 
     /*
-     * init the dir in which to store logs in - thows IllegalArgumentException
-     * if we can't get it.
+     * init the dir in which to store logs in - thows IllegalArgumentException if we can't
get it.
      */
     protected void start() throws IOException {
         if (!m_dir.isDirectory() && !m_dir.mkdirs()) {
@@ -69,15 +80,35 @@ public class LogStoreImpl implements Log
     /**
      * @see org.apache.ace.log.server.store.LogStore#get(org.apache.ace.log.Descriptor)
      */
-    public synchronized List<Event> get(Descriptor descriptor)
-            throws IOException {
+    public List<Event> get(Descriptor descriptor)
+        throws IOException {
+        try {
+            obtainLock(descriptor.getTargetID(), descriptor.getStoreID());
+            return getInternal(descriptor);
+        }
+        finally {
+            releaseLock(descriptor.getTargetID(), descriptor.getStoreID());
+        }
+    }
+
+    /**
+     * Retrieve the events that match the given descriptor. This method relies on external
locking, the caller should
+     * take care of that.
+     * 
+     * @param descriptor
+     *            the events to retrieve
+     * @return the events that match
+     * @throws IOException
+     *             if anything goes wrong
+     */
+    private List<Event> getInternal(Descriptor descriptor) throws IOException {
         final List<Event> result = new ArrayList<Event>();
         final SortedRangeSet set = descriptor.getRangeSet();
         BufferedReader in = null;
         try {
             File log = new File(new File(m_dir,
-                    targetIDToFilename(descriptor.getTargetID())),
-                    String.valueOf(descriptor.getStoreID()));
+                targetIDToFilename(descriptor.getTargetID())),
+                String.valueOf(descriptor.getStoreID()));
             if (!log.isFile()) {
                 return result;
             }
@@ -85,12 +116,13 @@ public class LogStoreImpl implements Log
             String file = log.getAbsolutePath();
             long counter = 0;
             for (String line = in.readLine(); line != null; line = in
-                    .readLine()) {
+                .readLine()) {
                 Event event = new Event(line);
                 long id = event.getID();
                 if ((counter != -1) && ++counter == id) {
 
-                } else {
+                }
+                else {
                     counter = -1;
                 }
                 if (set.contains(id)) {
@@ -99,15 +131,16 @@ public class LogStoreImpl implements Log
             }
             if (counter < 1) {
                 m_fileToID.remove(file);
-            } else {
+            }
+            else {
                 m_fileToID.put(file, counter);
             }
-        } 
+        }
         finally {
             if (in != null) {
                 try {
                     in.close();
-                } 
+                }
                 catch (Exception ex) {
                     // Not much we can do
                 }
@@ -116,23 +149,21 @@ public class LogStoreImpl implements Log
         return result;
     }
 
-    private final Map<String, Long> m_fileToID = new HashMap<String, Long>();
-
     /**
      * @see org.apache.ace.log.server.store.LogStore#getDescriptor(String, long)
      */
-    public synchronized Descriptor getDescriptor(String targetID, long logID)
-            throws IOException {
+    public Descriptor getDescriptor(String targetID, long logID)
+        throws IOException {
         Long high = m_fileToID.get(new File(new File(m_dir,
-                targetIDToFilename(targetID)), String.valueOf(logID))
-                .getAbsolutePath());
+            targetIDToFilename(targetID)), String.valueOf(logID))
+            .getAbsolutePath());
         if (high != null) {
             Range r = new Range(1, high);
             return new Descriptor(targetID, logID, new SortedRangeSet(
-                    r.toRepresentation()));
+                r.toRepresentation()));
         }
         List<Event> events = get(new Descriptor(targetID, logID,
-                SortedRangeSet.FULL_SET));
+            SortedRangeSet.FULL_SET));
 
         long[] idsArray = new long[events.size()];
         int i = 0;
@@ -146,7 +177,7 @@ public class LogStoreImpl implements Log
      * @see org.apache.ace.log.server.store.LogStore#getDescriptors(String)
      */
     public List<Descriptor> getDescriptors(String targetID)
-            throws IOException {
+        throws IOException {
         File dir = new File(m_dir, targetIDToFilename(targetID));
         List<Descriptor> result = new ArrayList<Descriptor>();
         if (!dir.isDirectory()) {
@@ -178,13 +209,20 @@ public class LogStoreImpl implements Log
         Map<String, Map<Long, List<Event>>> sorted = sort(events);
         for (String targetID : sorted.keySet()) {
             for (Long logID : sorted.get(targetID).keySet()) {
-                put(targetID, logID, sorted.get(targetID).get(logID));
+                try {
+                    obtainLock(targetID, logID);
+                    put(targetID, logID, sorted.get(targetID).get(logID));
+                }
+                finally {
+                    releaseLock(targetID, logID);
+                }
             }
         }
     }
 
     /**
-     * Add a list of events to the log of the given ids.
+     * Add a list of events to the log of the given ids. This method relies on external locking,
the caller should take
+     * care of that.
      * 
      * @param targetID
      *            the id of the target to append to its log.
@@ -195,8 +233,8 @@ public class LogStoreImpl implements Log
      * @throws java.io.IOException
      *             in case of any error.
      */
-    protected synchronized void put(String targetID, Long logID,
-            List<Event> list) throws IOException {
+    protected void put(String targetID, Long logID,
+        List<Event> list) throws IOException {
         if ((list == null) || (list.size() == 0)) {
             // nothing to add, so return
             return;
@@ -206,7 +244,7 @@ public class LogStoreImpl implements Log
         // 2. we need to insert events in the existing file (meaning we have to
         // rewrite basically the whole file)
         String file = new File(new File(m_dir, targetIDToFilename(targetID)),
-                String.valueOf(logID)).getAbsolutePath();
+            String.valueOf(logID)).getAbsolutePath();
         Long highest = m_fileToID.get(file);
         boolean cached = false;
         if (highest != null) {
@@ -216,15 +254,20 @@ public class LogStoreImpl implements Log
         }
         List<Event> events = null;
         if (!cached) {
-            events = get(new Descriptor(targetID, logID,
-                    SortedRangeSet.FULL_SET));
+            events = getInternal(new Descriptor(targetID, logID,
+                SortedRangeSet.FULL_SET));
 
             // remove duplicates first
             list.removeAll(events);
         }
 
-        if (list.size() == 0) {
-            // nothing to add anymore, so return
+        boolean removeEvents = false;
+        if (m_maxEvents > 0 && m_maxEvents < list.size() + events.size()) {
+            removeEvents = true;
+        }
+
+        if (!removeEvents && list.size() == 0) {
+            // nothing to add or remove anymore, so return
             return;
         }
 
@@ -234,26 +277,34 @@ public class LogStoreImpl implements Log
             if (!dir.isDirectory() && !dir.mkdirs()) {
                 throw new IOException("Unable to create backup store.");
             }
-            if (cached
-                    || ((events.size() == 0) || (events.get(events.size() - 1)
-                            .getID() < list.get(0).getID()))) {
-                // we can append to the existing file
+            if (!removeEvents && (cached
+                || ((events.size() == 0) || (events.get(events.size() - 1)
+                .getID() < list.get(0).getID())))) {
+                // we can append to the existing file without need to remove records
                 out = new PrintWriter(new FileWriter(new File(dir,
-                        logID.toString()), true));
-            } else {
+                    logID.toString()), true));
+            }
+            else {
                 // we have to merge the lists
                 list.addAll(events);
                 // and sort
                 Collections.sort(list);
+                // and remove if necessary
+                for (int i = 0; i < m_maxEvents; i++) {
+                    if (list.size() > 0) {
+                        list.remove(0);
+                    }
+                }
                 out = new PrintWriter(new FileWriter(new File(dir,
-                        logID.toString())));
+                    logID.toString())));
             }
             long high = 0;
             for (Event event : list) {
                 out.println(event.toRepresentation());
                 if (high < event.getID()) {
                     high = event.getID();
-                } else {
+                }
+                else {
                     high = Long.MAX_VALUE;
                 }
                 // send (eventadmin)event about a new (log)event being stored
@@ -264,14 +315,15 @@ public class LogStoreImpl implements Log
             }
             if ((cached) && (high < Long.MAX_VALUE)) {
                 m_fileToID.put(file, new Long(high));
-            } else {
+            }
+            else {
                 m_fileToID.remove(file);
             }
-        } 
+        }
         finally {
             try {
                 out.close();
-            } 
+            }
             catch (Exception ex) {
                 // Not much we can do
             }
@@ -279,20 +331,18 @@ public class LogStoreImpl implements Log
     }
 
     /**
-     * Sort the given list of events into a map of maps according to the
-     * targetID and the logID of each event.
+     * Sort the given list of events into a map of maps according to the targetID and the
logID of each event.
      * 
      * @param events
      *            a list of events to sort.
-     * @return a map of maps that maps target ids to a map that maps log ids to
-     *         a list of events that have those ids.
+     * @return a map of maps that maps target ids to a map that maps log ids to a list of
events that have those ids.
      */
     @SuppressWarnings("boxing")
     protected Map<String, Map<Long, List<Event>>> sort(List<Event>
events) {
         Map<String, Map<Long, List<Event>>> result = new HashMap<String,
Map<Long, List<Event>>>();
         for (Event event : events) {
             Map<Long, List<Event>> target = result
-                    .get(event.getTargetID());
+                .get(event.getTargetID());
 
             if (target == null) {
                 target = new HashMap<Long, List<Event>>();
@@ -316,7 +366,7 @@ public class LogStoreImpl implements Log
     private <T> T notNull(T target) throws IOException {
         if (target == null) {
             throw new IOException(
-                    "Unknown IO error while trying to access the store.");
+                "Unknown IO error while trying to access the store.");
         }
         return target;
     }
@@ -331,7 +381,7 @@ public class LogStoreImpl implements Log
         String result = null;
         try {
             result = new String(bytes, "UTF-8");
-        } 
+        }
         catch (UnsupportedEncodingException e) {
             // UTF-8 is a mandatory encoding; this will never happen.
         }
@@ -346,15 +396,124 @@ public class LogStoreImpl implements Log
                 String hexValue = Integer.toHexString(b.intValue());
                 if (hexValue.length() % 2 == 0) {
                     result.append(hexValue);
-                } else {
+                }
+                else {
                     result.append('0').append(hexValue);
                 }
             }
-        } 
+        }
         catch (UnsupportedEncodingException e) {
             // UTF-8 is a mandatory encoding; this will never happen.
         }
 
         return result.toString();
     }
-}
\ No newline at end of file
+
+    @SuppressWarnings("rawtypes")
+    @Override
+    public void updated(Dictionary settings) throws ConfigurationException {
+        if (settings != null) {
+            String maximumNumberOfEvents = (String) settings.get(MAXIMUM_NUMBER_OF_EVENTS);
+            if (maximumNumberOfEvents != null) {
+                try {
+                    m_maxEvents = Integer.parseInt(maximumNumberOfEvents);
+                }
+                catch (NumberFormatException nfe) {
+                    throw new ConfigurationException(MAXIMUM_NUMBER_OF_EVENTS, "is not a
number");
+                }
+            }
+        }
+    }
+
+    @Override
+    public void clean() throws IOException {
+        // check if we event might have to cleanup anything
+        if (m_maxEvents <= 0) {
+            return;
+        }
+        // create a list of unique targets and their logs
+        Map<String, Set<Long>> allTargetsAndLogs = new HashMap<String, Set<Long>>();
+        for (Descriptor descriptor : getDescriptors()) {
+            Set<Long> logs = allTargetsAndLogs.get(descriptor.getTargetID());
+            if (logs == null) {
+                logs = new HashSet<Long>();
+                allTargetsAndLogs.put(descriptor.getTargetID(), logs);
+            }
+            logs.add(descriptor.getStoreID());
+        }
+
+        // cleanup per log
+        for (String targetID : allTargetsAndLogs.keySet()) {
+            for (Long logId : allTargetsAndLogs.get(targetID)) {
+                clean(targetID, logId);
+            }
+        }
+    }
+
+    private void clean(String targetID, Long logID) throws IOException {
+        try {
+            obtainLock(targetID, logID);
+            List<Event> events = getInternal(new Descriptor(targetID, logID, SortedRangeSet.FULL_SET));
+            if (events.size() > m_maxEvents) {
+                for (int i = 0; i < m_maxEvents; i++) {
+                    if (events.size() > 0) {
+                        events.remove(0);
+                    }
+                }
+            }
+            put(targetID, logID, events);
+        }
+        finally {
+            releaseLock(targetID, logID);
+        }
+    }
+
+    private void obtainLock(String targetID, long logID) throws IOException {
+        Set<Long> newLockedLogs = new HashSet<Long>();
+        Set<Long> lockedLogs = m_locks.putIfAbsent(targetID, newLockedLogs);
+        if (lockedLogs == null) {
+            lockedLogs = newLockedLogs;
+        }
+        boolean alreadyLocked;
+
+        synchronized (lockedLogs) {
+            alreadyLocked = lockedLogs.contains(logID);
+            if (!alreadyLocked) {
+                // lock it now, we are working on it
+                lockedLogs.add(logID);
+            }
+        }
+
+        // try to obtain the lock if we could not lock it on the first try
+        if (alreadyLocked) {
+            int nrOfTries = 1;
+            while (alreadyLocked && nrOfTries < 10) {
+                try {
+                    Thread.sleep(20);
+                }
+                catch (InterruptedException e) {
+                    break;
+                }
+                nrOfTries++;
+                synchronized (lockedLogs) {
+                    alreadyLocked = lockedLogs.contains(logID);
+                    if (!alreadyLocked) {
+                        // lock it now, we are working on it
+                        lockedLogs.add(logID);
+                    }
+                }
+            }
+            // if the log is still locked throw an exception
+            if (alreadyLocked) {
+                throw new IOException("Could not obtain a lock for the store " + logID +
" of target " + targetID);
+            }
+        }
+    }
+
+    private void releaseLock(String targetID, Long logID) {
+        Set<Long> lockedLogs = m_locks.get(targetID);
+        synchronized (lockedLogs) {
+            lockedLogs.remove(logID);
+        }
+    }
+}

Modified: ace/trunk/org.apache.ace.log/src/org/apache/ace/log/server/store/mongo/Activator.java
URL: http://svn.apache.org/viewvc/ace/trunk/org.apache.ace.log/src/org/apache/ace/log/server/store/mongo/Activator.java?rev=1537630&r1=1537629&r2=1537630&view=diff
==============================================================================
--- ace/trunk/org.apache.ace.log/src/org/apache/ace/log/server/store/mongo/Activator.java
(original)
+++ ace/trunk/org.apache.ace.log/src/org/apache/ace/log/server/store/mongo/Activator.java
Thu Oct 31 20:43:14 2013
@@ -36,6 +36,8 @@ import org.osgi.service.event.EventAdmin
 import org.osgi.service.log.LogService;
 
 public class Activator extends DependencyActivatorBase implements ManagedServiceFactory {
+    public static final String PID = "org.apache.ace.log.server.store.mongo";
+
     private static final String LOG_NAME = "name";
 
     private DependencyManager m_manager;
@@ -83,7 +85,8 @@ public class Activator extends Dependenc
                 .setInterface(LogStore.class.getName(), props)
                 .setImplementation(new MongoLogStore(name))
                 .add(createServiceDependency().setService(EventAdmin.class).setRequired(false))
-                .add(createServiceDependency().setService(MongoDBService.class).setRequired(true));
+                .add(createServiceDependency().setService(MongoDBService.class).setRequired(true))
+                .add(createConfigurationDependency().setPid(PID));
             m_instances.put(pid, service);
             m_manager.add(service);
         } else {

Modified: ace/trunk/org.apache.ace.log/src/org/apache/ace/log/server/store/mongo/MongoLogStore.java
URL: http://svn.apache.org/viewvc/ace/trunk/org.apache.ace.log/src/org/apache/ace/log/server/store/mongo/MongoLogStore.java?rev=1537630&r1=1537629&r2=1537630&view=diff
==============================================================================
--- ace/trunk/org.apache.ace.log/src/org/apache/ace/log/server/store/mongo/MongoLogStore.java
(original)
+++ ace/trunk/org.apache.ace.log/src/org/apache/ace/log/server/store/mongo/MongoLogStore.java
Thu Oct 31 20:43:14 2013
@@ -2,6 +2,7 @@ package org.apache.ace.log.server.store.
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Dictionary;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -15,6 +16,8 @@ import org.apache.ace.feedback.Event;
 import org.apache.ace.log.server.store.LogStore;
 import org.apache.ace.range.Range;
 import org.apache.ace.range.SortedRangeSet;
+import org.osgi.service.cm.ConfigurationException;
+import org.osgi.service.cm.ManagedService;
 
 import com.mongodb.BasicDBObject;
 import com.mongodb.DBCollection;
@@ -23,9 +26,12 @@ import com.mongodb.DBObject;
 import com.mongodb.MapReduceCommand.OutputType;
 import com.mongodb.MapReduceOutput;
 
-public class MongoLogStore implements LogStore {
+public class MongoLogStore implements LogStore, ManagedService {
+    private static final String MAXIMUM_NUMBER_OF_EVENTS = "MaxEvents";
+
     private final String m_logname;
     private volatile MongoDBService m_mongoDBService;
+    private int m_maxEvents = 0;
 
     public MongoLogStore(String logname) {
         this.m_logname = logname;
@@ -91,6 +97,7 @@ public class MongoLogStore implements Lo
 
     @Override
     public void put(List<Event> events) throws IOException {
+        // TODO : if m_max_events > 0 then make sure there are no more than m_maxEvents
         DBCollection collection = m_mongoDBService.getDB().getCollection(m_logname);
 
         for (Event event : events) {
@@ -142,5 +149,24 @@ public class MongoLogStore implements Lo
     public List<Descriptor> getDescriptors() throws IOException {
         return getDescriptors(null);
     }
+    
+    @SuppressWarnings("rawtypes")
+    @Override
+    public void updated(Dictionary settings) throws ConfigurationException {
+        if (settings != null) {
+            String maximumNumberOfEvents = (String) settings.get(MAXIMUM_NUMBER_OF_EVENTS);
+            if (maximumNumberOfEvents != null) {
+                try {
+                    m_maxEvents = Integer.parseInt(maximumNumberOfEvents);
+                } catch (NumberFormatException nfe) {
+                    throw new ConfigurationException(MAXIMUM_NUMBER_OF_EVENTS, "is not a
number");
+                }
+            }
+        }
+    }
 
+    @Override
+    public void clean() throws IOException {
+        // TODO : if m_max_events > 0 then remove all events from the mongo store where
there are more than m_maxEvents
+    }
 }

Modified: ace/trunk/org.apache.ace.log/test/org/apache/ace/log/server/servlet/LogServletTest.java
URL: http://svn.apache.org/viewvc/ace/trunk/org.apache.ace.log/test/org/apache/ace/log/server/servlet/LogServletTest.java?rev=1537630&r1=1537629&r2=1537630&view=diff
==============================================================================
--- ace/trunk/org.apache.ace.log/test/org/apache/ace/log/server/servlet/LogServletTest.java
(original)
+++ ace/trunk/org.apache.ace.log/test/org/apache/ace/log/server/servlet/LogServletTest.java
Thu Oct 31 20:43:14 2013
@@ -129,6 +129,9 @@ public class LogServletTest {
         public void put(List<Event> events) throws IOException {
             m_events = events;
         }
+        public void clean() throws IOException {
+            throw new UnsupportedOperationException("not implemented");
+        }
     }
 
     private class MockServletOutputStream extends ServletOutputStream {

Modified: ace/trunk/org.apache.ace.log/test/org/apache/ace/log/server/store/impl/ServerLogStoreTester.java
URL: http://svn.apache.org/viewvc/ace/trunk/org.apache.ace.log/test/org/apache/ace/log/server/store/impl/ServerLogStoreTester.java?rev=1537630&r1=1537629&r2=1537630&view=diff
==============================================================================
--- ace/trunk/org.apache.ace.log/test/org/apache/ace/log/server/store/impl/ServerLogStoreTester.java
(original)
+++ ace/trunk/org.apache.ace.log/test/org/apache/ace/log/server/store/impl/ServerLogStoreTester.java
Thu Oct 31 20:43:14 2013
@@ -23,10 +23,12 @@ import static org.apache.ace.test.utils.
 import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Dictionary;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Properties;
 import java.util.Set;
 
 import org.apache.ace.feedback.AuditEvent;
@@ -39,6 +41,8 @@ import org.testng.annotations.BeforeMeth
 import org.testng.annotations.Test;
 
 public class ServerLogStoreTester {
+    private static final String MAXIMUM_NUMBER_OF_EVENTS = "MaxEvents";
+
     private LogStoreImpl m_logStore;
     private File m_dir;
 
@@ -104,6 +108,92 @@ public class ServerLogStoreTester {
         assert m_logStore.getDescriptors(targetID).size() == 1 : "We expect to find a single
event: expected 1, found " + m_logStore.getDescriptors(targetID).size();
     }
 
+    @SuppressWarnings({ "rawtypes", "unchecked" })
+    @Test(groups = { TestUtils.UNIT })
+    public void testMaximumNumberOfEvents() throws Exception {
+        Dictionary settings = new Properties();
+        settings.put(MAXIMUM_NUMBER_OF_EVENTS, "1");
+        m_logStore.updated(settings);
+        
+        List<Event> events = new ArrayList<Event>();
+        for (String target : new String[] { "target"}) {
+            for (long log : new long[] { 1 }) {
+                for (long id : new long[] { 1, 2 }) {
+                    events.add(new Event(target, log, id, System.currentTimeMillis(), AuditEvent.FRAMEWORK_STARTED,
new HashMap<String, String>()));
+                }
+            }
+        }
+
+        m_logStore.put(events);
+
+        List<Descriptor> allDescriptors = m_logStore.getDescriptors();
+        assert allDescriptors.size() == 1 : "Expected only one descriptor, found: " + allDescriptors.size();
+        for (Descriptor range : allDescriptors) {
+            List<Descriptor> allLogsForTarget = m_logStore.getDescriptors(range.getTargetID());
+            for (Descriptor range2 : allLogsForTarget) {
+                List<Event> getEvents = m_logStore.get(m_logStore.getDescriptor(range2.getTargetID(),
range2.getStoreID()));
+                assert getEvents.size() == 1 : "Only one event expected, found " + getEvents.size();
+            }
+        }
+    }
+
+    @SuppressWarnings({ "rawtypes", "unchecked" })
+    @Test(groups = { TestUtils.UNIT })
+    public void testMaximumNumberOfEventsMultipleLogs() throws Exception {
+        Dictionary settings = new Properties();
+        settings.put(MAXIMUM_NUMBER_OF_EVENTS, "1");
+        m_logStore.updated(settings);
+        
+        List<Event> events = new ArrayList<Event>();
+        for (String target : new String[] { "target"}) {
+            for (long log : new long[] { 1,2 }) {
+                for (long id : new long[] { 1, 2 }) {
+                    events.add(new Event(target, log, id, System.currentTimeMillis(), AuditEvent.FRAMEWORK_STARTED,
new HashMap<String, String>()));
+                }
+            }
+        }
+        
+        m_logStore.put(events);
+        List<Descriptor> allDescriptors = m_logStore.getDescriptors();
+        assert allDescriptors.size() == 2 : "Expected two descriptor, found: " + allDescriptors.size();
+        for (Descriptor range : allDescriptors) {
+            List<Descriptor> allLogsForTarget = m_logStore.getDescriptors(range.getTargetID());
+            for (Descriptor range2 : allLogsForTarget) {
+                List<Event> getEvents = m_logStore.get(m_logStore.getDescriptor(range2.getTargetID(),
range2.getStoreID()));
+                assert getEvents.size() == 1 : "Only one event expected, found " + getEvents.size();
+            }
+        }
+    }
+
+    @SuppressWarnings({ "rawtypes", "unchecked" })
+    @Test(groups = { TestUtils.UNIT })
+    public void testClean() throws Exception {
+        List<Event> events = new ArrayList<Event>();
+        for (String target : new String[] { "target"}) {
+            for (long log : new long[] { 1,2 }) {
+                for (long id : new long[] { 1, 2 }) {
+                    events.add(new Event(target, log, id, System.currentTimeMillis(), AuditEvent.FRAMEWORK_STARTED,
new HashMap<String, String>()));
+                }
+            }
+        }
+        m_logStore.put(events);
+
+        Dictionary settings = new Properties();
+        settings.put(MAXIMUM_NUMBER_OF_EVENTS, "1");
+        m_logStore.updated(settings);
+        
+        m_logStore.clean();
+        List<Descriptor> allDescriptors = m_logStore.getDescriptors();
+        assert allDescriptors.size() == 2 : "Expected two descriptor, found: " + allDescriptors.size();
+        for (Descriptor range : allDescriptors) {
+            List<Descriptor> allLogsForTarget = m_logStore.getDescriptors(range.getTargetID());
+            for (Descriptor range2 : allLogsForTarget) {
+                List<Event> getEvents = m_logStore.get(m_logStore.getDescriptor(range2.getTargetID(),
range2.getStoreID()));
+                assert getEvents.size() == 1 : "Only one event expected, found " + getEvents.size();
+            }
+        }
+    }
+    
     private void delete(File root) {
         if (root.isDirectory()) {
             for (File child : root.listFiles()) {

Added: ace/trunk/run-server-allinone/conf/org.apache.ace.log.server.store.filebased.cfg
URL: http://svn.apache.org/viewvc/ace/trunk/run-server-allinone/conf/org.apache.ace.log.server.store.filebased.cfg?rev=1537630&view=auto
==============================================================================
--- ace/trunk/run-server-allinone/conf/org.apache.ace.log.server.store.filebased.cfg (added)
+++ ace/trunk/run-server-allinone/conf/org.apache.ace.log.server.store.filebased.cfg Thu Oct
31 20:43:14 2013
@@ -0,0 +1 @@
+MaxEvents=0
\ No newline at end of file

Added: ace/trunk/run-server/conf/org.apache.ace.log.server.store.filebased.cfg
URL: http://svn.apache.org/viewvc/ace/trunk/run-server/conf/org.apache.ace.log.server.store.filebased.cfg?rev=1537630&view=auto
==============================================================================
--- ace/trunk/run-server/conf/org.apache.ace.log.server.store.filebased.cfg (added)
+++ ace/trunk/run-server/conf/org.apache.ace.log.server.store.filebased.cfg Thu Oct 31 20:43:14
2013
@@ -0,0 +1 @@
+MaxEvents=0
\ No newline at end of file



Mime
View raw message