incubator-isis-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rmatth...@apache.org
Subject svn commit: r1125813 - in /incubator/isis/trunk/runtimes/dflt/objectstores/nosql/src: main/java/org/apache/isis/runtimes/dflt/objectstores/nosql/file/server/ test/java/org/apache/isis/runtimes/dflt/objectstores/nosql/file/server/
Date Sat, 21 May 2011 20:48:34 GMT
Author: rmatthews
Date: Sat May 21 20:48:33 2011
New Revision: 1125813

URL: http://svn.apache.org/viewvc?rev=1125813&view=rev
Log:
Final changes of OS for first release.

Completed the recovery log transfer and replay feature.

Added:
    incubator/isis/trunk/runtimes/dflt/objectstores/nosql/src/main/java/org/apache/isis/runtimes/dflt/objectstores/nosql/file/server/LogRange.java
  (with props)
Modified:
    incubator/isis/trunk/runtimes/dflt/objectstores/nosql/src/main/java/org/apache/isis/runtimes/dflt/objectstores/nosql/file/server/FileServer.java
    incubator/isis/trunk/runtimes/dflt/objectstores/nosql/src/main/java/org/apache/isis/runtimes/dflt/objectstores/nosql/file/server/LogWriter.java
    incubator/isis/trunk/runtimes/dflt/objectstores/nosql/src/main/java/org/apache/isis/runtimes/dflt/objectstores/nosql/file/server/Util.java
    incubator/isis/trunk/runtimes/dflt/objectstores/nosql/src/test/java/org/apache/isis/runtimes/dflt/objectstores/nosql/file/server/DataReaderTest.java
    incubator/isis/trunk/runtimes/dflt/objectstores/nosql/src/test/java/org/apache/isis/runtimes/dflt/objectstores/nosql/file/server/DataWriterTest.java
    incubator/isis/trunk/runtimes/dflt/objectstores/nosql/src/test/java/org/apache/isis/runtimes/dflt/objectstores/nosql/file/server/FileServerTest.java
    incubator/isis/trunk/runtimes/dflt/objectstores/nosql/src/test/java/org/apache/isis/runtimes/dflt/objectstores/nosql/file/server/LogWriterTest.java

Modified: incubator/isis/trunk/runtimes/dflt/objectstores/nosql/src/main/java/org/apache/isis/runtimes/dflt/objectstores/nosql/file/server/FileServer.java
URL: http://svn.apache.org/viewvc/incubator/isis/trunk/runtimes/dflt/objectstores/nosql/src/main/java/org/apache/isis/runtimes/dflt/objectstores/nosql/file/server/FileServer.java?rev=1125813&r1=1125812&r2=1125813&view=diff
==============================================================================
--- incubator/isis/trunk/runtimes/dflt/objectstores/nosql/src/main/java/org/apache/isis/runtimes/dflt/objectstores/nosql/file/server/FileServer.java
(original)
+++ incubator/isis/trunk/runtimes/dflt/objectstores/nosql/src/main/java/org/apache/isis/runtimes/dflt/objectstores/nosql/file/server/FileServer.java
Sat May 21 20:48:33 2011
@@ -77,7 +77,7 @@ public class FileServer {
 
         final Options options = new Options();
         options.addOption("h", "help", false, "Show this help");
-        options.addOption("m", "mode", true, "mode: normal | recovery | secondary");
+        options.addOption("m", "mode", true, "mode: normal | secondary | recovery | archive");
 
         CommandLineParser parser = new BasicParser();
         CommandLine cmd = parser.parse(options, args);
@@ -92,6 +92,9 @@ public class FileServer {
         if ("recovery".equals(mode)) {
             final FileServer fileServer = new FileServer();
             fileServer.startRecovery(cmd.getArgList());
+        } else if ("archive".equals(mode)) {
+            final FileServer fileServer = new FileServer();
+            fileServer.startArchive(cmd.getArgList());
         } else if ("secondary".equals(mode)) {
             final FileServer fileServer = new FileServer();
             fileServer.startSecondary();
@@ -126,8 +129,9 @@ public class FileServer {
             final String data = config.getString("fileserver.data");
             final String services = config.getString("fileserver.services");
             final String logs = config.getString("fileserver.logs");
+            final String archive = config.getString("fileserver.archive");
 
-            Util.setDirectory(data, services, logs);
+            Util.setDirectory(data, services, logs, archive);
             server = new FileServerProcessor();
         } catch (final ConfigurationException e) {
             LOG.error("configuration failure", e);
@@ -178,10 +182,13 @@ public class FileServer {
             socket.setSoTimeout(connectionTimeout);
             LOG.info("listenting on " + socket.getInetAddress().getHostAddress() + " port
" + socket.getLocalPort());
             LOG.debug("listenting on " + socket);
-            long lastRecoveryFile = lastRecoveryFile();
-            File file = Util.logFile(lastRecoveryFile);
-            LOG.info("replaying last recovery file: " + file.getAbsolutePath());
-            recover(file);
+            LogRange logFileRange = Util.logFileRange();
+            if (!logFileRange.noLogFile()) {
+                long lastRecoveryFile = logFileRange.getLast();
+                File file = Util.logFile(lastRecoveryFile);
+                LOG.info("replaying last recovery file: " + file.getAbsolutePath());
+                recover(file);
+            }
             server.startup();
         } catch (final UnknownHostException e) {
             LOG.error("Unknown host " + serviceHost, e);
@@ -416,11 +423,12 @@ public class FileServer {
 
     private void startRecovery(List list) {
         LOG.info("starting recovery");
-        long lastId = lastRecoveryFile();
-        if (lastId < 0) {
+        final LogRange logFileRange = Util.logFileRange();
+        if (logFileRange.noLogFile()) {
             System.err.println("No recovery files found");
             System.exit(0);
         }
+        long lastId = logFileRange.getLast();
         LOG.info("last log file is " + Util.logFile(lastId).getName());
 
         long startId = lastId;
@@ -433,8 +441,8 @@ public class FileServer {
                 endId = Long.valueOf((String) list.get(1));
             }
         }
-        if (startId > lastId || endId > lastId) {
-            System.err.println("File IDs invalid: they must be less than or equal to " +
lastId);
+        if (startId < logFileRange.getFirst() || startId > lastId || endId > lastId)
{
+            System.err.println("File IDs invalid: they must be between " + logFileRange.getFirst()
+ " and " + lastId);
             System.exit(0);
         }
         if (startId > endId) {
@@ -448,20 +456,46 @@ public class FileServer {
             LOG.info("recovering data from " + file.getName());
             recover(file);
         }
+        LOG.info("recovery complete");
     }
 
-    private long lastRecoveryFile() {
-        long lastId = -1;
-        while (Util.logFile(lastId + 1).exists()) {
-            lastId++;
+
+    private void startArchive(List list) {
+        LOG.info("starting archiving");
+        final LogRange logFileRange = Util.logFileRange();
+        if (logFileRange.noLogFile()) {
+            System.err.println("No recovery files found");
+            System.exit(0);
         }
-        return lastId;
-    }
+        long lastId = logFileRange.getLast();
+        LOG.info("last log file is " + Util.logFile(lastId).getName());
+
+        long endId = lastId - 1;
+
+        int size = list.size();
+        if (size > 0) {
+            endId = Long.valueOf((String) list.get(0));
+        }
+        if (endId >= lastId) {
+            System.err.println("File ID invalid: they must be less that " + lastId);
+            System.exit(0);
+        }
+        long startId = logFileRange.getFirst();
+        for (long id = startId; id <= endId; id++) {
+            File file = Util.logFile(id);
+            LOG.info("moving " + file.getName());
+            File destination = Util.archiveLogFile(id);
+            file.renameTo(destination);
+        }
+        LOG.info("archive complete");
 
+    }
+    
     private void startSecondary() {
         String serviceHost = config.getString("fileserver.sync-host", DEFAULT_HOST);
         int servicePort = config.getInt("fileserver.sync-port", DEFAULT_SYNC_PORT);
 
+        Util.ensureDirectoryExists();
         ServerSocket socket = null;
         try {
             LOG.debug("setting up syncing socket on " + serviceHost + ":" + servicePort);
@@ -494,7 +528,8 @@ public class FileServer {
                 return;
             }
 
-            long lastId = lastRecoveryFile();
+            LogRange logFileRange = Util.logFileRange();
+            long lastId = logFileRange.noLogFile() ? -1 : logFileRange.getLast();
             output.writeLong(lastId);
             do {
                 if (input.readByte() != RECOVERY_LOG) {
@@ -502,7 +537,7 @@ public class FileServer {
                 }
                 crc32.reset();
                 long logId = input.readLong();
-                File file = Util.logFile(logId);
+                File file = Util.tmpLogFile(logId);
                 LOG.info("syncing recovery file: " + file.getName());
                 BufferedOutputStream fileOutput = new BufferedOutputStream(new FileOutputStream(file));
 
@@ -521,7 +556,8 @@ public class FileServer {
                 }
 
                 recover(file);
-
+                File renameTo = Util.logFile(logId);
+                file.renameTo(renameTo);
             } while (true);
         } catch (final NoSqlStoreException e) {
             LOG.error("file server failure", e);
@@ -574,7 +610,7 @@ public class FileServer {
         String header;
         while ((header = reader.readLine()) != null) {
             if (header.startsWith("#transaction ended")) {
-                LOG.info("transaction read in (ending " + reader.getLineNumber() + ")");
+                LOG.debug("transaction read in (ending " + reader.getLineNumber() + ")");
                 content.writeData();
                 reader.readLine();
                 return;

Added: incubator/isis/trunk/runtimes/dflt/objectstores/nosql/src/main/java/org/apache/isis/runtimes/dflt/objectstores/nosql/file/server/LogRange.java
URL: http://svn.apache.org/viewvc/incubator/isis/trunk/runtimes/dflt/objectstores/nosql/src/main/java/org/apache/isis/runtimes/dflt/objectstores/nosql/file/server/LogRange.java?rev=1125813&view=auto
==============================================================================
--- incubator/isis/trunk/runtimes/dflt/objectstores/nosql/src/main/java/org/apache/isis/runtimes/dflt/objectstores/nosql/file/server/LogRange.java
(added)
+++ incubator/isis/trunk/runtimes/dflt/objectstores/nosql/src/main/java/org/apache/isis/runtimes/dflt/objectstores/nosql/file/server/LogRange.java
Sat May 21 20:48:33 2011
@@ -0,0 +1,42 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing,
+ *  software distributed under the License is distributed on an
+ *  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ *  KIND, either express or implied.  See the License for the
+ *  specific language governing permissions and limitations
+ *  under the License.
+ */
+package org.apache.isis.runtimes.dflt.objectstores.nosql.file.server;
+
+class LogRange {
+    private long first = Long.MAX_VALUE;
+    private long last = 0;
+   
+    public long getFirst() {
+        return first;
+    }
+    
+    public long getLast() {
+        return last;
+    }
+
+    public boolean noLogFile() {
+        return first == Long.MAX_VALUE && last == 0;
+    }
+
+    public void add(long sequence) {
+        last = Math.max(last, sequence);
+        first = Math.min(first, sequence);
+    }
+}
+

Propchange: incubator/isis/trunk/runtimes/dflt/objectstores/nosql/src/main/java/org/apache/isis/runtimes/dflt/objectstores/nosql/file/server/LogRange.java
------------------------------------------------------------------------------
    svn:mime-type = text/plain

Modified: incubator/isis/trunk/runtimes/dflt/objectstores/nosql/src/main/java/org/apache/isis/runtimes/dflt/objectstores/nosql/file/server/LogWriter.java
URL: http://svn.apache.org/viewvc/incubator/isis/trunk/runtimes/dflt/objectstores/nosql/src/main/java/org/apache/isis/runtimes/dflt/objectstores/nosql/file/server/LogWriter.java?rev=1125813&r1=1125812&r2=1125813&view=diff
==============================================================================
--- incubator/isis/trunk/runtimes/dflt/objectstores/nosql/src/main/java/org/apache/isis/runtimes/dflt/objectstores/nosql/file/server/LogWriter.java
(original)
+++ incubator/isis/trunk/runtimes/dflt/objectstores/nosql/src/main/java/org/apache/isis/runtimes/dflt/objectstores/nosql/file/server/LogWriter.java
Sat May 21 20:48:33 2011
@@ -32,7 +32,7 @@ public class LogWriter {
 
     private DataOutputStream writer;
     private boolean startNewFile = false;
-    private int nextLogIdToWrite;
+    private long nextLogIdToWrite;
 
     public void startNewFile() {
         // don't start new file if old one is empty
@@ -102,7 +102,11 @@ public class LogWriter {
     }
 
     private void openNewFile() {
-        File file = findNextFile();
+        nextLogIdToWrite++;
+        File file = Util.logFile(nextLogIdToWrite);
+        if (file.exists()) {
+            throw new NoSqlStoreException("Log file already exists");
+        }
         openFile(file);
     }
 
@@ -115,18 +119,8 @@ public class LogWriter {
         }
     }
 
-    private File findNextFile() {
-        File file;
-        do {
-            nextLogIdToWrite++;
-            file = Util.logFile(nextLogIdToWrite);
-        } while (file.exists());
-        return file;
-    }
-
     public void startup() {
-        findNextFile();
-        nextLogIdToWrite--;
+        nextLogIdToWrite = Util.logFileRange().getLast();
         startNewFile();
         if (!startNewFile) {
             File file = Util.logFile(nextLogIdToWrite);

Modified: incubator/isis/trunk/runtimes/dflt/objectstores/nosql/src/main/java/org/apache/isis/runtimes/dflt/objectstores/nosql/file/server/Util.java
URL: http://svn.apache.org/viewvc/incubator/isis/trunk/runtimes/dflt/objectstores/nosql/src/main/java/org/apache/isis/runtimes/dflt/objectstores/nosql/file/server/Util.java?rev=1125813&r1=1125812&r2=1125813&view=diff
==============================================================================
--- incubator/isis/trunk/runtimes/dflt/objectstores/nosql/src/main/java/org/apache/isis/runtimes/dflt/objectstores/nosql/file/server/Util.java
(original)
+++ incubator/isis/trunk/runtimes/dflt/objectstores/nosql/src/main/java/org/apache/isis/runtimes/dflt/objectstores/nosql/file/server/Util.java
Sat May 21 20:48:33 2011
@@ -30,6 +30,7 @@ import java.nio.charset.Charset;
 import org.apache.isis.runtimes.dflt.objectstores.nosql.NoSqlStoreException;
 import org.apache.log4j.Logger;
 
+
 public class Util {
 
     private static final Logger LOG = Logger.getLogger(Util.class);
@@ -37,6 +38,7 @@ public class Util {
     private static final String DEFAULT_DIRECTORY = "data";
     private static final String SERVICES_DIRECTORY = "services";
     private static final String LOGS_DIRECTORY = "logs";
+    private static final String LOGS_ARCHIVE_DIRECTORY = "archive";
     public static final String ABORT = "abort";
     public static final String OK = "ok";
     public static final String READ_ERROR = "Read error";
@@ -46,12 +48,14 @@ public class Util {
     private static File dataDirectory = new File(DEFAULT_DIRECTORY);
     private static File serviceDirectory = new File(DEFAULT_DIRECTORY, SERVICES_DIRECTORY);
     private static File logDirectory = new File(DEFAULT_DIRECTORY, LOGS_DIRECTORY);
+    private static File logArchiveDirectory = new File(DEFAULT_DIRECTORY, LOGS_ARCHIVE_DIRECTORY);
 
-    static void setDirectory(final String data, final String services, final String logs)
{
+    static void setDirectory(final String data, final String services, final String logs,
final String archive) {
         final String directory = data == null ? DEFAULT_DIRECTORY : data;
         Util.dataDirectory = new File(directory);
         Util.serviceDirectory = new File(directory, services == null ? SERVICES_DIRECTORY
: services);
         Util.logDirectory = new File(directory, logs == null ? LOGS_DIRECTORY : logs);
+        Util.logArchiveDirectory = new File(directory, archive == null ? LOGS_ARCHIVE_DIRECTORY
: archive);
     }
 
     static void ensureDirectoryExists() {
@@ -61,6 +65,9 @@ public class Util {
         if (!logDirectory.exists()) {
             logDirectory.mkdirs();
         }
+        if (!logArchiveDirectory.exists()) {
+            logArchiveDirectory.mkdirs();
+        }
     }
 
     public static boolean isPopulated() {
@@ -160,6 +167,30 @@ public class Util {
         return new File(logDirectory, "recovery" + id + ".log");
     }
 
+    static File tmpLogFile(final long id) {
+        return new File(logDirectory, "recovery" + id + ".log.tmp");
+    }
+
+    public static File archiveLogFile(long id) {
+        return new File(logArchiveDirectory, "recovery" + id + ".log");
+    }
+
+    static LogRange logFileRange() {
+        LogRange logRange = new LogRange();
+        File[] listFiles = logDirectory.listFiles();
+        if (listFiles != null) {
+            for (File file : listFiles) {
+                String name = file.getName();
+                String substring = name.substring(8, name.length() - 4);
+                try {
+                    long sequence = Long.parseLong(substring);
+                    logRange.add(sequence);
+                } catch (NumberFormatException ignore) {}
+            }
+        }
+        return logRange;
+    }
+
     static final char DELETE = 'D';
 
     public static final Charset ENCODING = Charset.forName("utf-8");
@@ -282,7 +313,6 @@ public class Util {
         };
     }
 
-
     public static void closeSafely(final FileOutputStream output) {
         if (output != null) {
             try {

Modified: incubator/isis/trunk/runtimes/dflt/objectstores/nosql/src/test/java/org/apache/isis/runtimes/dflt/objectstores/nosql/file/server/DataReaderTest.java
URL: http://svn.apache.org/viewvc/incubator/isis/trunk/runtimes/dflt/objectstores/nosql/src/test/java/org/apache/isis/runtimes/dflt/objectstores/nosql/file/server/DataReaderTest.java?rev=1125813&r1=1125812&r2=1125813&view=diff
==============================================================================
--- incubator/isis/trunk/runtimes/dflt/objectstores/nosql/src/test/java/org/apache/isis/runtimes/dflt/objectstores/nosql/file/server/DataReaderTest.java
(original)
+++ incubator/isis/trunk/runtimes/dflt/objectstores/nosql/src/test/java/org/apache/isis/runtimes/dflt/objectstores/nosql/file/server/DataReaderTest.java
Sat May 21 20:48:33 2011
@@ -37,7 +37,7 @@ public class DataReaderTest {
 
     @BeforeClass
     public static void setUp() throws Exception {
-        Util.setDirectory("target/test", "services", "logs");
+        Util.setDirectory("target/test", "services", "logs", "archive");
         Util.ensureDirectoryExists();
         new File("target/test/type").mkdir();
     }

Modified: incubator/isis/trunk/runtimes/dflt/objectstores/nosql/src/test/java/org/apache/isis/runtimes/dflt/objectstores/nosql/file/server/DataWriterTest.java
URL: http://svn.apache.org/viewvc/incubator/isis/trunk/runtimes/dflt/objectstores/nosql/src/test/java/org/apache/isis/runtimes/dflt/objectstores/nosql/file/server/DataWriterTest.java?rev=1125813&r1=1125812&r2=1125813&view=diff
==============================================================================
--- incubator/isis/trunk/runtimes/dflt/objectstores/nosql/src/test/java/org/apache/isis/runtimes/dflt/objectstores/nosql/file/server/DataWriterTest.java
(original)
+++ incubator/isis/trunk/runtimes/dflt/objectstores/nosql/src/test/java/org/apache/isis/runtimes/dflt/objectstores/nosql/file/server/DataWriterTest.java
Sat May 21 20:48:33 2011
@@ -43,7 +43,7 @@ public class DataWriterTest {
     @Before
     public void setUp() throws Exception {
         Logger.getRootLogger().setLevel(Level.OFF);
-        Util.setDirectory(TARGET_DIRECTORY, "services", "logs");
+        Util.setDirectory(TARGET_DIRECTORY, "services", "logs", "archive");
         Util.ensureDirectoryExists();
     }
 

Modified: incubator/isis/trunk/runtimes/dflt/objectstores/nosql/src/test/java/org/apache/isis/runtimes/dflt/objectstores/nosql/file/server/FileServerTest.java
URL: http://svn.apache.org/viewvc/incubator/isis/trunk/runtimes/dflt/objectstores/nosql/src/test/java/org/apache/isis/runtimes/dflt/objectstores/nosql/file/server/FileServerTest.java?rev=1125813&r1=1125812&r2=1125813&view=diff
==============================================================================
--- incubator/isis/trunk/runtimes/dflt/objectstores/nosql/src/test/java/org/apache/isis/runtimes/dflt/objectstores/nosql/file/server/FileServerTest.java
(original)
+++ incubator/isis/trunk/runtimes/dflt/objectstores/nosql/src/test/java/org/apache/isis/runtimes/dflt/objectstores/nosql/file/server/FileServerTest.java
Sat May 21 20:48:33 2011
@@ -51,7 +51,7 @@ public class FileServerTest {
 
     @BeforeClass
     public static void setUp() throws Exception {
-        Util.setDirectory("target/test", "services", "logs");
+        Util.setDirectory("target/test", "services", "logs", "archive");
         Util.ensureDirectoryExists();
         new File("target/test/type").mkdir();
     }

Modified: incubator/isis/trunk/runtimes/dflt/objectstores/nosql/src/test/java/org/apache/isis/runtimes/dflt/objectstores/nosql/file/server/LogWriterTest.java
URL: http://svn.apache.org/viewvc/incubator/isis/trunk/runtimes/dflt/objectstores/nosql/src/test/java/org/apache/isis/runtimes/dflt/objectstores/nosql/file/server/LogWriterTest.java?rev=1125813&r1=1125812&r2=1125813&view=diff
==============================================================================
--- incubator/isis/trunk/runtimes/dflt/objectstores/nosql/src/test/java/org/apache/isis/runtimes/dflt/objectstores/nosql/file/server/LogWriterTest.java
(original)
+++ incubator/isis/trunk/runtimes/dflt/objectstores/nosql/src/test/java/org/apache/isis/runtimes/dflt/objectstores/nosql/file/server/LogWriterTest.java
Sat May 21 20:48:33 2011
@@ -43,7 +43,7 @@ public class LogWriterTest {
 
     @BeforeClass
     public static void directory() {
-        Util.setDirectory("target/test", "services", "logs");
+        Util.setDirectory("target/test", "services", "logs", "archive");
         Util.ensureDirectoryExists();
     }
 



Mime
View raw message