incubator-accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e..@apache.org
Subject svn commit: r1241112 - in /incubator/accumulo/branches/1.4/src/server/src: main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java test/java/org/apache/accumulo/server/gc/ test/java/org/apache/accumulo/server/gc/TestConfirmDeletes.java
Date Mon, 06 Feb 2012 19:03:35 GMT
Author: ecn
Date: Mon Feb  6 19:03:35 2012
New Revision: 1241112

URL: http://svn.apache.org/viewvc?rev=1241112&view=rev
Log:
ACCUMULO-373 fix delete-directory confirmation:
 * refactor gc to allow running confirmDeletes() under a mock instance
 * add unit tests to check directory and file references
 * fix the flag used to to find directory entries on the gc's use of the TabletScanner()
 

Added:
    incubator/accumulo/branches/1.4/src/server/src/test/java/org/apache/accumulo/server/gc/
    incubator/accumulo/branches/1.4/src/server/src/test/java/org/apache/accumulo/server/gc/TestConfirmDeletes.java
  (with props)
Modified:
    incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java

Modified: incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java?rev=1241112&r1=1241111&r2=1241112&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
(original)
+++ incubator/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
Mon Feb  6 19:03:35 2012
@@ -33,14 +33,17 @@ import java.util.concurrent.TimeUnit;
 
 import org.apache.accumulo.core.Constants;
 import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
 import org.apache.accumulo.core.client.BatchWriter;
 import org.apache.accumulo.core.client.Connector;
 import org.apache.accumulo.core.client.Instance;
 import org.apache.accumulo.core.client.IsolatedScanner;
 import org.apache.accumulo.core.client.MutationsRejectedException;
 import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
 import org.apache.accumulo.core.client.impl.ScannerImpl;
 import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
 import org.apache.accumulo.core.conf.Property;
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.KeyExtent;
@@ -50,16 +53,16 @@ import org.apache.accumulo.core.data.Val
 import org.apache.accumulo.core.file.FileOperations;
 import org.apache.accumulo.core.file.FileUtil;
 import org.apache.accumulo.core.gc.thrift.GCMonitorService;
+import org.apache.accumulo.core.gc.thrift.GCMonitorService.Iface;
 import org.apache.accumulo.core.gc.thrift.GCStatus;
 import org.apache.accumulo.core.gc.thrift.GcCycleStats;
-import org.apache.accumulo.core.gc.thrift.GCMonitorService.Iface;
 import org.apache.accumulo.core.master.state.tables.TableState;
 import org.apache.accumulo.core.security.thrift.AuthInfo;
 import org.apache.accumulo.core.util.CachedConfiguration;
 import org.apache.accumulo.core.util.ColumnFQ;
 import org.apache.accumulo.core.util.ServerServices;
-import org.apache.accumulo.core.util.UtilWaitThread;
 import org.apache.accumulo.core.util.ServerServices.Service;
+import org.apache.accumulo.core.util.UtilWaitThread;
 import org.apache.accumulo.core.zookeeper.ZooUtil;
 import org.apache.accumulo.server.Accumulo;
 import org.apache.accumulo.server.ServerConstants;
@@ -122,6 +125,17 @@ public class SimpleGarbageCollector impl
   public static void main(String[] args) throws UnknownHostException, IOException {
     Accumulo.init("gc");
     SimpleGarbageCollector gc = new SimpleGarbageCollector(args);
+    
+    FileSystem fs;
+    try {
+      fs = TraceFileSystem.wrap(FileUtil.getFileSystem(CachedConfiguration.getInstance(),
ServerConfiguration.getSiteConfiguration()));
+    } catch (IOException e) {
+      String str = "Can't get default file system";
+      log.fatal(str, e);
+      throw new IllegalStateException(str, e);
+    }
+    gc.init(fs, HdfsZooInstance.getInstance(), SecurityConstants.getSystemCredentials(),
ServerConfiguration.getSystemConfiguration());
+    Accumulo.enableTracing(gc.address, "gc");
     gc.run();
   }
   
@@ -138,8 +152,6 @@ public class SimpleGarbageCollector impl
     opts.addOption(optAddress);
     
     try {
-      fs = TraceFileSystem.wrap(FileUtil.getFileSystem(CachedConfiguration.getInstance(),
ServerConfiguration.getSiteConfiguration()));
-      ;
       commandLine = new BasicParser().parse(opts, args);
       if (commandLine.getArgs().length != 0)
         throw new ParseException("Extraneous arguments");
@@ -152,18 +164,17 @@ public class SimpleGarbageCollector impl
       String str = "Can't parse the command line options";
       log.fatal(str, e);
       throw new IllegalArgumentException(str, e);
-    } catch (IOException e) {
-      String str = "Can't get default file system";
-      log.fatal(str, e);
-      throw new IllegalStateException(str, e);
     }
-    
-    instance = HdfsZooInstance.getInstance();
-    credentials = SecurityConstants.getSystemCredentials();
-    
-    gcStartDelay = ServerConfiguration.getSystemConfiguration().getTimeInMillis(Property.GC_CYCLE_START);
-    long gcDelay = ServerConfiguration.getSystemConfiguration().getTimeInMillis(Property.GC_CYCLE_DELAY);
-    numDeleteThreads = ServerConfiguration.getSystemConfiguration().getCount(Property.GC_DELETE_THREADS);
+  }
+  
+  public void init(FileSystem fs, Instance instance, AuthInfo credentials, AccumuloConfiguration
conf) {
+    this.fs = fs;
+    this.instance = instance;
+    this.credentials = credentials;
+    
+    gcStartDelay = conf.getTimeInMillis(Property.GC_CYCLE_START);
+    long gcDelay = conf.getTimeInMillis(Property.GC_CYCLE_DELAY);
+    numDeleteThreads = conf.getCount(Property.GC_DELETE_THREADS);
     log.info("start delay: " + (offline ? 0 + " sec (offline)" : gcStartDelay + " milliseconds"));
     log.info("time delay: " + gcDelay + " milliseconds");
     log.info("safemode: " + safemode);
@@ -171,7 +182,6 @@ public class SimpleGarbageCollector impl
     log.info("verbose: " + verbose);
     log.info("memory threshold: " + CANDIDATE_MEMORY_PERCENTAGE + " of " + Runtime.getRuntime().maxMemory()
+ " bytes");
     log.info("delete threads: " + numDeleteThreads);
-    Accumulo.enableTracing(address, "gc");
   }
   
   private void run() {
@@ -419,7 +429,7 @@ public class SimpleGarbageCollector impl
    * This method removes candidates from the candidate list under two conditions: 1. They
are in the same folder as a bulk processing file, if that option is
    * selected 2. They are still in use in the file column family in the METADATA table
    */
-  private void confirmDeletes(SortedSet<String> candidates) throws AccumuloException
{
+  public void confirmDeletes(SortedSet<String> candidates) throws AccumuloException
{
     
     Scanner scanner;
     if (offline) {
@@ -429,7 +439,13 @@ public class SimpleGarbageCollector impl
         throw new IllegalStateException("Unable to create offline metadata scanner", e);
       }
     } else {
-      scanner = new IsolatedScanner(new ScannerImpl(instance, credentials, Constants.METADATA_TABLE_ID,
Constants.NO_AUTHS));
+      try {
+        scanner = new IsolatedScanner(instance.getConnector(credentials).createScanner(Constants.METADATA_TABLE_NAME,
Constants.NO_AUTHS));
+      } catch (AccumuloSecurityException ex) {
+        throw new AccumuloException(ex);
+      } catch (TableNotFoundException ex) {
+        throw new AccumuloException(ex);
+      }
     }
     
     // skip candidates that are in a bulk processing folder
@@ -469,7 +485,7 @@ public class SimpleGarbageCollector impl
     scanner.fetchColumnFamily(Constants.METADATA_SCANFILE_COLUMN_FAMILY);
     ColumnFQ.fetch(scanner, Constants.METADATA_DIRECTORY_COLUMN);
     
-    TabletIterator tabletIterator = new TabletIterator(scanner, Constants.METADATA_KEYSPACE,
false, false);
+    TabletIterator tabletIterator = new TabletIterator(scanner, Constants.METADATA_KEYSPACE,
false, true);
     
     while (tabletIterator.hasNext()) {
       Map<Key,Value> tabletKeyValues = tabletIterator.next();
@@ -486,7 +502,6 @@ public class SimpleGarbageCollector impl
             String table = new String(KeyExtent.tableOfMetadataRow(entry.getKey().getRow()));
             delete = "/" + table + cf;
           }
-          
           // WARNING: This line is EXTREMELY IMPORTANT.
           // You MUST REMOVE candidates that are still in use
           if (candidates.remove(delete))

Added: incubator/accumulo/branches/1.4/src/server/src/test/java/org/apache/accumulo/server/gc/TestConfirmDeletes.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/server/src/test/java/org/apache/accumulo/server/gc/TestConfirmDeletes.java?rev=1241112&view=auto
==============================================================================
--- incubator/accumulo/branches/1.4/src/server/src/test/java/org/apache/accumulo/server/gc/TestConfirmDeletes.java
(added)
+++ incubator/accumulo/branches/1.4/src/server/src/test/java/org/apache/accumulo/server/gc/TestConfirmDeletes.java
Mon Feb  6 19:03:35 2012
@@ -0,0 +1,111 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.gc;
+
+import java.nio.ByteBuffer;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import junit.framework.Assert;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.DefaultConfiguration;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.thrift.AuthInfo;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+
+/**
+ * 
+ */
+public class TestConfirmDeletes {
+  
+  AuthInfo auth = new AuthInfo("root", ByteBuffer.wrap("secret".getBytes()), "instance");
+
+  SortedSet<String> singletonSet(String s) {
+    SortedSet<String> result = new TreeSet<String>();
+    result.add(s);
+    return result;
+  }
+
+  @Test
+  public void test() throws Exception {
+    
+    // have a directory reference
+    String metadata[] = {"1636< last:3353986642a66eb 192.168.117.9:9997", "1636< srv:dir
/default_tablet", "1636< srv:flush 2",
+        "1636< srv:lock tservers/192.168.117.9:9997/zlock-0000000000$3353986642a66eb",
"1636< srv:time M1328505870023", "1636< ~tab:~pr \0",};
+    
+    SortedSet<String> candidates = singletonSet("/1636/default_tablet");
+    test1(metadata, candidates);
+    Assert.assertEquals(0, candidates.size());
+    
+    // have no file reference
+    candidates = singletonSet("/1636/default_tablet/someFile");
+    test1(metadata, candidates);
+    Assert.assertEquals(1, candidates.size());
+    
+    // have a file reference
+    metadata = new String[] {"1636< file:/default_tablet/someFile 10,100", "1636< last:3353986642a66eb
192.168.117.9:9997", "1636< srv:dir /default_tablet",
+        "1636< srv:flush 2", "1636< srv:lock tservers/192.168.117.9:9997/zlock-0000000000$3353986642a66eb",
"1636< srv:time M1328505870023",
+        "1636< ~tab:~pr \0",};
+    test1(metadata, candidates);
+    Assert.assertEquals(0, candidates.size());
+    
+    // have an indirect file reference
+    candidates = singletonSet("/1636/default_tablet/someFile");
+    metadata = new String[] {"1636< file:../default_tablet/someFile 10,100", "1636<
last:3353986642a66eb 192.168.117.9:9997", "1636< srv:dir /default_tablet",
+        "1636< srv:flush 2", "1636< srv:lock tservers/192.168.117.9:9997/zlock-0000000000$3353986642a66eb",
"1636< srv:time M1328505870023",
+        "1636< ~tab:~pr \0",};
+    test1(metadata, candidates);
+    Assert.assertEquals(0, candidates.size());
+
+  }
+  
+  private void test1(String[] metadata, SortedSet<String> candidates) throws Exception
{
+    Instance instance = new MockInstance("mockabyebaby");
+    FileSystem fs = FileSystem.getLocal(CachedConfiguration.getInstance());
+    AccumuloConfiguration aconf = DefaultConfiguration.getInstance();
+    
+    load(instance, metadata);
+
+    SimpleGarbageCollector gc = new SimpleGarbageCollector(new String[] {});
+    gc.init(fs, instance, auth, aconf);
+    gc.confirmDeletes(candidates);
+  }
+  
+  private void load(Instance instance, String[] metadata) throws Exception {
+    Connector conn = instance.getConnector(auth);
+    BatchWriter bw = conn.createBatchWriter(Constants.METADATA_TABLE_NAME, 1000, 1000, 1);
+    for (String line : metadata) {
+      String[] parts = line.split(" ");
+      String[] columnParts = parts[1].split(":");
+      Mutation m = new Mutation(parts[0]);
+      m.put(new Text(columnParts[0]), new Text(columnParts[1]), new Value(parts[2].getBytes()));
+      bw.addMutation(m);
+    }
+    bw.close();
+  }
+}

Propchange: incubator/accumulo/branches/1.4/src/server/src/test/java/org/apache/accumulo/server/gc/TestConfirmDeletes.java
------------------------------------------------------------------------------
    svn:mime-type = text/plain



Mime
View raw message