incubator-accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e..@apache.org
Subject svn commit: r1241115 - in /incubator/accumulo/trunk: ./ src/core/ src/server/ src/server/src/main/java/org/apache/accumulo/server/gc/ src/server/src/test/java/org/apache/accumulo/server/gc/
Date Mon, 06 Feb 2012 19:06:33 GMT
Author: ecn
Date: Mon Feb  6 19:06:32 2012
New Revision: 1241115

URL: http://svn.apache.org/viewvc?rev=1241115&view=rev
Log:
ACCUMULO-373 merge to trunk

Added:
    incubator/accumulo/trunk/src/server/src/test/java/org/apache/accumulo/server/gc/
      - copied from r1241113, incubator/accumulo/branches/1.4/src/server/src/test/java/org/apache/accumulo/server/gc/
    incubator/accumulo/trunk/src/server/src/test/java/org/apache/accumulo/server/gc/TestConfirmDeletes.java
      - copied unchanged from r1241113, incubator/accumulo/branches/1.4/src/server/src/test/java/org/apache/accumulo/server/gc/TestConfirmDeletes.java
Modified:
    incubator/accumulo/trunk/   (props changed)
    incubator/accumulo/trunk/src/core/   (props changed)
    incubator/accumulo/trunk/src/server/   (props changed)
    incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java

Propchange: incubator/accumulo/trunk/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Feb  6 19:06:32 2012
@@ -1,3 +1,3 @@
 /incubator/accumulo/branches/1.3:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006,1227215,1227231,1227611,1228195,1230180,1230736,1231043,1236873
 /incubator/accumulo/branches/1.3.5rc:1209938
-/incubator/accumulo/branches/1.4:1201902-1240351
+/incubator/accumulo/branches/1.4:1201902-1241113

Propchange: incubator/accumulo/trunk/src/core/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Feb  6 19:06:32 2012
@@ -1,3 +1,3 @@
-/incubator/accumulo/branches/1.3.5rc/src/core:1209938
 /incubator/accumulo/branches/1.3/src/core:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006,1227215
-/incubator/accumulo/branches/1.4/src/core:1201902-1240351
+/incubator/accumulo/branches/1.3.5rc/src/core:1209938
+/incubator/accumulo/branches/1.4/src/core:1201902-1241113

Propchange: incubator/accumulo/trunk/src/server/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Feb  6 19:06:32 2012
@@ -1,3 +1,3 @@
-/incubator/accumulo/branches/1.3.5rc/src/server:1209938
 /incubator/accumulo/branches/1.3/src/server:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006,1227215,1227231,1227611
-/incubator/accumulo/branches/1.4/src/server:1201902-1240351
+/incubator/accumulo/branches/1.3.5rc/src/server:1209938
+/incubator/accumulo/branches/1.4/src/server:1201902-1241113

Modified: incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java?rev=1241115&r1=1241114&r2=1241115&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
(original)
+++ incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
Mon Feb  6 19:06:32 2012
@@ -33,14 +33,17 @@ import java.util.concurrent.TimeUnit;
 
 import org.apache.accumulo.core.Constants;
 import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
 import org.apache.accumulo.core.client.BatchWriter;
 import org.apache.accumulo.core.client.Connector;
 import org.apache.accumulo.core.client.Instance;
 import org.apache.accumulo.core.client.IsolatedScanner;
 import org.apache.accumulo.core.client.MutationsRejectedException;
 import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
 import org.apache.accumulo.core.client.impl.ScannerImpl;
 import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
 import org.apache.accumulo.core.conf.Property;
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.KeyExtent;
@@ -50,16 +53,16 @@ import org.apache.accumulo.core.data.Val
 import org.apache.accumulo.core.file.FileOperations;
 import org.apache.accumulo.core.file.FileUtil;
 import org.apache.accumulo.core.gc.thrift.GCMonitorService;
+import org.apache.accumulo.core.gc.thrift.GCMonitorService.Iface;
 import org.apache.accumulo.core.gc.thrift.GCStatus;
 import org.apache.accumulo.core.gc.thrift.GcCycleStats;
-import org.apache.accumulo.core.gc.thrift.GCMonitorService.Iface;
 import org.apache.accumulo.core.master.state.tables.TableState;
 import org.apache.accumulo.core.security.thrift.AuthInfo;
 import org.apache.accumulo.core.util.CachedConfiguration;
 import org.apache.accumulo.core.util.ColumnFQ;
 import org.apache.accumulo.core.util.ServerServices;
-import org.apache.accumulo.core.util.UtilWaitThread;
 import org.apache.accumulo.core.util.ServerServices.Service;
+import org.apache.accumulo.core.util.UtilWaitThread;
 import org.apache.accumulo.core.zookeeper.ZooUtil;
 import org.apache.accumulo.server.Accumulo;
 import org.apache.accumulo.server.ServerConstants;
@@ -122,6 +125,17 @@ public class SimpleGarbageCollector impl
   public static void main(String[] args) throws UnknownHostException, IOException {
     Accumulo.init("gc");
     SimpleGarbageCollector gc = new SimpleGarbageCollector(args);
+    
+    FileSystem fs;
+    try {
+      fs = TraceFileSystem.wrap(FileUtil.getFileSystem(CachedConfiguration.getInstance(),
ServerConfiguration.getSiteConfiguration()));
+    } catch (IOException e) {
+      String str = "Can't get default file system";
+      log.fatal(str, e);
+      throw new IllegalStateException(str, e);
+    }
+    gc.init(fs, HdfsZooInstance.getInstance(), SecurityConstants.getSystemCredentials(),
ServerConfiguration.getSystemConfiguration());
+    Accumulo.enableTracing(gc.address, "gc");
     gc.run();
   }
   
@@ -138,8 +152,6 @@ public class SimpleGarbageCollector impl
     opts.addOption(optAddress);
     
     try {
-      fs = TraceFileSystem.wrap(FileUtil.getFileSystem(CachedConfiguration.getInstance(),
ServerConfiguration.getSiteConfiguration()));
-      ;
       commandLine = new BasicParser().parse(opts, args);
       if (commandLine.getArgs().length != 0)
         throw new ParseException("Extraneous arguments");
@@ -152,18 +164,17 @@ public class SimpleGarbageCollector impl
       String str = "Can't parse the command line options";
       log.fatal(str, e);
       throw new IllegalArgumentException(str, e);
-    } catch (IOException e) {
-      String str = "Can't get default file system";
-      log.fatal(str, e);
-      throw new IllegalStateException(str, e);
     }
-    
-    instance = HdfsZooInstance.getInstance();
-    credentials = SecurityConstants.getSystemCredentials();
-    
-    gcStartDelay = ServerConfiguration.getSystemConfiguration().getTimeInMillis(Property.GC_CYCLE_START);
-    long gcDelay = ServerConfiguration.getSystemConfiguration().getTimeInMillis(Property.GC_CYCLE_DELAY);
-    numDeleteThreads = ServerConfiguration.getSystemConfiguration().getCount(Property.GC_DELETE_THREADS);
+  }
+  
+  public void init(FileSystem fs, Instance instance, AuthInfo credentials, AccumuloConfiguration
conf) {
+    this.fs = fs;
+    this.instance = instance;
+    this.credentials = credentials;
+    
+    gcStartDelay = conf.getTimeInMillis(Property.GC_CYCLE_START);
+    long gcDelay = conf.getTimeInMillis(Property.GC_CYCLE_DELAY);
+    numDeleteThreads = conf.getCount(Property.GC_DELETE_THREADS);
     log.info("start delay: " + (offline ? 0 + " sec (offline)" : gcStartDelay + " milliseconds"));
     log.info("time delay: " + gcDelay + " milliseconds");
     log.info("safemode: " + safemode);
@@ -171,7 +182,6 @@ public class SimpleGarbageCollector impl
     log.info("verbose: " + verbose);
     log.info("memory threshold: " + CANDIDATE_MEMORY_PERCENTAGE + " of " + Runtime.getRuntime().maxMemory()
+ " bytes");
     log.info("delete threads: " + numDeleteThreads);
-    Accumulo.enableTracing(address, "gc");
   }
   
   private void run() {
@@ -419,7 +429,7 @@ public class SimpleGarbageCollector impl
    * This method removes candidates from the candidate list under two conditions: 1. They
are in the same folder as a bulk processing file, if that option is
    * selected 2. They are still in use in the file column family in the METADATA table
    */
-  private void confirmDeletes(SortedSet<String> candidates) throws AccumuloException
{
+  public void confirmDeletes(SortedSet<String> candidates) throws AccumuloException
{
     
     Scanner scanner;
     if (offline) {
@@ -429,7 +439,13 @@ public class SimpleGarbageCollector impl
         throw new IllegalStateException("Unable to create offline metadata scanner", e);
       }
     } else {
-      scanner = new IsolatedScanner(new ScannerImpl(instance, credentials, Constants.METADATA_TABLE_ID,
Constants.NO_AUTHS));
+      try {
+        scanner = new IsolatedScanner(instance.getConnector(credentials).createScanner(Constants.METADATA_TABLE_NAME,
Constants.NO_AUTHS));
+      } catch (AccumuloSecurityException ex) {
+        throw new AccumuloException(ex);
+      } catch (TableNotFoundException ex) {
+        throw new AccumuloException(ex);
+      }
     }
     
     // skip candidates that are in a bulk processing folder
@@ -469,7 +485,7 @@ public class SimpleGarbageCollector impl
     scanner.fetchColumnFamily(Constants.METADATA_SCANFILE_COLUMN_FAMILY);
     ColumnFQ.fetch(scanner, Constants.METADATA_DIRECTORY_COLUMN);
     
-    TabletIterator tabletIterator = new TabletIterator(scanner, Constants.METADATA_KEYSPACE,
false, false);
+    TabletIterator tabletIterator = new TabletIterator(scanner, Constants.METADATA_KEYSPACE,
false, true);
     
     while (tabletIterator.hasNext()) {
       Map<Key,Value> tabletKeyValues = tabletIterator.next();
@@ -486,7 +502,6 @@ public class SimpleGarbageCollector impl
             String table = new String(KeyExtent.tableOfMetadataRow(entry.getKey().getRow()));
             delete = "/" + table + cf;
           }
-          
           // WARNING: This line is EXTREMELY IMPORTANT.
           // You MUST REMOVE candidates that are still in use
           if (candidates.remove(delete))



Mime
View raw message