Return-Path: X-Original-To: apmail-incubator-accumulo-commits-archive@minotaur.apache.org Delivered-To: apmail-incubator-accumulo-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 3C08F98B5 for ; Mon, 6 Feb 2012 19:06:55 +0000 (UTC) Received: (qmail 40889 invoked by uid 500); 6 Feb 2012 19:06:55 -0000 Delivered-To: apmail-incubator-accumulo-commits-archive@incubator.apache.org Received: (qmail 40856 invoked by uid 500); 6 Feb 2012 19:06:54 -0000 Mailing-List: contact accumulo-commits-help@incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: accumulo-dev@incubator.apache.org Delivered-To: mailing list accumulo-commits@incubator.apache.org Received: (qmail 40849 invoked by uid 99); 6 Feb 2012 19:06:54 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 06 Feb 2012 19:06:54 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 06 Feb 2012 19:06:53 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 5BEDD23888D2; Mon, 6 Feb 2012 19:06:33 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1241115 - in /incubator/accumulo/trunk: ./ src/core/ src/server/ src/server/src/main/java/org/apache/accumulo/server/gc/ src/server/src/test/java/org/apache/accumulo/server/gc/ Date: Mon, 06 Feb 2012 19:06:33 -0000 To: accumulo-commits@incubator.apache.org From: ecn@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120206190633.5BEDD23888D2@eris.apache.org> Author: ecn Date: Mon Feb 6 19:06:32 2012 New Revision: 1241115 URL: http://svn.apache.org/viewvc?rev=1241115&view=rev Log: ACCUMULO-373 merge to trunk Added: incubator/accumulo/trunk/src/server/src/test/java/org/apache/accumulo/server/gc/ - copied from r1241113, incubator/accumulo/branches/1.4/src/server/src/test/java/org/apache/accumulo/server/gc/ incubator/accumulo/trunk/src/server/src/test/java/org/apache/accumulo/server/gc/TestConfirmDeletes.java - copied unchanged from r1241113, incubator/accumulo/branches/1.4/src/server/src/test/java/org/apache/accumulo/server/gc/TestConfirmDeletes.java Modified: incubator/accumulo/trunk/ (props changed) incubator/accumulo/trunk/src/core/ (props changed) incubator/accumulo/trunk/src/server/ (props changed) incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java Propchange: incubator/accumulo/trunk/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Mon Feb 6 19:06:32 2012 @@ -1,3 +1,3 @@ /incubator/accumulo/branches/1.3:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006,1227215,1227231,1227611,1228195,1230180,1230736,1231043,1236873 /incubator/accumulo/branches/1.3.5rc:1209938 -/incubator/accumulo/branches/1.4:1201902-1240351 +/incubator/accumulo/branches/1.4:1201902-1241113 Propchange: incubator/accumulo/trunk/src/core/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Mon Feb 6 19:06:32 2012 @@ -1,3 +1,3 @@ -/incubator/accumulo/branches/1.3.5rc/src/core:1209938 /incubator/accumulo/branches/1.3/src/core:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006,1227215 -/incubator/accumulo/branches/1.4/src/core:1201902-1240351 +/incubator/accumulo/branches/1.3.5rc/src/core:1209938 +/incubator/accumulo/branches/1.4/src/core:1201902-1241113 Propchange: incubator/accumulo/trunk/src/server/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Mon Feb 6 19:06:32 2012 @@ -1,3 +1,3 @@ -/incubator/accumulo/branches/1.3.5rc/src/server:1209938 /incubator/accumulo/branches/1.3/src/server:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006,1227215,1227231,1227611 -/incubator/accumulo/branches/1.4/src/server:1201902-1240351 +/incubator/accumulo/branches/1.3.5rc/src/server:1209938 +/incubator/accumulo/branches/1.4/src/server:1201902-1241113 Modified: incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java?rev=1241115&r1=1241114&r2=1241115&view=diff ============================================================================== --- incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java (original) +++ incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java Mon Feb 6 19:06:32 2012 @@ -33,14 +33,17 @@ import java.util.concurrent.TimeUnit; import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.BatchWriter; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.Instance; import org.apache.accumulo.core.client.IsolatedScanner; import org.apache.accumulo.core.client.MutationsRejectedException; import org.apache.accumulo.core.client.Scanner; +import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.client.impl.ScannerImpl; import org.apache.accumulo.core.client.impl.Tables; +import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.KeyExtent; @@ -50,16 +53,16 @@ import org.apache.accumulo.core.data.Val import org.apache.accumulo.core.file.FileOperations; import org.apache.accumulo.core.file.FileUtil; import org.apache.accumulo.core.gc.thrift.GCMonitorService; +import org.apache.accumulo.core.gc.thrift.GCMonitorService.Iface; import org.apache.accumulo.core.gc.thrift.GCStatus; import org.apache.accumulo.core.gc.thrift.GcCycleStats; -import org.apache.accumulo.core.gc.thrift.GCMonitorService.Iface; import org.apache.accumulo.core.master.state.tables.TableState; import org.apache.accumulo.core.security.thrift.AuthInfo; import org.apache.accumulo.core.util.CachedConfiguration; import org.apache.accumulo.core.util.ColumnFQ; import org.apache.accumulo.core.util.ServerServices; -import org.apache.accumulo.core.util.UtilWaitThread; import org.apache.accumulo.core.util.ServerServices.Service; +import org.apache.accumulo.core.util.UtilWaitThread; import org.apache.accumulo.core.zookeeper.ZooUtil; import org.apache.accumulo.server.Accumulo; import org.apache.accumulo.server.ServerConstants; @@ -122,6 +125,17 @@ public class SimpleGarbageCollector impl public static void main(String[] args) throws UnknownHostException, IOException { Accumulo.init("gc"); SimpleGarbageCollector gc = new SimpleGarbageCollector(args); + + FileSystem fs; + try { + fs = TraceFileSystem.wrap(FileUtil.getFileSystem(CachedConfiguration.getInstance(), ServerConfiguration.getSiteConfiguration())); + } catch (IOException e) { + String str = "Can't get default file system"; + log.fatal(str, e); + throw new IllegalStateException(str, e); + } + gc.init(fs, HdfsZooInstance.getInstance(), SecurityConstants.getSystemCredentials(), ServerConfiguration.getSystemConfiguration()); + Accumulo.enableTracing(gc.address, "gc"); gc.run(); } @@ -138,8 +152,6 @@ public class SimpleGarbageCollector impl opts.addOption(optAddress); try { - fs = TraceFileSystem.wrap(FileUtil.getFileSystem(CachedConfiguration.getInstance(), ServerConfiguration.getSiteConfiguration())); - ; commandLine = new BasicParser().parse(opts, args); if (commandLine.getArgs().length != 0) throw new ParseException("Extraneous arguments"); @@ -152,18 +164,17 @@ public class SimpleGarbageCollector impl String str = "Can't parse the command line options"; log.fatal(str, e); throw new IllegalArgumentException(str, e); - } catch (IOException e) { - String str = "Can't get default file system"; - log.fatal(str, e); - throw new IllegalStateException(str, e); } - - instance = HdfsZooInstance.getInstance(); - credentials = SecurityConstants.getSystemCredentials(); - - gcStartDelay = ServerConfiguration.getSystemConfiguration().getTimeInMillis(Property.GC_CYCLE_START); - long gcDelay = ServerConfiguration.getSystemConfiguration().getTimeInMillis(Property.GC_CYCLE_DELAY); - numDeleteThreads = ServerConfiguration.getSystemConfiguration().getCount(Property.GC_DELETE_THREADS); + } + + public void init(FileSystem fs, Instance instance, AuthInfo credentials, AccumuloConfiguration conf) { + this.fs = fs; + this.instance = instance; + this.credentials = credentials; + + gcStartDelay = conf.getTimeInMillis(Property.GC_CYCLE_START); + long gcDelay = conf.getTimeInMillis(Property.GC_CYCLE_DELAY); + numDeleteThreads = conf.getCount(Property.GC_DELETE_THREADS); log.info("start delay: " + (offline ? 0 + " sec (offline)" : gcStartDelay + " milliseconds")); log.info("time delay: " + gcDelay + " milliseconds"); log.info("safemode: " + safemode); @@ -171,7 +182,6 @@ public class SimpleGarbageCollector impl log.info("verbose: " + verbose); log.info("memory threshold: " + CANDIDATE_MEMORY_PERCENTAGE + " of " + Runtime.getRuntime().maxMemory() + " bytes"); log.info("delete threads: " + numDeleteThreads); - Accumulo.enableTracing(address, "gc"); } private void run() { @@ -419,7 +429,7 @@ public class SimpleGarbageCollector impl * This method removes candidates from the candidate list under two conditions: 1. They are in the same folder as a bulk processing file, if that option is * selected 2. They are still in use in the file column family in the METADATA table */ - private void confirmDeletes(SortedSet candidates) throws AccumuloException { + public void confirmDeletes(SortedSet candidates) throws AccumuloException { Scanner scanner; if (offline) { @@ -429,7 +439,13 @@ public class SimpleGarbageCollector impl throw new IllegalStateException("Unable to create offline metadata scanner", e); } } else { - scanner = new IsolatedScanner(new ScannerImpl(instance, credentials, Constants.METADATA_TABLE_ID, Constants.NO_AUTHS)); + try { + scanner = new IsolatedScanner(instance.getConnector(credentials).createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS)); + } catch (AccumuloSecurityException ex) { + throw new AccumuloException(ex); + } catch (TableNotFoundException ex) { + throw new AccumuloException(ex); + } } // skip candidates that are in a bulk processing folder @@ -469,7 +485,7 @@ public class SimpleGarbageCollector impl scanner.fetchColumnFamily(Constants.METADATA_SCANFILE_COLUMN_FAMILY); ColumnFQ.fetch(scanner, Constants.METADATA_DIRECTORY_COLUMN); - TabletIterator tabletIterator = new TabletIterator(scanner, Constants.METADATA_KEYSPACE, false, false); + TabletIterator tabletIterator = new TabletIterator(scanner, Constants.METADATA_KEYSPACE, false, true); while (tabletIterator.hasNext()) { Map tabletKeyValues = tabletIterator.next(); @@ -486,7 +502,6 @@ public class SimpleGarbageCollector impl String table = new String(KeyExtent.tableOfMetadataRow(entry.getKey().getRow())); delete = "/" + table + cf; } - // WARNING: This line is EXTREMELY IMPORTANT. // You MUST REMOVE candidates that are still in use if (candidates.remove(delete))