accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ctubb...@apache.org
Subject [11/11] accumulo git commit: Merge branch '1.7'
Date Thu, 07 May 2015 18:54:15 GMT
Merge branch '1.7'

This closes #35


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/bc3dc3b8
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/bc3dc3b8
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/bc3dc3b8

Branch: refs/heads/master
Commit: bc3dc3b8de0bbc1f48760a413089b5ef04620c28
Parents: 6acfd41 b932755
Author: Christopher Tubbs <ctubbsii@apache.org>
Authored: Thu May 7 14:48:54 2015 -0400
Committer: Christopher Tubbs <ctubbsii@apache.org>
Committed: Thu May 7 14:48:54 2015 -0400

----------------------------------------------------------------------
 .../core/client/ClientConfiguration.java        |  4 +-
 .../core/client/ClientSideIteratorScanner.java  |  4 +-
 .../core/client/impl/ActiveCompactionImpl.java  |  1 -
 .../core/client/impl/ConditionalWriterImpl.java |  3 +-
 .../core/client/impl/OfflineIterator.java       |  2 +-
 .../core/client/impl/ThriftScanner.java         |  2 +-
 .../core/client/impl/ThriftTransportPool.java   | 13 +++--
 .../core/client/lexicoder/PairLexicoder.java    |  3 +-
 .../lexicoder/impl/AbstractLexicoder.java       |  3 +-
 .../core/client/mapred/AbstractInputFormat.java | 11 ++---
 .../client/mapred/AccumuloOutputFormat.java     |  2 +-
 .../core/client/mapred/InputFormatBase.java     | 20 ++++----
 .../client/mapred/impl/BatchInputSplit.java     |  3 +-
 .../client/mapreduce/AbstractInputFormat.java   | 25 +++++-----
 .../client/mapreduce/AccumuloInputFormat.java   |  2 +-
 .../core/client/mapreduce/InputFormatBase.java  | 21 ++++----
 .../core/client/mapreduce/RangeInputSplit.java  |  2 +-
 .../client/mapreduce/impl/BatchInputSplit.java  | 19 ++++----
 .../core/client/mapreduce/impl/SplitUtils.java  |  8 ++--
 .../core/client/mock/MockScannerBase.java       |  7 +--
 .../client/security/tokens/DelegationToken.java |  3 +-
 .../accumulo/core/compaction/PatternType.java   |  2 +-
 .../accumulo/core/compaction/SizeType.java      |  2 +-
 .../accumulo/core/compaction/StringType.java    |  2 +-
 .../apache/accumulo/core/compaction/Type.java   |  2 +-
 .../accumulo/core/compaction/UIntType.java      |  2 +-
 .../accumulo/core/conf/ConfigurationDocGen.java |  8 ++--
 .../org/apache/accumulo/core/conf/Property.java | 10 ++--
 .../apache/accumulo/core/data/KeyExtent.java    | 15 +++---
 .../org/apache/accumulo/core/data/Mutation.java |  9 ++--
 .../org/apache/accumulo/core/data/TabletId.java |  2 +
 .../accumulo/core/data/impl/TabletIdImpl.java   | 10 ++--
 .../core/file/DispatchingFileFactory.java       |  2 +-
 .../core/file/blockfile/cache/HeapSize.java     |  1 -
 .../file/blockfile/cache/LruBlockCache.java     | 17 +++----
 .../apache/accumulo/core/file/rfile/RFile.java  |  3 +-
 .../core/iterators/AggregatingIterator.java     |  1 -
 .../accumulo/core/iterators/IteratorUtil.java   |  3 +-
 .../accumulo/core/iterators/LongCombiner.java   |  4 +-
 .../core/iterators/SortedKeyValueIterator.java  | 10 ++--
 .../core/iterators/system/MapFileIterator.java  |  1 -
 .../core/iterators/system/VisibilityFilter.java |  1 -
 .../core/iterators/user/BigDecimalCombiner.java |  3 +-
 .../core/iterators/user/RegExFilter.java        |  1 -
 .../iterators/user/RowEncodingIterator.java     |  2 +-
 .../iterators/user/TransformingIterator.java    |  1 -
 .../core/metadata/schema/MetadataSchema.java    |  2 +-
 .../accumulo/core/rpc/SaslConnectionParams.java |  7 ++-
 .../core/rpc/UGIAssumingTransportFactory.java   |  2 +-
 .../CachingHDFSSecretKeyEncryptionStrategy.java | 12 ++---
 .../security/crypto/CryptoModuleFactory.java    |  8 ++--
 .../NonCachingSecretKeyEncryptionStrategy.java  | 14 +++---
 .../org/apache/accumulo/core/util/Help.java     |  2 +-
 .../java/org/apache/accumulo/core/util/Jar.java |  2 +-
 .../accumulo/core/util/UtilWaitThread.java      |  1 -
 .../core/util/format/FormatterFactory.java      |  1 -
 .../apache/accumulo/core/volume/VolumeImpl.java |  1 -
 .../apache/accumulo/core/zookeeper/ZooUtil.java |  1 -
 .../core/client/ClientConfigurationTest.java    |  4 +-
 .../client/lexicoder/BytesLexicoderTest.java    |  2 +-
 .../client/lexicoder/DateLexicoderTest.java     |  2 +-
 .../client/lexicoder/StringLexicoderTest.java   |  2 +-
 .../client/lexicoder/TextLexicoderTest.java     |  2 +-
 .../lexicoder/impl/AbstractLexicoderTest.java   |  7 +--
 .../client/lexicoder/impl/ByteUtilsTest.java    |  4 +-
 .../core/client/mock/MockNamespacesTest.java    |  7 +--
 .../apache/accumulo/core/data/MutationTest.java | 16 +++----
 .../iterators/FirstEntryInRowIteratorTest.java  |  3 +-
 .../iterators/aggregation/NumSummationTest.java |  3 +-
 .../iterators/user/RowDeletingIteratorTest.java |  2 +-
 examples/simple/pom.xml                         |  9 ++++
 .../client/CountingVerifyingReceiver.java       |  2 +-
 .../examples/simple/client/RowOperations.java   |  1 -
 .../simple/mapreduce/TokenFileWordCount.java    |  1 -
 .../examples/simple/reservations/ARS.java       |  1 -
 .../examples/simple/dirlist/CountTest.java      |  1 -
 .../org/apache/accumulo/fate/AdminUtil.java     |  2 +-
 .../org/apache/accumulo/fate/AgeOffStore.java   |  1 -
 .../java/org/apache/accumulo/fate/Fate.java     |  1 -
 .../apache/accumulo/fate/util/AddressUtil.java  |  1 -
 .../accumulo/fate/util/UtilWaitThread.java      |  1 -
 .../zookeeper/DistributedReadWriteLock.java     |  1 -
 .../apache/accumulo/fate/zookeeper/Retry.java   |  1 -
 .../fate/zookeeper/TransactionWatcher.java      |  1 -
 .../accumulo/fate/zookeeper/ZooCache.java       | 13 ++---
 .../accumulo/fate/util/AddressUtilTest.java     |  1 -
 maven-plugin/src/it/plugin-test/pom.xml         |  7 +++
 pom.xml                                         | 50 ++++++++++++++++++++
 .../org/apache/accumulo/proxy/ProxyServer.java  |  3 +-
 server/base/pom.xml                             |  9 ++++
 .../org/apache/accumulo/server/TabletLevel.java |  4 +-
 .../server/client/ClientServiceHandler.java     |  4 +-
 .../server/master/state/MetaDataStateStore.java |  2 +-
 .../server/master/state/TabletStateStore.java   |  7 +--
 .../master/state/ZooTabletStateStore.java       |  5 +-
 .../server/problems/ProblemReports.java         |  2 +-
 .../AuthenticationTokenKeyManager.java          |  3 +-
 .../ZooAuthenticationKeyDistributor.java        |  1 -
 .../server/util/MasterMetadataUtil.java         |  6 +--
 .../accumulo/server/util/MetadataTableUtil.java |  8 ++--
 .../accumulo/server/util/TabletIterator.java    |  9 ----
 .../iterators/MetadataBulkLoadFilterTest.java   |  2 +-
 .../security/handler/ZKAuthenticatorTest.java   |  2 +-
 server/gc/pom.xml                               | 11 +++++
 .../org/apache/accumulo/gc/GCExecutable.java    |  2 +-
 .../gc/GarbageCollectWriteAheadLogs.java        | 40 ++++++++--------
 .../CloseWriteAheadLogReferences.java           | 18 ++++---
 server/master/pom.xml                           | 11 +++++
 .../accumulo/master/TabletGroupWatcher.java     | 13 ++---
 .../accumulo/master/tableOps/ChooseDir.java     |  2 +-
 .../accumulo/master/tableOps/CleanUp.java       |  9 ++--
 .../master/tableOps/CleanUpBulkImport.java      |  2 +-
 .../accumulo/master/tableOps/CloneInfo.java     |  2 +-
 .../accumulo/master/tableOps/CloneMetadata.java |  2 +-
 .../master/tableOps/CloneZookeeper.java         |  2 +-
 .../master/tableOps/CompactionDriver.java       |  2 +-
 .../master/tableOps/CompleteBulkImport.java     |  2 +-
 .../accumulo/master/tableOps/CreateDir.java     |  2 +-
 .../master/tableOps/CreateImportDir.java        |  2 +-
 .../accumulo/master/tableOps/ExportInfo.java    |  2 +-
 .../master/tableOps/FinishCancelCompaction.java |  2 +-
 .../master/tableOps/FinishCloneTable.java       |  2 +-
 .../master/tableOps/FinishCreateNamespace.java  |  2 +-
 .../master/tableOps/FinishCreateTable.java      |  2 +-
 .../master/tableOps/FinishImportTable.java      |  2 +-
 .../tableOps/ImportPopulateZookeeper.java       |  2 +-
 .../master/tableOps/ImportSetupPermissions.java |  2 +-
 .../master/tableOps/ImportedTableInfo.java      |  2 +-
 .../accumulo/master/tableOps/NamespaceInfo.java |  2 +-
 .../master/tableOps/PopulateMetadata.java       |  2 +-
 .../master/tableOps/PopulateZookeeper.java      |  2 +-
 .../PopulateZookeeperWithNamespace.java         |  2 +-
 .../accumulo/master/tableOps/TableInfo.java     |  2 +-
 .../master/tableOps/WriteExportFiles.java       |  2 +-
 server/monitor/pom.xml                          |  9 ++++
 .../monitor/servlets/trace/NullScanner.java     |  3 +-
 .../accumulo/monitor/ShowTraceLinkTypeTest.java |  3 +-
 server/native/pom.xml                           |  7 +++
 server/tracer/pom.xml                           | 11 +++++
 .../accumulo/tracer/AsyncSpanReceiver.java      |  4 +-
 .../apache/accumulo/tracer/TraceTableStats.java | 30 +++++-------
 server/tserver/pom.xml                          |  9 ++++
 .../server/GarbageCollectionLogger.java         |  3 +-
 .../accumulo/tserver/MemKeyComparator.java      |  2 +-
 .../tserver/MemKeyConversionIterator.java       |  2 +-
 .../PartialMutationSkippingIterator.java        |  2 +-
 .../tserver/TabletIteratorEnvironment.java      |  2 +-
 .../apache/accumulo/tserver/TabletServer.java   | 27 ++++-------
 .../EverythingCompactionStrategy.java           |  2 +-
 .../apache/accumulo/tserver/log/DfsLogger.java  |  8 ++--
 .../tserver/log/TabletServerLogger.java         |  4 +-
 .../accumulo/tserver/tablet/MinorCompactor.java |  2 +-
 .../accumulo/tserver/tablet/SplitInfo.java      |  7 +--
 .../apache/accumulo/tserver/tablet/Tablet.java  | 44 ++++++-----------
 .../org/apache/accumulo/shell/ShellOptions.java |  1 -
 .../start/classloader/AccumuloClassLoader.java  |  4 +-
 .../accumulo/test/EstimateInMemMapOverhead.java |  1 -
 .../test/InMemoryMapMemoryUsageTest.java        |  2 +-
 .../accumulo/test/IntObjectMemoryUsageTest.java |  2 +-
 .../apache/accumulo/test/MemoryUsageTest.java   |  2 +-
 .../accumulo/test/MutationMemoryUsageTest.java  |  2 +-
 .../accumulo/test/TextMemoryUsageTest.java      |  2 +-
 .../test/continuous/ContinuousBatchWalker.java  | 10 ----
 .../accumulo/test/continuous/HistData.java      |  2 +-
 .../test/continuous/PrintScanTimeHistogram.java |  2 +-
 .../accumulo/test/functional/AuthsIterator.java |  2 +-
 .../accumulo/test/randomwalk/Environment.java   |  2 +-
 .../test/randomwalk/concurrent/Replication.java | 14 +++---
 .../accumulo/harness/AccumuloClusterIT.java     |  3 +-
 .../apache/accumulo/proxy/SimpleProxyBase.java  |  2 +-
 .../test/BalanceWithOfflineTableIT.java         |  8 ++--
 .../accumulo/test/GarbageCollectWALIT.java      |  2 +-
 .../apache/accumulo/test/TableOperationsIT.java |  4 +-
 .../java/org/apache/accumulo/test/VolumeIT.java |  2 +-
 .../test/functional/AccumuloInputFormatIT.java  | 14 +++---
 .../accumulo/test/functional/ConcurrencyIT.java | 17 ++++---
 .../accumulo/test/functional/KerberosIT.java    |  6 +--
 .../test/functional/KerberosProxyIT.java        |  8 ++--
 .../accumulo/test/functional/WALSunnyDayIT.java | 16 +++----
 .../test/performance/RollWALPerformanceIT.java  | 14 ++----
 .../performance/metadata/FastBulkImportIT.java  |  5 +-
 .../test/replication/ReplicationIT.java         |  4 +-
 182 files changed, 529 insertions(+), 498 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/bc3dc3b8/examples/simple/pom.xml
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/bc3dc3b8/pom.xml
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/bc3dc3b8/server/base/pom.xml
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/bc3dc3b8/server/base/src/main/java/org/apache/accumulo/server/util/MasterMetadataUtil.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/MasterMetadataUtil.java
index 0a30314,29745d5..da2948d
--- a/server/base/src/main/java/org/apache/accumulo/server/util/MasterMetadataUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/MasterMetadataUtil.java
@@@ -70,7 -68,7 +70,7 @@@ public class MasterMetadataUtil 
    private static final Logger log = LoggerFactory.getLogger(MasterMetadataUtil.class);
  
    public static void addNewTablet(ClientContext context, KeyExtent extent, String path,
TServerInstance location, Map<FileRef,DataFileValue> datafileSizes,
-       Multimap<Long, FileRef> bulkLoadedFiles, String time, long lastFlushID, long
lastCompactID, ZooLock zooLock) {
 -      Map<FileRef,Long> bulkLoadedFiles, String time, long lastFlushID, long lastCompactID,
ZooLock zooLock) {
++      Multimap<Long,FileRef> bulkLoadedFiles, String time, long lastFlushID, long
lastCompactID, ZooLock zooLock) {
      Mutation m = extent.getPrevRowUpdateMutation();
  
      TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(path.getBytes(UTF_8)));
@@@ -89,9 -87,9 +89,9 @@@
        m.put(DataFileColumnFamily.NAME, entry.getKey().meta(), new Value(entry.getValue().encode()));
      }
  
-     for (Entry<Long, FileRef> entry : bulkLoadedFiles.entries()) {
 -    for (Entry<FileRef,Long> entry : bulkLoadedFiles.entrySet()) {
 -      byte[] tidBytes = Long.toString(entry.getValue()).getBytes();
 -      m.put(TabletsSection.BulkFileColumnFamily.NAME, entry.getKey().meta(), new Value(tidBytes));
++    for (Entry<Long,FileRef> entry : bulkLoadedFiles.entries()) {
 +      byte[] tidBytes = Long.toString(entry.getKey()).getBytes();
 +      m.put(TabletsSection.BulkFileColumnFamily.NAME, entry.getValue().meta(), new Value(tidBytes));
      }
  
      MetadataTableUtil.update(context, zooLock, m, extent);

http://git-wip-us.apache.org/repos/asf/accumulo/blob/bc3dc3b8/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
----------------------------------------------------------------------
diff --cc server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
index 58ca528,f53f6a5..5d5415d
--- a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
@@@ -926,9 -924,9 +926,9 @@@ public class MetadataTableUtil 
      }
    }
  
-   public static Multimap<Long, FileRef> getBulkFilesLoaded(ClientContext context,
KeyExtent extent) throws IOException {
 -  public static Map<FileRef,Long> getBulkFilesLoaded(ClientContext context, KeyExtent
extent) throws IOException {
++  public static Multimap<Long,FileRef> getBulkFilesLoaded(ClientContext context, KeyExtent
extent) throws IOException {
      Text metadataRow = extent.getMetadataEntry();
-     Multimap<Long, FileRef> ret = HashMultimap.create();
 -    Map<FileRef,Long> ret = new HashMap<FileRef,Long>();
++    Multimap<Long,FileRef> ret = HashMultimap.create();
  
      VolumeManager fs = VolumeManagerImpl.get();
      Scanner scanner = new ScannerImpl(context, extent.isMeta() ? RootTable.ID : MetadataTable.ID,
Authorizations.EMPTY);

http://git-wip-us.apache.org/repos/asf/accumulo/blob/bc3dc3b8/server/gc/pom.xml
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/bc3dc3b8/server/master/pom.xml
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/bc3dc3b8/server/monitor/pom.xml
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/bc3dc3b8/server/native/pom.xml
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/bc3dc3b8/server/tracer/pom.xml
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/bc3dc3b8/server/tserver/pom.xml
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/bc3dc3b8/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/SplitInfo.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/SplitInfo.java
index d866d99,f8f2183..f2111c7
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/SplitInfo.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/SplitInfo.java
@@@ -41,9 -39,8 +41,10 @@@ final public class SplitInfo 
    private final long initFlushID;
    private final long initCompactID;
    private final TServerInstance lastLocation;
-   private final Multimap<Long, FileRef> bulkImported;
++  private final Multimap<Long,FileRef> bulkImported;
  
-   SplitInfo(String d, SortedMap<FileRef,DataFileValue> dfv, String time, long initFlushID,
long initCompactID, TServerInstance lastLocation, Multimap<Long, FileRef> bulkImported)
{
 -  SplitInfo(String d, SortedMap<FileRef,DataFileValue> dfv, String time, long initFlushID,
long initCompactID, TServerInstance lastLocation) {
++  SplitInfo(String d, SortedMap<FileRef,DataFileValue> dfv, String time, long initFlushID,
long initCompactID, TServerInstance lastLocation,
++      Multimap<Long,FileRef> bulkImported) {
      this.dir = d;
      this.datafiles = dfv;
      this.time = time;
@@@ -77,8 -73,4 +78,8 @@@
      return lastLocation;
    }
  
-   public Multimap<Long, FileRef> getBulkImported() {
++  public Multimap<Long,FileRef> getBulkImported() {
 +    return bulkImported;
 +  }
 +
  }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/bc3dc3b8/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
----------------------------------------------------------------------
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index 0b2d5e3,17864be..5de3236
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@@ -253,9 -243,6 +253,8 @@@ public class Tablet implements TabletCo
  
    private final ConfigurationObserver configObserver;
  
-   private final Cache<Long, List<FileRef> > bulkImported = CacheBuilder.newBuilder().expireAfterAccess(4,
TimeUnit.HOURS).build();
- 
++  private final Cache<Long,List<FileRef>> bulkImported = CacheBuilder.newBuilder().expireAfterAccess(4,
TimeUnit.HOURS).build();
 +
    private final int logId;
  
    @Override
@@@ -324,8 -311,8 +323,8 @@@
    }
  
    private Tablet(TabletServer tabletServer, Text location, KeyExtent extent, TabletResourceManager
trm, SortedMap<FileRef,DataFileValue> datafiles,
-       String time, long initFlushID, long initCompactID, TServerInstance lastLocation, Multimap<Long,
FileRef> bulkImported) throws IOException {
 -      String time, long initFlushID, long initCompactID, TServerInstance lastLocation) throws
IOException {
 -    this(tabletServer, extent, location, trm, NO_LOG_ENTRIES, datafiles, time, lastLocation,
new HashSet<FileRef>(), initFlushID, initCompactID);
++      String time, long initFlushID, long initCompactID, TServerInstance lastLocation, Multimap<Long,FileRef>
bulkImported) throws IOException {
 +    this(tabletServer, extent, location, trm, NO_LOG_ENTRIES, datafiles, time, lastLocation,
new HashSet<FileRef>(), initFlushID, initCompactID, bulkImported);
    }
  
    private static String lookupTime(AccumuloConfiguration conf, KeyExtent extent, SortedMap<Key,Value>
tabletsKeyValues) {
@@@ -447,45 -445,20 +446,30 @@@
      return null;
    }
  
-   private static Multimap<Long, FileRef> lookupBulkImported(SortedMap<Key,Value>
tabletsKeyValues, VolumeManager fs) {
-     Multimap<Long, FileRef> result = HashMultimap.create();
++  private static Multimap<Long,FileRef> lookupBulkImported(SortedMap<Key,Value>
tabletsKeyValues, VolumeManager fs) {
++    Multimap<Long,FileRef> result = HashMultimap.create();
 +    for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
 +      if (entry.getKey().getColumnFamily().compareTo(BulkFileColumnFamily.NAME) == 0) {
 +        result.put(Long.decode(entry.getValue().toString()), new FileRef(fs, entry.getKey()));
 +      }
 +    }
 +    return result;
 +  }
 +
    public Tablet(TabletServer tabletServer, KeyExtent extent, Text location, TabletResourceManager
trm, SortedMap<Key,Value> tabletsKeyValues)
        throws IOException {
-     this(tabletServer, extent, location, trm,
-         lookupLogEntries(tabletsKeyValues, tabletServer, extent),
-         lookupDatafiles(tabletServer, tabletServer.getFileSystem(), extent, tabletsKeyValues),
-         lookupTime(tabletServer.getConfiguration(), extent, tabletsKeyValues),
-         lookupLastServer(tabletsKeyValues),
-         lookupScanFiles(tabletsKeyValues, tabletServer.getFileSystem()),
-         lookupFlushID(tabletsKeyValues),
-         lookupCompactID(tabletsKeyValues),
-         lookupBulkImported(tabletsKeyValues, tabletServer.getFileSystem()));
 -    this(tabletServer, extent, location, trm, lookupLogEntries(tabletServer, extent, tabletsKeyValues),
lookupDatafiles(tabletServer,
 -        tabletServer.getFileSystem(), extent, tabletsKeyValues), lookupTime(tabletServer.getConfiguration(),
extent, tabletsKeyValues), lookupLastServer(
 -        extent, tabletsKeyValues), lookupScanFiles(extent, tabletsKeyValues, tabletServer.getFileSystem()),
lookupFlushID(extent, tabletsKeyValues),
 -        lookupCompactID(extent, tabletsKeyValues));
++    this(tabletServer, extent, location, trm, lookupLogEntries(tabletsKeyValues, tabletServer,
extent), lookupDatafiles(tabletServer,
++        tabletServer.getFileSystem(), extent, tabletsKeyValues), lookupTime(tabletServer.getConfiguration(),
extent, tabletsKeyValues),
++        lookupLastServer(tabletsKeyValues), lookupScanFiles(tabletsKeyValues, tabletServer.getFileSystem()),
lookupFlushID(tabletsKeyValues),
++        lookupCompactID(tabletsKeyValues), lookupBulkImported(tabletsKeyValues, tabletServer.getFileSystem()));
    }
  
    /**
     * yet another constructor - this one allows us to avoid costly lookups into the Metadata
table if we already know the files we need - as at split time
     */
-   private Tablet(
-       final TabletServer tabletServer,
-       final KeyExtent extent,
-       final Text location,
-       final TabletResourceManager trm,
-       final List<LogEntry> rawLogEntries,
-       final SortedMap<FileRef,DataFileValue> rawDatafiles,
-       String time,
-       final TServerInstance lastLocation,
-       final Set<FileRef> scanFiles,
-       final long initFlushID,
-       final long initCompactID,
-       final Multimap<Long, FileRef> bulkImported) throws IOException {
+   private Tablet(final TabletServer tabletServer, final KeyExtent extent, final Text location,
final TabletResourceManager trm,
+       final List<LogEntry> rawLogEntries, final SortedMap<FileRef,DataFileValue>
rawDatafiles, String time, final TServerInstance lastLocation,
 -      Set<FileRef> scanFiles, long initFlushID, long initCompactID) throws IOException
{
++      final Set<FileRef> scanFiles, final long initFlushID, final long initCompactID,
final Multimap<Long,FileRef> bulkImported) throws IOException {
  
      TableConfiguration tblConf = tabletServer.getTableConfiguration(extent);
      if (null == tblConf) {
@@@ -675,8 -646,8 +659,8 @@@
          currentLogs.add(new DfsLogger(tabletServer.getServerConfig(), logEntry.filename,
logEntry.getColumnQualifier().toString()));
        }
  
-       log.info("Write-Ahead Log recovery complete for " + this.extent + " (" + entriesUsedOnTablet.get()
+ " mutations applied, " + getTabletMemory().getNumEntries()
 -      log.info("Write-Ahead Log recovery complete for " + this.extent + " (" + count[0]
+ " mutations applied, " + getTabletMemory().getNumEntries()
--          + " entries created)");
++      log.info("Write-Ahead Log recovery complete for " + this.extent + " (" + entriesUsedOnTablet.get()
+ " mutations applied, "
++          + getTabletMemory().getNumEntries() + " entries created)");
      }
  
      String contextName = tableConfiguration.get(Property.TABLE_CLASSPATH);
@@@ -2295,7 -2266,7 +2279,7 @@@
        // it is possible that some of the bulk loading flags will be deleted after being
read below because the bulk load
        // finishes.... therefore split could propagate load flags for a finished bulk load...
there is a special iterator
        // on the metadata table to clean up this type of garbage
-       Multimap<Long, FileRef> bulkLoadedFiles = MetadataTableUtil.getBulkFilesLoaded(getTabletServer(),
extent);
 -      Map<FileRef,Long> bulkLoadedFiles = MetadataTableUtil.getBulkFilesLoaded(getTabletServer(),
extent);
++      Multimap<Long,FileRef> bulkLoadedFiles = MetadataTableUtil.getBulkFilesLoaded(getTabletServer(),
extent);
  
        MetadataTableUtil.splitTablet(high, extent.getPrevEndRow(), splitRatio, getTabletServer(),
getTabletServer().getLock());
        MasterMetadataUtil.addNewTablet(getTabletServer(), low, lowDirectory, getTabletServer().getTabletSession(),
lowDatafileSizes, bulkLoadedFiles, time,

http://git-wip-us.apache.org/repos/asf/accumulo/blob/bc3dc3b8/test/src/test/java/org/apache/accumulo/test/performance/metadata/FastBulkImportIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/performance/metadata/FastBulkImportIT.java
index d1ecfe9,0000000..05c907c
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/performance/metadata/FastBulkImportIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/performance/metadata/FastBulkImportIT.java
@@@ -1,104 -1,0 +1,103 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test.performance.metadata;
 +
 +import static org.junit.Assert.assertTrue;
 +
 +import java.util.SortedSet;
 +import java.util.TreeSet;
 +
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.conf.AccumuloConfiguration;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.file.FileOperations;
 +import org.apache.accumulo.core.file.FileSKVWriter;
 +import org.apache.accumulo.core.file.rfile.RFile;
 +import org.apache.accumulo.core.util.CachedConfiguration;
 +import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
 +import org.apache.accumulo.test.functional.ConfigurableMacIT;
 +import org.apache.hadoop.conf.Configuration;
 +import org.apache.hadoop.fs.FileSystem;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.hadoop.io.Text;
 +import org.junit.Test;
 +
 +// ACCUMULO-3327
 +public class FastBulkImportIT extends ConfigurableMacIT {
 +
 +  @Override
 +  protected int defaultTimeoutSeconds() {
 +    return 60;
 +  }
 +
- 
 +  @Override
 +  protected void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
 +    cfg.setNumTservers(3);
 +    cfg.setProperty(Property.TSERV_BULK_ASSIGNMENT_THREADS, "5");
 +    cfg.setProperty(Property.TSERV_BULK_PROCESS_THREADS, "5");
 +    cfg.setProperty(Property.TABLE_MAJC_RATIO, "9999");
 +    cfg.setProperty(Property.TABLE_FILE_MAX, "9999");
 +  }
 +
- 
 +  @Test
 +  public void test() throws Exception {
 +    log.info("Creating table");
 +    final String tableName = getUniqueNames(1)[0];
 +    final Connector c = getConnector();
 +    c.tableOperations().create(tableName);
 +    log.info("Adding splits");
 +    SortedSet<Text> splits = new TreeSet<>();
 +    for (int i = 1; i < 0xfff; i += 7) {
 +      splits.add(new Text(Integer.toHexString(i)));
 +    }
 +    c.tableOperations().addSplits(tableName, splits);
 +
 +    log.info("Creating bulk import files");
 +    FileSystem fs = getCluster().getFileSystem();
 +    Path basePath = getCluster().getTemporaryPath();
 +    CachedConfiguration.setInstance(fs.getConf());
 +
 +    Path base = new Path(basePath, "testBulkFail_" + tableName);
 +    fs.delete(base, true);
 +    fs.mkdirs(base);
 +    Path bulkFailures = new Path(base, "failures");
 +    Path files = new Path(base, "files");
 +    fs.mkdirs(bulkFailures);
 +    fs.mkdirs(files);
 +    for (int i = 0; i < 100; i++) {
-       FileSKVWriter writer = FileOperations.getInstance().openWriter(files.toString() +
"/bulk_" + i + "." + RFile.EXTENSION, fs, fs.getConf(), AccumuloConfiguration.getDefaultConfiguration());
++      FileSKVWriter writer = FileOperations.getInstance().openWriter(files.toString() +
"/bulk_" + i + "." + RFile.EXTENSION, fs, fs.getConf(),
++          AccumuloConfiguration.getDefaultConfiguration());
 +      writer.startDefaultLocalityGroup();
 +      for (int j = 0x100; j < 0xfff; j += 3) {
 +        writer.append(new Key(Integer.toHexString(j)), new Value(new byte[0]));
 +      }
 +      writer.close();
 +    }
 +    log.info("Waiting for balance");
 +    c.instanceOperations().waitForBalance();
 +
 +    log.info("Bulk importing files");
 +    long now = System.currentTimeMillis();
 +    c.tableOperations().importDirectory(tableName, files.toString(), bulkFailures.toString(),
true);
 +    double diffSeconds = (System.currentTimeMillis() - now) / 1000.;
 +    log.info(String.format("Import took %.2f seconds", diffSeconds));
 +    assertTrue(diffSeconds < 30);
 +  }
 +
 +}


Mime
View raw message