From commits-return-22839-archive-asf-public=cust-asf.ponee.io@accumulo.apache.org Thu Apr 18 21:14:30 2019 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [207.244.88.153]) by mx-eu-01.ponee.io (Postfix) with SMTP id A23F118061A for ; Thu, 18 Apr 2019 23:14:29 +0200 (CEST) Received: (qmail 89569 invoked by uid 500); 18 Apr 2019 21:14:29 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 89560 invoked by uid 99); 18 Apr 2019 21:14:29 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 18 Apr 2019 21:14:29 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id 43FCF879EE; Thu, 18 Apr 2019 21:14:28 +0000 (UTC) Date: Thu, 18 Apr 2019 21:14:28 +0000 To: "commits@accumulo.apache.org" Subject: [accumulo] branch master updated: Avoid locking in tserver when reading online tablets (#1100) MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Message-ID: <155562206782.12130.5071222008872549698@gitbox.apache.org> From: kturner@apache.org X-Git-Host: gitbox.apache.org X-Git-Repo: accumulo X-Git-Refname: refs/heads/master X-Git-Reftype: branch X-Git-Oldrev: 519203f28e91c9852baf5b1d372db1f19ddabcbb X-Git-Newrev: 8b52f3bb63d1c8942886b4c2d85b742bc68a047d X-Git-Rev: 8b52f3bb63d1c8942886b4c2d85b742bc68a047d X-Git-NotificationType: ref_changed_plus_diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git The following commit(s) were added to refs/heads/master by this push: new 8b52f3b Avoid locking in tserver when reading online tablets (#1100) 8b52f3b is described below commit 8b52f3bb63d1c8942886b4c2d85b742bc68a047d Author: Keith Turner AuthorDate: Thu Apr 18 17:14:23 2019 -0400 Avoid locking in tserver when reading online tablets (#1100) Profiling Accumulo during running a performance test showed lots of lock contention for the map of online tablets. Most operations need to read this map. This change creates a read only snapshot of the online tablets that is used to avoid lock contention. --- .../org/apache/accumulo/tserver/OnlineTablets.java | 57 ++++++++++ .../org/apache/accumulo/tserver/TabletServer.java | 124 ++++++++------------- .../tserver/metrics/TabletServerMetricsUtil.java | 28 ++--- .../tserver/tablet/BulkImportCacheCleaner.java | 4 +- 4 files changed, 120 insertions(+), 93 deletions(-) diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/OnlineTablets.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/OnlineTablets.java new file mode 100644 index 0000000..57e7be0 --- /dev/null +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/OnlineTablets.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.accumulo.tserver; + +import java.util.SortedMap; +import java.util.TreeMap; + +import org.apache.accumulo.core.dataImpl.KeyExtent; +import org.apache.accumulo.tserver.tablet.Tablet; + +import com.google.common.collect.ImmutableSortedMap; + +/* + * The set of online tablets is frequently read by many threads and infrequently updated. This + * class exists to create a simple wrapper that keeps an immutable snapshot up to date. Many + * threads can access the snapshot without interfering with each other. + */ +public class OnlineTablets { + private volatile ImmutableSortedMap snapshot = ImmutableSortedMap.of(); + private final SortedMap onlineTablets = new TreeMap(); + + public synchronized void put(KeyExtent ke, Tablet t) { + onlineTablets.put(ke, t); + snapshot = ImmutableSortedMap.copyOf(onlineTablets); + } + + public synchronized void remove(KeyExtent ke) { + onlineTablets.remove(ke); + snapshot = ImmutableSortedMap.copyOf(onlineTablets); + } + + public synchronized void split(KeyExtent oldTablet, Tablet newTablet1, Tablet newTablet2) { + onlineTablets.remove(oldTablet); + onlineTablets.put(newTablet1.getExtent(), newTablet1); + onlineTablets.put(newTablet2.getExtent(), newTablet2); + snapshot = ImmutableSortedMap.copyOf(onlineTablets); + } + + ImmutableSortedMap snapshot() { + return snapshot; + } +} diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java index f4508ff..63cfdf8 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java @@ -30,7 +30,6 @@ import java.security.PrivilegedExceptionAction; import java.security.SecureRandom; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; @@ -314,8 +313,7 @@ public class TabletServer implements Runnable { private final ServerContext context; private final VolumeManager fs; - private final SortedMap onlineTablets = Collections - .synchronizedSortedMap(new TreeMap()); + private final OnlineTablets onlineTablets = new OnlineTablets(); private final SortedSet unopenedTablets = Collections .synchronizedSortedSet(new TreeSet()); private final SortedSet openingTablets = Collections @@ -367,15 +365,13 @@ public class TabletServer implements Runnable { this.replWorker = new ReplicationWorker(context, fs); this.statsKeeper = new TabletStatsKeeper(); SimpleTimer.getInstance(aconf).schedule(() -> { - synchronized (onlineTablets) { - long now = System.currentTimeMillis(); - for (Tablet tablet : onlineTablets.values()) - try { - tablet.updateRates(now); - } catch (Exception ex) { - log.error("Error updating rates for {}", tablet.getExtent(), ex); - } - } + long now = System.currentTimeMillis(); + for (Tablet tablet : getOnlineTablets().values()) + try { + tablet.updateRates(now); + } catch (Exception ex) { + log.error("Error updating rates for {}", tablet.getExtent(), ex); + } }, 5000, 5000); final long walogMaxSize = aconf.getAsBytes(Property.TSERV_WALOG_MAX_SIZE); @@ -501,7 +497,7 @@ public class TabletServer implements Runnable { fileRefMap.put(new FileRef(path.toString(), path), mapping.getValue()); } - Tablet importTablet = onlineTablets.get(new KeyExtent(tke)); + Tablet importTablet = getOnlineTablet(new KeyExtent(tke)); if (importTablet == null) { failures.add(tke); @@ -542,7 +538,7 @@ public class TabletServer implements Runnable { fileRefMap.put(new FileRef(path.toString(), path), mapping.getValue()); } - Tablet importTablet = onlineTablets.get(new KeyExtent(tke)); + Tablet importTablet = getOnlineTablet(new KeyExtent(tke)); if (importTablet != null) { try { @@ -608,7 +604,7 @@ public class TabletServer implements Runnable { if (waitForWrites) writeTracker.waitForWrites(TabletType.type(extent)); - Tablet tablet = onlineTablets.get(extent); + Tablet tablet = getOnlineTablet(extent); if (tablet == null) throw new NotServingTabletException(textent); @@ -693,7 +689,7 @@ public class TabletServer implements Runnable { } } catch (CancellationException ce) { sessionManager.removeSession(scanID); - Tablet tablet = onlineTablets.get(scanSession.extent); + Tablet tablet = getOnlineTablet(scanSession.extent); if (tablet == null || tablet.isClosed()) throw new NotServingTabletException(scanSession.extent.toThrift()); else @@ -924,7 +920,7 @@ public class TabletServer implements Runnable { Tables.getNamespaceId(context, tableId))) { long t2 = System.currentTimeMillis(); us.authTimes.addStat(t2 - t1); - us.currentTablet = onlineTablets.get(keyExtent); + us.currentTablet = getOnlineTablet(keyExtent); if (us.currentTablet != null) { us.queuedMutations.put(us.currentTablet, new ArrayList<>()); } else { @@ -1224,7 +1220,7 @@ public class TabletServer implements Runnable { throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED); final KeyExtent keyExtent = new KeyExtent(tkeyExtent); - final Tablet tablet = onlineTablets.get(new KeyExtent(keyExtent)); + final Tablet tablet = getOnlineTablet(new KeyExtent(keyExtent)); if (tablet == null) { throw new NotServingTabletException(tkeyExtent); } @@ -1303,7 +1299,7 @@ public class TabletServer implements Runnable { while (iter.hasNext()) { final Entry> entry = iter.next(); - final Tablet tablet = onlineTablets.get(entry.getKey()); + final Tablet tablet = getOnlineTablet(entry.getKey()); if (tablet == null || tablet.isClosed()) { for (ServerConditionalMutation scm : entry.getValue()) @@ -1350,7 +1346,7 @@ public class TabletServer implements Runnable { try (TraceScope prepSpan = Trace.startSpan("prep")) { long t1 = System.currentTimeMillis(); for (Entry> entry : es) { - final Tablet tablet = onlineTablets.get(entry.getKey()); + final Tablet tablet = getOnlineTablet(entry.getKey()); if (tablet == null || tablet.isClosed() || sessionCanceled) { for (ServerConditionalMutation scm : entry.getValue()) results.add(new TCMResult(scm.getID(), TCMStatus.IGNORED)); @@ -1566,7 +1562,7 @@ public class TabletServer implements Runnable { KeyExtent keyExtent = new KeyExtent(tkeyExtent); - Tablet tablet = onlineTablets.get(keyExtent); + Tablet tablet = getOnlineTablet(keyExtent); if (tablet == null) { throw new NotServingTabletException(tkeyExtent); } @@ -1591,14 +1587,10 @@ public class TabletServer implements Runnable { @Override public List getTabletStats(TInfo tinfo, TCredentials credentials, String tableId) { - TreeMap onlineTabletsCopy; - synchronized (onlineTablets) { - onlineTabletsCopy = new TreeMap<>(onlineTablets); - } List result = new ArrayList<>(); TableId text = TableId.of(tableId); KeyExtent start = new KeyExtent(text, new Text(), null); - for (Entry entry : onlineTabletsCopy.tailMap(start).entrySet()) { + for (Entry entry : getOnlineTablets().tailMap(start).entrySet()) { KeyExtent ke = entry.getKey(); if (ke.getTableId().compareTo(text) == 0) { Tablet tablet = entry.getValue(); @@ -1691,7 +1683,8 @@ public class TabletServer implements Runnable { Set unopenedOverlapping = KeyExtent.findOverlapping(extent, unopenedTablets); Set openingOverlapping = KeyExtent.findOverlapping(extent, openingTablets); - Set onlineOverlapping = KeyExtent.findOverlapping(extent, onlineTablets); + Set onlineOverlapping = KeyExtent.findOverlapping(extent, + onlineTablets.snapshot()); Set all = new HashSet<>(); all.addAll(unopenedOverlapping); @@ -1702,7 +1695,7 @@ public class TabletServer implements Runnable { // ignore any tablets that have recently split, for error logging for (KeyExtent e2 : onlineOverlapping) { - Tablet tablet = onlineTablets.get(e2); + Tablet tablet = getOnlineTablet(e2); if (System.currentTimeMillis() - tablet.getSplitCreationTime() < RECENTLY_SPLIT_MILLIES) { all.remove(e2); @@ -1736,7 +1729,7 @@ public class TabletServer implements Runnable { @Override public void run() { ah.run(); - if (onlineTablets.containsKey(extent)) { + if (onlineTablets.snapshot().containsKey(extent)) { log.info("Root tablet loaded: {}", extent); } else { log.info("Root tablet failed to load"); @@ -1784,10 +1777,9 @@ public class TabletServer implements Runnable { KeyExtent ke = new KeyExtent(TableId.of(tableId), ByteBufferUtil.toText(endRow), ByteBufferUtil.toText(startRow)); - synchronized (onlineTablets) { - for (Tablet tablet : onlineTablets.values()) - if (ke.overlaps(tablet.getExtent())) - tabletsToFlush.add(tablet); + for (Tablet tablet : getOnlineTablets().values()) { + if (ke.overlaps(tablet.getExtent())) + tabletsToFlush.add(tablet); } Long flushID = null; @@ -1818,7 +1810,7 @@ public class TabletServer implements Runnable { throw new RuntimeException(e); } - Tablet tablet = onlineTablets.get(new KeyExtent(textent)); + Tablet tablet = getOnlineTablet(new KeyExtent(textent)); if (tablet != null) { log.info("Flushing {}", tablet.getExtent()); try { @@ -1886,7 +1878,7 @@ public class TabletServer implements Runnable { KeyExtent ke = new KeyExtent(textent); - Tablet tablet = onlineTablets.get(ke); + Tablet tablet = getOnlineTablet(ke); if (tablet != null) { tablet.chopFiles(); } @@ -1906,10 +1898,10 @@ public class TabletServer implements Runnable { ByteBufferUtil.toText(startRow)); ArrayList tabletsToCompact = new ArrayList<>(); - synchronized (onlineTablets) { - for (Tablet tablet : onlineTablets.values()) - if (ke.overlaps(tablet.getExtent())) - tabletsToCompact.add(tablet); + + for (Tablet tablet : getOnlineTablets().values()) { + if (ke.overlaps(tablet.getExtent())) + tabletsToCompact.add(tablet); } Pair compactionInfo = null; @@ -2106,10 +2098,6 @@ public class TabletServer implements Runnable { return totalQueuedMutationSize.addAndGet(additionalMutationSize); } - public Tablet getOnlineTablet(KeyExtent extent) { - return onlineTablets.get(extent); - } - public Session getSession(long sessionId) { return sessionManager.getSession(sessionId); } @@ -2132,21 +2120,13 @@ public class TabletServer implements Runnable { sleepUninterruptibly(getConfiguration().getTimeInMillis(Property.TSERV_MAJC_DELAY), TimeUnit.MILLISECONDS); - TreeMap copyOnlineTablets = new TreeMap<>(); - - synchronized (onlineTablets) { - copyOnlineTablets.putAll(onlineTablets); // avoid - // concurrent - // modification - } - List closedCopy; synchronized (closedLogs) { closedCopy = copyClosedLogs(closedLogs); } - Iterator> iter = copyOnlineTablets.entrySet().iterator(); + Iterator> iter = getOnlineTablets().entrySet().iterator(); // bail early now if we're shutting down while (iter.hasNext()) { @@ -2225,11 +2205,8 @@ public class TabletServer implements Runnable { statsKeeper.saveMajorMinorTimes(tablet.getTabletStats()); // lose the reference to the old tablet and open two new ones - synchronized (onlineTablets) { - onlineTablets.remove(tablet.getExtent()); - onlineTablets.put(newTablets[0].getExtent(), newTablets[0]); - onlineTablets.put(newTablets[1].getExtent(), newTablets[1]); - } + onlineTablets.split(tablet.getExtent(), newTablets[0], newTablets[1]); + // tell the master enqueueMasterMessage(new SplitReportMessage(tablet.getExtent(), newTablets[0].getExtent(), new Text("/" + newTablets[0].getLocation().getName()), newTablets[1].getExtent(), @@ -2279,8 +2256,8 @@ public class TabletServer implements Runnable { } } synchronized (onlineTablets) { - if (onlineTablets.containsKey(extent)) { - t = onlineTablets.get(extent); + if (onlineTablets.snapshot().containsKey(extent)) { + t = onlineTablets.snapshot().get(extent); } } @@ -2375,7 +2352,8 @@ public class TabletServer implements Runnable { // check Set unopenedOverlapping = KeyExtent.findOverlapping(extent, unopenedTablets); Set openingOverlapping = KeyExtent.findOverlapping(extent, openingTablets); - Set onlineOverlapping = KeyExtent.findOverlapping(extent, onlineTablets); + Set onlineOverlapping = KeyExtent.findOverlapping(extent, + onlineTablets.snapshot()); if (openingOverlapping.contains(extent) || onlineOverlapping.contains(extent)) return; @@ -3141,13 +3119,7 @@ public class TabletServer implements Runnable { SimpleTimer.getInstance(aconf).schedule(gcDebugTask, 0, TIME_BETWEEN_GC_CHECKS); Runnable constraintTask = () -> { - ArrayList tablets; - - synchronized (onlineTablets) { - tablets = new ArrayList<>(onlineTablets.values()); - } - - for (Tablet tablet : tablets) { + for (Tablet tablet : getOnlineTablets().values()) { tablet.checkConstraints(); } }; @@ -3159,13 +3131,9 @@ public class TabletServer implements Runnable { long start = System.currentTimeMillis(); TabletServerStatus result = new TabletServerStatus(); - Map onlineTabletsCopy; - synchronized (this.onlineTablets) { - onlineTabletsCopy = new HashMap<>(this.onlineTablets); - } final Map tables = new HashMap<>(); - onlineTabletsCopy.forEach((ke, tablet) -> { + getOnlineTablets().forEach((ke, tablet) -> { String tableId = ke.getTableId().canonical(); TableInfo table = tables.get(tableId); if (table == null) { @@ -3342,10 +3310,12 @@ public class TabletServer implements Runnable { }; } - public Collection getOnlineTablets() { - synchronized (onlineTablets) { - return new ArrayList<>(onlineTablets.values()); - } + public SortedMap getOnlineTablets() { + return onlineTablets.snapshot(); + } + + public Tablet getOnlineTablet(KeyExtent extent) { + return onlineTablets.snapshot().get(extent); } public VolumeManager getFileSystem() { @@ -3436,7 +3406,7 @@ public class TabletServer implements Runnable { } ReferencedRemover refRemover = candidates -> { - for (Tablet tablet : getOnlineTablets()) { + for (Tablet tablet : getOnlineTablets().values()) { tablet.removeInUseLogs(candidates); if (candidates.isEmpty()) { break; diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerMetricsUtil.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerMetricsUtil.java index e31653d..4c33d95 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerMetricsUtil.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/metrics/TabletServerMetricsUtil.java @@ -34,7 +34,7 @@ public class TabletServerMetricsUtil { public long getEntries() { long result = 0; - for (Tablet tablet : tserver.getOnlineTablets()) { + for (Tablet tablet : tserver.getOnlineTablets().values()) { result += tablet.getNumEntries(); } return result; @@ -42,7 +42,7 @@ public class TabletServerMetricsUtil { public long getEntriesInMemory() { long result = 0; - for (Tablet tablet : tserver.getOnlineTablets()) { + for (Tablet tablet : tserver.getOnlineTablets().values()) { result += tablet.getNumEntriesInMemory(); } return result; @@ -50,7 +50,7 @@ public class TabletServerMetricsUtil { public double getIngest() { double result = 0; - for (Tablet tablet : tserver.getOnlineTablets()) { + for (Tablet tablet : tserver.getOnlineTablets().values()) { result += tablet.ingestRate(); } return result; @@ -58,7 +58,7 @@ public class TabletServerMetricsUtil { public double getIngestByteRate() { double result = 0; - for (Tablet tablet : tserver.getOnlineTablets()) { + for (Tablet tablet : tserver.getOnlineTablets().values()) { result += tablet.ingestByteRate(); } return result; @@ -66,7 +66,7 @@ public class TabletServerMetricsUtil { public double getQueryRate() { double result = 0; - for (Tablet tablet : tserver.getOnlineTablets()) { + for (Tablet tablet : tserver.getOnlineTablets().values()) { result += tablet.queryRate(); } return result; @@ -74,7 +74,7 @@ public class TabletServerMetricsUtil { public double getQueryByteRate() { double result = 0; - for (Tablet tablet : tserver.getOnlineTablets()) { + for (Tablet tablet : tserver.getOnlineTablets().values()) { result += tablet.queryByteRate(); } return result; @@ -82,7 +82,7 @@ public class TabletServerMetricsUtil { public double getScannedRate() { double result = 0; - for (Tablet tablet : tserver.getOnlineTablets()) { + for (Tablet tablet : tserver.getOnlineTablets().values()) { result += tablet.scanRate(); } return result; @@ -90,7 +90,7 @@ public class TabletServerMetricsUtil { public int getMajorCompactions() { int result = 0; - for (Tablet tablet : tserver.getOnlineTablets()) { + for (Tablet tablet : tserver.getOnlineTablets().values()) { if (tablet.isMajorCompactionRunning()) result++; } @@ -99,7 +99,7 @@ public class TabletServerMetricsUtil { public int getMajorCompactionsQueued() { int result = 0; - for (Tablet tablet : tserver.getOnlineTablets()) { + for (Tablet tablet : tserver.getOnlineTablets().values()) { if (tablet.isMajorCompactionQueued()) result++; } @@ -108,7 +108,7 @@ public class TabletServerMetricsUtil { public int getMinorCompactions() { int result = 0; - for (Tablet tablet : tserver.getOnlineTablets()) { + for (Tablet tablet : tserver.getOnlineTablets().values()) { if (tablet.isMinorCompactionRunning()) result++; } @@ -117,7 +117,7 @@ public class TabletServerMetricsUtil { public int getMinorCompactionsQueued() { int result = 0; - for (Tablet tablet : tserver.getOnlineTablets()) { + for (Tablet tablet : tserver.getOnlineTablets().values()) { if (tablet.isMinorCompactionQueued()) result++; } @@ -125,7 +125,7 @@ public class TabletServerMetricsUtil { } public int getOnlineCount() { - return tserver.getOnlineTablets().size(); + return tserver.getOnlineTablets().values().size(); } public int getOpeningCount() { @@ -134,7 +134,7 @@ public class TabletServerMetricsUtil { public long getQueries() { long result = 0; - for (Tablet tablet : tserver.getOnlineTablets()) { + for (Tablet tablet : tserver.getOnlineTablets().values()) { result += tablet.totalQueries(); } return result; @@ -159,7 +159,7 @@ public class TabletServerMetricsUtil { public double getAverageFilesPerTablet() { int count = 0; long result = 0; - for (Tablet tablet : tserver.getOnlineTablets()) { + for (Tablet tablet : tserver.getOnlineTablets().values()) { result += tablet.getDatafiles().size(); count++; } diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/BulkImportCacheCleaner.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/BulkImportCacheCleaner.java index 68444cd..e982be3 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/BulkImportCacheCleaner.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/BulkImportCacheCleaner.java @@ -39,7 +39,7 @@ public class BulkImportCacheCleaner implements Runnable { public void run() { // gather the list of transactions the tablets have cached final Set tids = new HashSet<>(); - for (Tablet tablet : server.getOnlineTablets()) { + for (Tablet tablet : server.getOnlineTablets().values()) { tids.addAll(tablet.getBulkIngestedFiles().keySet()); } try { @@ -49,7 +49,7 @@ public class BulkImportCacheCleaner implements Runnable { // remove any that are still alive tids.removeAll(allTransactionsAlive); // cleanup any memory of these transactions - for (Tablet tablet : server.getOnlineTablets()) { + for (Tablet tablet : server.getOnlineTablets().values()) { tablet.cleanupBulkLoadedFiles(tids); } } catch (KeeperException | InterruptedException e) {