Return-Path: X-Original-To: apmail-accumulo-commits-archive@www.apache.org Delivered-To: apmail-accumulo-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id BD91210883 for ; Fri, 13 Dec 2013 22:56:09 +0000 (UTC) Received: (qmail 1435 invoked by uid 500); 13 Dec 2013 22:56:09 -0000 Delivered-To: apmail-accumulo-commits-archive@accumulo.apache.org Received: (qmail 1335 invoked by uid 500); 13 Dec 2013 22:56:09 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 1226 invoked by uid 99); 13 Dec 2013 22:56:09 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 13 Dec 2013 22:56:09 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 262208B892A; Fri, 13 Dec 2013 22:56:09 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: ctubbsii@apache.org To: commits@accumulo.apache.org Date: Fri, 13 Dec 2013 22:56:10 -0000 Message-Id: In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [2/3] git commit: ACCUMULO-2010 Remove Closeable from interfaces ACCUMULO-2010 Remove Closeable from interfaces This gets around compiler warnings about resource leaks. This reverts back to the old API semantics, where we are not advising that clients close Instance objects. If it is determined in the future that we should be advising clients to do this, and therefore want to re-introduce the Closeable interface to Instance and the ZooKeeper objects, it should be accompanied by changes that actually close these resources in our own code. We should not be recommending this without following our own recommendation, and adding Closeable to the interface is a recommendation to call the close method. Also clean up additional resource warnings. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/335f693a Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/335f693a Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/335f693a Branch: refs/heads/1.5.1-SNAPSHOT Commit: 335f693a4045d2c2501e2ed6ece0493734093143 Parents: 07a0433 Author: Christopher Tubbs Authored: Fri Dec 13 17:15:13 2013 -0500 Committer: Christopher Tubbs Committed: Fri Dec 13 17:15:13 2013 -0500 ---------------------------------------------------------------------- .../apache/accumulo/core/client/Instance.java | 26 +- .../core/file/rfile/bcfile/PrintInfo.java | 8 +- .../accumulo/core/zookeeper/ZooCache.java | 110 +++--- .../accumulo/core/zookeeper/ZooReader.java | 39 ++- .../apache/accumulo/core/data/RangeTest.java | 337 ++++++++++--------- .../simple/filedata/FileDataIngest.java | 71 ++-- .../accumulo/server/logger/LogReader.java | 21 +- .../accumulo/server/logger/LogService.java | 83 +++-- .../accumulo/server/test/MidPointPerfTest2.java | 118 +++---- .../test/continuous/UndefinedAnalyzer.java | 260 +++++++------- .../accumulo/server/util/SendLogToChainsaw.java | 131 ++++--- 11 files changed, 631 insertions(+), 573 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/335f693a/src/core/src/main/java/org/apache/accumulo/core/client/Instance.java ---------------------------------------------------------------------- diff --git a/src/core/src/main/java/org/apache/accumulo/core/client/Instance.java b/src/core/src/main/java/org/apache/accumulo/core/client/Instance.java index 3b2af18..b3ed056 100644 --- a/src/core/src/main/java/org/apache/accumulo/core/client/Instance.java +++ b/src/core/src/main/java/org/apache/accumulo/core/client/Instance.java @@ -16,7 +16,6 @@ */ package org.apache.accumulo.core.client; -import java.io.Closeable; import java.nio.ByteBuffer; import java.util.List; @@ -27,49 +26,49 @@ import org.apache.accumulo.core.security.thrift.AuthInfo; * This class represents the information a client needs to know to connect to an instance of accumulo. * */ -public interface Instance extends Closeable { +public interface Instance { /** * Returns the location of the tablet server that is serving the root tablet. * * @return location in "hostname:port" form */ public abstract String getRootTabletLocation(); - + /** * Returns the location(s) of the accumulo master and any redundant servers. * * @return a list of locations in "hostname:port" form */ public abstract List getMasterLocations(); - + /** * Returns a unique string that identifies this instance of accumulo. * * @return a UUID */ public abstract String getInstanceID(); - + /** * Returns the instance name given at system initialization time. * * @return current instance name */ public abstract String getInstanceName(); - + /** * Returns a comma-separated list of zookeeper servers the instance is using. * * @return the zookeeper servers this instance is using in "hostname:port" form */ public abstract String getZooKeepers(); - + /** * Returns the zookeeper connection timeout. * * @return the configured timeout to connect to zookeeper */ public abstract int getZooKeepersSessionTimeOut(); - + /** * Returns a connection to accumulo. * @@ -84,7 +83,7 @@ public interface Instance extends Closeable { * when a user's credentials are invalid */ public abstract Connector getConnector(String user, byte[] pass) throws AccumuloException, AccumuloSecurityException; - + /** * Returns a connection to accumulo. * @@ -97,7 +96,7 @@ public interface Instance extends Closeable { * when a user's credentials are invalid */ public abstract Connector getConnector(AuthInfo auth) throws AccumuloException, AccumuloSecurityException; - + /** * Returns a connection to accumulo. * @@ -112,7 +111,7 @@ public interface Instance extends Closeable { * when a user's credentials are invalid */ public abstract Connector getConnector(String user, ByteBuffer pass) throws AccumuloException, AccumuloSecurityException; - + /** * Returns a connection to this instance of accumulo. * @@ -132,16 +131,15 @@ public interface Instance extends Closeable { * Closes up the instance to free up all associated resources. You should try to reuse an Instance as much as you can because there is some location caching * stored which will enhance performance. */ - @Override public abstract void close(); - + /** * Returns the AccumuloConfiguration to use when interacting with this instance. * * @return the AccumuloConfiguration that specifies properties related to interacting with this instance */ public abstract AccumuloConfiguration getConfiguration(); - + /** * Set the AccumuloConfiguration to use when interacting with this instance. * http://git-wip-us.apache.org/repos/asf/accumulo/blob/335f693a/src/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/PrintInfo.java ---------------------------------------------------------------------- diff --git a/src/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/PrintInfo.java b/src/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/PrintInfo.java index 88e32d1..a29a53e 100644 --- a/src/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/PrintInfo.java +++ b/src/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/PrintInfo.java @@ -32,9 +32,9 @@ public class PrintInfo { public static void printMetaBlockInfo(Configuration conf, FileSystem fs, Path path) throws Exception { FSDataInputStream fsin = fs.open(path); BCFile.Reader bcfr = new BCFile.Reader(fsin, fs.getFileStatus(path).getLen(), conf); - + Set> es = bcfr.metaIndex.index.entrySet(); - + for (Entry entry : es) { PrintStream out = System.out; out.println("Meta block : " + entry.getKey()); @@ -43,8 +43,10 @@ public class PrintInfo { out.println(" Compression type : " + entry.getValue().getCompressionAlgorithm().getName()); out.println(); } + + bcfr.close(); } - + public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); @SuppressWarnings("deprecation") http://git-wip-us.apache.org/repos/asf/accumulo/blob/335f693a/src/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooCache.java ---------------------------------------------------------------------- diff --git a/src/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooCache.java b/src/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooCache.java index 1d55f6c..cbb3918 100644 --- a/src/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooCache.java +++ b/src/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooCache.java @@ -18,7 +18,6 @@ package org.apache.accumulo.core.zookeeper; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; -import java.io.Closeable; import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.IOException; @@ -41,29 +40,29 @@ import org.apache.zookeeper.data.Stat; * Caches values stored in zookeeper and keeps them up to date as they change in zookeeper. * */ -public class ZooCache implements Closeable { +public class ZooCache { private static final Logger log = Logger.getLogger(ZooCache.class); - + private ZCacheWatcher watcher = new ZCacheWatcher(); private Watcher externalWatcher = null; - + private HashMap cache; private HashMap statCache; private HashMap> childrenCache; - + private ZooReader zReader; - + private ZooKeeper getZooKeeper() { return zReader.getZooKeeper(); } - + private class ZCacheWatcher implements Watcher { @Override public void process(WatchedEvent event) { - + if (log.isTraceEnabled()) log.trace(event); - + switch (event.getType()) { case NodeDataChanged: case NodeChildrenChanged: @@ -92,21 +91,21 @@ public class ZooCache implements Closeable { default: log.warn("Unhandled: " + event); } - + if (externalWatcher != null) { externalWatcher.process(event); } } } - + public ZooCache(String zooKeepers, int sessionTimeout) { this(zooKeepers, sessionTimeout, null); } - + public ZooCache(String zooKeepers, int sessionTimeout, Watcher watcher) { this(new ZooReader(zooKeepers, sessionTimeout), watcher); } - + public ZooCache(ZooReader reader, Watcher watcher) { this.zReader = reader; this.cache = new HashMap(); @@ -114,23 +113,23 @@ public class ZooCache implements Closeable { this.childrenCache = new HashMap>(); this.externalWatcher = watcher; } - + private static interface ZooRunnable { void run(ZooKeeper zooKeeper) throws KeeperException, InterruptedException; } - + private synchronized void retry(ZooRunnable op) { - + int sleepTime = 100; - + while (true) { - + ZooKeeper zooKeeper = getZooKeeper(); - + try { op.run(zooKeeper); return; - + } catch (KeeperException e) { if (e.code() == Code.NONODE) { log.error("Looked up non existant node in cache " + e.getPath(), e); @@ -141,7 +140,7 @@ public class ZooCache implements Closeable { } catch (ConcurrentModificationException e) { log.debug("Zookeeper was modified, will retry"); } - + try { // do not hold lock while sleeping wait(sleepTime); @@ -150,20 +149,20 @@ public class ZooCache implements Closeable { } if (sleepTime < 10000) sleepTime = (int) (sleepTime + sleepTime * Math.random()); - + } } - + public synchronized List getChildren(final String zPath) { - + ZooRunnable zr = new ZooRunnable() { - + @Override public void run(ZooKeeper zooKeeper) throws KeeperException, InterruptedException { - + if (childrenCache.containsKey(zPath)) return; - + try { List children = zooKeeper.getChildren(zPath, watcher); childrenCache.put(zPath, children); @@ -173,31 +172,31 @@ public class ZooCache implements Closeable { } } } - + }; - + retry(zr); - + List children = childrenCache.get(zPath); if (children == null) { return null; } return Collections.unmodifiableList(children); } - + public synchronized byte[] get(final String zPath) { return get(zPath, null); } - + public synchronized byte[] get(final String zPath, Stat stat) { ZooRunnable zr = new ZooRunnable() { - + @Override public void run(ZooKeeper zooKeeper) throws KeeperException, InterruptedException { - + if (cache.containsKey(zPath)) return; - + /* * The following call to exists() is important, since we are caching that a node does not exist. Once the node comes into existance, it will be added to * the cache. But this notification of a node coming into existance will only be given if exists() was previously called. @@ -205,11 +204,11 @@ public class ZooCache implements Closeable { * If the call to exists() is bypassed and only getData() is called with a special case that looks for Code.NONODE in the KeeperException, then * non-existance can not be cached. */ - + Stat stat = zooKeeper.exists(zPath, watcher); - + byte[] data = null; - + if (stat == null) { if (log.isTraceEnabled()) log.trace("zookeeper did not contain " + zPath); @@ -228,11 +227,11 @@ public class ZooCache implements Closeable { log.trace("putting " + zPath + " " + (data == null ? null : new String(data)) + " in cache"); put(zPath, data, stat); } - + }; - + retry(zr); - + if (stat != null) { Stat cstat = statCache.get(zPath); if (cstat != null) { @@ -241,26 +240,26 @@ public class ZooCache implements Closeable { DataOutputStream dos = new DataOutputStream(baos); cstat.write(dos); dos.close(); - + ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray()); DataInputStream dis = new DataInputStream(bais); stat.readFields(dis); - + dis.close(); } catch (IOException e) { throw new RuntimeException(e); } } } - + return cache.get(zPath); } - + private synchronized void put(String zPath, byte[] data, Stat stat) { cache.put(zPath, data); statCache.put(zPath, stat); } - + private synchronized void remove(String zPath) { if (log.isTraceEnabled()) log.trace("removing " + zPath + " from cache"); @@ -268,36 +267,36 @@ public class ZooCache implements Closeable { childrenCache.remove(zPath); statCache.remove(zPath); } - + public synchronized void clear() { cache.clear(); childrenCache.clear(); statCache.clear(); } - + public synchronized void clear(String zPath) { - + for (Iterator i = cache.keySet().iterator(); i.hasNext();) { String path = i.next(); if (path.startsWith(zPath)) i.remove(); } - + for (Iterator i = childrenCache.keySet().iterator(); i.hasNext();) { String path = i.next(); if (path.startsWith(zPath)) i.remove(); } - + for (Iterator i = statCache.keySet().iterator(); i.hasNext();) { String path = i.next(); if (path.startsWith(zPath)) i.remove(); } } - + private static Map instances = new HashMap(); - + public static synchronized ZooCache getInstance(String zooKeepers, int sessionTimeout) { String key = zooKeepers + ":" + sessionTimeout; ZooCache zc = instances.get(key); @@ -305,11 +304,10 @@ public class ZooCache implements Closeable { zc = new ZooCache(zooKeepers, sessionTimeout); instances.put(key, zc); } - + return zc; } - - @Override + public void close() { cache.clear(); statCache.clear(); http://git-wip-us.apache.org/repos/asf/accumulo/blob/335f693a/src/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooReader.java ---------------------------------------------------------------------- diff --git a/src/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooReader.java b/src/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooReader.java index ab02034..aabc0f2 100644 --- a/src/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooReader.java +++ b/src/core/src/main/java/org/apache/accumulo/core/zookeeper/ZooReader.java @@ -16,67 +16,66 @@ */ package org.apache.accumulo.core.zookeeper; -import java.io.Closeable; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import org.apache.accumulo.core.client.Instance; +import org.apache.zookeeper.AsyncCallback.VoidCallback; import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.KeeperException.Code; import org.apache.zookeeper.Watcher; import org.apache.zookeeper.ZooKeeper; -import org.apache.zookeeper.AsyncCallback.VoidCallback; -import org.apache.zookeeper.KeeperException.Code; import org.apache.zookeeper.data.Stat; -public class ZooReader implements IZooReader, Closeable { - +public class ZooReader implements IZooReader { + protected String keepers; protected int timeout; - + protected ZooKeeper getSession(String keepers, int timeout, String auth) { return ZooSession.getSession(keepers, timeout, auth); } - + protected ZooKeeper getZooKeeper() { return getSession(keepers, timeout, null); } - + @Override public byte[] getData(String zPath, Stat stat) throws KeeperException, InterruptedException { return getZooKeeper().getData(zPath, false, stat); } - + @Override public Stat getStatus(String zPath) throws KeeperException, InterruptedException { return getZooKeeper().exists(zPath, false); } - + @Override public Stat getStatus(String zPath, Watcher watcher) throws KeeperException, InterruptedException { return getZooKeeper().exists(zPath, watcher); } - + @Override public List getChildren(String zPath) throws KeeperException, InterruptedException { return getZooKeeper().getChildren(zPath, false); } - + @Override public List getChildren(String zPath, Watcher watcher) throws KeeperException, InterruptedException { return getZooKeeper().getChildren(zPath, watcher); } - + @Override public boolean exists(String zPath) throws KeeperException, InterruptedException { return getZooKeeper().exists(zPath, false) != null; } - + @Override public boolean exists(String zPath, Watcher watcher) throws KeeperException, InterruptedException { return getZooKeeper().exists(zPath, watcher) != null; } - + @Override public void sync(final String path) throws KeeperException, InterruptedException { final AtomicInteger rc = new AtomicInteger(); @@ -89,7 +88,8 @@ public class ZooReader implements IZooReader, Closeable { waiter.set(true); waiter.notifyAll(); } - }}, null); + } + }, null); synchronized (waiter) { while (!waiter.get()) waiter.wait(); @@ -99,19 +99,18 @@ public class ZooReader implements IZooReader, Closeable { throw KeeperException.create(code); } } - + public ZooReader(String keepers, int timeout) { this.keepers = keepers; this.timeout = timeout; } - + public ZooReader(Instance instance) { this(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut()); } /** - * Closes this reader. If closure of the underlying session is interrupted, - * this method sets the calling thread's interrupt status. + * Closes this reader. If closure of the underlying session is interrupted, this method sets the calling thread's interrupt status. */ public void close() { try { http://git-wip-us.apache.org/repos/asf/accumulo/blob/335f693a/src/core/src/test/java/org/apache/accumulo/core/data/RangeTest.java ---------------------------------------------------------------------- diff --git a/src/core/src/test/java/org/apache/accumulo/core/data/RangeTest.java b/src/core/src/test/java/org/apache/accumulo/core/data/RangeTest.java index 68d9731..1f9a40d 100644 --- a/src/core/src/test/java/org/apache/accumulo/core/data/RangeTest.java +++ b/src/core/src/test/java/org/apache/accumulo/core/data/RangeTest.java @@ -35,554 +35,554 @@ public class RangeTest extends TestCase { Key ik1 = null; if (k1 != null) ik1 = new Key(new Text(k1), 0l); - + Key ik2 = null; if (k2 != null) ik2 = new Key(new Text(k2), 0l); - + return new Range(ik1, ik2); } - + private List nrl(Range... ranges) { return Arrays.asList(ranges); } - + private void check(List rl, List expected) { HashSet s1 = new HashSet(rl); HashSet s2 = new HashSet(expected); - + assertTrue("got : " + rl + " expected : " + expected, s1.equals(s2)); } - + public void testMergeOverlapping1() { List rl = nrl(nr("a", "c"), nr("a", "b")); List expected = nrl(nr("a", "c")); check(Range.mergeOverlapping(rl), expected); } - + public void testMergeOverlapping2() { List rl = nrl(nr("a", "c"), nr("d", "f")); List expected = nrl(nr("a", "c"), nr("d", "f")); check(Range.mergeOverlapping(rl), expected); } - + public void testMergeOverlapping3() { List rl = nrl(nr("a", "e"), nr("b", "f"), nr("c", "r"), nr("g", "j"), nr("t", "x")); List expected = nrl(nr("a", "r"), nr("t", "x")); check(Range.mergeOverlapping(rl), expected); } - + public void testMergeOverlapping4() { List rl = nrl(nr("a", "e"), nr("b", "f"), nr("c", "r"), nr("g", "j")); List expected = nrl(nr("a", "r")); check(Range.mergeOverlapping(rl), expected); } - + public void testMergeOverlapping5() { List rl = nrl(nr("a", "e")); List expected = nrl(nr("a", "e")); check(Range.mergeOverlapping(rl), expected); } - + public void testMergeOverlapping6() { List rl = nrl(); List expected = nrl(); check(Range.mergeOverlapping(rl), expected); } - + public void testMergeOverlapping7() { List rl = nrl(nr("a", "e"), nr("g", "q"), nr("r", "z")); List expected = nrl(nr("a", "e"), nr("g", "q"), nr("r", "z")); check(Range.mergeOverlapping(rl), expected); } - + public void testMergeOverlapping8() { List rl = nrl(nr("a", "c"), nr("a", "c")); List expected = nrl(nr("a", "c")); check(Range.mergeOverlapping(rl), expected); } - + public void testMergeOverlapping9() { List rl = nrl(nr(null, null)); List expected = nrl(nr(null, null)); check(Range.mergeOverlapping(rl), expected); } - + public void testMergeOverlapping10() { List rl = nrl(nr(null, null), nr("a", "c")); List expected = nrl(nr(null, null)); check(Range.mergeOverlapping(rl), expected); } - + public void testMergeOverlapping11() { List rl = nrl(nr("a", "c"), nr(null, null)); List expected = nrl(nr(null, null)); check(Range.mergeOverlapping(rl), expected); } - + public void testMergeOverlapping12() { List rl = nrl(nr("b", "d"), nr("c", null)); List expected = nrl(nr("b", null)); check(Range.mergeOverlapping(rl), expected); } - + public void testMergeOverlapping13() { List rl = nrl(nr("b", "d"), nr("a", null)); List expected = nrl(nr("a", null)); check(Range.mergeOverlapping(rl), expected); } - + public void testMergeOverlapping14() { List rl = nrl(nr("b", "d"), nr("e", null)); List expected = nrl(nr("b", "d"), nr("e", null)); check(Range.mergeOverlapping(rl), expected); } - + public void testMergeOverlapping15() { List rl = nrl(nr("b", "d"), nr("e", null), nr("c", "f")); List expected = nrl(nr("b", null)); check(Range.mergeOverlapping(rl), expected); } - + public void testMergeOverlapping16() { List rl = nrl(nr("b", "d"), nr("f", null), nr("c", "e")); List expected = nrl(nr("b", "e"), nr("f", null)); check(Range.mergeOverlapping(rl), expected); } - + public void testMergeOverlapping17() { List rl = nrl(nr("b", "d"), nr("r", null), nr("c", "e"), nr("g", "t")); List expected = nrl(nr("b", "e"), nr("g", null)); check(Range.mergeOverlapping(rl), expected); } - + public void testMergeOverlapping18() { List rl = nrl(nr(null, "d"), nr("r", null), nr("c", "e"), nr("g", "t")); List expected = nrl(nr(null, "e"), nr("g", null)); check(Range.mergeOverlapping(rl), expected); } - + public void testMergeOverlapping19() { List rl = nrl(nr(null, "d"), nr("r", null), nr("c", "e"), nr("g", "t"), nr("d", "h")); List expected = nrl(nr(null, null)); check(Range.mergeOverlapping(rl), expected); } - + public void testMergeOverlapping20() { - + List rl = nrl(new Range(new Text("a"), true, new Text("b"), false), new Range(new Text("b"), false, new Text("c"), false)); List expected = nrl(new Range(new Text("a"), true, new Text("b"), false), new Range(new Text("b"), false, new Text("c"), false)); check(Range.mergeOverlapping(rl), expected); - + rl = nrl(new Range(new Text("a"), true, new Text("b"), false), new Range(new Text("b"), true, new Text("c"), false)); expected = nrl(new Range(new Text("a"), true, new Text("c"), false)); check(Range.mergeOverlapping(rl), expected); - + rl = nrl(new Range(new Text("a"), true, new Text("b"), true), new Range(new Text("b"), false, new Text("c"), false)); expected = nrl(new Range(new Text("a"), true, new Text("c"), false)); check(Range.mergeOverlapping(rl), expected); - + rl = nrl(new Range(new Text("a"), true, new Text("b"), true), new Range(new Text("b"), true, new Text("c"), false)); expected = nrl(new Range(new Text("a"), true, new Text("c"), false)); check(Range.mergeOverlapping(rl), expected); - + } - + public void testMergeOverlapping22() { - + Range ke1 = new KeyExtent(new Text("tab1"), new Text("Bank"), null).toMetadataRange(); Range ke2 = new KeyExtent(new Text("tab1"), new Text("Fails"), new Text("Bank")).toMetadataRange(); Range ke3 = new KeyExtent(new Text("tab1"), new Text("Sam"), new Text("Fails")).toMetadataRange(); Range ke4 = new KeyExtent(new Text("tab1"), new Text("bails"), new Text("Sam")).toMetadataRange(); Range ke5 = new KeyExtent(new Text("tab1"), null, new Text("bails")).toMetadataRange(); - + List rl = nrl(ke1, ke2, ke3, ke4, ke5); List expected = nrl(new KeyExtent(new Text("tab1"), null, null).toMetadataRange()); check(Range.mergeOverlapping(rl), expected); - + rl = nrl(ke1, ke2, ke4, ke5); expected = nrl(new KeyExtent(new Text("tab1"), new Text("Fails"), null).toMetadataRange(), new KeyExtent(new Text("tab1"), null, new Text("Sam")).toMetadataRange()); check(Range.mergeOverlapping(rl), expected); - + rl = nrl(ke2, ke3, ke4, ke5); expected = nrl(new KeyExtent(new Text("tab1"), null, new Text("Bank")).toMetadataRange()); check(Range.mergeOverlapping(rl), expected); - + rl = nrl(ke1, ke2, ke3, ke4); expected = nrl(new KeyExtent(new Text("tab1"), new Text("bails"), null).toMetadataRange()); check(Range.mergeOverlapping(rl), expected); - + rl = nrl(ke2, ke3, ke4); expected = nrl(new KeyExtent(new Text("tab1"), new Text("bails"), new Text("Bank")).toMetadataRange()); check(Range.mergeOverlapping(rl), expected); } - + public void testMergeOverlapping21() { for (boolean b1 : new boolean[] {true, false}) for (boolean b2 : new boolean[] {true, false}) for (boolean b3 : new boolean[] {true, false}) for (boolean b4 : new boolean[] {true, false}) { - + // System.out.println("b1:"+b1+" b2:"+b2+" b3:"+b3+" b4:"+b4); - + List rl = nrl(new Range(new Key(new Text("a")), b1, new Key(new Text("m")), b2), new Range(new Key(new Text("b")), b3, new Key(new Text("n")), b4)); List expected = nrl(new Range(new Key(new Text("a")), b1, new Key(new Text("n")), b4)); check(Range.mergeOverlapping(rl), expected); - + rl = nrl(new Range(new Key(new Text("a")), b1, new Key(new Text("m")), b2), new Range(new Key(new Text("a")), b3, new Key(new Text("n")), b4)); expected = nrl(new Range(new Key(new Text("a")), b1 || b3, new Key(new Text("n")), b4)); check(Range.mergeOverlapping(rl), expected); - + rl = nrl(new Range(new Key(new Text("a")), b1, new Key(new Text("n")), b2), new Range(new Key(new Text("b")), b3, new Key(new Text("n")), b4)); expected = nrl(new Range(new Key(new Text("a")), b1, new Key(new Text("n")), b2 || b4)); check(Range.mergeOverlapping(rl), expected); - + rl = nrl(new Range(new Key(new Text("a")), b1, new Key(new Text("n")), b2), new Range(new Key(new Text("a")), b3, new Key(new Text("n")), b4)); expected = nrl(new Range(new Key(new Text("a")), b1 || b3, new Key(new Text("n")), b2 || b4)); check(Range.mergeOverlapping(rl), expected); } - + } - + public void testEqualsNull() { - + assertTrue(nr(null, "d").equals(nr(null, "d"))); - + assertTrue(nr(null, null).equals(nr(null, null))); - + assertTrue(nr("a", null).equals(nr("a", null))); - + assertFalse(nr(null, "d").equals(nr("a", "d"))); assertFalse(nr("a", "d").equals(nr(null, "d"))); - + assertFalse(nr(null, null).equals(nr("a", "d"))); assertFalse(nr("a", "d").equals(nr(null, null))); - + assertFalse(nr("a", null).equals(nr("a", "d"))); assertFalse(nr("a", "d").equals(nr("a", null))); } - + public void testEquals() { assertFalse(nr("b", "d").equals(nr("a", "d"))); assertFalse(nr("a", "d").equals(nr("b", "d"))); - + assertFalse(nr("x", "y").equals(nr("a", "d"))); assertFalse(nr("a", "d").equals(nr("x", "y"))); - + assertFalse(nr("a", "z").equals(nr("a", "d"))); assertFalse(nr("a", "d").equals(nr("a", "z"))); - + assertTrue(nr("a", "z").equals(nr("a", "z"))); } - + public void testRow1() { Range rowRange = new Range(new Text("r1")); - + assertTrue(rowRange.contains(new Key(new Text("r1")))); assertTrue(rowRange.contains(new Key(new Text("r1"), new Text("cf1")))); assertTrue(rowRange.contains(new Key(new Text("r1"), new Text("cf1"), new Text("cq1")))); - + assertFalse(rowRange.contains(new Key(new Text("r1")).followingKey(PartialKey.ROW))); assertFalse(rowRange.contains(new Key(new Text("r11")))); assertFalse(rowRange.contains(new Key(new Text("r0")))); } - + public void testRow2() { Range rowRange = new Range(new Text("r1"), new Text("r2")); - + assertTrue(rowRange.contains(new Key(new Text("r1")))); assertTrue(rowRange.contains(new Key(new Text("r1"), new Text("cf1")))); assertTrue(rowRange.contains(new Key(new Text("r1"), new Text("cf1"), new Text("cq1")))); - + assertTrue(rowRange.contains(new Key(new Text("r1")).followingKey(PartialKey.ROW))); assertTrue(rowRange.contains(new Key(new Text("r11")))); - + assertTrue(rowRange.contains(new Key(new Text("r2")))); assertTrue(rowRange.contains(new Key(new Text("r2"), new Text("cf1")))); assertTrue(rowRange.contains(new Key(new Text("r2"), new Text("cf1"), new Text("cq1")))); - + assertFalse(rowRange.contains(new Key(new Text("r0")))); assertFalse(rowRange.contains(new Key(new Text("r2")).followingKey(PartialKey.ROW))); } - + public void testRow3() { Range rowRange = new Range(new Text("r1"), false, new Text("r2"), false); - + assertFalse(rowRange.contains(new Key(new Text("r1")))); assertFalse(rowRange.contains(new Key(new Text("r1"), new Text("cf1")))); assertFalse(rowRange.contains(new Key(new Text("r1"), new Text("cf1"), new Text("cq1")))); - + assertTrue(rowRange.contains(new Key(new Text("r1")).followingKey(PartialKey.ROW))); assertTrue(rowRange.contains(new Key(new Text("r11")))); - + assertFalse(rowRange.contains(new Key(new Text("r2")))); assertFalse(rowRange.contains(new Key(new Text("r2"), new Text("cf1")))); assertFalse(rowRange.contains(new Key(new Text("r2"), new Text("cf1"), new Text("cq1")))); - + assertFalse(rowRange.contains(new Key(new Text("r0")))); assertFalse(rowRange.contains(new Key(new Text("r2")).followingKey(PartialKey.ROW))); } - + public void testRow4() { Range rowRange = new Range(new Text("r1"), true, new Text("r2"), false); - + assertTrue(rowRange.contains(new Key(new Text("r1")))); assertTrue(rowRange.contains(new Key(new Text("r1"), new Text("cf1")))); assertTrue(rowRange.contains(new Key(new Text("r1"), new Text("cf1"), new Text("cq1")))); - + assertTrue(rowRange.contains(new Key(new Text("r1")).followingKey(PartialKey.ROW))); assertTrue(rowRange.contains(new Key(new Text("r11")))); - + assertFalse(rowRange.contains(new Key(new Text("r2")))); assertFalse(rowRange.contains(new Key(new Text("r2"), new Text("cf1")))); assertFalse(rowRange.contains(new Key(new Text("r2"), new Text("cf1"), new Text("cq1")))); - + assertFalse(rowRange.contains(new Key(new Text("r0")))); assertFalse(rowRange.contains(new Key(new Text("r2")).followingKey(PartialKey.ROW))); } - + public void testRow5() { Range rowRange = new Range(new Text("r1"), false, new Text("r2"), true); - + assertFalse(rowRange.contains(new Key(new Text("r1")))); assertFalse(rowRange.contains(new Key(new Text("r1"), new Text("cf1")))); assertFalse(rowRange.contains(new Key(new Text("r1"), new Text("cf1"), new Text("cq1")))); - + assertTrue(rowRange.contains(new Key(new Text("r1")).followingKey(PartialKey.ROW))); assertTrue(rowRange.contains(new Key(new Text("r11")))); - + assertTrue(rowRange.contains(new Key(new Text("r2")))); assertTrue(rowRange.contains(new Key(new Text("r2"), new Text("cf1")))); assertTrue(rowRange.contains(new Key(new Text("r2"), new Text("cf1"), new Text("cq1")))); - + assertFalse(rowRange.contains(new Key(new Text("r0")))); assertFalse(rowRange.contains(new Key(new Text("r2")).followingKey(PartialKey.ROW))); } - + public void testRow6() { Range rowRange = new Range(new Text("r1"), true, null, true); - + assertTrue(rowRange.contains(new Key(new Text("r1")))); assertTrue(rowRange.contains(new Key(new Text("r1"), new Text("cf1")))); assertTrue(rowRange.contains(new Key(new Text("r1"), new Text("cf1"), new Text("cq1")))); - + assertTrue(rowRange.contains(new Key(new Text("r1")).followingKey(PartialKey.ROW))); assertTrue(rowRange.contains(new Key(new Text("r11")))); - + assertTrue(rowRange.contains(new Key(new Text("r2")))); assertTrue(rowRange.contains(new Key(new Text("r2"), new Text("cf1")))); assertTrue(rowRange.contains(new Key(new Text("r2"), new Text("cf1"), new Text("cq1")))); - + assertFalse(rowRange.contains(new Key(new Text("r0")))); assertTrue(rowRange.contains(new Key(new Text("r2")).followingKey(PartialKey.ROW))); } - + public void testRow7() { Range rowRange = new Range(null, true, new Text("r2"), true); - + assertTrue(rowRange.contains(new Key(new Text("r1")))); assertTrue(rowRange.contains(new Key(new Text("r1"), new Text("cf1")))); assertTrue(rowRange.contains(new Key(new Text("r1"), new Text("cf1"), new Text("cq1")))); - + assertTrue(rowRange.contains(new Key(new Text("r1")).followingKey(PartialKey.ROW))); assertTrue(rowRange.contains(new Key(new Text("r11")))); - + assertTrue(rowRange.contains(new Key(new Text("r2")))); assertTrue(rowRange.contains(new Key(new Text("r2"), new Text("cf1")))); assertTrue(rowRange.contains(new Key(new Text("r2"), new Text("cf1"), new Text("cq1")))); - + assertTrue(rowRange.contains(new Key(new Text("r0")))); assertFalse(rowRange.contains(new Key(new Text("r2")).followingKey(PartialKey.ROW))); } - + public void testRow8() { Range rowRange = new Range((Text) null); - + assertTrue(rowRange.contains(new Key(new Text("r1")))); assertTrue(rowRange.contains(new Key(new Text("r1"), new Text("cf1")))); assertTrue(rowRange.contains(new Key(new Text("r1"), new Text("cf1"), new Text("cq1")))); - + assertTrue(rowRange.contains(new Key(new Text("r1")).followingKey(PartialKey.ROW))); assertTrue(rowRange.contains(new Key(new Text("r11")))); - + assertTrue(rowRange.contains(new Key(new Text("r2")))); assertTrue(rowRange.contains(new Key(new Text("r2"), new Text("cf1")))); assertTrue(rowRange.contains(new Key(new Text("r2"), new Text("cf1"), new Text("cq1")))); - + assertTrue(rowRange.contains(new Key(new Text("r0")))); assertTrue(rowRange.contains(new Key(new Text("r2")).followingKey(PartialKey.ROW))); } - + private static Range nr(String r1, boolean r1i, String r2, boolean r2i) { Text tr1 = null; Text tr2 = null; - + if (r1 != null) tr1 = new Text(r1); - + if (r2 != null) tr2 = new Text(r2); - + return new Range(tr1, r1i, tr2, r2i); - + } - + private static Key nk(String r) { return new Key(new Text(r)); } - + public void testClip1() { Range fence = nr("a", false, "c", false); - + runClipTest(fence, nr("a", false, "c", false), nr("a", false, "c", false)); runClipTest(fence, nr("a", true, "c", false), nr("a", false, "c", false)); runClipTest(fence, nr("a", false, "c", true), nr("a", false, "c", false)); runClipTest(fence, nr("a", true, "c", true), nr("a", false, "c", false)); - + fence = nr("a", true, "c", false); - + runClipTest(fence, nr("a", false, "c", false), nr("a", false, "c", false)); runClipTest(fence, nr("a", true, "c", false), nr("a", true, "c", false)); runClipTest(fence, nr("a", false, "c", true), nr("a", false, "c", false)); runClipTest(fence, nr("a", true, "c", true), nr("a", true, "c", false)); - + fence = nr("a", false, "c", true); - + runClipTest(fence, nr("a", false, "c", false), nr("a", false, "c", false)); runClipTest(fence, nr("a", true, "c", false), nr("a", false, "c", false)); runClipTest(fence, nr("a", false, "c", true), nr("a", false, "c", true)); runClipTest(fence, nr("a", true, "c", true), nr("a", false, "c", true)); - + fence = nr("a", true, "c", true); - + runClipTest(fence, nr("a", false, "c", false), nr("a", false, "c", false)); runClipTest(fence, nr("a", true, "c", false), nr("a", true, "c", false)); runClipTest(fence, nr("a", false, "c", true), nr("a", false, "c", true)); runClipTest(fence, nr("a", true, "c", true), nr("a", true, "c", true)); } - + public void testClip2() { Range fence = nr("a", false, "c", false); - + runClipTest(fence, nr(null, true, null, true), nr("a", false, "c", false)); runClipTest(fence, nr(null, true, "c", true), nr("a", false, "c", false)); runClipTest(fence, nr("a", true, null, true), nr("a", false, "c", false)); runClipTest(fence, nr("a", true, "c", true), nr("a", false, "c", false)); } - + public void testClip3() { Range fence = nr("a", false, "c", false); - + runClipTest(fence, nr("0", false, "z", false), nr("a", false, "c", false)); runClipTest(fence, nr("0", true, "z", false), nr("a", false, "c", false)); runClipTest(fence, nr("0", false, "z", true), nr("a", false, "c", false)); runClipTest(fence, nr("0", true, "z", true), nr("a", false, "c", false)); - + runClipTest(fence, nr("0", false, "b", false), nr("a", false, "b", false)); runClipTest(fence, nr("0", true, "b", false), nr("a", false, "b", false)); runClipTest(fence, nr("0", false, "b", true), nr("a", false, "b", true)); runClipTest(fence, nr("0", true, "b", true), nr("a", false, "b", true)); - + runClipTest(fence, nr("a1", false, "z", false), nr("a1", false, "c", false)); runClipTest(fence, nr("a1", true, "z", false), nr("a1", true, "c", false)); runClipTest(fence, nr("a1", false, "z", true), nr("a1", false, "c", false)); runClipTest(fence, nr("a1", true, "z", true), nr("a1", true, "c", false)); - + runClipTest(fence, nr("a1", false, "b", false), nr("a1", false, "b", false)); runClipTest(fence, nr("a1", true, "b", false), nr("a1", true, "b", false)); runClipTest(fence, nr("a1", false, "b", true), nr("a1", false, "b", true)); runClipTest(fence, nr("a1", true, "b", true), nr("a1", true, "b", true)); } - + public void testClip4() { Range fence = new Range(nk("c"), false, nk("n"), false); - + runClipTest(fence, new Range(nk("a"), false, nk("c"), false)); runClipTest(fence, new Range(nk("a"), false, nk("c"), true)); runClipTest(fence, new Range(nk("n"), false, nk("r"), false)); runClipTest(fence, new Range(nk("n"), true, nk("r"), false)); runClipTest(fence, new Range(nk("a"), true, nk("b"), false)); runClipTest(fence, new Range(nk("a"), true, nk("b"), true)); - + fence = new Range(nk("c"), true, nk("n"), true); - + runClipTest(fence, new Range(nk("a"), false, nk("c"), false)); runClipTest(fence, new Range(nk("a"), false, nk("c"), true), new Range(nk("c"), true, nk("c"), true)); runClipTest(fence, new Range(nk("n"), false, nk("r"), false)); runClipTest(fence, new Range(nk("n"), true, nk("r"), false), new Range(nk("n"), true, nk("n"), true)); runClipTest(fence, new Range(nk("q"), false, nk("r"), false)); runClipTest(fence, new Range(nk("q"), true, nk("r"), false)); - + fence = nr("b", true, "b", true); - + runClipTest(fence, nr("b", false, "c", false)); runClipTest(fence, nr("b", true, "c", false), nr("b", true, "b", true)); runClipTest(fence, nr("a", false, "b", false)); runClipTest(fence, nr("a", false, "b", true), nr("b", true, "b", true)); - + } - + public void testBug1() { - + // unit test related to a bug that was observed (bug was not in range, but want to ensure the following works) - + // clip caught the scanner going to a tablet passed the end of the scan range Range fence = new Range(new Text("10<"), false, new Text("~"), true); - + Key k1 = new Key(new Text("10<"), new Text("~tab"), new Text("~pr")); Range range = new Range(k1, true, k1.followingKey(PartialKey.ROW), false); - + runClipTest(fence, range); - + // scanner was not handling edge case properly... Range scanRange = new Range(new Key("10;007cdc5b0".getBytes(), "~tab".getBytes(), "~pr".getBytes(), "".getBytes(), 130962, false), false, new Key(new Text( "10<")).followingKey(PartialKey.ROW), false); // below is the proper check the scanner now does instead of just comparing the row bytes scanRange.afterEndKey(new Key(new Text("10<")).followingKey(PartialKey.ROW)); } - + private void runClipTest(Range fence, Range range) { try { fence.clip(range); assertFalse(true); } catch (IllegalArgumentException e) { - + } - + } - + private void runClipTest(Range fence, Range range, Range expected) { Range clipped = fence.clip(range); assertEquals(expected, clipped); } - + private static Key nk(String r, String cf, String cq) { return new Key(new Text(r), new Text(cf), new Text(cq)); } - + private static Key nk(String r, String cf, String cq, String cv) { return new Key(new Text(r), new Text(cf), new Text(cq), new Text(cv)); } - + private static Column nc(String cf, String cq) { return new Column(cf.getBytes(), cq == null ? null : cq.getBytes(), null); } - + private static Column nc(String cf) { return nc(cf, null); } - + private static Range nr(String row) { return new Range(new Text(row)); } - + public void testBound1() { Range range1 = nr("row1"); - + Range range2 = range1.bound(nc("b"), nc("e")); - + assertFalse(range2.contains(nk("row1"))); assertFalse(range2.contains(nk("row1", "a", "z"))); assertTrue(range2.contains(nk("row1", "b", ""))); @@ -593,19 +593,19 @@ public class RangeTest extends TestCase { assertFalse(range2.contains(nk("row1", "e", "").followingKey(PartialKey.ROW_COLFAM))); assertFalse(range2.contains(nk("row1", "f", ""))); assertFalse(range2.contains(nk("row1", "f", "z"))); - + } - + public void testBound2() { Range range1 = new Range(nk("row1", "b", "x"), true, nk("row1", "f", "x"), true); - + Range range2 = range1.bound(nc("a"), nc("g")); assertEquals(range1, range2); assertFalse(range2.contains(nk("row1", "a", "x"))); assertTrue(range2.contains(nk("row1", "b", "x"))); assertTrue(range2.contains(nk("row1", "f", "x"))); assertFalse(range2.contains(nk("row1", "g", ""))); - + Range range3 = range1.bound(nc("c"), nc("d")); assertFalse(range3.contains(nk("row1", "b", "x"))); assertTrue(range3.contains(nk("row1", "c", ""))); @@ -614,7 +614,7 @@ public class RangeTest extends TestCase { assertTrue(range3.contains(nk("row1", "d", "z"))); assertFalse(range3.contains(nk("row1", "e", ""))); assertFalse(range3.contains(nk("row1", "f", "x"))); - + Range range4 = range1.bound(nc("c", "w"), nc("d", "z")); assertFalse(range4.contains(nk("row1", "b", "x"))); assertTrue(range4.contains(nk("row1", "c", "w"))); @@ -625,14 +625,14 @@ public class RangeTest extends TestCase { assertFalse(range4.contains(nk("row1", "d", "{", ""))); assertFalse(range4.contains(nk("row1", "d", "z", "a").followingKey(PartialKey.ROW_COLFAM_COLQUAL))); assertFalse(range4.contains(nk("row1", "f", "x"))); - + Range range5 = range1.bound(nc("b", "w"), nc("f", "z")); assertEquals(range1, range5); assertFalse(range5.contains(nk("row1", "b", "w"))); assertTrue(range5.contains(nk("row1", "b", "x"))); assertTrue(range5.contains(nk("row1", "f", "x"))); assertFalse(range5.contains(nk("row1", "f", "z"))); - + Range range6 = range1.bound(nc("b", "y"), nc("f", "w")); assertFalse(range6.contains(nk("row1", "b", "x"))); assertTrue(range6.contains(nk("row1", "b", "y"))); @@ -640,7 +640,7 @@ public class RangeTest extends TestCase { assertTrue(range6.contains(nk("row1", "f", "w", "a"))); assertFalse(range6.contains(nk("row1", "f", "w").followingKey(PartialKey.ROW_COLFAM_COLQUAL))); assertFalse(range6.contains(nk("row1", "f", "x"))); - + Range range7 = range1.bound(nc("a", "y"), nc("g", "w")); assertEquals(range1, range7); assertFalse(range7.contains(nk("row1", "b", "w"))); @@ -648,26 +648,26 @@ public class RangeTest extends TestCase { assertTrue(range7.contains(nk("row1", "f", "x"))); assertFalse(range7.contains(nk("row1", "f", "z"))); } - + public void testString() { Range r1 = new Range(new Text("r1")); Range r2 = new Range("r1"); assertEquals(r1, r2); - + r1 = new Range(new Text("r1"), new Text("r2")); r2 = new Range("r1", "r2"); assertEquals(r1, r2); - + r1 = new Range(new Text("r1"), false, new Text("r2"), true); r2 = new Range("r1", false, "r2", true); assertEquals(r1, r2); - + r1 = new Range(new Text("r1"), true, new Text("r2"), false); r2 = new Range("r1", true, "r2", false); assertEquals(r1, r2); - + } - + public void testExactRange() { Range r = Range.exact("abc"); assertTrue(r.contains(new Key("abc"))); @@ -675,31 +675,31 @@ public class RangeTest extends TestCase { assertFalse(r.contains(new Key("abcd"))); assertFalse(r.contains(new Key("abb"))); assertFalse(r.contains(new Key("abd"))); - + r = Range.exact("abc", "def"); assertTrue(r.contains(new Key("abc", "def", "ghi"))); assertFalse(r.contains(new Key("abc", "defg"))); assertFalse(r.contains(new Key("abc", "dee"))); assertFalse(r.contains(new Key("abc", "deg"))); - + r = Range.exact("abc", "def", "ghi"); assertTrue(r.contains(new Key("abc", "def", "ghi", "j&k"))); assertFalse(r.contains(new Key("abc", "def", "ghij"))); assertFalse(r.contains(new Key("abc", "def", "ghh"))); assertFalse(r.contains(new Key("abc", "def", "ghj"))); - + r = Range.exact("abc", "def", "ghi", "j&k"); assertTrue(r.contains(new Key("abc", "def", "ghi", "j&k", 7l))); assertFalse(r.contains(new Key("abc", "def", "ghi", "j&kl"))); assertFalse(r.contains(new Key("abc", "def", "ghi", "j&j"))); assertFalse(r.contains(new Key("abc", "def", "ghi", "j&l"))); - + r = Range.exact("abc", "def", "ghi", "j&k", 7l); assertTrue(r.contains(new Key("abc", "def", "ghi", "j&k", 7l))); assertFalse(r.contains(new Key("abc", "def", "ghi", "j&k", 6l))); assertFalse(r.contains(new Key("abc", "def", "ghi", "j&k", 8l))); } - + public void testPrefixRange() { Range r = Range.prefix("abc"); assertTrue(r.contains(new Key("abc"))); @@ -707,59 +707,59 @@ public class RangeTest extends TestCase { assertTrue(r.contains(new Key("abcd"))); assertFalse(r.contains(new Key("abb"))); assertFalse(r.contains(new Key("abd"))); - + r = Range.prefix("abc", "def"); assertTrue(r.contains(new Key("abc", "def", "ghi"))); assertTrue(r.contains(new Key("abc", "defg"))); assertFalse(r.contains(new Key("abc", "dee"))); assertFalse(r.contains(new Key("abc", "deg"))); - + r = Range.prefix("abc", "def", "ghi"); assertTrue(r.contains(new Key("abc", "def", "ghi", "j&k"))); assertTrue(r.contains(new Key("abc", "def", "ghij"))); assertFalse(r.contains(new Key("abc", "def", "ghh"))); assertFalse(r.contains(new Key("abc", "def", "ghj"))); - + r = Range.prefix("abc", "def", "ghi", "j&k"); assertTrue(r.contains(new Key("abc", "def", "ghi", "j&k", 7l))); assertTrue(r.contains(new Key("abc", "def", "ghi", "j&kl"))); assertFalse(r.contains(new Key("abc", "def", "ghi", "j&j"))); assertFalse(r.contains(new Key("abc", "def", "ghi", "j&l"))); - + r = Range.prefix(makeText((byte) 0x07, (byte) 0xff)); assertTrue(r.contains(new Key(makeText((byte) 0x07, (byte) 0xff)))); assertTrue(r.contains(new Key(makeText((byte) 0x07, (byte) 0xff, (byte) 0x00)))); assertFalse(r.contains(new Key(makeText((byte) 0x07, (byte) 0xfe)))); assertFalse(r.contains(new Key(makeText((byte) 0x08)))); - + r = Range.prefix(makeText((byte) 0xff)); assertTrue(r.isInfiniteStopKey()); assertTrue(r.contains(new Key(makeText((byte) 0xff)))); assertTrue(r.contains(new Key(makeText((byte) 0xff, (byte) 0x07)))); - + r = Range.prefix(new Text("abc"), makeText((byte) 0xff)); assertTrue(r.contains(new Key(new Text("abc"), makeText((byte) 0xff)))); assertTrue(r.contains(new Key(new Text("abc"), makeText((byte) 0xff, (byte) 0x07)))); assertFalse(r.contains(new Key(new Text("abcd")))); assertFalse(r.contains(new Key(new Text("abd")))); - + r = Range.prefix(new Text("abc"), new Text("def"), makeText((byte) 0xff)); assertTrue(r.contains(new Key(new Text("abc"), new Text("def"), makeText((byte) 0xff)))); assertTrue(r.contains(new Key(new Text("abc"), new Text("def"), makeText((byte) 0xff, (byte) 0x07)))); assertFalse(r.contains(new Key(new Text("abc"), new Text("defg")))); assertFalse(r.contains(new Key(new Text("abc"), new Text("deg")))); - + r = Range.prefix(new Text("abc"), new Text("def"), new Text("ghi"), makeText((byte) 0xff)); assertTrue(r.contains(new Key(new Text("abc"), new Text("def"), new Text("ghi"), makeText((byte) 0xff)))); assertTrue(r.contains(new Key(new Text("abc"), new Text("def"), new Text("ghi"), makeText((byte) 0xff, (byte) 0x07)))); assertFalse(r.contains(new Key(new Text("abc"), new Text("def"), new Text("ghij")))); assertFalse(r.contains(new Key(new Text("abc"), new Text("def"), new Text("ghj")))); } - + public static Text makeText(byte... b) { return new Text(b); } - + public void testPrefix() { assertEquals(Range.followingPrefix(makeText((byte) 0x07)), new Text(makeText((byte) 0x08))); assertEquals(Range.followingPrefix(makeText((byte) 0xfe)), new Text(makeText((byte) 0xff))); @@ -813,6 +813,7 @@ public class RangeTest extends TestCase { Range r = new Range(new Key(new Text("soup")), true, false, new Key(new Text("nuts")), true, false); TRange tr = r.toThrift(); try { + @SuppressWarnings("unused") Range r2 = new Range(tr); fail("Thrift constructor allowed invalid range"); } catch (IllegalArgumentException exc) { http://git-wip-us.apache.org/repos/asf/accumulo/blob/335f693a/src/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/FileDataIngest.java ---------------------------------------------------------------------- diff --git a/src/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/FileDataIngest.java b/src/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/FileDataIngest.java index 4ee1063..2fd0623 100644 --- a/src/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/FileDataIngest.java +++ b/src/examples/simple/src/main/java/org/apache/accumulo/examples/simple/filedata/FileDataIngest.java @@ -44,13 +44,13 @@ public class FileDataIngest { public static final String REFS_FILE_EXT = "filext"; public static final ByteSequence CHUNK_CF_BS = new ArrayByteSequence(CHUNK_CF.getBytes(), 0, CHUNK_CF.getLength()); public static final ByteSequence REFS_CF_BS = new ArrayByteSequence(REFS_CF.getBytes(), 0, REFS_CF.getLength()); - + int chunkSize; byte[] chunkSizeBytes; byte[] buf; MessageDigest md5digest; ColumnVisibility cv; - + public FileDataIngest(int chunkSize, ColumnVisibility colvis) { this.chunkSize = chunkSize; chunkSizeBytes = intToBytes(chunkSize); @@ -62,13 +62,13 @@ public class FileDataIngest { } cv = colvis; } - + public String insertFileData(String filename, BatchWriter bw) throws MutationsRejectedException, IOException { if (chunkSize == 0) return ""; md5digest.reset(); String uid = hexString(md5digest.digest(filename.getBytes())); - + // read through file once, calculating hashes md5digest.reset(); InputStream fis = new FileInputStream(filename); @@ -80,10 +80,10 @@ public class FileDataIngest { numRead = fis.read(buf); } fis.close(); - + String hash = hexString(md5digest.digest()); Text row = new Text(hash); - + // write info to accumulo Mutation m = new Mutation(row); m.put(REFS_CF, KeyUtil.buildNullSepText(uid, REFS_ORIG_FILE), cv, new Value(filename.getBytes())); @@ -91,30 +91,35 @@ public class FileDataIngest { if (fext != null) m.put(REFS_CF, KeyUtil.buildNullSepText(uid, REFS_FILE_EXT), cv, new Value(fext.getBytes())); bw.addMutation(m); - + // read through file again, writing chunks to accumulo - fis = new FileInputStream(filename); - numRead = fis.read(buf); int chunkCount = 0; - while (numRead >= 0) { - while (numRead < buf.length) { - int moreRead = fis.read(buf, numRead, buf.length - numRead); - if (moreRead > 0) - numRead += moreRead; - else if (moreRead < 0) - break; - } - m = new Mutation(row); - Text chunkCQ = new Text(chunkSizeBytes); - chunkCQ.append(intToBytes(chunkCount), 0, 4); - m.put(CHUNK_CF, chunkCQ, cv, new Value(buf, 0, numRead)); - bw.addMutation(m); - if (chunkCount == Integer.MAX_VALUE) - throw new RuntimeException("too many chunks for file " + filename + ", try raising chunk size"); - chunkCount++; + try { + fis = new FileInputStream(filename); numRead = fis.read(buf); + while (numRead >= 0) { + while (numRead < buf.length) { + int moreRead = fis.read(buf, numRead, buf.length - numRead); + if (moreRead > 0) + numRead += moreRead; + else if (moreRead < 0) + break; + } + m = new Mutation(row); + Text chunkCQ = new Text(chunkSizeBytes); + chunkCQ.append(intToBytes(chunkCount), 0, 4); + m.put(CHUNK_CF, chunkCQ, cv, new Value(buf, 0, numRead)); + bw.addMutation(m); + if (chunkCount == Integer.MAX_VALUE) + throw new RuntimeException("too many chunks for file " + filename + ", try raising chunk size"); + chunkCount++; + numRead = fis.read(buf); + } + } finally { + if (fis != null) { + fis.close(); + } } - fis.close(); m = new Mutation(row); Text chunkCQ = new Text(chunkSizeBytes); chunkCQ.append(intToBytes(chunkCount), 0, 4); @@ -122,14 +127,14 @@ public class FileDataIngest { bw.addMutation(m); return hash; } - + public static int bytesToInt(byte[] b, int offset) { if (b.length <= offset + 3) throw new NumberFormatException("couldn't pull integer from bytes at offset " + offset); int i = (((b[offset] & 255) << 24) + ((b[offset + 1] & 255) << 16) + ((b[offset + 2] & 255) << 8) + ((b[offset + 3] & 255) << 0)); return i; } - + public static byte[] intToBytes(int l) { byte[] b = new byte[4]; b[0] = (byte) (l >>> 24); @@ -138,13 +143,13 @@ public class FileDataIngest { b[3] = (byte) (l >>> 0); return b; } - + private static String getExt(String filename) { if (filename.indexOf(".") == -1) return null; return filename.substring(filename.lastIndexOf(".") + 1); } - + public String hexString(byte[] bytes) { StringBuilder sb = new StringBuilder(); for (byte b : bytes) { @@ -152,14 +157,14 @@ public class FileDataIngest { } return sb.toString(); } - + public static void main(String[] args) throws Exception { if (args.length < 8) { System.out.println("usage: " + FileDataIngest.class.getSimpleName() + " { }"); System.exit(1); } - + String instance = args[0]; String zooKeepers = args[1]; String user = args[2]; @@ -167,7 +172,7 @@ public class FileDataIngest { String dataTable = args[4]; ColumnVisibility colvis = new ColumnVisibility(args[5]); int chunkSize = Integer.parseInt(args[6]); - + Connector conn = new ZooKeeperInstance(instance, zooKeepers).getConnector(user, pass.getBytes()); if (!conn.tableOperations().exists(dataTable)) { conn.tableOperations().create(dataTable); http://git-wip-us.apache.org/repos/asf/accumulo/blob/335f693a/src/server/src/main/java/org/apache/accumulo/server/logger/LogReader.java ---------------------------------------------------------------------- diff --git a/src/server/src/main/java/org/apache/accumulo/server/logger/LogReader.java b/src/server/src/main/java/org/apache/accumulo/server/logger/LogReader.java index bf8c160..4eb6289 100644 --- a/src/server/src/main/java/org/apache/accumulo/server/logger/LogReader.java +++ b/src/server/src/main/java/org/apache/accumulo/server/logger/LogReader.java @@ -108,14 +108,26 @@ public class LogReader { if (fs.isFile(path)) { // read log entries from a simple hdfs file org.apache.hadoop.io.SequenceFile.Reader reader = new SequenceFile.Reader(fs, new Path(file), conf); - while (reader.next(key, value)) { - printLogEvent(key, value, row, rowMatcher, ke, tabletIds, max); + try { + while (reader.next(key, value)) { + printLogEvent(key, value, row, rowMatcher, ke, tabletIds, max); + } + } finally { + if (reader != null) { + reader.close(); + } } } else if (local.isFile(path)) { // read log entries from a simple file org.apache.hadoop.io.SequenceFile.Reader reader = new SequenceFile.Reader(local, new Path(file), conf); - while (reader.next(key, value)) { - printLogEvent(key, value, row, rowMatcher, ke, tabletIds, max); + try { + while (reader.next(key, value)) { + printLogEvent(key, value, row, rowMatcher, ke, tabletIds, max); + } + } finally { + if (reader != null) { + reader.close(); + } } } else { try { @@ -129,6 +141,7 @@ public class LogReader { while (input.next(key, value)) { printLogEvent(key, value, row, rowMatcher, ke, tabletIds, max); } + input.close(); } } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/335f693a/src/server/src/main/java/org/apache/accumulo/server/logger/LogService.java ---------------------------------------------------------------------- diff --git a/src/server/src/main/java/org/apache/accumulo/server/logger/LogService.java b/src/server/src/main/java/org/apache/accumulo/server/logger/LogService.java index ebfc287..6fabd95 100644 --- a/src/server/src/main/java/org/apache/accumulo/server/logger/LogService.java +++ b/src/server/src/main/java/org/apache/accumulo/server/logger/LogService.java @@ -92,34 +92,34 @@ import org.apache.zookeeper.Watcher.Event.KeeperState; */ public class LogService implements MutationLogger.Iface, Watcher { static final org.apache.log4j.Logger LOG = org.apache.log4j.Logger.getLogger(LogService.class); - + private Configuration conf; private Authenticator authenticator; private TServer service; private LogWriter writer_; private MutationLogger.Iface writer; private String ephemeralNode; - + enum ShutdownState { STARTED, REGISTERED, WAITING_FOR_HALT, HALT }; - + private ShutdownState shutdownState = ShutdownState.STARTED; - + synchronized void switchState(ShutdownState state) { LOG.info("Switching from " + shutdownState + " to " + state); shutdownState = state; } - + synchronized private void closedCheck() throws LoggerClosedException { if (!shutdownState.equals(ShutdownState.REGISTERED)) throw new LoggerClosedException(); } - + private List fileLocks = new ArrayList(); - + private final String addressString; - + public static void main(String[] args) throws Exception { LogService logService; SecurityUtil.serverLogin(); @@ -136,16 +136,16 @@ public class LogService implements MutationLogger.Iface, Watcher { LOG.error("Unexpected exception, exiting.", ex); } } - + public LogService(String[] args) throws UnknownHostException, KeeperException, InterruptedException, IOException { try { Accumulo.init("logger"); } catch (UnknownHostException e1) { LOG.error("Error reading logging configuration"); } - + FileSystemMonitor.start(Property.LOGGER_MONITOR_FS); - + conf = CachedConfiguration.getInstance(); FileSystem fs = null; try { @@ -168,34 +168,45 @@ public class LogService implements MutationLogger.Iface, Watcher { } rootDirs.add(root); } - + for (String root : rootDirs) { File rootFile = new File(root); rootFile.mkdirs(); + @SuppressWarnings("resource") FileOutputStream lockOutputStream = new FileOutputStream(root + "/.lock"); FileLock fileLock = lockOutputStream.getChannel().tryLock(); if (fileLock == null) - throw new IOException("Failed to acquire lock file"); + try { + throw new IOException("Failed to acquire lock file"); + } finally { + if (lockOutputStream != null) + lockOutputStream.close(); + } fileLocks.add(fileLock); - + try { File test = new File(root, "test_writable"); if (!test.mkdir()) throw new RuntimeException("Unable to write to write-ahead log directory " + root); test.delete(); } catch (Throwable t) { - LOG.fatal("Unable to write to write-ahead log directory", t); - throw new RuntimeException(t); + try { + LOG.fatal("Unable to write to write-ahead log directory", t); + throw new RuntimeException(t); + } finally { + if (lockOutputStream != null) + lockOutputStream.close(); + } } LOG.info("Storing recovery logs at " + root); } - + authenticator = ZKAuthenticator.getInstance(); int poolSize = ServerConfiguration.getSystemConfiguration().getCount(Property.LOGGER_COPY_THREADPOOL_SIZE); boolean archive = ServerConfiguration.getSystemConfiguration().getBoolean(Property.LOGGER_ARCHIVE); AccumuloConfiguration acuConf = ServerConfiguration.getSystemConfiguration(); writer_ = new LogWriter(acuConf, fs, rootDirs, HdfsZooInstance.getInstance().getInstanceID(), poolSize, archive); - + // call before putting this service online removeIncompleteCopies(acuConf, fs, rootDirs); @@ -241,7 +252,7 @@ public class LogService implements MutationLogger.Iface, Watcher { this.switchState(ShutdownState.REGISTERED); Accumulo.enableTracing(address.getHostName(), "logger"); } - + /** * @param acuConf * @param fs @@ -265,7 +276,7 @@ public class LogService implements MutationLogger.Iface, Watcher { } } } - + // look for .recovered that are not finished for (String walog : walogs) { Path path = new Path(ServerConstants.getRecoveryDir() + "/" + walog + ".recovered"); @@ -292,7 +303,7 @@ public class LogService implements MutationLogger.Iface, Watcher { } Runtime.getRuntime().halt(0); } - + void registerInZooKeeper(String zooDir) { try { IZooReaderWriter zoo = ZooReaderWriter.getInstance(); @@ -305,7 +316,7 @@ public class LogService implements MutationLogger.Iface, Watcher { throw new RuntimeException("Unexpected error creating zookeeper entry " + zooDir); } } - + private void checkForSystemPrivs(String request, AuthInfo credentials) throws ThriftSecurityException { try { if (!authenticator.hasSystemPermission(credentials, credentials.user, SystemPermission.SYSTEM)) { @@ -317,13 +328,13 @@ public class LogService implements MutationLogger.Iface, Watcher { throw e.asThriftException(); } } - + @Override public void close(TInfo info, long id) throws NoSuchLogIDException, LoggerClosedException, TException { closedCheck(); writer.close(info, id); } - + @Override public LogCopyInfo startCopy(TInfo info, AuthInfo credentials, String localLog, String fullyQualifiedFileName, boolean sort) throws ThriftSecurityException, TException { @@ -332,50 +343,50 @@ public class LogService implements MutationLogger.Iface, Watcher { lci.loggerZNode = ephemeralNode; return lci; } - + @Override public LogFile create(TInfo info, AuthInfo credentials, String tserverSession) throws ThriftSecurityException, LoggerClosedException, TException { checkForSystemPrivs("create", credentials); closedCheck(); return writer.create(info, credentials, tserverSession); } - + @Override public void log(TInfo info, long id, long seq, int tid, TMutation mutation) throws NoSuchLogIDException, LoggerClosedException, TException { closedCheck(); writer.log(info, id, seq, tid, mutation); } - + @Override public void logManyTablets(TInfo info, long id, List mutations) throws NoSuchLogIDException, LoggerClosedException, TException { closedCheck(); writer.logManyTablets(info, id, mutations); } - + @Override public void minorCompactionFinished(TInfo info, long id, long seq, int tid, String fqfn) throws NoSuchLogIDException, LoggerClosedException, TException { closedCheck(); writer.minorCompactionFinished(info, id, seq, tid, fqfn); } - + @Override public void minorCompactionStarted(TInfo info, long id, long seq, int tid, String fqfn) throws NoSuchLogIDException, LoggerClosedException, TException { closedCheck(); writer.minorCompactionStarted(info, id, seq, tid, fqfn); } - + @Override public void defineTablet(TInfo info, long id, long seq, int tid, TKeyExtent tablet) throws NoSuchLogIDException, LoggerClosedException, TException { closedCheck(); writer.defineTablet(info, id, seq, tid, tablet); } - + @Override public void process(WatchedEvent event) { LOG.debug("event " + event.getPath() + " " + event.getType() + " " + event.getState()); if (event.getState() == KeeperState.Disconnected) return; - + if (event.getState() == KeeperState.Expired) { LOG.warn("Logger lost zookeeper registration at " + event.getPath()); service.stop(); @@ -400,13 +411,13 @@ public class LogService implements MutationLogger.Iface, Watcher { service.stop(); } } - + @Override public List getClosedLogs(TInfo info, AuthInfo credentials) throws ThriftSecurityException, TException { checkForSystemPrivs("getClosedLogs", credentials); return writer.getClosedLogs(info, credentials); } - + @Override public void remove(TInfo info, AuthInfo credentials, List files) throws TException { try { @@ -416,7 +427,7 @@ public class LogService implements MutationLogger.Iface, Watcher { LOG.error(ex, ex); } } - + @Override public void beginShutdown(TInfo tinfo, AuthInfo credentials) throws TException { try { @@ -427,7 +438,7 @@ public class LogService implements MutationLogger.Iface, Watcher { LOG.error(ex, ex); } } - + @Override public void halt(TInfo tinfo, AuthInfo credentials) throws TException { try { http://git-wip-us.apache.org/repos/asf/accumulo/blob/335f693a/src/server/src/main/java/org/apache/accumulo/server/test/MidPointPerfTest2.java ---------------------------------------------------------------------- diff --git a/src/server/src/main/java/org/apache/accumulo/server/test/MidPointPerfTest2.java b/src/server/src/main/java/org/apache/accumulo/server/test/MidPointPerfTest2.java index 2029b82..f55fcab 100644 --- a/src/server/src/main/java/org/apache/accumulo/server/test/MidPointPerfTest2.java +++ b/src/server/src/main/java/org/apache/accumulo/server/test/MidPointPerfTest2.java @@ -41,59 +41,60 @@ import org.apache.hadoop.io.WritableComparable; */ @Deprecated class MultipleIndexIterator2 { - + private MySequenceFile.Reader[] readers; private boolean[] hasNextKey; private Key[] nextKey; private int currentMin = 0; - + int findMinimum() { int minIndex = -1; for (int i = 0; i < nextKey.length; i++) { if (!hasNextKey[i]) { continue; } - + if (minIndex == -1) { minIndex = i; } else if (nextKey[i].compareTo(nextKey[minIndex]) < 0) { minIndex = i; } } - + return minIndex; } - + MultipleIndexIterator2(Configuration conf, FileSystem fs, List paths) throws IOException, InstantiationException, IllegalAccessException { readers = new MySequenceFile.Reader[paths.size()]; - + int ri = 0; for (Path path : paths) { + @SuppressWarnings("resource") MySequenceFile.Reader index = new MySequenceFile.Reader(fs, path, conf); readers[ri++] = index; } - + hasNextKey = new boolean[readers.length]; nextKey = new Key[readers.length]; for (int i = 0; i < readers.length; i++) { nextKey[i] = (Key) readers[i].getKeyClass().newInstance(); hasNextKey[i] = readers[i].next(nextKey[i]); } - + currentMin = findMinimum(); } - + boolean hasNext() { return currentMin >= 0; } - + WritableComparable next() { if (currentMin < 0) { throw new RuntimeException("There is no next"); } - + WritableComparable ret = nextKey[currentMin]; - + try { nextKey[currentMin] = (Key) readers[currentMin].getKeyClass().newInstance(); hasNextKey[currentMin] = readers[currentMin].next(nextKey[currentMin]); @@ -104,15 +105,15 @@ class MultipleIndexIterator2 { } catch (IOException e) { throw new RuntimeException(e); } - + currentMin = findMinimum(); - + return ret; } - + public void close() { currentMin = -1; - + for (Reader reader : readers) { try { reader.close(); @@ -136,27 +137,27 @@ public class MidPointPerfTest2 { private static int maxFiles; private static String tmpDir; private static boolean createTestData = false; - + public static void main(String[] args) { - + for (String string : args) { System.out.println("arg : " + string); } - + if (args[0].equals("-ctd")) { dir = args[1]; numFiles = Integer.parseInt(args[2]); numEntries = Integer.parseInt(args[3]); min = Integer.parseInt(args[4]); max = Integer.parseInt(args[5]); - + createTestData = true; } else { dir = args[0]; maxFiles = Integer.parseInt(args[1]); tmpDir = args[2]; } - + try { if (createTestData) { Random r = new Random(); @@ -168,124 +169,125 @@ public class MidPointPerfTest2 { throw new RuntimeException(e); } } - + private static void timeIterate(String dir, int maxFiles, String tmpDir) throws Exception { Configuration conf = CachedConfiguration.getInstance(); FileSystem fs = FileSystem.get(conf); - + FileStatus[] files = fs.globStatus(new Path(dir + "/*/index")); ArrayList paths = new ArrayList(files.length); - + for (FileStatus fileStatus : files) { paths.add(fileStatus.getPath()); } - + long t1 = System.currentTimeMillis(); ArrayList rpaths = reduceFiles(conf, fs, paths, maxFiles, tmpDir, 0); long t2 = System.currentTimeMillis(); - + MultipleIndexIterator2 mii = new MultipleIndexIterator2(conf, fs, rpaths); - + int count = 0; while (mii.hasNext()) { mii.next(); count++; } - + long t3 = System.currentTimeMillis(); - + System.out.printf("reduce time : %6.2f secs \n", (t2 - t1) / 1000.0); System.out.printf("iterate time : %6.2f secs \n", (t3 - t2) / 1000.0); System.out.printf("total time : %6.2f secs \n", (t3 - t1) / 1000.0); - + System.out.println("count " + count); } - + private static ArrayList reduceFiles(Configuration conf, FileSystem fs, ArrayList paths, int maxFiles, String tmpDir, int pass) throws IOException, InstantiationException, IllegalAccessException { if (paths.size() <= maxFiles) { return paths; } - + String newDir = String.format("%s/pass_%04d", tmpDir, pass); fs.mkdirs(new Path(newDir)); - + int start = 0; - + ArrayList outFiles = new ArrayList(); - + int count = 0; - + while (start < paths.size()) { int end = Math.min(maxFiles + start, paths.size()); List inFiles = paths.subList(start, end); - + start = end; - + Path outFile = new Path(String.format("%s/index_%04d", newDir, count++)); outFiles.add(outFile); - + long t1 = System.currentTimeMillis(); - + MySequenceFile.Writer writer = MySequenceFile.createWriter(fs, conf, outFile, Key.class, LongWritable.class, MySequenceFile.CompressionType.BLOCK); MultipleIndexIterator2 mii = new MultipleIndexIterator2(conf, fs, inFiles); - + while (mii.hasNext()) { writer.append(mii.next(), new LongWritable(0)); } - + mii.close(); writer.close(); - + long t2 = System.currentTimeMillis(); - + System.out.printf("out : %s num in : %d time : %6.2f secs\n", outFile, inFiles.size(), (t2 - t1) / 1000.0); } - + return reduceFiles(conf, fs, outFiles, maxFiles, tmpDir, pass + 1); } - + static class CompareKeys implements Comparator { public CompareKeys() {} - + public boolean equals(Key a, Key b) { return a.equals(b); } - + + @Override public int compare(Key a, Key b) { return a.compareTo(b); } } - + private static void createTestData(String dir, int numFiles, int numEntries, int min, int max, Random r) throws Exception { Configuration conf = CachedConfiguration.getInstance(); FileSystem fs = FileSystem.get(conf); - + for (int i = 0; i < numFiles; i++) { String newDir = String.format("%s/" + MyMapFile.EXTENSION + "_%06d", dir, i); fs.mkdirs(new Path(newDir)); - + List keys = new ArrayList(); - + for (int j = 0; j < numEntries; j++) { String row = String.format("row_%010d", r.nextInt() % (max - min) + min); Key key1 = new Key(new Text(row), new Text(String.format("cf_%03d", r.nextInt() % 100)), new Text(String.format("cf_%05d", r.nextInt() % 10000))); keys.add(key1); } - + Collections.sort(keys, new CompareKeys()); - + MySequenceFile.Writer writer = MySequenceFile.createWriter(fs, conf, new Path(newDir + "/index"), Key.class, LongWritable.class, MySequenceFile.CompressionType.BLOCK); - + System.out.println(new Path(newDir + "/index")); - + for (Key key : keys) { writer.append(key, new LongWritable(0)); } - + writer.close(); } } - + }