incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [34/45] git commit: Some more refactoring and adding more tests for the CacheDirectory.
Date Sat, 28 Sep 2013 02:12:20 GMT
Some more refactoring and adding more tests for the CacheDirectory.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/890be501
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/890be501
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/890be501

Branch: refs/heads/apache-blur-0.2
Commit: 890be5016797d91d1d7e685c584307924e3c7def
Parents: 28be85a
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Fri Sep 20 14:08:52 2013 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Fri Sep 20 14:08:52 2013 -0400

----------------------------------------------------------------------
 .../cachevalue/UnsafeCacheValue.java            |   2 +-
 .../apache/blur/store/BaseDirectoryTest.java    | 273 --------------
 .../blur/store/BaseDirectoryTestSuite.java      | 375 +++++++++++++++++++
 .../blur/store/CacheDirectoryTestSuite.java     |  72 ++++
 .../apache/blur/store/HdfsDirectoryTest.java    |  41 --
 .../blur/store/HdfsDirectoryTestSuite.java      |  41 ++
 .../blur/store/SoftlinkHdfsDirectoryTest.java   |  43 ---
 .../store/SoftlinkHdfsDirectoryTestSuite.java   |  43 +++
 .../blur/store/blockcache/BlockCacheTest.java   |   4 +-
 9 files changed, 535 insertions(+), 359 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/890be501/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/cachevalue/UnsafeCacheValue.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/cachevalue/UnsafeCacheValue.java
b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/cachevalue/UnsafeCacheValue.java
index 46af604..d8e1cad 100644
--- a/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/cachevalue/UnsafeCacheValue.java
+++ b/blur-store/src/main/java/org/apache/blur/store/blockcache_v2/cachevalue/UnsafeCacheValue.java
@@ -24,7 +24,7 @@ import sun.misc.Unsafe;
 @SuppressWarnings("serial")
 public class UnsafeCacheValue extends BaseCacheValue {
 
-  private static final int MINIMUM_SIZE = 1024;
+//  private static final int MINIMUM_SIZE = 1024;
 
   private static final Unsafe _unsafe;
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/890be501/blur-store/src/test/java/org/apache/blur/store/BaseDirectoryTest.java
----------------------------------------------------------------------
diff --git a/blur-store/src/test/java/org/apache/blur/store/BaseDirectoryTest.java b/blur-store/src/test/java/org/apache/blur/store/BaseDirectoryTest.java
deleted file mode 100644
index 666dcfe..0000000
--- a/blur-store/src/test/java/org/apache/blur/store/BaseDirectoryTest.java
+++ /dev/null
@@ -1,273 +0,0 @@
-package org.apache.blur.store;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.Random;
-import java.util.Set;
-
-import org.apache.blur.lucene.LuceneVersionConstant;
-import org.apache.blur.store.buffer.BufferStore;
-import org.apache.lucene.analysis.core.KeywordAnalyzer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field.Store;
-import org.apache.lucene.document.IntField;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.NumericRangeQuery;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.store.RAMDirectory;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-public abstract class BaseDirectoryTest {
-  protected static final File TMPDIR = new File(System.getProperty("blur.tmp.dir", "/tmp"));
-
-  protected static final int MAX_NUMBER_OF_WRITES = 10000;
-  protected static final int MIN_FILE_SIZE = 100;
-  protected static final int MAX_FILE_SIZE = 100000;
-  protected static final int MIN_BUFFER_SIZE = 1;
-  protected static final int MAX_BUFFER_SIZE = 5000;
-  protected static final int MAX_NUMBER_OF_READS = 10000;
-  protected Directory directory;
-  protected File file;
-  protected long seed;
-  protected Random random;
-
-  @Before
-  public void setUp() throws IOException {
-    BufferStore.init(128, 128);
-    file = new File(TMPDIR, "hdfsdirectorytest");
-    rm(file);
-    seed = new Random().nextLong();
-    random = new Random(seed);
-    setupDirectory();
-  }
-
-  @After
-  public void tearDown() {
-    print(file, "");
-  }
-
-  private void print(File f, String buf) {
-    if (f.isDirectory()) {
-      System.out.println(buf + "\\" + f.getName());
-      for (File fl : f.listFiles()) {
-        if (fl.getName().startsWith(".")) {
-          continue;
-        }
-        print(fl, buf + " ");
-      }
-    } else {
-      System.out.println(buf + f.getName() + " " + f.length());
-    }
-  }
-
-  protected abstract void setupDirectory() throws IOException;
-
-  @Test
-  public void testWritingAndReadingAFile() throws IOException {
-
-    IndexOutput output = directory.createOutput("testing.test", IOContext.DEFAULT);
-    output.writeInt(12345);
-    output.flush();
-    output.close();
-
-    IndexInput input = directory.openInput("testing.test", IOContext.DEFAULT);
-    assertEquals(12345, input.readInt());
-    input.close();
-
-    String[] listAll = directory.listAll();
-    assertEquals(1, listAll.length);
-    assertEquals("testing.test", listAll[0]);
-
-    assertEquals(4, directory.fileLength("testing.test"));
-
-    IndexInput input1 = directory.openInput("testing.test", IOContext.DEFAULT);
-
-    IndexInput input2 = (IndexInput) input1.clone();
-    assertEquals(12345, input2.readInt());
-    input2.close();
-
-    assertEquals(12345, input1.readInt());
-    input1.close();
-
-    assertFalse(directory.fileExists("testing.test.other"));
-    assertTrue(directory.fileExists("testing.test"));
-    directory.deleteFile("testing.test");
-    assertFalse(directory.fileExists("testing.test"));
-  }
-
-  @Test
-  public void testEOF() throws IOException {
-    Directory fsDir = new RAMDirectory();
-    String name = "test.eof";
-    createFile(name, fsDir, directory);
-    long fsLength = fsDir.fileLength(name);
-    long hdfsLength = directory.fileLength(name);
-    assertEquals(fsLength, hdfsLength);
-    testEof(name, fsDir, fsLength);
-    testEof(name, directory, hdfsLength);
-  }
-
-  private void testEof(String name, Directory directory, long length) throws IOException
{
-    IndexInput input = directory.openInput(name, IOContext.DEFAULT);
-    input.seek(length);
-    try {
-      input.readByte();
-      fail("should throw eof");
-    } catch (IOException e) {
-    }
-  }
-
-  @Test
-  public void testWrites() throws IOException {
-    int i = 0;
-    try {
-      Set<String> names = new HashSet<String>();
-      for (; i < 10; i++) {
-        Directory fsDir = new RAMDirectory();
-        String name = getName();
-        System.out.println("Working on pass [" + i + "] seed [" + seed + "] contains [" +
names.contains(name) + "]");
-        names.add(name);
-        createFile(name, fsDir, directory);
-        assertInputsEquals(name, fsDir, directory);
-        fsDir.close();
-      }
-    } catch (Exception e) {
-      e.printStackTrace();
-      fail("Test failed with seed [" + seed + "] on pass [" + i + "]");
-    }
-  }
-
-  @Test
-  public void testCreateIndex() throws IOException {
-    IndexWriterConfig conf = new IndexWriterConfig(LuceneVersionConstant.LUCENE_VERSION,
new KeywordAnalyzer());
-    IndexWriter writer = new IndexWriter(directory, conf);
-    int numDocs = 1000;
-    DirectoryReader reader = null;
-    for (int i = 0; i < 100; i++) {
-      if (reader == null) {
-        reader = DirectoryReader.open(writer, true);
-      } else {
-        DirectoryReader old = reader;
-        reader = DirectoryReader.openIfChanged(old, writer, true);
-        if (reader == null) {
-          reader = old;
-        } else {
-          old.close();
-        }
-      }
-      assertEquals(i * numDocs, reader.numDocs());
-      IndexSearcher searcher = new IndexSearcher(reader);
-      NumericRangeQuery<Integer> query = NumericRangeQuery.newIntRange("id", 42, 42,
true, true);
-      TopDocs topDocs = searcher.search(query, 10);
-      assertEquals(i, topDocs.totalHits);
-      addDocuments(writer, numDocs);
-    }
-    writer.close(false);
-    reader.close();
-  }
-
-  private void addDocuments(IndexWriter writer, int numDocs) throws IOException {
-    for (int i = 0; i < numDocs; i++) {
-      writer.addDocument(getDoc(i));
-    }
-  }
-
-  private Document getDoc(int i) {
-    Document document = new Document();
-    document.add(new IntField("id", i, Store.YES));
-    return document;
-  }
-
-  private void assertInputsEquals(String name, Directory fsDir, Directory hdfs) throws IOException
{
-    int reads = random.nextInt(MAX_NUMBER_OF_READS);
-    IndexInput fsInput = fsDir.openInput(name, IOContext.DEFAULT);
-    IndexInput hdfsInput = hdfs.openInput(name, IOContext.DEFAULT);
-    assertEquals(fsInput.length(), hdfsInput.length());
-    int fileLength = (int) fsInput.length();
-    for (int i = 0; i < reads; i++) {
-      byte[] fsBuf = new byte[random.nextInt(Math.min(MAX_BUFFER_SIZE - MIN_BUFFER_SIZE,
fileLength)) + MIN_BUFFER_SIZE];
-      byte[] hdfsBuf = new byte[fsBuf.length];
-      int offset = random.nextInt(fsBuf.length);
-      int length = random.nextInt(fsBuf.length - offset);
-      int pos = random.nextInt(fileLength - length);
-      fsInput.seek(pos);
-      fsInput.readBytes(fsBuf, offset, length);
-      hdfsInput.seek(pos);
-      hdfsInput.readBytes(hdfsBuf, offset, length);
-      for (int f = offset; f < length; f++) {
-        if (fsBuf[f] != hdfsBuf[f]) {
-          fail();
-        }
-      }
-    }
-    fsInput.close();
-    hdfsInput.close();
-  }
-
-  private void createFile(String name, Directory fsDir, Directory hdfs) throws IOException
{
-    int writes = random.nextInt(MAX_NUMBER_OF_WRITES);
-    int fileLength = random.nextInt(MAX_FILE_SIZE - MIN_FILE_SIZE) + MIN_FILE_SIZE;
-    IndexOutput fsOutput = fsDir.createOutput(name, IOContext.DEFAULT);
-    fsOutput.setLength(fileLength);
-    IndexOutput hdfsOutput = hdfs.createOutput(name, IOContext.DEFAULT);
-    hdfsOutput.setLength(fileLength);
-    for (int i = 0; i < writes; i++) {
-      byte[] buf = new byte[random.nextInt(Math.min(MAX_BUFFER_SIZE - MIN_BUFFER_SIZE, fileLength))
+ MIN_BUFFER_SIZE];
-      random.nextBytes(buf);
-      int offset = random.nextInt(buf.length);
-      int length = random.nextInt(buf.length - offset);
-      fsOutput.writeBytes(buf, offset, length);
-      hdfsOutput.writeBytes(buf, offset, length);
-    }
-    fsOutput.close();
-    hdfsOutput.close();
-  }
-
-  private String getName() {
-    return Long.toString(Math.abs(random.nextLong()));
-  }
-
-  public static void rm(File file) {
-    if (!file.exists()) {
-      return;
-    }
-    if (file.isDirectory()) {
-      for (File f : file.listFiles()) {
-        rm(f);
-      }
-    }
-    file.delete();
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/890be501/blur-store/src/test/java/org/apache/blur/store/BaseDirectoryTestSuite.java
----------------------------------------------------------------------
diff --git a/blur-store/src/test/java/org/apache/blur/store/BaseDirectoryTestSuite.java b/blur-store/src/test/java/org/apache/blur/store/BaseDirectoryTestSuite.java
new file mode 100644
index 0000000..48364ca
--- /dev/null
+++ b/blur-store/src/test/java/org/apache/blur/store/BaseDirectoryTestSuite.java
@@ -0,0 +1,375 @@
+package org.apache.blur.store;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Random;
+import java.util.Set;
+
+import org.apache.blur.lucene.LuceneVersionConstant;
+import org.apache.blur.store.blockcache.LastModified;
+import org.apache.blur.store.buffer.BufferStore;
+import org.apache.lucene.analysis.core.KeywordAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.document.IntField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.NumericRangeQuery;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.Lock;
+import org.apache.lucene.store.LockFactory;
+import org.apache.lucene.store.RAMDirectory;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public abstract class BaseDirectoryTestSuite {
+  protected static final File TMPDIR = new File(System.getProperty("blur.tmp.dir", "/tmp"));
+
+  protected static final int MAX_NUMBER_OF_WRITES = 10000;
+  protected static final int MIN_FILE_SIZE = 100;
+  protected static final int MAX_FILE_SIZE = 100000;
+  protected static final int MIN_BUFFER_SIZE = 1;
+  protected static final int MAX_BUFFER_SIZE = 5000;
+  protected static final int MAX_NUMBER_OF_READS = 10000;
+  protected Directory directory;
+  protected File file;
+  protected long seed;
+  protected Random random;
+
+  @Before
+  public void setUp() throws IOException {
+    BufferStore.init(128, 128);
+    file = new File(TMPDIR, "hdfsdirectorytest");
+    rm(file);
+    seed = new Random().nextLong();
+    random = new Random(seed);
+    setupDirectory();
+  }
+
+  @After
+  public void tearDown() {
+    print(file, "");
+  }
+
+  private void print(File f, String buf) {
+    if (f.isDirectory()) {
+      System.out.println(buf + "\\" + f.getName());
+      for (File fl : f.listFiles()) {
+        if (fl.getName().startsWith(".")) {
+          continue;
+        }
+        print(fl, buf + " ");
+      }
+    } else {
+      System.out.println(buf + f.getName() + " " + f.length());
+    }
+  }
+
+  protected abstract void setupDirectory() throws IOException;
+
+  @Test
+  public void testWritingAndReadingAFile() throws IOException {
+
+    IndexOutput output = directory.createOutput("testing.test", IOContext.DEFAULT);
+    output.writeInt(12345);
+    output.flush();
+    output.close();
+
+    IndexInput input = directory.openInput("testing.test", IOContext.DEFAULT);
+    assertEquals(12345, input.readInt());
+    input.close();
+
+    String[] listAll = directory.listAll();
+    assertEquals(1, listAll.length);
+    assertEquals("testing.test", listAll[0]);
+
+    assertEquals(4, directory.fileLength("testing.test"));
+
+    IndexInput input1 = directory.openInput("testing.test", IOContext.DEFAULT);
+
+    IndexInput input2 = (IndexInput) input1.clone();
+    assertEquals(12345, input2.readInt());
+    input2.close();
+
+    assertEquals(12345, input1.readInt());
+    input1.close();
+
+    assertFalse(directory.fileExists("testing.test.other"));
+    assertTrue(directory.fileExists("testing.test"));
+    directory.deleteFile("testing.test");
+    assertFalse(directory.fileExists("testing.test"));
+  }
+
+  @Test
+  public void testEOF() throws IOException {
+    Directory fsDir = new RAMDirectory();
+    String name = "test.eof";
+    createFile(name, fsDir, directory);
+    long fsLength = fsDir.fileLength(name);
+    long hdfsLength = directory.fileLength(name);
+    assertEquals(fsLength, hdfsLength);
+    testEof(name, fsDir, fsLength);
+    testEof(name, directory, hdfsLength);
+  }
+
+  private void testEof(String name, Directory directory, long length) throws IOException
{
+    IndexInput input = directory.openInput(name, IOContext.DEFAULT);
+    try {
+      input.seek(length);
+      input.readByte();
+      fail("should throw eof");
+    } catch (IOException e) {
+    }
+  }
+
+  @Test
+  public void testWrites() throws IOException {
+    int i = 0;
+    try {
+      Set<String> names = new HashSet<String>();
+      for (; i < 10; i++) {
+        Directory fsDir = new RAMDirectory();
+        String name = getName();
+        System.out.println("Working on pass [" + i + "] seed [" + seed + "] contains [" +
names.contains(name) + "]");
+        names.add(name);
+        createFile(name, fsDir, directory);
+        assertInputsEquals(name, fsDir, directory);
+        fsDir.close();
+      }
+    } catch (Exception e) {
+      e.printStackTrace();
+      fail("Test failed with seed [" + seed + "] on pass [" + i + "]");
+    }
+  }
+
+  @Test
+  public void testCreateIndex() throws IOException {
+    IndexWriterConfig conf = new IndexWriterConfig(LuceneVersionConstant.LUCENE_VERSION,
new KeywordAnalyzer());
+    IndexWriter writer = new IndexWriter(directory, conf);
+    int numDocs = 1000;
+    DirectoryReader reader = null;
+    for (int i = 0; i < 100; i++) {
+      if (reader == null) {
+        reader = DirectoryReader.open(writer, true);
+      } else {
+        DirectoryReader old = reader;
+        reader = DirectoryReader.openIfChanged(old, writer, true);
+        if (reader == null) {
+          reader = old;
+        } else {
+          old.close();
+        }
+      }
+      assertEquals(i * numDocs, reader.numDocs());
+      IndexSearcher searcher = new IndexSearcher(reader);
+      NumericRangeQuery<Integer> query = NumericRangeQuery.newIntRange("id", 42, 42,
true, true);
+      TopDocs topDocs = searcher.search(query, 10);
+      assertEquals(i, topDocs.totalHits);
+      addDocuments(writer, numDocs);
+    }
+    writer.close(false);
+    reader.close();
+  }
+
+  private void addDocuments(IndexWriter writer, int numDocs) throws IOException {
+    for (int i = 0; i < numDocs; i++) {
+      writer.addDocument(getDoc(i));
+    }
+  }
+
+  private Document getDoc(int i) {
+    Document document = new Document();
+    document.add(new IntField("id", i, Store.YES));
+    return document;
+  }
+
+  private void assertInputsEquals(String name, Directory fsDir, Directory hdfs) throws IOException
{
+    int reads = random.nextInt(MAX_NUMBER_OF_READS);
+    IndexInput fsInput = fsDir.openInput(name, IOContext.DEFAULT);
+    IndexInput hdfsInput = hdfs.openInput(name, IOContext.DEFAULT);
+    assertEquals(fsInput.length(), hdfsInput.length());
+    int fileLength = (int) fsInput.length();
+    for (int i = 0; i < reads; i++) {
+      byte[] fsBuf = new byte[random.nextInt(Math.min(MAX_BUFFER_SIZE - MIN_BUFFER_SIZE,
fileLength)) + MIN_BUFFER_SIZE];
+      byte[] hdfsBuf = new byte[fsBuf.length];
+      int offset = random.nextInt(fsBuf.length);
+      int length = random.nextInt(fsBuf.length - offset);
+      int pos = random.nextInt(fileLength - length);
+      fsInput.seek(pos);
+      fsInput.readBytes(fsBuf, offset, length);
+      hdfsInput.seek(pos);
+      hdfsInput.readBytes(hdfsBuf, offset, length);
+      for (int f = offset; f < length; f++) {
+        if (fsBuf[f] != hdfsBuf[f]) {
+          fail();
+        }
+      }
+    }
+    fsInput.close();
+    hdfsInput.close();
+  }
+
+  private void createFile(String name, Directory fsDir, Directory hdfs) throws IOException
{
+    int writes = random.nextInt(MAX_NUMBER_OF_WRITES);
+    int fileLength = random.nextInt(MAX_FILE_SIZE - MIN_FILE_SIZE) + MIN_FILE_SIZE;
+    IndexOutput fsOutput = fsDir.createOutput(name, IOContext.DEFAULT);
+    fsOutput.setLength(fileLength);
+    IndexOutput hdfsOutput = hdfs.createOutput(name, IOContext.DEFAULT);
+    hdfsOutput.setLength(fileLength);
+    for (int i = 0; i < writes; i++) {
+      byte[] buf = new byte[random.nextInt(Math.min(MAX_BUFFER_SIZE - MIN_BUFFER_SIZE, fileLength))
+ MIN_BUFFER_SIZE];
+      random.nextBytes(buf);
+      int offset = random.nextInt(buf.length);
+      int length = random.nextInt(buf.length - offset);
+      fsOutput.writeBytes(buf, offset, length);
+      hdfsOutput.writeBytes(buf, offset, length);
+    }
+    fsOutput.close();
+    hdfsOutput.close();
+  }
+
+  private String getName() {
+    return Long.toString(Math.abs(random.nextLong()));
+  }
+
+  public static void rm(File file) {
+    if (!file.exists()) {
+      return;
+    }
+    if (file.isDirectory()) {
+      for (File f : file.listFiles()) {
+        rm(f);
+      }
+    }
+    file.delete();
+  }
+
+  public static Directory wrapLastModified(Directory dir) {
+    return new DirectoryLastModified(dir);
+  }
+
+  public static class DirectoryLastModified extends Directory implements LastModified {
+
+    private Directory _directory;
+
+    public DirectoryLastModified(Directory dir) {
+      _directory = dir;
+    }
+
+    @Override
+    public long getFileModified(String name) throws IOException {
+      if (_directory instanceof FSDirectory) {
+        File fileDir = ((FSDirectory) _directory).getDirectory();
+        return new File(fileDir, name).lastModified();
+      }
+      throw new RuntimeException("not impl");
+    }
+
+    @Override
+    public String[] listAll() throws IOException {
+      return _directory.listAll();
+    }
+
+    @Override
+    public void deleteFile(String name) throws IOException {
+      _directory.deleteFile(name);
+    }
+
+    @Override
+    public long fileLength(String name) throws IOException {
+      return _directory.fileLength(name);
+    }
+
+    @Override
+    public IndexOutput createOutput(String name, IOContext context) throws IOException {
+      return _directory.createOutput(name, context);
+    }
+
+    @Override
+    public void sync(Collection<String> names) throws IOException {
+      _directory.sync(names);
+    }
+
+    @Override
+    public IndexInput openInput(String name, IOContext context) throws IOException {
+      return _directory.openInput(name, context);
+    }
+
+    @Override
+    public Lock makeLock(String name) {
+      return _directory.makeLock(name);
+    }
+
+    @Override
+    public void clearLock(String name) throws IOException {
+      _directory.clearLock(name);
+    }
+
+    @Override
+    public void setLockFactory(LockFactory lockFactory) throws IOException {
+      _directory.setLockFactory(lockFactory);
+    }
+
+    @Override
+    public LockFactory getLockFactory() {
+      return _directory.getLockFactory();
+    }
+
+    @Override
+    public String getLockID() {
+      return _directory.getLockID();
+    }
+
+    @Override
+    public void copy(Directory to, String src, String dest, IOContext context) throws IOException
{
+      _directory.copy(to, src, dest, context);
+    }
+
+    @Override
+    public IndexInputSlicer createSlicer(String name, IOContext context) throws IOException
{
+      return _directory.createSlicer(name, context);
+    }
+
+    @Override
+    public boolean fileExists(String name) throws IOException {
+      return _directory.fileExists(name);
+    }
+
+    @Override
+    public void close() throws IOException {
+      _directory.close();
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/890be501/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuite.java
----------------------------------------------------------------------
diff --git a/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuite.java b/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuite.java
new file mode 100644
index 0000000..c2baa1e
--- /dev/null
+++ b/blur-store/src/test/java/org/apache/blur/store/CacheDirectoryTestSuite.java
@@ -0,0 +1,72 @@
+package org.apache.blur.store;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.blur.store.blockcache_v2.BaseCache;
+import org.apache.blur.store.blockcache_v2.BaseCache.STORE;
+import org.apache.blur.store.blockcache_v2.Cache;
+import org.apache.blur.store.blockcache_v2.CacheDirectory;
+import org.apache.blur.store.blockcache_v2.FileNameBlockSize;
+import org.apache.blur.store.blockcache_v2.FileNameFilter;
+import org.apache.blur.store.buffer.BufferStore;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.junit.Test;
+
+public class CacheDirectoryTestSuite extends BaseDirectoryTestSuite {
+
+  @Override
+  protected void setupDirectory() throws IOException {
+    int totalNumberOfBytes = 1000000;
+    int fileBufferSize = 129;
+    final int blockSize = 137;
+    FileNameBlockSize fileNameBlockSize = new FileNameBlockSize() {
+      @Override
+      public int getBlockSize(String directoryName, String fileName) {
+        return blockSize;
+      }
+    };
+    FileNameFilter writeFilter = new FileNameFilter() {
+      @Override
+      public boolean accept(String directoryName, String fileName) {
+        return true;
+      }
+    };
+    FileNameFilter readFilter = new FileNameFilter() {
+      @Override
+      public boolean accept(String directoryName, String fileName) {
+        // @TODO this needs to be enabled and issues resolved...
+        return false;
+      }
+    };
+
+    Cache cache = new BaseCache(totalNumberOfBytes, fileBufferSize, fileNameBlockSize, readFilter,
writeFilter,
+        STORE.ON_HEAP);
+    Directory dir = FSDirectory.open(new File(file, "cache"));
+
+    BufferStore.init(128, 128);
+    directory = new CacheDirectory("test", wrapLastModified(dir), cache);
+  }
+
+  @Test
+  public void runsTheTests() {
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/890be501/blur-store/src/test/java/org/apache/blur/store/HdfsDirectoryTest.java
----------------------------------------------------------------------
diff --git a/blur-store/src/test/java/org/apache/blur/store/HdfsDirectoryTest.java b/blur-store/src/test/java/org/apache/blur/store/HdfsDirectoryTest.java
deleted file mode 100644
index 26501c8..0000000
--- a/blur-store/src/test/java/org/apache/blur/store/HdfsDirectoryTest.java
+++ /dev/null
@@ -1,41 +0,0 @@
-package org.apache.blur.store;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import java.io.File;
-import java.io.IOException;
-import java.net.URI;
-
-import org.apache.blur.store.hdfs.HdfsDirectory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.junit.Test;
-
-public class HdfsDirectoryTest extends BaseDirectoryTest {
-
-  @Override
-  protected void setupDirectory() throws IOException {
-    URI uri = new File(file, "hdfs").toURI();
-    Path hdfsDirPath = new Path(uri.toString());
-    Configuration conf = new Configuration();
-    directory = new HdfsDirectory(conf, hdfsDirPath);
-  }
-
-  @Test
-  public void runsTheTests() {}
-  
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/890be501/blur-store/src/test/java/org/apache/blur/store/HdfsDirectoryTestSuite.java
----------------------------------------------------------------------
diff --git a/blur-store/src/test/java/org/apache/blur/store/HdfsDirectoryTestSuite.java b/blur-store/src/test/java/org/apache/blur/store/HdfsDirectoryTestSuite.java
new file mode 100644
index 0000000..7079c5c
--- /dev/null
+++ b/blur-store/src/test/java/org/apache/blur/store/HdfsDirectoryTestSuite.java
@@ -0,0 +1,41 @@
+package org.apache.blur.store;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+
+import org.apache.blur.store.hdfs.HdfsDirectory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.junit.Test;
+
+public class HdfsDirectoryTestSuite extends BaseDirectoryTestSuite {
+
+  @Override
+  protected void setupDirectory() throws IOException {
+    URI uri = new File(file, "hdfs").toURI();
+    Path hdfsDirPath = new Path(uri.toString());
+    Configuration conf = new Configuration();
+    directory = new HdfsDirectory(conf, hdfsDirPath);
+  }
+
+  @Test
+  public void runsTheTests() {}
+  
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/890be501/blur-store/src/test/java/org/apache/blur/store/SoftlinkHdfsDirectoryTest.java
----------------------------------------------------------------------
diff --git a/blur-store/src/test/java/org/apache/blur/store/SoftlinkHdfsDirectoryTest.java
b/blur-store/src/test/java/org/apache/blur/store/SoftlinkHdfsDirectoryTest.java
deleted file mode 100644
index 0ac2735..0000000
--- a/blur-store/src/test/java/org/apache/blur/store/SoftlinkHdfsDirectoryTest.java
+++ /dev/null
@@ -1,43 +0,0 @@
-package org.apache.blur.store;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import java.io.File;
-import java.io.IOException;
-import java.net.URI;
-
-import org.apache.blur.store.hdfs.SoftlinkHdfsDirectory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.junit.Test;
-
-public class SoftlinkHdfsDirectoryTest extends BaseDirectoryTest {
-
-  @Override
-  protected void setupDirectory() throws IOException {
-    URI uri = new File(file, "hdfs").toURI();
-    Path hdfsDirPath = new Path(uri.toString());
-    Path store = new Path(hdfsDirPath, "store");
-    Path link = new Path(hdfsDirPath, "link");
-    Configuration conf = new Configuration();
-    directory = new SoftlinkHdfsDirectory(conf, store, link);
-  }
-
-  @Test
-  public void runsTheTests() {}
-  
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/890be501/blur-store/src/test/java/org/apache/blur/store/SoftlinkHdfsDirectoryTestSuite.java
----------------------------------------------------------------------
diff --git a/blur-store/src/test/java/org/apache/blur/store/SoftlinkHdfsDirectoryTestSuite.java
b/blur-store/src/test/java/org/apache/blur/store/SoftlinkHdfsDirectoryTestSuite.java
new file mode 100644
index 0000000..d6648cf
--- /dev/null
+++ b/blur-store/src/test/java/org/apache/blur/store/SoftlinkHdfsDirectoryTestSuite.java
@@ -0,0 +1,43 @@
+package org.apache.blur.store;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+
+import org.apache.blur.store.hdfs.SoftlinkHdfsDirectory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.junit.Test;
+
+public class SoftlinkHdfsDirectoryTestSuite extends BaseDirectoryTestSuite {
+
+  @Override
+  protected void setupDirectory() throws IOException {
+    URI uri = new File(file, "hdfs").toURI();
+    Path hdfsDirPath = new Path(uri.toString());
+    Path store = new Path(hdfsDirPath, "store");
+    Path link = new Path(hdfsDirPath, "link");
+    Configuration conf = new Configuration();
+    directory = new SoftlinkHdfsDirectory(conf, store, link);
+  }
+
+  @Test
+  public void runsTheTests() {}
+  
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/890be501/blur-store/src/test/java/org/apache/blur/store/blockcache/BlockCacheTest.java
----------------------------------------------------------------------
diff --git a/blur-store/src/test/java/org/apache/blur/store/blockcache/BlockCacheTest.java
b/blur-store/src/test/java/org/apache/blur/store/blockcache/BlockCacheTest.java
index b1d9d12..0699189 100644
--- a/blur-store/src/test/java/org/apache/blur/store/blockcache/BlockCacheTest.java
+++ b/blur-store/src/test/java/org/apache/blur/store/blockcache/BlockCacheTest.java
@@ -116,9 +116,10 @@ public class BlockCacheTest {
 
   /**
    * Verify checking of buffer size limits against the cached block size.
+   * @throws IOException 
    */
   @Test
-  public void testLongBuffer() {
+  public void testLongBuffer() throws IOException {
     Random random = new Random();
     int blockSize = BlockCache._8K;
     int slabSize = blockSize * 1024;
@@ -138,6 +139,7 @@ public class BlockCacheTest {
     assertTrue(blockCache.store(blockCacheKey, 1, testData, 0, blockSize - 1));
     assertTrue(blockCache.store(blockCacheKey, 1, testData, blockSize, blockSize - 1));
     assertTrue(blockCache.store(blockCacheKey, 1, testData, blockSize * 2, blockSize - 1));
+    blockCache.close(); 
   }
 
   private static byte[] testData(Random random, int size, byte[] buf) {


Mime
View raw message