incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject git commit: Moved a directory and supporting class to the blur-store project.
Date Tue, 30 Oct 2012 14:05:11 GMT
Updated Branches:
  refs/heads/0.2-dev fdfd698a2 -> e50119e41


Moved a directory and supporting class to the blur-store project.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/e50119e4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/e50119e4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/e50119e4

Branch: refs/heads/0.2-dev
Commit: e50119e418be89ea03998981f6af6afcd22e11ce
Parents: fdfd698
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Tue Oct 30 10:04:32 2012 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Tue Oct 30 10:04:32 2012 -0400

----------------------------------------------------------------------
 .../indexserver/DistributedIndexServer.java        |    2 +-
 .../apache/blur/manager/writer/BlurNRTIndex.java   |    2 +
 .../manager/writer/DirectoryReferenceCounter.java  |  269 ---------------
 .../manager/writer/DirectoryReferenceFileGC.java   |   98 ------
 .../writer/DirectoryReferenceCounterTest.java      |    2 +
 .../blur/lucene/DirectoryReferenceCounter.java     |  269 +++++++++++++++
 .../blur/lucene/DirectoryReferenceFileGC.java      |   98 ++++++
 7 files changed, 372 insertions(+), 368 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e50119e4/src/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
b/src/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
index 8bbb75b..e347f5f 100644
--- a/src/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
+++ b/src/blur-core/src/main/java/org/apache/blur/manager/indexserver/DistributedIndexServer.java
@@ -42,6 +42,7 @@ import org.apache.blur.analysis.BlurAnalyzer;
 import org.apache.blur.concurrent.Executors;
 import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
+import org.apache.blur.lucene.DirectoryReferenceFileGC;
 import org.apache.blur.lucene.search.FairSimilarity;
 import org.apache.blur.manager.BlurFilterCache;
 import org.apache.blur.manager.clusterstatus.ClusterStatus;
@@ -51,7 +52,6 @@ import org.apache.blur.manager.writer.BlurIndexCloser;
 import org.apache.blur.manager.writer.BlurIndexReader;
 import org.apache.blur.manager.writer.BlurIndexRefresher;
 import org.apache.blur.manager.writer.BlurNRTIndex;
-import org.apache.blur.manager.writer.DirectoryReferenceFileGC;
 import org.apache.blur.metrics.BlurMetrics;
 import org.apache.blur.store.blockcache.BlockDirectory;
 import org.apache.blur.store.blockcache.Cache;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e50119e4/src/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
b/src/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
index 2f6e0c0..9c0f355 100644
--- a/src/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
+++ b/src/blur-core/src/main/java/org/apache/blur/manager/writer/BlurNRTIndex.java
@@ -28,6 +28,8 @@ import org.apache.blur.analysis.BlurAnalyzer;
 import org.apache.blur.index.IndexWriter;
 import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
+import org.apache.blur.lucene.DirectoryReferenceCounter;
+import org.apache.blur.lucene.DirectoryReferenceFileGC;
 import org.apache.blur.thrift.generated.Record;
 import org.apache.blur.thrift.generated.Row;
 import org.apache.hadoop.conf.Configuration;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e50119e4/src/blur-core/src/main/java/org/apache/blur/manager/writer/DirectoryReferenceCounter.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/manager/writer/DirectoryReferenceCounter.java
b/src/blur-core/src/main/java/org/apache/blur/manager/writer/DirectoryReferenceCounter.java
deleted file mode 100644
index 56f9caf..0000000
--- a/src/blur-core/src/main/java/org/apache/blur/manager/writer/DirectoryReferenceCounter.java
+++ /dev/null
@@ -1,269 +0,0 @@
-package org.apache.blur.manager.writer;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.blur.log.Log;
-import org.apache.blur.log.LogFactory;
-import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.store.Lock;
-import org.apache.lucene.store.LockFactory;
-
-public class DirectoryReferenceCounter extends Directory {
-
-  private final static Log LOG = LogFactory.getLog(DirectoryReferenceCounter.class);
-  private Directory directory;
-  private Map<String, AtomicInteger> refs = new ConcurrentHashMap<String, AtomicInteger>();
-  private DirectoryReferenceFileGC gc;
-
-  public DirectoryReferenceCounter(Directory directory, DirectoryReferenceFileGC gc) {
-    this.directory = directory;
-    this.gc = gc;
-  }
-
-  public void deleteFile(String name) throws IOException {
-    if (name.equals(IndexFileNames.SEGMENTS_GEN)) {
-      deleteFile(name);
-      return;
-    }
-    AtomicInteger counter = refs.get(name);
-    if (counter != null && counter.get() > 0) {
-      addToFileGC(name);
-    } else {
-      LOG.debug("Delete file [{0}]", name);
-      directory.deleteFile(name);
-    }
-  }
-
-  @Override
-  public IndexOutput createOutput(String name, IOContext context) throws IOException {
-    if (name.equals(IndexFileNames.SEGMENTS_GEN)) {
-      return directory.createOutput(name, context);
-    }
-    LOG.debug("Create file [{0}]", name);
-    AtomicInteger counter = refs.get(name);
-    if (counter != null) {
-      LOG.error("Unknown error while trying to create ref counter for [{0}] reference exists.",
name);
-      throw new IOException("Reference exists [" + name + "]");
-    }
-    counter = new AtomicInteger(0);
-    refs.put(name, counter);
-    return directory.createOutput(name, context);
-  }
-
-  @Override
-  public IndexInput openInput(String name, IOContext context) throws IOException {
-    IndexInput input = directory.openInput(name, context);
-    if (name.equals(IndexFileNames.SEGMENTS_GEN)) {
-      return input;
-    }
-    return wrap(name, input);
-  }
-
-  public static class RefIndexInput extends IndexInput {
-
-    private IndexInput input;
-    private AtomicInteger ref;
-    private boolean closed = false;
-
-    public RefIndexInput(String resourceDescription, IndexInput input, AtomicInteger ref)
{
-      super(resourceDescription);
-      this.input = input;
-      this.ref = ref;
-      ref.incrementAndGet();
-    }
-
-    @Override
-    protected void finalize() throws Throwable {
-      // Seems like not all the clones are being closed...
-      close();
-    }
-
-    @Override
-    public RefIndexInput clone() {
-      RefIndexInput ref = (RefIndexInput) super.clone();
-      ref.input = (IndexInput) input.clone();
-      ref.ref.incrementAndGet();
-      return ref;
-    }
-
-    @Override
-    public void close() throws IOException {
-      if (!closed) {
-        input.close();
-        ref.decrementAndGet();
-        closed = true;
-      }
-    }
-
-    @Override
-    public short readShort() throws IOException {
-      return input.readShort();
-    }
-
-    @Override
-    public int readInt() throws IOException {
-      return input.readInt();
-    }
-
-    @Override
-    public void seek(long pos) throws IOException {
-      input.seek(pos);
-    }
-
-    @Override
-    public int readVInt() throws IOException {
-      return input.readVInt();
-    }
-
-    @Override
-    public String toString() {
-      return input.toString();
-    }
-
-    @Override
-    public long readLong() throws IOException {
-      return input.readLong();
-    }
-
-    @Override
-    public long readVLong() throws IOException {
-      return input.readVLong();
-    }
-
-    @Override
-    public String readString() throws IOException {
-      return input.readString();
-    }
-
-    @Override
-    public long getFilePointer() {
-      return input.getFilePointer();
-    }
-
-    @Override
-    public byte readByte() throws IOException {
-      return input.readByte();
-    }
-
-    @Override
-    public void readBytes(byte[] b, int offset, int len) throws IOException {
-      input.readBytes(b, offset, len);
-    }
-
-    @Override
-    public void readBytes(byte[] b, int offset, int len, boolean useBuffer) throws IOException
{
-      input.readBytes(b, offset, len, useBuffer);
-    }
-
-    @Override
-    public long length() {
-      return input.length();
-    }
-
-    @Override
-    public Map<String, String> readStringStringMap() throws IOException {
-      return input.readStringStringMap();
-    }
-
-  }
-
-  @Override
-  public void sync(Collection<String> names) throws IOException {
-    directory.sync(names);
-  }
-
-  @Override
-  public void clearLock(String name) throws IOException {
-    directory.clearLock(name);
-  }
-
-  @Override
-  public void close() throws IOException {
-    directory.close();
-  }
-
-  @Override
-  public void setLockFactory(LockFactory lockFactory) throws IOException {
-    directory.setLockFactory(lockFactory);
-  }
-
-  @Override
-  public String getLockID() {
-    return directory.getLockID();
-  }
-
-  @Override
-  public void copy(Directory to, String src, String dest, IOContext context) throws IOException
{
-    directory.copy(to, src, dest, context);
-  }
-
-  @Override
-  public boolean fileExists(String name) throws IOException {
-    return directory.fileExists(name);
-  }
-
-  @Override
-  public long fileLength(String name) throws IOException {
-    return directory.fileLength(name);
-  }
-
-  @Override
-  public LockFactory getLockFactory() {
-    return directory.getLockFactory();
-  }
-
-  @Override
-  public String[] listAll() throws IOException {
-    return directory.listAll();
-  }
-
-  @Override
-  public Lock makeLock(String name) {
-    return directory.makeLock(name);
-  }
-
-  @Override
-  public String toString() {
-    return directory.toString();
-  }
-
-  private void addToFileGC(String name) {
-    if (gc != null) {
-      LOG.debug("Add file [{0}] to be GCed once refs are closed.", name);
-      gc.add(directory, name, refs);
-    }
-  }
-
-  private IndexInput wrap(String name, IndexInput input) {
-    AtomicInteger counter = refs.get(name);
-    if (counter == null) {
-      counter = new AtomicInteger();
-      refs.put(name, counter);
-    }
-    return new RefIndexInput(input.toString(), input, counter);
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e50119e4/src/blur-core/src/main/java/org/apache/blur/manager/writer/DirectoryReferenceFileGC.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/manager/writer/DirectoryReferenceFileGC.java
b/src/blur-core/src/main/java/org/apache/blur/manager/writer/DirectoryReferenceFileGC.java
deleted file mode 100644
index 0aa64f4..0000000
--- a/src/blur-core/src/main/java/org/apache/blur/manager/writer/DirectoryReferenceFileGC.java
+++ /dev/null
@@ -1,98 +0,0 @@
-package org.apache.blur.manager.writer;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import java.io.IOException;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Timer;
-import java.util.TimerTask;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.blur.log.Log;
-import org.apache.blur.log.LogFactory;
-import org.apache.lucene.store.Directory;
-
-
-public class DirectoryReferenceFileGC extends TimerTask {
-
-  private static final Log LOG = LogFactory.getLog(DirectoryReferenceFileGC.class);
-
-  private Timer _timer;
-  private long _delay = 5000;
-  private LinkedBlockingQueue<Value> _queue;
-
-  public static class Value {
-    public Value(Directory directory, String name, Map<String, AtomicInteger> refs)
{
-      this.directory = directory;
-      this.name = name;
-      this.refs = refs;
-    }
-
-    Directory directory;
-    String name;
-    Map<String, AtomicInteger> refs;
-
-    public boolean tryToDelete() throws IOException {
-      AtomicInteger counter = refs.get(name);
-      if (counter.get() <= 0) {
-        refs.remove(name);
-        LOG.debug("Removing file [{0}]", name);
-        directory.deleteFile(name);
-        return true;
-      } else {
-        LOG.debug("File [{0}] had too many refs [{1}]", name, counter.get());
-      }
-      return false;
-    }
-  }
-
-  public void init() {
-    _timer = new Timer("Blur-File-GC", true);
-    _timer.scheduleAtFixedRate(this, _delay, _delay);
-    _queue = new LinkedBlockingQueue<Value>();
-  }
-
-  public void add(Directory directory, String name, Map<String, AtomicInteger> refs)
{
-    try {
-      _queue.put(new Value(directory, name, refs));
-    } catch (InterruptedException e) {
-      throw new RuntimeException(e);
-    }
-  }
-
-  public void close() {
-    _timer.purge();
-    _timer.cancel();
-  }
-
-  @Override
-  public void run() {
-    Iterator<Value> iterator = _queue.iterator();
-    while (iterator.hasNext()) {
-      Value value = iterator.next();
-      try {
-        if (value.tryToDelete()) {
-          iterator.remove();
-        }
-      } catch (IOException e) {
-        LOG.error("Unknown error", e);
-      }
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e50119e4/src/blur-core/src/test/java/org/apache/blur/manager/writer/DirectoryReferenceCounterTest.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/test/java/org/apache/blur/manager/writer/DirectoryReferenceCounterTest.java
b/src/blur-core/src/test/java/org/apache/blur/manager/writer/DirectoryReferenceCounterTest.java
index c0106bf..3ee6629 100644
--- a/src/blur-core/src/test/java/org/apache/blur/manager/writer/DirectoryReferenceCounterTest.java
+++ b/src/blur-core/src/test/java/org/apache/blur/manager/writer/DirectoryReferenceCounterTest.java
@@ -27,6 +27,8 @@ import java.util.Collection;
 import java.util.Map;
 import java.util.concurrent.LinkedBlockingQueue;
 
+import org.apache.blur.lucene.DirectoryReferenceCounter;
+import org.apache.blur.lucene.DirectoryReferenceFileGC;
 import org.apache.lucene.analysis.core.KeywordAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e50119e4/src/blur-store/src/main/java/org/apache/blur/lucene/DirectoryReferenceCounter.java
----------------------------------------------------------------------
diff --git a/src/blur-store/src/main/java/org/apache/blur/lucene/DirectoryReferenceCounter.java
b/src/blur-store/src/main/java/org/apache/blur/lucene/DirectoryReferenceCounter.java
new file mode 100644
index 0000000..68a9487
--- /dev/null
+++ b/src/blur-store/src/main/java/org/apache/blur/lucene/DirectoryReferenceCounter.java
@@ -0,0 +1,269 @@
+package org.apache.blur.lucene;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.blur.log.Log;
+import org.apache.blur.log.LogFactory;
+import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.Lock;
+import org.apache.lucene.store.LockFactory;
+
+public class DirectoryReferenceCounter extends Directory {
+
+  private final static Log LOG = LogFactory.getLog(DirectoryReferenceCounter.class);
+  private Directory directory;
+  private Map<String, AtomicInteger> refs = new ConcurrentHashMap<String, AtomicInteger>();
+  private DirectoryReferenceFileGC gc;
+
+  public DirectoryReferenceCounter(Directory directory, DirectoryReferenceFileGC gc) {
+    this.directory = directory;
+    this.gc = gc;
+  }
+
+  public void deleteFile(String name) throws IOException {
+    if (name.equals(IndexFileNames.SEGMENTS_GEN)) {
+      deleteFile(name);
+      return;
+    }
+    AtomicInteger counter = refs.get(name);
+    if (counter != null && counter.get() > 0) {
+      addToFileGC(name);
+    } else {
+      LOG.debug("Delete file [{0}]", name);
+      directory.deleteFile(name);
+    }
+  }
+
+  @Override
+  public IndexOutput createOutput(String name, IOContext context) throws IOException {
+    if (name.equals(IndexFileNames.SEGMENTS_GEN)) {
+      return directory.createOutput(name, context);
+    }
+    LOG.debug("Create file [{0}]", name);
+    AtomicInteger counter = refs.get(name);
+    if (counter != null) {
+      LOG.error("Unknown error while trying to create ref counter for [{0}] reference exists.",
name);
+      throw new IOException("Reference exists [" + name + "]");
+    }
+    counter = new AtomicInteger(0);
+    refs.put(name, counter);
+    return directory.createOutput(name, context);
+  }
+
+  @Override
+  public IndexInput openInput(String name, IOContext context) throws IOException {
+    IndexInput input = directory.openInput(name, context);
+    if (name.equals(IndexFileNames.SEGMENTS_GEN)) {
+      return input;
+    }
+    return wrap(name, input);
+  }
+
+  public static class RefIndexInput extends IndexInput {
+
+    private IndexInput input;
+    private AtomicInteger ref;
+    private boolean closed = false;
+
+    public RefIndexInput(String resourceDescription, IndexInput input, AtomicInteger ref)
{
+      super(resourceDescription);
+      this.input = input;
+      this.ref = ref;
+      ref.incrementAndGet();
+    }
+
+    @Override
+    protected void finalize() throws Throwable {
+      // Seems like not all the clones are being closed...
+      close();
+    }
+
+    @Override
+    public RefIndexInput clone() {
+      RefIndexInput ref = (RefIndexInput) super.clone();
+      ref.input = (IndexInput) input.clone();
+      ref.ref.incrementAndGet();
+      return ref;
+    }
+
+    @Override
+    public void close() throws IOException {
+      if (!closed) {
+        input.close();
+        ref.decrementAndGet();
+        closed = true;
+      }
+    }
+
+    @Override
+    public short readShort() throws IOException {
+      return input.readShort();
+    }
+
+    @Override
+    public int readInt() throws IOException {
+      return input.readInt();
+    }
+
+    @Override
+    public void seek(long pos) throws IOException {
+      input.seek(pos);
+    }
+
+    @Override
+    public int readVInt() throws IOException {
+      return input.readVInt();
+    }
+
+    @Override
+    public String toString() {
+      return input.toString();
+    }
+
+    @Override
+    public long readLong() throws IOException {
+      return input.readLong();
+    }
+
+    @Override
+    public long readVLong() throws IOException {
+      return input.readVLong();
+    }
+
+    @Override
+    public String readString() throws IOException {
+      return input.readString();
+    }
+
+    @Override
+    public long getFilePointer() {
+      return input.getFilePointer();
+    }
+
+    @Override
+    public byte readByte() throws IOException {
+      return input.readByte();
+    }
+
+    @Override
+    public void readBytes(byte[] b, int offset, int len) throws IOException {
+      input.readBytes(b, offset, len);
+    }
+
+    @Override
+    public void readBytes(byte[] b, int offset, int len, boolean useBuffer) throws IOException
{
+      input.readBytes(b, offset, len, useBuffer);
+    }
+
+    @Override
+    public long length() {
+      return input.length();
+    }
+
+    @Override
+    public Map<String, String> readStringStringMap() throws IOException {
+      return input.readStringStringMap();
+    }
+
+  }
+
+  @Override
+  public void sync(Collection<String> names) throws IOException {
+    directory.sync(names);
+  }
+
+  @Override
+  public void clearLock(String name) throws IOException {
+    directory.clearLock(name);
+  }
+
+  @Override
+  public void close() throws IOException {
+    directory.close();
+  }
+
+  @Override
+  public void setLockFactory(LockFactory lockFactory) throws IOException {
+    directory.setLockFactory(lockFactory);
+  }
+
+  @Override
+  public String getLockID() {
+    return directory.getLockID();
+  }
+
+  @Override
+  public void copy(Directory to, String src, String dest, IOContext context) throws IOException
{
+    directory.copy(to, src, dest, context);
+  }
+
+  @Override
+  public boolean fileExists(String name) throws IOException {
+    return directory.fileExists(name);
+  }
+
+  @Override
+  public long fileLength(String name) throws IOException {
+    return directory.fileLength(name);
+  }
+
+  @Override
+  public LockFactory getLockFactory() {
+    return directory.getLockFactory();
+  }
+
+  @Override
+  public String[] listAll() throws IOException {
+    return directory.listAll();
+  }
+
+  @Override
+  public Lock makeLock(String name) {
+    return directory.makeLock(name);
+  }
+
+  @Override
+  public String toString() {
+    return directory.toString();
+  }
+
+  private void addToFileGC(String name) {
+    if (gc != null) {
+      LOG.debug("Add file [{0}] to be GCed once refs are closed.", name);
+      gc.add(directory, name, refs);
+    }
+  }
+
+  private IndexInput wrap(String name, IndexInput input) {
+    AtomicInteger counter = refs.get(name);
+    if (counter == null) {
+      counter = new AtomicInteger();
+      refs.put(name, counter);
+    }
+    return new RefIndexInput(input.toString(), input, counter);
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/e50119e4/src/blur-store/src/main/java/org/apache/blur/lucene/DirectoryReferenceFileGC.java
----------------------------------------------------------------------
diff --git a/src/blur-store/src/main/java/org/apache/blur/lucene/DirectoryReferenceFileGC.java
b/src/blur-store/src/main/java/org/apache/blur/lucene/DirectoryReferenceFileGC.java
new file mode 100644
index 0000000..b73efd7
--- /dev/null
+++ b/src/blur-store/src/main/java/org/apache/blur/lucene/DirectoryReferenceFileGC.java
@@ -0,0 +1,98 @@
+package org.apache.blur.lucene;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.blur.log.Log;
+import org.apache.blur.log.LogFactory;
+import org.apache.lucene.store.Directory;
+
+
+public class DirectoryReferenceFileGC extends TimerTask {
+
+  private static final Log LOG = LogFactory.getLog(DirectoryReferenceFileGC.class);
+
+  private Timer _timer;
+  private long _delay = 5000;
+  private LinkedBlockingQueue<Value> _queue;
+
+  public static class Value {
+    public Value(Directory directory, String name, Map<String, AtomicInteger> refs)
{
+      this.directory = directory;
+      this.name = name;
+      this.refs = refs;
+    }
+
+    Directory directory;
+    String name;
+    Map<String, AtomicInteger> refs;
+
+    public boolean tryToDelete() throws IOException {
+      AtomicInteger counter = refs.get(name);
+      if (counter.get() <= 0) {
+        refs.remove(name);
+        LOG.debug("Removing file [{0}]", name);
+        directory.deleteFile(name);
+        return true;
+      } else {
+        LOG.debug("File [{0}] had too many refs [{1}]", name, counter.get());
+      }
+      return false;
+    }
+  }
+
+  public void init() {
+    _timer = new Timer("Blur-File-GC", true);
+    _timer.scheduleAtFixedRate(this, _delay, _delay);
+    _queue = new LinkedBlockingQueue<Value>();
+  }
+
+  public void add(Directory directory, String name, Map<String, AtomicInteger> refs)
{
+    try {
+      _queue.put(new Value(directory, name, refs));
+    } catch (InterruptedException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  public void close() {
+    _timer.purge();
+    _timer.cancel();
+  }
+
+  @Override
+  public void run() {
+    Iterator<Value> iterator = _queue.iterator();
+    while (iterator.hasNext()) {
+      Value value = iterator.next();
+      try {
+        if (value.tryToDelete()) {
+          iterator.remove();
+        }
+      } catch (IOException e) {
+        LOG.error("Unknown error", e);
+      }
+    }
+  }
+}


Mime
View raw message