asterixdb-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Murtadha Hubail (Code Review)" <do-not-re...@asterix-gerrit.ics.uci.edu>
Subject Change in hyracks[master]: Add temp workspace files deletion to IOManager
Date Fri, 31 Jul 2015 14:46:58 GMT
Murtadha Hubail has uploaded a new change for review.

  https://asterix-gerrit.ics.uci.edu/337

Change subject: Add temp workspace files deletion to IOManager
......................................................................

Add temp workspace files deletion to IOManager

This change includes the following:
- Add a method to delete temp workspace files (WAF)
- Expose LSMComponents files suffixes to Asterix

Change-Id: I760074764755e7aee100ff33c14b13bf4b29ec2e
---
M hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/io/IIOManager.java
M hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/io/IOManager.java
M hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
M hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java
M hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexReplicationJob.java
M hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
M hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMIndexReplicationJob.java
M hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java
M hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java
9 files changed, 60 insertions(+), 20 deletions(-)


  git pull ssh://asterix-gerrit.ics.uci.edu:29418/hyracks refs/changes/37/337/1

diff --git a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/io/IIOManager.java
b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/io/IIOManager.java
index 1fd5a93..c4ab769 100644
--- a/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/io/IIOManager.java
+++ b/hyracks/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/io/IIOManager.java
@@ -50,4 +50,6 @@
     public void sync(IFileHandle fileHandle, boolean metadata) throws HyracksDataException;
 
     public void setExecutor(Executor executor);
+
+    public void deleteWorkspaceFiles();
 }
\ No newline at end of file
diff --git a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/io/IOManager.java
b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/io/IOManager.java
index b3f3e2a..45a572f 100644
--- a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/io/IOManager.java
+++ b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/io/IOManager.java
@@ -15,6 +15,7 @@
 package edu.uci.ics.hyracks.control.nc.io;
 
 import java.io.File;
+import java.io.FilenameFilter;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
@@ -31,12 +32,10 @@
 import edu.uci.ics.hyracks.api.io.IODeviceHandle;
 
 public class IOManager implements IIOManager {
+    private static final String WORKSPACE_FILE_SUFFIX = ".waf";
     private final List<IODeviceHandle> ioDevices;
-
     private Executor executor;
-
     private final List<IODeviceHandle> workAreaIODevices;
-
     private int workAreaDeviceIndex;
 
     public IOManager(List<IODeviceHandle> devices, Executor executor) throws HyracksException
{
@@ -163,7 +162,8 @@
         String waPath = dev.getWorkAreaPath();
         File waf;
         try {
-            waf = File.createTempFile(prefix, ".waf", new File(dev.getPath(), waPath));
+            waf = File.createTempFile(prefix, WORKSPACE_FILE_SUFFIX, new File(dev.getPath(),
waPath));
+            System.out.println("Dev Path: " + dev.getPath() + " - waPath: " + waPath);
         } catch (IOException e) {
             throw new HyracksDataException(e);
         }
@@ -252,4 +252,27 @@
             throw new HyracksDataException(e);
         }
     }
+
+    @Override
+    public void deleteWorkspaceFiles() {
+        for (IODeviceHandle ioDevice : workAreaIODevices) {
+            File workspaceFolder = new File(ioDevice.getPath(), ioDevice.getWorkAreaPath());
+            if (workspaceFolder.exists() && workspaceFolder.isDirectory()) {
+                File[] workspaceFiles = workspaceFolder.listFiles(WORKSPACE_FILES_FILTER);
+                for (File workspaceFile : workspaceFiles) {
+                    workspaceFile.delete();
+                }
+            }
+        }
+    }
+
+    private static final FilenameFilter WORKSPACE_FILES_FILTER = new FilenameFilter() {
+        public boolean accept(File dir, String name) {
+            if (name.endsWith(WORKSPACE_FILE_SUFFIX)) {
+                return true;
+            } else {
+                return false;
+            }
+        }
+    };
 }
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
index 41a49b5..48b5b70 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
@@ -36,7 +36,7 @@
 import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
 
 public class LSMBTreeFileManager extends AbstractLSMIndexFileManager {
-    private static final String BTREE_STRING = "b";
+    public static final String BTREE_STRING = "b";
 
     private final TreeIndexFactory<? extends ITreeIndex> btreeFactory;
 
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java
b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java
index cb7dca9..c69cddd 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java
@@ -35,8 +35,8 @@
 import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
 
 public class LSMBTreeWithBuddyFileManager extends AbstractLSMIndexFileManager {
-    private static final String BUDDY_BTREE_STRING = "buddy";
-    private static final String BTREE_STRING = "b";
+    public static final String BUDDY_BTREE_STRING = "buddy";
+    public static final String BTREE_STRING = "b";
 
     private final TreeIndexFactory<? extends ITreeIndex> btreeFactory;
     private final TreeIndexFactory<? extends ITreeIndex> buddyBtreeFactory;
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexReplicationJob.java
b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexReplicationJob.java
index 1791084..2a54687 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexReplicationJob.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/api/ILSMIndexReplicationJob.java
@@ -16,8 +16,13 @@
 
 import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
 import edu.uci.ics.hyracks.api.replication.IReplicationJob;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex;
 
 public interface ILSMIndexReplicationJob extends IReplicationJob {
 
     public void endReplication() throws HyracksDataException;
+
+    public ILSMIndexOperationContext getLSMIndexOperationContext();
+
+    public AbstractLSMIndex getLSMIndex();
 }
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
index 4a771ea..553b68c 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
@@ -42,7 +42,7 @@
 
 public abstract class AbstractLSMIndexFileManager implements ILSMIndexFileManager {
 
-    protected static final String SPLIT_STRING = "_";
+    public static final String SPLIT_STRING = "_";
     protected static final String BLOOM_FILTER_STRING = "f";
     protected static final String TRANSACTION_PREFIX = ".T";
 
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMIndexReplicationJob.java
b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMIndexReplicationJob.java
index 232bfb2..a32acf5 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMIndexReplicationJob.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/common/impls/LSMIndexReplicationJob.java
@@ -24,19 +24,29 @@
 public class LSMIndexReplicationJob extends AbstractReplicationJob implements ILSMIndexReplicationJob
{
 
     private final AbstractLSMIndex lsmIndex;
-    private final ILSMIndexOperationContext ctx;
-
-    public LSMIndexReplicationJob(AbstractLSMIndex lsmIndex, ILSMIndexOperationContext ctx,
+    private final ILSMIndexOperationContext operationContext;
+    
+    public LSMIndexReplicationJob(AbstractLSMIndex lsmIndex, ILSMIndexOperationContext operationContext,
             Set<String> filesToReplicate, ReplicationOperation operation, ReplicationExecutionType
executionType) {
         super(ReplicationJobType.LSM_COMPONENT, operation, executionType, filesToReplicate);
         this.lsmIndex = lsmIndex;
-        this.ctx = ctx;
+        this.operationContext = operationContext;
     }
-
+    
     @Override
     public void endReplication() throws HyracksDataException {
-        if (ctx != null) {
-            lsmIndex.lsmHarness.endReplication(ctx);
+        if (operationContext != null) {
+            lsmIndex.lsmHarness.endReplication(operationContext);
         }
     }
+    
+    @Override
+    public AbstractLSMIndex getLSMIndex(){
+        return lsmIndex;
+    }
+    
+    @Override
+    public ILSMIndexOperationContext getLSMIndexOperationContext(){
+        return operationContext;
+    }
 }
diff --git a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java
b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java
index c4398b7..9ddd59c 100644
--- a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java
+++ b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java
@@ -34,9 +34,9 @@
 
 // TODO: Refactor for better code sharing with other file managers.
 public class LSMInvertedIndexFileManager extends AbstractLSMIndexFileManager implements IInvertedIndexFileNameMapper
{
-    private static final String DICT_BTREE_SUFFIX = "b";
-    private static final String INVLISTS_SUFFIX = "i";
-    private static final String DELETED_KEYS_BTREE_SUFFIX = "d";
+    public static final String DICT_BTREE_SUFFIX = "b";
+    public static final String INVLISTS_SUFFIX = "i";
+    public static final String DELETED_KEYS_BTREE_SUFFIX = "d";
 
     // We only need a BTree factory because the inverted indexes consistency is validated
against its dictionary BTree.
     private final BTreeFactory btreeFactory;
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java
b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java
index 7e3fb06..9832dfc 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java
@@ -36,8 +36,8 @@
 import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
 
 public class LSMRTreeFileManager extends AbstractLSMIndexFileManager {
-    private static final String RTREE_STRING = "r";
-    private static final String BTREE_STRING = "b";
+    public static final String RTREE_STRING = "r";
+    public static final String BTREE_STRING = "b";
 
     private final TreeIndexFactory<? extends ITreeIndex> rtreeFactory;
     private final TreeIndexFactory<? extends ITreeIndex> btreeFactory;

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/337
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I760074764755e7aee100ff33c14b13bf4b29ec2e
Gerrit-PatchSet: 1
Gerrit-Project: hyracks
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail <hubailmor@gmail.com>

Mime
View raw message