asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amo...@apache.org
Subject [09/12] asterixdb git commit: Cleanup FileSplit and FileReference
Date Thu, 24 Nov 2016 06:30:13 GMT
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/PigletFileDataSink.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/PigletFileDataSink.java b/hyracks-fullstack/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/PigletFileDataSink.java
index d159f60..d0d94ac 100644
--- a/hyracks-fullstack/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/PigletFileDataSink.java
+++ b/hyracks-fullstack/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/PigletFileDataSink.java
@@ -22,7 +22,7 @@ import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSink;
 import org.apache.hyracks.algebricks.core.algebra.properties.FileSplitDomain;
 import org.apache.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
 import org.apache.hyracks.algebricks.core.algebra.properties.RandomPartitioningProperty;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
+import org.apache.hyracks.api.io.FileSplit;
 
 public class PigletFileDataSink implements IDataSink {
     private String file;

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/PigletFileDataSource.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/PigletFileDataSource.java b/hyracks-fullstack/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/PigletFileDataSource.java
index cb62a34..6a675fc 100644
--- a/hyracks-fullstack/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/PigletFileDataSource.java
+++ b/hyracks-fullstack/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/PigletFileDataSource.java
@@ -31,7 +31,7 @@ import org.apache.hyracks.algebricks.core.algebra.properties.INodeDomain;
 import org.apache.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
 import org.apache.hyracks.algebricks.core.algebra.properties.RandomPartitioningProperty;
 import org.apache.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
+import org.apache.hyracks.api.io.FileSplit;
 
 public class PigletFileDataSource implements IDataSource<String> {
     private final String file;

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/PigletMetadataProvider.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/PigletMetadataProvider.java b/hyracks-fullstack/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/PigletMetadataProvider.java
index 7bcb1d6..37906c1 100644
--- a/hyracks-fullstack/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/PigletMetadataProvider.java
+++ b/hyracks-fullstack/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/PigletMetadataProvider.java
@@ -47,6 +47,8 @@ import org.apache.hyracks.algebricks.runtime.writers.PrinterBasedWriterFactory;
 import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
 import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
 import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileSplit;
 import org.apache.hyracks.api.job.JobSpecification;
 import org.apache.hyracks.dataflow.common.data.marshalling.FloatSerializerDeserializer;
 import org.apache.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
@@ -58,7 +60,6 @@ import org.apache.hyracks.dataflow.common.data.parsers.UTF8StringParserFactory;
 import org.apache.hyracks.dataflow.std.file.ConstantFileSplitProvider;
 import org.apache.hyracks.dataflow.std.file.DelimitedDataTupleParserFactory;
 import org.apache.hyracks.dataflow.std.file.FileScanOperatorDescriptor;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
 import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
 import org.apache.hyracks.dataflow.std.file.ITupleParserFactory;
 
@@ -66,7 +67,7 @@ public class PigletMetadataProvider implements IMetadataProvider<String, String>
     private static final Map<FunctionIdentifier, PigletFunction> FN_MAP;
 
     static {
-        Map<FunctionIdentifier, PigletFunction> map = new HashMap<FunctionIdentifier, PigletFunction>();
+        Map<FunctionIdentifier, PigletFunction> map = new HashMap<>();
 
         map.put(AlgebricksBuiltinFunctions.EQ, new PigletFunction(AlgebricksBuiltinFunctions.EQ));
 
@@ -129,7 +130,7 @@ public class PigletMetadataProvider implements IMetadataProvider<String, String>
 
         IOperatorDescriptor scanner = new FileScanOperatorDescriptor(jobSpec, fsp, tpf, rDesc);
         AlgebricksAbsolutePartitionConstraint constraint = new AlgebricksAbsolutePartitionConstraint(locations);
-        return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(scanner, constraint);
+        return new Pair<>(scanner, constraint);
     }
 
     @Override
@@ -142,10 +143,15 @@ public class PigletMetadataProvider implements IMetadataProvider<String, String>
         for (int i = 0; i < fileSplits.length; ++i) {
             locations[i] = fileSplits[i].getNodeName();
         }
-        IPushRuntimeFactory prf = new SinkWriterRuntimeFactory(printColumns, printerFactories,
-                fileSplits[0].getLocalFile().getFile(), PrinterBasedWriterFactory.INSTANCE, inputDesc);
+        IPushRuntimeFactory prf;
+        try {
+            prf = new SinkWriterRuntimeFactory(printColumns, printerFactories,
+                    fileSplits[0].getFile(null), PrinterBasedWriterFactory.INSTANCE, inputDesc);
         AlgebricksAbsolutePartitionConstraint constraint = new AlgebricksAbsolutePartitionConstraint(locations);
-        return new Pair<IPushRuntimeFactory, AlgebricksPartitionConstraint>(prf, constraint);
+        return new Pair<>(prf, constraint);
+        } catch (HyracksDataException e) {
+            throw new AlgebricksException(e);
+        }
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/algebricks/algebricks-tests/src/test/java/org/apache/hyracks/algebricks/tests/pushruntime/PushRuntimeTest.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/algebricks/algebricks-tests/src/test/java/org/apache/hyracks/algebricks/tests/pushruntime/PushRuntimeTest.java b/hyracks-fullstack/algebricks/algebricks-tests/src/test/java/org/apache/hyracks/algebricks/tests/pushruntime/PushRuntimeTest.java
index 1276518..af6ddca 100644
--- a/hyracks-fullstack/algebricks/algebricks-tests/src/test/java/org/apache/hyracks/algebricks/tests/pushruntime/PushRuntimeTest.java
+++ b/hyracks-fullstack/algebricks/algebricks-tests/src/test/java/org/apache/hyracks/algebricks/tests/pushruntime/PushRuntimeTest.java
@@ -64,7 +64,7 @@ import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
 import org.apache.hyracks.api.dataflow.value.IMissingWriterFactory;
 import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
 import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
-import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.FileSplit;
 import org.apache.hyracks.api.job.JobSpecification;
 import org.apache.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
 import org.apache.hyracks.data.std.primitive.IntegerPointable;
@@ -80,7 +80,6 @@ import org.apache.hyracks.dataflow.std.connectors.OneToOneConnectorDescriptor;
 import org.apache.hyracks.dataflow.std.file.ConstantFileSplitProvider;
 import org.apache.hyracks.dataflow.std.file.DelimitedDataTupleParserFactory;
 import org.apache.hyracks.dataflow.std.file.FileScanOperatorDescriptor;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
 import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
 import org.apache.hyracks.dataflow.std.file.LineFileWriteOperatorDescriptor;
 import org.apache.hyracks.dataflow.std.group.preclustered.PreclusteredGroupOperatorDescriptor;
@@ -181,8 +180,8 @@ public class PushRuntimeTest {
 
         // the scanner
         FileSplit[] intFileSplits = new FileSplit[1];
-        intFileSplits[0] = new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID,
-                new FileReference(new File("data/simple/int-part1.tbl")));
+        intFileSplits[0] = new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID, new File("data/simple/int-part1.tbl")
+                .getAbsolutePath(), false);
         IFileSplitProvider intSplitProvider = new ConstantFileSplitProvider(intFileSplits);
         RecordDescriptor intScannerDesc = new RecordDescriptor(
                 new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE });
@@ -264,8 +263,8 @@ public class PushRuntimeTest {
 
         // the scanner
         FileSplit[] fileSplits = new FileSplit[1];
-        fileSplits[0] = new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID,
-                new FileReference(new File("data/tpch0.001/customer.tbl")));
+        fileSplits[0] = new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID, new File("data/tpch0.001/customer.tbl")
+                .getAbsolutePath(), false);
         IFileSplitProvider splitProvider = new ConstantFileSplitProvider(fileSplits);
 
         RecordDescriptor scannerDesc = new RecordDescriptor(new ISerializerDeserializer[] {
@@ -347,7 +346,7 @@ public class PushRuntimeTest {
         // the scanner
         FileSplit[] fileSplits = new FileSplit[1];
         fileSplits[0] = new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID,
-                new FileReference(new File("data/tpch0.001/customer-part1.tbl")));
+                new File("data/tpch0.001/customer-part1.tbl").getAbsolutePath(), false);
         IFileSplitProvider splitProvider = new ConstantFileSplitProvider(fileSplits);
         RecordDescriptor scannerDesc = new RecordDescriptor(new ISerializerDeserializer[] {
                 IntegerSerializerDeserializer.INSTANCE, new UTF8StringSerializerDeserializer(),
@@ -399,7 +398,7 @@ public class PushRuntimeTest {
         // the scanner
         FileSplit[] fileSplits = new FileSplit[1];
         fileSplits[0] = new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID,
-                new FileReference(new File("data/tpch0.001/customer.tbl")));
+                new File("data/tpch0.001/customer.tbl").getAbsolutePath(), false);
         IFileSplitProvider splitProvider = new ConstantFileSplitProvider(fileSplits);
         RecordDescriptor scannerDesc = new RecordDescriptor(new ISerializerDeserializer[] {
                 IntegerSerializerDeserializer.INSTANCE, new UTF8StringSerializerDeserializer(),
@@ -584,7 +583,7 @@ public class PushRuntimeTest {
         }
 
         FileSplit[] inputSplits = new FileSplit[] {
-                new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID, new FileReference(inputFile)) };
+                new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID, inputFile.getAbsolutePath(), false) };
 
         DelimitedDataTupleParserFactory stringParser = new DelimitedDataTupleParserFactory(
                 new IValueParserFactory[] { UTF8StringParserFactory.INSTANCE }, '\u0000');
@@ -605,7 +604,7 @@ public class PushRuntimeTest {
         IOperatorDescriptor outputOp[] = new IOperatorDescriptor[outputFile.length];
         for (int i = 0; i < outputArity; i++) {
             outputOp[i] = new LineFileWriteOperatorDescriptor(spec, new FileSplit[] {
-                    new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID, new FileReference(outputFile[i])) });
+                    new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID, outputFile[i].getAbsolutePath(), false) });
             PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, outputOp[i],
                     new String[] { AlgebricksHyracksIntegrationUtil.NC1_ID });
         }
@@ -643,7 +642,7 @@ public class PushRuntimeTest {
         }
 
         FileSplit[] inputSplits = new FileSplit[] {
-                new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID, new FileReference(inputFiles[0])) };
+                new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID, inputFiles[0].getAbsolutePath(), false) };
         IFileSplitProvider intSplitProvider = new ConstantFileSplitProvider(inputSplits);
 
         RecordDescriptor scannerDesc = new RecordDescriptor(
@@ -667,7 +666,7 @@ public class PushRuntimeTest {
         IOperatorDescriptor outputOp[] = new IOperatorDescriptor[outputFile.length];
         for (int i = 0; i < outputArity; i++) {
             outputOp[i] = new LineFileWriteOperatorDescriptor(spec, new FileSplit[] {
-                    new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID, new FileReference(outputFile[i])) });
+                    new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID, outputFile[i].getAbsolutePath(), false) });
             PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, outputOp[i],
                     new String[] { AlgebricksHyracksIntegrationUtil.NC1_ID });
         }
@@ -693,8 +692,8 @@ public class PushRuntimeTest {
 
         // the scanner
         FileSplit[] fileSplits = new FileSplit[1];
-        fileSplits[0] = new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID,
-                new FileReference(new File("data/tpch0.001/nation.tbl")));
+        fileSplits[0] = new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID, new File("data/tpch0.001/nation.tbl")
+                .getAbsolutePath(), false);
         IFileSplitProvider splitProvider = new ConstantFileSplitProvider(fileSplits);
         RecordDescriptor scannerDesc = new RecordDescriptor(new ISerializerDeserializer[] {
                 IntegerSerializerDeserializer.INSTANCE, new UTF8StringSerializerDeserializer(),
@@ -803,7 +802,7 @@ public class PushRuntimeTest {
         // the scanner
         FileSplit[] fileSplits = new FileSplit[1];
         fileSplits[0] = new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID,
-                new FileReference(new File("data/tpch0.001/customer.tbl")));
+                new File("data/tpch0.001/customer.tbl").getAbsolutePath(), false);
         IFileSplitProvider splitProvider = new ConstantFileSplitProvider(fileSplits);
         RecordDescriptor scannerDesc = new RecordDescriptor(new ISerializerDeserializer[] {
                 IntegerSerializerDeserializer.INSTANCE, new UTF8StringSerializerDeserializer(),

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileReference.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileReference.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileReference.java
index 8893adb..12a138a 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileReference.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileReference.java
@@ -21,20 +21,21 @@ package org.apache.hyracks.api.io;
 import java.io.File;
 import java.io.Serializable;
 
+/**
+ * A device handle and a relative path.
+ * Used to identify a file in the local Node Controller.
+ * Only used for files which are stored inside an IO device.
+ */
 public final class FileReference implements Serializable {
     private static final long serialVersionUID = 1L;
-
     private final File file;
     private final IODeviceHandle dev;
+    private final String path;
 
-    public FileReference(IODeviceHandle dev, String devRelPath) {
-        file = new File(dev.getPath(), devRelPath);
+    public FileReference(IODeviceHandle dev, String path) {
+        file = new File(dev.getMount(), path);
         this.dev = dev;
-    }
-
-    public FileReference(File file) {
-        this.file = file;
-        this.dev = null;
+        this.path = path;
     }
 
     public File getFile() {
@@ -63,7 +64,26 @@ public final class FileReference implements Serializable {
         return file.hashCode();
     }
 
-    public void delete() {
-        file.delete();
+    /**
+     * Delete the file
+     *
+     * @return true if file was deleted, false, otherwise
+     */
+    public boolean delete() {
+        return file.delete();
+    }
+
+    /**
+     * @return the relative path
+     */
+    public String getRelativePath() {
+        return path;
+    }
+
+    /**
+     * @return the absolute path
+     */
+    public String getAbsolutePath() {
+        return file.getAbsolutePath();
     }
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileSplit.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileSplit.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileSplit.java
new file mode 100644
index 0000000..bd4567b
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileSplit.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.api.io;
+
+import java.io.File;
+import java.io.Serializable;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+/**
+ * A node and a path (Can be relative: inside the IO device or absolute inside or outside IO devices)
+ * Used to identify a file/dir across the cluster.
+ */
+public class FileSplit implements Serializable {
+    private static final long serialVersionUID = 1L;
+    private final String node;
+    private final String path;
+    private final boolean managed;
+
+    /**
+     * Constructor
+     *
+     * @param node
+     * @param path
+     * @param maanged
+     */
+    public FileSplit(String node, String path, boolean relative) {
+        this.node = node;
+        this.path = path;
+        this.managed = relative;
+    }
+
+    @Override
+    public String toString() {
+        return "Node: " + node + " " + (managed ? "managed" : "absolute") + " path: " + path;
+    }
+
+    /**
+     * @return the path
+     */
+    public String getPath() {
+        return path;
+    }
+
+    /**
+     * @return true if relative
+     */
+    public boolean isManaged() {
+        return managed;
+    }
+
+    /**
+     * Get the local file represented by this split
+     *
+     * @param ioManager
+     * @return
+     * @throws HyracksDataException
+     */
+    public File getFile(IIOManager ioManager) throws HyracksDataException {
+        return managed ? getFileRef(ioManager).getFile() : new File(path);
+    }
+
+    /**
+     * Get the file reference for the split
+     *
+     * @param ioManager
+     * @return
+     * @throws HyracksDataException
+     */
+    public FileReference getFileRef(IIOManager ioManager) throws HyracksDataException {
+        return ioManager.getFileRef(path, managed);
+    }
+
+    /**
+     * @return the node
+     */
+    public String getNodeName() {
+        return node;
+    }
+}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IFileDeviceComputer.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IFileDeviceComputer.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IFileDeviceComputer.java
new file mode 100644
index 0000000..e75efd4
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IFileDeviceComputer.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.api.io;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+/**
+ * Computes the device destination for a file from its relative path
+ */
+@FunctionalInterface
+public interface IFileDeviceComputer {
+
+    /**
+     * Compute the device from the relative path
+     *
+     * @param relativePath
+     * @return
+     */
+    IODeviceHandle compute(String relativePath) throws HyracksDataException;
+
+}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java
index 63709ed..0b3f71a 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java
@@ -66,5 +66,15 @@ public interface IIOManager {
      * @param relativePath
      * @return A file reference based on the mounting point of {@code ioDeviceId} and the passed {@code relativePath}
      */
-    public FileReference getAbsoluteFileRef(int ioDeviceId, String relativePath);
+    public FileReference getFileRef(int ioDeviceId, String relativePath);
+
+    /**
+     * A file reference based on the mounting point of {@code ioDeviceId} and the passed {@code relativePath}
+     *
+     * @param path
+     * @param relative
+     * @return
+     * @throws HyracksDataException
+     */
+    public FileReference getFileRef(String path, boolean relative) throws HyracksDataException;
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IODeviceHandle.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IODeviceHandle.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IODeviceHandle.java
index 1fcff90..552fbeb 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IODeviceHandle.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IODeviceHandle.java
@@ -20,32 +20,114 @@ package org.apache.hyracks.api.io;
 
 import java.io.File;
 import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.StringTokenizer;
 
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+/**
+ * Represents an IO device
+ */
 public class IODeviceHandle implements Serializable {
     private static final long serialVersionUID = 1L;
+    /**
+     * Absolute mount point
+     */
+    private final File mount;
+    /**
+     * relative workspace
+     */
+    private final String workspace;
+
+    /**
+     * @param mount
+     *            The device root
+     * @param workspace
+     *            The relative workspace inside the device
+     */
+    public IODeviceHandle(File mount, String workspace) {
+        this.mount = mount;
+        this.workspace = workspace == null ? null
+                : workspace.endsWith(File.separator) ? workspace.substring(0, workspace.length() - 1)
+                        : workspace;
+    }
+
+    public File getMount() {
+        return mount;
+    }
+
+    public String getWorkspace() {
+        return workspace;
+    }
 
-    private final File path;
+    /**
+     * Create a file reference
+     *
+     * @param relPath
+     *            the relative path
+     * @return
+     */
+    public FileReference createFileRef(String relPath) {
+        return new FileReference(this, relPath);
+    }
 
-    private final String workAreaPath;
+    /**
+     * Get handles for IO devices
+     *
+     * @param ioDevices
+     *            comma separated list of devices
+     * @return
+     */
+    public static List<IODeviceHandle> getDevices(String ioDevices) {
+        List<IODeviceHandle> devices = new ArrayList<>();
+        StringTokenizer tok = new StringTokenizer(ioDevices, ",");
+        while (tok.hasMoreElements()) {
+            String devPath = tok.nextToken().trim();
+            devices.add(new IODeviceHandle(new File(devPath), "."));
+        }
+        return devices;
+    }
 
-    public IODeviceHandle(File path, String workAreaPath) {
-        this.path = path;
-        workAreaPath = workAreaPath.trim();
-        if (workAreaPath.endsWith(File.separator)) {
-            workAreaPath = workAreaPath.substring(0, workAreaPath.length() - 1);
+    /**
+     * @param absolutePath
+     * @return the relative path
+     * @throws HyracksDataException
+     */
+    public String getRelativePath(String absolutePath) throws HyracksDataException {
+        if (absolutePath.indexOf(mount.getAbsolutePath()) != 0) {
+            throw new HyracksDataException(
+                    "Passed path: " + absolutePath + " is not inside the device " + mount.getAbsolutePath());
         }
-        this.workAreaPath = workAreaPath;
+        return absolutePath.substring(mount.getAbsolutePath().length());
     }
 
-    public File getPath() {
-        return path;
+    /**
+     * determinea if the device contains a file with the passed relative path
+     * @param relPath
+     * @return true if it contains, false, otherwise
+     */
+    public boolean contains(String relPath) {
+        return new File(mount, relPath).exists();
     }
 
-    public String getWorkAreaPath() {
-        return workAreaPath;
+    @Override
+    public boolean equals(Object o) {
+        if (o == this) {
+            return true;
+        } else if (o instanceof IODeviceHandle) {
+            return mount.getAbsolutePath().equals(((IODeviceHandle) o).getMount().getAbsolutePath());
+        }
+        return false;
     }
 
-    public FileReference createFileReference(String relPath) {
-        return new FileReference(this, relPath);
+    @Override
+    public int hashCode() {
+        return mount.getAbsolutePath().hashCode();
+    }
+
+    @Override
+    public String toString() {
+        return "mount: " + mount.getAbsolutePath() + ((workspace == null) ? "" : ", workspace: " + workspace);
     }
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/DefaultDeviceComputer.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/DefaultDeviceComputer.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/DefaultDeviceComputer.java
new file mode 100644
index 0000000..bacb608
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/DefaultDeviceComputer.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.control.nc.io;
+
+import java.io.File;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.IFileDeviceComputer;
+import org.apache.hyracks.api.io.IODeviceHandle;
+
+public class DefaultDeviceComputer implements IFileDeviceComputer {
+    private final IOManager ioManager;
+    private AtomicInteger next = new AtomicInteger(0);
+
+    public DefaultDeviceComputer(IOManager ioManager) {
+        this.ioManager = ioManager;
+    }
+
+    @Override
+    public IODeviceHandle compute(String relPath) throws HyracksDataException {
+        String path = relPath;
+        // if number of devices is 1, we return the device
+        if (ioManager.getIODevices().size() == 1) {
+            return ioManager.getIODevices().get(0);
+        }
+        // check if it exists already on a device
+        int nextSeparator = path.lastIndexOf(File.separator);
+        while (nextSeparator > 0) {
+            for (IODeviceHandle dev : ioManager.getIODevices()) {
+                if (dev.contains(path)) {
+                    return dev;
+                }
+            }
+            path = path.substring(0, nextSeparator);
+            nextSeparator = path.lastIndexOf(File.separator);
+        }
+        // one last attempt
+        for (IODeviceHandle dev : ioManager.getIODevices()) {
+            if (dev.contains(path)) {
+                return dev;
+            }
+        }
+        // not on any device, round robin assignment
+        return ioManager.getIODevices().get(next.getAndIncrement() % ioManager.getIODevices().size());
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
index 6588f3b..bdb283a 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
@@ -31,17 +31,29 @@ import java.util.concurrent.Executor;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.api.exceptions.HyracksException;
 import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IFileDeviceComputer;
 import org.apache.hyracks.api.io.IFileHandle;
 import org.apache.hyracks.api.io.IIOFuture;
 import org.apache.hyracks.api.io.IIOManager;
 import org.apache.hyracks.api.io.IODeviceHandle;
 
 public class IOManager implements IIOManager {
+    /*
+     * Constants
+     */
     private static final String WORKSPACE_FILE_SUFFIX = ".waf";
+    private static final FilenameFilter WORKSPACE_FILES_FILTER = (dir, name) -> name.endsWith(WORKSPACE_FILE_SUFFIX);
+    /*
+     * Finals
+     */
     private final List<IODeviceHandle> ioDevices;
+    private final List<IODeviceHandle> workspaces;
+    /*
+     * Mutables
+     */
     private Executor executor;
-    private final List<IODeviceHandle> workAreaIODevices;
-    private int workAreaDeviceIndex;
+    private int workspaceIndex;
+    private IFileDeviceComputer deviceComputer;
 
     public IOManager(List<IODeviceHandle> devices, Executor executor) throws HyracksException {
         this(devices);
@@ -50,19 +62,21 @@ public class IOManager implements IIOManager {
 
     public IOManager(List<IODeviceHandle> devices) throws HyracksException {
         this.ioDevices = Collections.unmodifiableList(devices);
-        workAreaIODevices = new ArrayList<>();
+        workspaces = new ArrayList<>();
         for (IODeviceHandle d : ioDevices) {
-            if (d.getWorkAreaPath() != null) {
-                new File(d.getPath(), d.getWorkAreaPath()).mkdirs();
-                workAreaIODevices.add(d);
+            if (d.getWorkspace() != null) {
+                new File(d.getMount(), d.getWorkspace()).mkdirs();
+                workspaces.add(d);
             }
         }
-        if (workAreaIODevices.isEmpty()) {
+        if (workspaces.isEmpty()) {
             throw new HyracksException("No devices with work areas found");
         }
-        workAreaDeviceIndex = 0;
+        workspaceIndex = 0;
+        deviceComputer = new DefaultDeviceComputer(this);
     }
 
+    @Override
     public void setExecutor(Executor executor) {
         this.executor = executor;
     }
@@ -200,16 +214,16 @@ public class IOManager implements IIOManager {
     }
 
     public synchronized FileReference createWorkspaceFile(String prefix) throws HyracksDataException {
-        IODeviceHandle dev = workAreaIODevices.get(workAreaDeviceIndex);
-        workAreaDeviceIndex = (workAreaDeviceIndex + 1) % workAreaIODevices.size();
-        String waPath = dev.getWorkAreaPath();
+        IODeviceHandle dev = workspaces.get(workspaceIndex);
+        workspaceIndex = (workspaceIndex + 1) % workspaces.size();
+        String waPath = dev.getWorkspace();
         File waf;
         try {
-            waf = File.createTempFile(prefix, WORKSPACE_FILE_SUFFIX, new File(dev.getPath(), waPath));
+            waf = File.createTempFile(prefix, WORKSPACE_FILE_SUFFIX, new File(dev.getMount(), waPath));
         } catch (IOException e) {
             throw new HyracksDataException(e);
         }
-        return dev.createFileReference(waPath + File.separator + waf.getName());
+        return dev.createFileRef(waPath + File.separator + waf.getName());
     }
 
     private abstract class AsyncRequest implements IIOFuture, Runnable {
@@ -302,8 +316,8 @@ public class IOManager implements IIOManager {
 
     @Override
     public void deleteWorkspaceFiles() {
-        for (IODeviceHandle ioDevice : workAreaIODevices) {
-            File workspaceFolder = new File(ioDevice.getPath(), ioDevice.getWorkAreaPath());
+        for (IODeviceHandle ioDevice : workspaces) {
+            File workspaceFolder = new File(ioDevice.getMount(), ioDevice.getWorkspace());
             if (workspaceFolder.exists() && workspaceFolder.isDirectory()) {
                 File[] workspaceFiles = workspaceFolder.listFiles(WORKSPACE_FILES_FILTER);
                 for (File workspaceFile : workspaceFiles) {
@@ -313,15 +327,31 @@ public class IOManager implements IIOManager {
         }
     }
 
-    private static final FilenameFilter WORKSPACE_FILES_FILTER = new FilenameFilter() {
-        public boolean accept(File dir, String name) {
-            return name.endsWith(WORKSPACE_FILE_SUFFIX);
-        }
-    };
-
     @Override
-    public FileReference getAbsoluteFileRef(int ioDeviceId, String relativePath) {
+    public FileReference getFileRef(int ioDeviceId, String relativePath) {
         IODeviceHandle devHandle = ioDevices.get(ioDeviceId);
         return new FileReference(devHandle, relativePath);
     }
+
+    @Override
+    public FileReference getFileRef(String path, boolean relative) throws HyracksDataException {
+        if (relative) {
+            return new FileReference(deviceComputer.compute(path), path);
+        }
+        IODeviceHandle devHandle = getDevice(path);
+        if (devHandle == null) {
+            throw new HyracksDataException("The file with absolute path: " + path + " is outside all io devices");
+        }
+        String relativePath = devHandle.getRelativePath(path);
+        return new FileReference(devHandle, relativePath);
+    }
+
+    public IODeviceHandle getDevice(String fullPath) {
+        for (IODeviceHandle d : ioDevices) {
+            if (fullPath.startsWith(d.getMount().getAbsolutePath())) {
+                return d;
+            }
+        }
+        return null;
+    }
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/AbstractDeserializedFileScanOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/AbstractDeserializedFileScanOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/AbstractDeserializedFileScanOperatorDescriptor.java
index 189a3a1..6703b0f 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/AbstractDeserializedFileScanOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/AbstractDeserializedFileScanOperatorDescriptor.java
@@ -26,6 +26,7 @@ import org.apache.hyracks.api.dataflow.IOperatorNodePushable;
 import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
 import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileSplit;
 import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
 import org.apache.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
 import org.apache.hyracks.dataflow.std.base.IOpenableDataWriterOperator;
@@ -69,7 +70,7 @@ public abstract class AbstractDeserializedFileScanOperatorDescriptor extends Abs
             RecordDescriptor desc = recordDescriptors[0];
             IRecordReader reader;
             try {
-                reader = createRecordReader(split.getLocalFile().getFile(), desc);
+                reader = createRecordReader(split.getFile(null), desc);
             } catch (Exception e) {
                 throw new HyracksDataException(e);
             }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/AbstractFileWriteOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/AbstractFileWriteOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/AbstractFileWriteOperatorDescriptor.java
index 6b0ee4d..5028ccc 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/AbstractFileWriteOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/AbstractFileWriteOperatorDescriptor.java
@@ -23,6 +23,7 @@ import org.apache.hyracks.api.dataflow.IOpenableDataWriter;
 import org.apache.hyracks.api.dataflow.IOperatorNodePushable;
 import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileSplit;
 import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
 import org.apache.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
 import org.apache.hyracks.dataflow.std.base.IOpenableDataWriterOperator;

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/ConstantFileSplitProvider.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/ConstantFileSplitProvider.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/ConstantFileSplitProvider.java
index 028a91a..abd8d2c 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/ConstantFileSplitProvider.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/ConstantFileSplitProvider.java
@@ -18,6 +18,8 @@
  */
 package org.apache.hyracks.dataflow.std.file;
 
+import org.apache.hyracks.api.io.FileSplit;
+
 public class ConstantFileSplitProvider implements IFileSplitProvider {
     private static final long serialVersionUID = 1L;
     private final FileSplit[] splits;

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FileRemoveOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FileRemoveOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FileRemoveOperatorDescriptor.java
index c3883e8..94c5a68 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FileRemoveOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FileRemoveOperatorDescriptor.java
@@ -29,6 +29,7 @@ import org.apache.hyracks.api.dataflow.IOperatorNodePushable;
 import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
 import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileSplit;
 import org.apache.hyracks.api.io.IIOManager;
 import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
 import org.apache.hyracks.dataflow.std.base.AbstractOperatorNodePushable;
@@ -61,8 +62,6 @@ public class FileRemoveOperatorDescriptor extends AbstractSingleActivityOperator
     public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
             IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
         final FileSplit split = fileSplitProvider.getFileSplits()[partition];
-        final String path = split.getLocalFile().getFile().getPath();
-        final int deviceId = split.getIODeviceId();
         final IIOManager ioManager = ctx.getIOManager();
         return new AbstractOperatorNodePushable() {
 
@@ -73,7 +72,8 @@ public class FileRemoveOperatorDescriptor extends AbstractSingleActivityOperator
 
             @Override
             public void initialize() throws HyracksDataException {
-                File f = ioManager.getAbsoluteFileRef(deviceId, path).getFile();
+                // will only work for files inside the io devices
+                File f = ioManager.getFileRef(split.getPath(), split.isManaged()).getFile();
                 if (quietly) {
                     FileUtils.deleteQuietly(f);
                 } else {

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FileScanOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FileScanOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FileScanOperatorDescriptor.java
index 5a3d776..311f79e 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FileScanOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FileScanOperatorDescriptor.java
@@ -28,6 +28,8 @@ import org.apache.hyracks.api.dataflow.IOperatorNodePushable;
 import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
 import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileSplit;
+import org.apache.hyracks.api.io.IIOManager;
 import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
 import org.apache.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
 import org.apache.hyracks.dataflow.std.base.AbstractUnaryOutputSourceOperatorNodePushable;
@@ -52,10 +54,11 @@ public class FileScanOperatorDescriptor extends AbstractSingleActivityOperatorDe
             IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
         final FileSplit split = fileSplitProvider.getFileSplits()[partition];
         final ITupleParser tp = tupleParserFactory.createTupleParser(ctx);
+        final IIOManager ioManager = ctx.getIOManager();
         return new AbstractUnaryOutputSourceOperatorNodePushable() {
             @Override
             public void initialize() throws HyracksDataException {
-                File f = split.getLocalFile().getFile();
+                File f = split.getFile(ioManager);
                 try {
                     writer.open();
                     InputStream in;

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FileSplit.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FileSplit.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FileSplit.java
deleted file mode 100644
index b201acd..0000000
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FileSplit.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.dataflow.std.file;
-
-import java.io.File;
-import java.io.Serializable;
-
-import org.apache.hyracks.api.io.FileReference;
-
-public class FileSplit implements Serializable {
-    private static final long serialVersionUID = 1L;
-    private final FileReference file;
-    private final int ioDeviceId;
-    private final int partition;
-    private final String nodeName;
-
-    public FileSplit(String nodeName, FileReference file) {
-        this.nodeName = nodeName;
-        this.file = file;
-        this.ioDeviceId = 0;
-        this.partition = -1;
-    }
-
-    public FileSplit(String nodeName, FileReference file, int ioDeviceId, int partition) {
-        this.nodeName = nodeName;
-        this.file = file;
-        this.ioDeviceId = ioDeviceId;
-        this.partition = partition;
-    }
-
-    public FileSplit(String nodeName, String path, int ioDeviceId) {
-        this.nodeName = nodeName;
-        this.file = new FileReference(new File(path));
-        this.ioDeviceId = ioDeviceId;
-        this.partition = -1;
-    }
-
-    public FileSplit(String nodeName, String path) {
-        this.nodeName = nodeName;
-        this.file = new FileReference(new File(path));
-        this.ioDeviceId = 0;
-        this.partition = -1;
-    }
-
-    public String getNodeName() {
-        return nodeName;
-    }
-
-    public FileReference getLocalFile() {
-        return file;
-    }
-
-    public int getIODeviceId() {
-        return ioDeviceId;
-    }
-
-    public int getPartition() {
-        return partition;
-    }
-
-    @Override
-    public String toString() {
-        return file.toString();
-    }
-}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FrameFileWriterOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FrameFileWriterOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FrameFileWriterOperatorDescriptor.java
index ae9585e..e398bc0 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FrameFileWriterOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FrameFileWriterOperatorDescriptor.java
@@ -28,6 +28,8 @@ import org.apache.hyracks.api.context.IHyracksTaskContext;
 import org.apache.hyracks.api.dataflow.IOperatorNodePushable;
 import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileSplit;
+import org.apache.hyracks.api.io.IIOManager;
 import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
 import org.apache.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
 import org.apache.hyracks.dataflow.std.base.AbstractUnaryInputSinkOperatorNodePushable;
@@ -46,13 +48,14 @@ public class FrameFileWriterOperatorDescriptor extends AbstractSingleActivityOpe
     public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
             IRecordDescriptorProvider recordDescProvider, final int partition, int nPartitions) {
         final FileSplit[] splits = fileSplitProvider.getFileSplits();
+        final IIOManager ioManager = ctx.getIOManager();
         return new AbstractUnaryInputSinkOperatorNodePushable() {
             private OutputStream out;
 
             @Override
             public void open() throws HyracksDataException {
                 try {
-                    out = new FileOutputStream(splits[partition].getLocalFile().getFile());
+                    out = new FileOutputStream(splits[partition].getFile(ioManager));
                 } catch (FileNotFoundException e) {
                     throw new HyracksDataException(e);
                 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/IFileSplitProvider.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/IFileSplitProvider.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/IFileSplitProvider.java
index cf2e923..a85e76e 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/IFileSplitProvider.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/IFileSplitProvider.java
@@ -20,6 +20,8 @@ package org.apache.hyracks.dataflow.std.file;
 
 import java.io.Serializable;
 
+import org.apache.hyracks.api.io.FileSplit;
+
 public interface IFileSplitProvider extends Serializable {
     public FileSplit[] getFileSplits();
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/LineFileWriteOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/LineFileWriteOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/LineFileWriteOperatorDescriptor.java
index 1a19a2b..1186283 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/LineFileWriteOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/LineFileWriteOperatorDescriptor.java
@@ -22,6 +22,7 @@ import java.io.File;
 import java.io.FileOutputStream;
 import java.io.OutputStream;
 
+import org.apache.hyracks.api.io.FileSplit;
 import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
 
 public class LineFileWriteOperatorDescriptor extends AbstractFileWriteOperatorDescriptor {
@@ -59,6 +60,6 @@ public class LineFileWriteOperatorDescriptor extends AbstractFileWriteOperatorDe
 
     @Override
     protected IRecordWriter createRecordWriter(FileSplit fileSplit, int index) throws Exception {
-        return new LineWriterImpl(fileSplit.getLocalFile().getFile(), columns, separator);
+        return new LineWriterImpl(fileSplit.getFile(null), columns, separator);
     }
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/PlainFileWriterOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/PlainFileWriterOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/PlainFileWriterOperatorDescriptor.java
index 33ac540..8df7fa7 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/PlainFileWriterOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/PlainFileWriterOperatorDescriptor.java
@@ -29,6 +29,8 @@ import org.apache.hyracks.api.dataflow.IOperatorNodePushable;
 import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
 import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileSplit;
+import org.apache.hyracks.api.io.IIOManager;
 import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
 import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
 import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream;
@@ -72,6 +74,7 @@ public class PlainFileWriterOperatorDescriptor extends AbstractSingleActivityOpe
             throws HyracksDataException {
         // Output files
         final FileSplit[] splits = fileSplitProvider.getFileSplits();
+        IIOManager ioManager = ctx.getIOManager();
         // Frame accessor
         final FrameTupleAccessor frameTupleAccessor = new FrameTupleAccessor(
                 recordDescProvider.getInputRecordDescriptor(getActivityId(), 0));
@@ -87,7 +90,7 @@ public class PlainFileWriterOperatorDescriptor extends AbstractSingleActivityOpe
             @Override
             public void open() throws HyracksDataException {
                 try {
-                    out = new BufferedWriter(new FileWriter(splits[partition].getLocalFile().getFile()));
+                    out = new BufferedWriter(new FileWriter(splits[partition].getFile(ioManager)));
                     bbis = new ByteBufferInputStream();
                     di = new DataInputStream(bbis);
                 } catch (Exception e) {

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/RecordFileScanOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/RecordFileScanOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/RecordFileScanOperatorDescriptor.java
index e807301..97f12b5 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/RecordFileScanOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/RecordFileScanOperatorDescriptor.java
@@ -25,6 +25,7 @@ import java.io.FileInputStream;
 import java.io.IOException;
 
 import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.io.FileSplit;
 import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
 
 public class RecordFileScanOperatorDescriptor extends AbstractDeserializedFileScanOperatorDescriptor {

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/group/sort/SortGroupByOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/group/sort/SortGroupByOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/group/sort/SortGroupByOperatorDescriptor.java
index 16d2158..da5b4a8 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/group/sort/SortGroupByOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/group/sort/SortGroupByOperatorDescriptor.java
@@ -48,6 +48,7 @@ import org.apache.hyracks.dataflow.std.sort.ISorter;
  */
 public class SortGroupByOperatorDescriptor extends AbstractSorterOperatorDescriptor {
 
+    private static final long serialVersionUID = 1L;
     private final int[] groupFields;
     private final IAggregatorDescriptorFactory mergeAggregatorFactory;
     private final IAggregatorDescriptorFactory partialAggregatorFactory;
@@ -102,6 +103,8 @@ public class SortGroupByOperatorDescriptor extends AbstractSorterOperatorDescrip
     @Override
     public AbstractSorterOperatorDescriptor.SortActivity getSortActivity(ActivityId id) {
         return new AbstractSorterOperatorDescriptor.SortActivity(id) {
+            private static final long serialVersionUID = 1L;
+
             @Override
             protected AbstractSortRunGenerator getRunGenerator(IHyracksTaskContext ctx,
                     IRecordDescriptorProvider recordDescriptorProvider) throws HyracksDataException {
@@ -116,6 +119,7 @@ public class SortGroupByOperatorDescriptor extends AbstractSorterOperatorDescrip
     @Override
     public AbstractSorterOperatorDescriptor.MergeActivity getMergeActivity(ActivityId id) {
         return new AbstractSorterOperatorDescriptor.MergeActivity(id) {
+            private static final long serialVersionUID = 1L;
 
             @Override
             protected AbstractExternalSortRunMerger getSortRunMerger(IHyracksTaskContext ctx,

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/intersect/IntersectOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/intersect/IntersectOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/intersect/IntersectOperatorDescriptor.java
index 98807eb..a213ca1 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/intersect/IntersectOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/intersect/IntersectOperatorDescriptor.java
@@ -51,6 +51,7 @@ import org.apache.hyracks.dataflow.std.base.AbstractUnaryOutputOperatorNodePusha
  */
 public class IntersectOperatorDescriptor extends AbstractOperatorDescriptor {
 
+    private static final long serialVersionUID = 1L;
     private final int[][] projectFields;
     private final INormalizedKeyComputerFactory firstKeyNormalizerFactory;
     private final IBinaryComparatorFactory[] comparatorFactory;
@@ -110,6 +111,8 @@ public class IntersectOperatorDescriptor extends AbstractOperatorDescriptor {
 
     private class IntersectActivity extends AbstractActivityNode {
 
+        private static final long serialVersionUID = 1L;
+
         public IntersectActivity(ActivityId activityId) {
             super(activityId);
         }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/sort/ExternalSortOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/sort/ExternalSortOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/sort/ExternalSortOperatorDescriptor.java
index 36d0c1b..8597ed6 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/sort/ExternalSortOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/sort/ExternalSortOperatorDescriptor.java
@@ -64,6 +64,8 @@ public class ExternalSortOperatorDescriptor extends AbstractSorterOperatorDescri
     @Override
     public AbstractSorterOperatorDescriptor.SortActivity getSortActivity(ActivityId id) {
         return new AbstractSorterOperatorDescriptor.SortActivity(id) {
+            private static final long serialVersionUID = 1L;
+
             @Override
             protected AbstractSortRunGenerator getRunGenerator(IHyracksTaskContext ctx,
                     IRecordDescriptorProvider recordDescProvider) throws HyracksDataException {
@@ -76,6 +78,8 @@ public class ExternalSortOperatorDescriptor extends AbstractSorterOperatorDescri
     @Override
     public AbstractSorterOperatorDescriptor.MergeActivity getMergeActivity(ActivityId id) {
         return new AbstractSorterOperatorDescriptor.MergeActivity(id) {
+            private static final long serialVersionUID = 1L;
+
             @Override
             protected ExternalSortRunMerger getSortRunMerger(IHyracksTaskContext ctx,
                     IRecordDescriptorProvider recordDescProvider, IFrameWriter writer, ISorter sorter,

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/sort/TopKSorterOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/sort/TopKSorterOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/sort/TopKSorterOperatorDescriptor.java
index 5df09c6..218faaf 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/sort/TopKSorterOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/sort/TopKSorterOperatorDescriptor.java
@@ -35,6 +35,7 @@ import org.apache.hyracks.dataflow.common.io.GeneratedRunFileReader;
 
 public class TopKSorterOperatorDescriptor extends AbstractSorterOperatorDescriptor {
 
+    private static final long serialVersionUID = 1L;
     private final int topK;
 
     public TopKSorterOperatorDescriptor(IOperatorDescriptorRegistry spec, int framesLimit, int topK, int[] sortFields,
@@ -47,6 +48,8 @@ public class TopKSorterOperatorDescriptor extends AbstractSorterOperatorDescript
     @Override
     public SortActivity getSortActivity(ActivityId id) {
         return new SortActivity(id) {
+            private static final long serialVersionUID = 1L;
+
             @Override
             protected AbstractSortRunGenerator getRunGenerator(IHyracksTaskContext ctx,
                     IRecordDescriptorProvider recordDescProvider) {
@@ -60,6 +63,8 @@ public class TopKSorterOperatorDescriptor extends AbstractSorterOperatorDescript
     @Override
     public MergeActivity getMergeActivity(ActivityId id) {
         return new MergeActivity(id) {
+            private static final long serialVersionUID = 1L;
+
             @Override
             protected ExternalSortRunMerger getSortRunMerger(IHyracksTaskContext ctx,
                     IRecordDescriptorProvider recordDescProvider, IFrameWriter writer, ISorter sorter,

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/InsertPipelineExample.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/InsertPipelineExample.java b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/InsertPipelineExample.java
index aa386a8..c6e81a4 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/InsertPipelineExample.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/InsertPipelineExample.java
@@ -19,9 +19,6 @@
 
 package org.apache.hyracks.examples.btree.client;
 
-import org.kohsuke.args4j.CmdLineParser;
-import org.kohsuke.args4j.Option;
-
 import org.apache.hyracks.api.client.HyracksConnection;
 import org.apache.hyracks.api.client.IHyracksClientConnection;
 import org.apache.hyracks.api.constraints.PartitionConstraintHelper;
@@ -54,6 +51,8 @@ import org.apache.hyracks.storage.am.common.dataflow.TreeIndexInsertUpdateDelete
 import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallbackFactory;
 import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
 import org.apache.hyracks.storage.common.IStorageManagerInterface;
+import org.kohsuke.args4j.CmdLineParser;
+import org.kohsuke.args4j.Option;
 
 // This example will insert tuples into the primary and secondary index using an insert pipeline
 
@@ -79,6 +78,9 @@ public class InsertPipelineExample {
 
         @Option(name = "-frame-size", usage = "Hyracks frame size (default: 32768)", required = false)
         public int frameSize = 32768;
+
+        @Option(name = "-relative", usage = "Whether the tree file names are relative", required = false)
+        public boolean relative = true;
     }
 
     public static void main(String[] args) throws Exception {
@@ -148,7 +150,8 @@ public class InsertPipelineExample {
         int[] primaryFieldPermutation = { 2, 1, 3, 4 }; // map field 2 of input
                                                         // tuple to field 0 of
                                                         // B-Tree tuple, etc.
-        IFileSplitProvider primarySplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.primaryBTreeName);
+        IFileSplitProvider primarySplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.primaryBTreeName,
+                options.relative);
 
         IIndexDataflowHelperFactory dataflowHelperFactory = new BTreeDataflowHelperFactory(true);
 
@@ -175,7 +178,7 @@ public class InsertPipelineExample {
         // tuple
         int[] secondaryFieldPermutation = { 1, 2 };
         IFileSplitProvider secondarySplitProvider = JobHelper.createFileSplitProvider(splitNCs,
-                options.secondaryBTreeName);
+                options.secondaryBTreeName, true);
         // create operator descriptor
         TreeIndexInsertUpdateDeleteOperatorDescriptor secondaryInsert = new TreeIndexInsertUpdateDeleteOperatorDescriptor(
                 spec, recDesc, storageManager, lcManagerProvider, secondarySplitProvider, secondaryTypeTraits,

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/JobHelper.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/JobHelper.java b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/JobHelper.java
index a189b40..05143a6 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/JobHelper.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/JobHelper.java
@@ -19,25 +19,22 @@
 
 package org.apache.hyracks.examples.btree.client;
 
-import java.io.File;
-
 import org.apache.hyracks.api.constraints.PartitionConstraintHelper;
 import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
-import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.FileSplit;
 import org.apache.hyracks.api.job.JobSpecification;
 import org.apache.hyracks.dataflow.std.file.ConstantFileSplitProvider;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
 import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
 
 public class JobHelper {
-    public static IFileSplitProvider createFileSplitProvider(String[] splitNCs, String btreeFileName) {
+    public static IFileSplitProvider createFileSplitProvider(String[] splitNCs, String btreeFileName,
+            boolean relative) {
         FileSplit[] fileSplits = new FileSplit[splitNCs.length];
         for (int i = 0; i < splitNCs.length; ++i) {
             String fileName = btreeFileName + "." + splitNCs[i];
-            fileSplits[i] = new FileSplit(splitNCs[i], new FileReference(new File(fileName)));
+            fileSplits[i] = new FileSplit(splitNCs[i], fileName, relative);
         }
-        IFileSplitProvider splitProvider = new ConstantFileSplitProvider(fileSplits);
-        return splitProvider;
+        return new ConstantFileSplitProvider(fileSplits);
     }
 
     public static void createPartitionConstraint(JobSpecification spec, IOperatorDescriptor op, String[] splitNCs) {

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/PrimaryIndexBulkLoadExample.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/PrimaryIndexBulkLoadExample.java b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/PrimaryIndexBulkLoadExample.java
index eaf9109..c455c0d 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/PrimaryIndexBulkLoadExample.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/PrimaryIndexBulkLoadExample.java
@@ -18,9 +18,6 @@
  */
 package org.apache.hyracks.examples.btree.client;
 
-import org.kohsuke.args4j.CmdLineParser;
-import org.kohsuke.args4j.Option;
-
 import org.apache.hyracks.api.client.HyracksConnection;
 import org.apache.hyracks.api.client.IHyracksClientConnection;
 import org.apache.hyracks.api.constraints.PartitionConstraintHelper;
@@ -52,6 +49,8 @@ import org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider;
 import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
 import org.apache.hyracks.storage.am.common.dataflow.TreeIndexBulkLoadOperatorDescriptor;
 import org.apache.hyracks.storage.common.IStorageManagerInterface;
+import org.kohsuke.args4j.CmdLineParser;
+import org.kohsuke.args4j.Option;
 
 // This example will load a primary index from randomly generated data
 
@@ -77,6 +76,9 @@ public class PrimaryIndexBulkLoadExample {
 
         @Option(name = "-frame-size", usage = "Hyracks frame size (default: 32768)", required = false)
         public int frameSize = 32768;
+
+        @Option(name = "-relative", usage = "Whether the tree file names are relative", required = false)
+        public boolean relative = true;
     }
 
     public static void main(String[] args) throws Exception {
@@ -152,7 +154,8 @@ public class PrimaryIndexBulkLoadExample {
         int[] fieldPermutation = { 2, 1, 3, 4 }; // map field 2 of input tuple
                                                  // to field 0 of B-Tree tuple,
                                                  // etc.
-        IFileSplitProvider btreeSplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.btreeName);
+        IFileSplitProvider btreeSplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.btreeName,
+                options.relative);
         IIndexDataflowHelperFactory dataflowHelperFactory = new BTreeDataflowHelperFactory(true);
         TreeIndexBulkLoadOperatorDescriptor btreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(spec, recDesc,
                 storageManager, lcManagerProvider, btreeSplitProvider, typeTraits, comparatorFactories, null,

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/PrimaryIndexSearchExample.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/PrimaryIndexSearchExample.java b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/PrimaryIndexSearchExample.java
index 6b22681..1867a80 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/PrimaryIndexSearchExample.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/PrimaryIndexSearchExample.java
@@ -20,9 +20,6 @@ package org.apache.hyracks.examples.btree.client;
 
 import java.io.DataOutput;
 
-import org.kohsuke.args4j.CmdLineParser;
-import org.kohsuke.args4j.Option;
-
 import org.apache.hyracks.api.client.HyracksConnection;
 import org.apache.hyracks.api.client.IHyracksClientConnection;
 import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
@@ -50,6 +47,8 @@ import org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider;
 import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
 import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallbackFactory;
 import org.apache.hyracks.storage.common.IStorageManagerInterface;
+import org.kohsuke.args4j.CmdLineParser;
+import org.kohsuke.args4j.Option;
 
 // This example will perform an ordered scan on the primary index
 // i.e. a range-search for [-infinity, +infinity]
@@ -70,6 +69,9 @@ public class PrimaryIndexSearchExample {
 
         @Option(name = "-frame-size", usage = "Hyracks frame size (default: 32768)", required = false)
         public int frameSize = 32768;
+
+        @Option(name = "-relative", usage = "Whether the tree file names are relative", required = false)
+        public boolean relative = true;
     }
 
     public static void main(String[] args) throws Exception {
@@ -142,7 +144,8 @@ public class PrimaryIndexSearchExample {
         int[] highKeyFields = { 1 }; // low key is in field 1 of tuples going
                                      // into search op
 
-        IFileSplitProvider btreeSplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.btreeName);
+        IFileSplitProvider btreeSplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.btreeName,
+                options.relative);
         IIndexDataflowHelperFactory dataflowHelperFactory = new BTreeDataflowHelperFactory(true);
         BTreeSearchOperatorDescriptor btreeSearchOp = new BTreeSearchOperatorDescriptor(spec, recDesc, storageManager,
                 lcManagerProvider, btreeSplitProvider, typeTraits, comparatorFactories, null, lowKeyFields,

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/SecondaryIndexBulkLoadExample.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/SecondaryIndexBulkLoadExample.java b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/SecondaryIndexBulkLoadExample.java
index 9b85b8b..2acc204 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/SecondaryIndexBulkLoadExample.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/SecondaryIndexBulkLoadExample.java
@@ -18,9 +18,6 @@
  */
 package org.apache.hyracks.examples.btree.client;
 
-import org.kohsuke.args4j.CmdLineParser;
-import org.kohsuke.args4j.Option;
-
 import org.apache.hyracks.api.client.HyracksConnection;
 import org.apache.hyracks.api.client.IHyracksClientConnection;
 import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
@@ -47,6 +44,8 @@ import org.apache.hyracks.storage.am.common.dataflow.TreeIndexBulkLoadOperatorDe
 import org.apache.hyracks.storage.am.common.dataflow.TreeIndexDiskOrderScanOperatorDescriptor;
 import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallbackFactory;
 import org.apache.hyracks.storage.common.IStorageManagerInterface;
+import org.kohsuke.args4j.CmdLineParser;
+import org.kohsuke.args4j.Option;
 
 // This example will load a secondary index with <key, primary-index key> pairs
 // We require an existing primary index built with PrimaryIndexBulkLoadExample
@@ -73,6 +72,9 @@ public class SecondaryIndexBulkLoadExample {
 
         @Option(name = "-frame-size", usage = "Hyracks frame size (default: 32768)", required = false)
         public int frameSize = 32768;
+
+        @Option(name = "-relative", usage = "Whether the tree file names are relative", required = false)
+        public boolean relative = true;
     }
 
     public static void main(String[] args) throws Exception {
@@ -123,7 +125,8 @@ public class SecondaryIndexBulkLoadExample {
         comparatorFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
 
         // use a disk-order scan to read primary index
-        IFileSplitProvider primarySplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.primaryBTreeName);
+        IFileSplitProvider primarySplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.primaryBTreeName,
+                options.relative);
         IIndexDataflowHelperFactory dataflowHelperFactory = new BTreeDataflowHelperFactory(true);
         TreeIndexDiskOrderScanOperatorDescriptor btreeScanOp = new TreeIndexDiskOrderScanOperatorDescriptor(spec,
                 recDesc, storageManager, lcManagerProvider, primarySplitProvider, primaryTypeTraits,
@@ -146,7 +149,8 @@ public class SecondaryIndexBulkLoadExample {
         // the B-Tree expects its keyfields to be at the front of its input
         // tuple
         int[] fieldPermutation = { 1, 0 };
-        IFileSplitProvider btreeSplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.secondaryBTreeName);
+        IFileSplitProvider btreeSplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.secondaryBTreeName,
+                options.relative);
         TreeIndexBulkLoadOperatorDescriptor btreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(spec, null,
                 storageManager, lcManagerProvider, btreeSplitProvider, secondaryTypeTraits, comparatorFactories, null,
                 fieldPermutation, 0.7f, false, 1000L, true, dataflowHelperFactory);

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/SecondaryIndexSearchExample.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/SecondaryIndexSearchExample.java b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/SecondaryIndexSearchExample.java
index 94ae0d3..5e140ce 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/SecondaryIndexSearchExample.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/org/apache/hyracks/examples/btree/client/SecondaryIndexSearchExample.java
@@ -20,9 +20,6 @@ package org.apache.hyracks.examples.btree.client;
 
 import java.io.DataOutput;
 
-import org.kohsuke.args4j.CmdLineParser;
-import org.kohsuke.args4j.Option;
-
 import org.apache.hyracks.api.client.HyracksConnection;
 import org.apache.hyracks.api.client.IHyracksClientConnection;
 import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
@@ -50,6 +47,8 @@ import org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider;
 import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
 import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallbackFactory;
 import org.apache.hyracks.storage.common.IStorageManagerInterface;
+import org.kohsuke.args4j.CmdLineParser;
+import org.kohsuke.args4j.Option;
 
 // This example will perform range search on the secondary index
 // and then retrieve the corresponding source records from the primary index
@@ -73,6 +72,9 @@ public class SecondaryIndexSearchExample {
 
         @Option(name = "-frame-size", usage = "Hyracks frame size (default: 32768)", required = false)
         public int frameSize = 32768;
+
+        @Option(name = "-relative", usage = "Whether the tree file names are relative", required = false)
+        public boolean relative = true;
     }
 
     public static void main(String[] args) throws Exception {
@@ -168,7 +170,7 @@ public class SecondaryIndexSearchExample {
                                               // index search op
 
         IFileSplitProvider secondarySplitProvider = JobHelper.createFileSplitProvider(splitNCs,
-                options.secondaryBTreeName);
+                options.secondaryBTreeName, options.relative);
         IIndexDataflowHelperFactory dataflowHelperFactory = new BTreeDataflowHelperFactory(true);
         BTreeSearchOperatorDescriptor secondarySearchOp = new BTreeSearchOperatorDescriptor(spec, secondaryRecDesc,
                 storageManager, lcManagerProvider, secondarySplitProvider, secondaryTypeTraits,
@@ -186,7 +188,8 @@ public class SecondaryIndexSearchExample {
                                             // going into primary index search
                                             // op
 
-        IFileSplitProvider primarySplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.primaryBTreeName);
+        IFileSplitProvider primarySplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.primaryBTreeName,
+                options.relative);
         BTreeSearchOperatorDescriptor primarySearchOp = new BTreeSearchOperatorDescriptor(spec, primaryRecDesc,
                 storageManager, lcManagerProvider, primarySplitProvider, primaryTypeTraits, primaryComparatorFactories,
                 null, primaryLowKeyFields, primaryHighKeyFields, true, true, dataflowHelperFactory, false, false, null,


Mime
View raw message