asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amo...@apache.org
Subject [4/4] incubator-asterixdb git commit: Refactored External Data
Date Wed, 23 Dec 2015 06:17:54 GMT
Refactored External Data

This change re-arrange asterix module's order. asterix-
external-data is moved in front of asterix-metadata.

Change-Id: I46b60b5e1cc37fd59adc0dd89f374d96502091b2
Reviewed-on: https://asterix-gerrit.ics.uci.edu/559
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: abdullah alamoudi <bamousaa@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/commit/486e47ad
Tree: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/tree/486e47ad
Diff: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/diff/486e47ad

Branch: refs/heads/master
Commit: 486e47ad7f341eb9828061158ca47a9c7302d60e
Parents: fc7272c
Author: Abdullah Alamoudi <bamousaa@gmail.com>
Authored: Wed Dec 23 08:17:57 2015 +0300
Committer: abdullah alamoudi <bamousaa@gmail.com>
Committed: Tue Dec 22 22:12:57 2015 -0800

----------------------------------------------------------------------
 .../physical/ExternalDataLookupPOperator.java   |  11 +-
 .../optimizer/rules/am/AccessMethodUtils.java   |  12 +-
 .../asterix/aql/translator/QueryTranslator.java |   4 +-
 .../file/ExternalIndexingOperations.java        |   6 +-
 .../org/apache/asterix/file/FeedOperations.java |   2 +-
 .../apache/asterix/file/IndexOperations.java    |   2 +-
 .../file/SecondaryBTreeOperationsHelper.java    |   2 +-
 .../file/SecondaryIndexOperationsHelper.java    |   4 +-
 .../file/SecondaryRTreeOperationsHelper.java    |   2 +-
 .../bootstrap/AsterixGlobalRecoveryManager.java |   2 +-
 .../bootstrap/ExternalLibraryBootstrap.java     |   2 +-
 asterix-external-data/pom.xml                   | 344 ++++++++--------
 .../adapter/factory/CNNFeedAdapterFactory.java  |   1 -
 .../adapter/factory/HDFSAdapterFactory.java     |  90 ++---
 .../factory/HDFSIndexingAdapterFactory.java     |  14 +-
 .../adapter/factory/HiveAdapterFactory.java     |  24 +-
 .../adapter/factory/IAdapterFactory.java        |  98 +++++
 .../factory/IControlledAdapterFactory.java      |  36 ++
 .../adapter/factory/IFeedAdapterFactory.java    |  29 ++
 .../factory/NCFileSystemAdapterFactory.java     |  20 +-
 .../PullBasedAzureTwitterAdapterFactory.java    |  45 +--
 .../factory/PullBasedTwitterAdapterFactory.java |   1 -
 .../factory/PushBasedTwitterAdapterFactory.java |   1 -
 .../adapter/factory/RSSFeedAdapterFactory.java  |   3 +-
 .../factory/StreamBasedAdapterFactory.java      |   4 +-
 .../dataset/adapter/ClientBasedFeedAdapter.java |   3 +-
 .../external/dataset/adapter/HDFSAdapter.java   |   2 +-
 .../dataset/adapter/HDFSIndexingAdapter.java    |   7 +-
 .../dataset/adapter/IControlledAdapter.java     |  65 +++
 .../dataset/adapter/PullBasedAdapter.java       |   7 +-
 .../external/feeds/FeedPolicyEnforcer.java      |  50 +++
 .../external/feeds/IPullBasedFeedAdapter.java   |  34 ++
 .../asterix/external/indexing/ExternalFile.java | 133 ++++++
 .../indexing/ExternalFileIndexAccessor.java     | 151 +++++++
 .../indexing/FilesIndexDescription.java         | 104 +++++
 .../external/indexing/IndexingConstants.java    | 205 ++++++++++
 .../AdmOrDelimitedControlledTupleParser.java    |  31 +-
 .../dataflow/FileIndexTupleTranslator.java      |  31 +-
 .../indexing/dataflow/HDFSLookupAdapter.java    |   4 +-
 .../dataflow/HDFSLookupAdapterFactory.java      | 105 +----
 .../dataflow/RCFileControlledTupleParser.java   |  24 +-
 .../dataflow/SeqOrTxtControlledTupleParser.java |  25 +-
 .../input/AbstractHDFSLookupInputStream.java    |   4 +-
 .../input/GenericFileAwareRecordReader.java     |   2 +-
 .../indexing/input/RCFileDataReader.java        |   2 +-
 .../indexing/input/RCFileLookupReader.java      |   9 +-
 .../input/SequenceFileLookupInputStream.java    |  16 +-
 .../input/SequenceFileLookupReader.java         |   7 +-
 .../input/TextFileLookupInputStream.java        |   3 +-
 .../indexing/input/TextFileLookupReader.java    |  12 +-
 .../indexing/input/TextualDataReader.java       |   2 +-
 ...rnalIndexBulkModifyOperatorNodePushable.java |   2 +-
 .../ExternalLoopkupOperatorDiscriptor.java      | 141 +++++++
 .../external/library/ExternalFunction.java      |   1 -
 .../library/ExternalLibraryManager.java         |  58 +++
 .../adapter/TestTypedAdapterFactory.java        |   3 +-
 .../aql/statement/SubscribeFeedStatement.java   |   2 +-
 asterix-metadata/pom.xml                        | 138 +++----
 .../asterix/metadata/MetadataManager.java       |   2 +-
 .../apache/asterix/metadata/MetadataNode.java   |   2 +-
 .../asterix/metadata/api/IMetadataManager.java  |   2 +-
 .../asterix/metadata/api/IMetadataNode.java     |   2 +-
 .../metadata/bootstrap/MetadataBootstrap.java   |   4 +-
 .../metadata/declared/AqlMetadataProvider.java  | 402 +++++++++++--------
 .../declared/FieldExtractingAdapterFactory.java |   4 +-
 .../declared/PKGeneratingAdapterFactory.java    |   2 +-
 .../asterix/metadata/entities/ExternalFile.java | 146 -------
 .../ExternalFileTupleTranslator.java            |   2 +-
 .../external/ExternalFileIndexAccessor.java     | 148 -------
 .../ExternalLoopkupOperatorDiscriptor.java      | 137 -------
 .../external/FilesIndexDescription.java         | 106 -----
 .../metadata/external/IAdapterFactory.java      |  98 -----
 .../metadata/external/IControlledAdapter.java   |  65 ---
 .../external/IControlledAdapterFactory.java     |  34 --
 .../metadata/external/IndexingConstants.java    | 207 ----------
 .../feeds/AbstractFeedDatasourceAdapter.java    |   1 +
 .../ExternalDataScanOperatorDescriptor.java     |   2 +-
 .../feeds/FeedIntakeOperatorDescriptor.java     |   3 +-
 .../feeds/FeedIntakeOperatorNodePushable.java   |   1 +
 .../feeds/FeedMetaComputeNodePushable.java      |   1 +
 .../metadata/feeds/FeedMetaNodePushable.java    |   1 +
 .../feeds/FeedMetaStoreNodePushable.java        |   1 +
 .../metadata/feeds/FeedPolicyEnforcer.java      |  50 ---
 .../apache/asterix/metadata/feeds/FeedUtil.java |   5 +-
 .../metadata/feeds/IFeedAdapterFactory.java     |  30 --
 .../metadata/feeds/IPullBasedFeedAdapter.java   |  34 --
 .../metadata/feeds/ITypedAdapterFactory.java    |   2 +-
 .../functions/ExternalLibraryManager.java       |  58 ---
 .../asterix/metadata/utils/DatasetUtils.java    |  12 +-
 .../ExternalFilesIndexOperatorDescriptor.java   |   4 +-
 .../data/GenericSocketFeedAdapterFactory.java   |   4 +-
 ...ControlledFileSystemBasedAdapterFactory.java |   4 +-
 .../data/SocketClientAdapterFactory.java        |   2 +-
 .../data/TwitterFirehoseFeedAdapterFactory.java |   2 +-
 94 files changed, 1823 insertions(+), 1899 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/ExternalDataLookupPOperator.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/ExternalDataLookupPOperator.java b/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/ExternalDataLookupPOperator.java
index ac8e775..9b85674 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/ExternalDataLookupPOperator.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/ExternalDataLookupPOperator.java
@@ -21,12 +21,11 @@ package org.apache.asterix.algebra.operators.physical;
 import java.util.ArrayList;
 import java.util.List;
 
-import org.apache.asterix.external.indexing.dataflow.HDFSLookupAdapterFactory;
 import org.apache.asterix.metadata.declared.AqlDataSource;
+import org.apache.asterix.metadata.declared.AqlDataSource.AqlDataSourceType;
 import org.apache.asterix.metadata.declared.AqlMetadataProvider;
 import org.apache.asterix.metadata.declared.AqlSourceId;
 import org.apache.asterix.metadata.declared.DatasetDataSource;
-import org.apache.asterix.metadata.declared.AqlDataSource.AqlDataSourceType;
 import org.apache.asterix.metadata.entities.Dataset;
 import org.apache.asterix.metadata.entities.Index;
 import org.apache.asterix.om.functions.AsterixBuiltinFunctions;
@@ -70,7 +69,8 @@ public class ExternalDataLookupPOperator extends AbstractScanPOperator {
     private boolean retainNull;
 
     public ExternalDataLookupPOperator(AqlSourceId datasetId, Dataset dataset, ARecordType recordType,
-            Index secondaryIndex, List<LogicalVariable> ridVarList, boolean requiresBroadcast, boolean retainInput, boolean retainNull) {
+            Index secondaryIndex, List<LogicalVariable> ridVarList, boolean requiresBroadcast, boolean retainInput,
+            boolean retainNull) {
         this.datasetId = datasetId;
         this.dataset = dataset;
         this.recordType = recordType;
@@ -123,7 +123,7 @@ public class ExternalDataLookupPOperator extends AbstractScanPOperator {
     @Override
     public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
             IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
-            throws AlgebricksException {
+                    throws AlgebricksException {
         ExternalDataLookupOperator edabro = (ExternalDataLookupOperator) op;
         ILogicalExpression expr = edabro.getExpressionRef().getValue();
         if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
@@ -144,7 +144,7 @@ public class ExternalDataLookupPOperator extends AbstractScanPOperator {
         }
 
         AqlMetadataProvider metadataProvider = (AqlMetadataProvider) context.getMetadataProvider();
-        Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> externalLoopup = HDFSLookupAdapterFactory
+        Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> externalLoopup = AqlMetadataProvider
                 .buildExternalDataLookupRuntime(builder.getJobSpec(), dataset, secondaryIndex, ridIndexes, retainInput,
                         typeEnv, outputVars, opSchema, context, metadataProvider, retainNull);
         builder.contributeHyracksOperator(edabro, externalLoopup.first);
@@ -164,6 +164,7 @@ public class ExternalDataLookupPOperator extends AbstractScanPOperator {
         return keyIndexes;
     }
 
+    @Override
     public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
             IPhysicalPropertiesVector reqdByParent) {
         if (requiresBroadcast) {

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
index d2c36cc..1ff3df6 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
@@ -29,11 +29,12 @@ import org.apache.asterix.algebra.operators.physical.ExternalDataLookupPOperator
 import org.apache.asterix.common.config.DatasetConfig.DatasetType;
 import org.apache.asterix.common.config.DatasetConfig.IndexType;
 import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.external.indexing.IndexingConstants;
 import org.apache.asterix.lang.common.util.FunctionUtil;
 import org.apache.asterix.metadata.declared.AqlSourceId;
 import org.apache.asterix.metadata.entities.Dataset;
+import org.apache.asterix.metadata.entities.ExternalDatasetDetails;
 import org.apache.asterix.metadata.entities.Index;
-import org.apache.asterix.metadata.external.IndexingConstants;
 import org.apache.asterix.metadata.utils.DatasetUtils;
 import org.apache.asterix.om.base.ABoolean;
 import org.apache.asterix.om.base.AInt32;
@@ -278,7 +279,8 @@ public class AccessMethodUtils {
                     throws AlgebricksException {
         int numPrimaryKeys = 0;
         if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
-            numPrimaryKeys = IndexingConstants.getRIDSize(dataset);
+            numPrimaryKeys = IndexingConstants
+                    .getRIDSize(((ExternalDatasetDetails) dataset.getDatasetDetails()).getProperties());
         } else {
             numPrimaryKeys = DatasetUtils.getPartitioningKeys(dataset).size();
         }
@@ -293,7 +295,8 @@ public class AccessMethodUtils {
             ILogicalOperator unnestMapOp) {
         int numPrimaryKeys;
         if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
-            numPrimaryKeys = IndexingConstants.getRIDSize(dataset);
+            numPrimaryKeys = IndexingConstants
+                    .getRIDSize(((ExternalDatasetDetails) dataset.getDatasetDetails()).getProperties());
         } else {
             numPrimaryKeys = DatasetUtils.getPartitioningKeys(dataset).size();
         }
@@ -567,7 +570,8 @@ public class AccessMethodUtils {
     }
 
     private static void appendExternalRecPrimaryKeys(Dataset dataset, List<Object> target) throws AsterixException {
-        int numPrimaryKeys = IndexingConstants.getRIDSize(dataset);
+        int numPrimaryKeys = IndexingConstants
+                .getRIDSize(((ExternalDatasetDetails) dataset.getDatasetDetails()).getProperties());
         for (int i = 0; i < numPrimaryKeys; i++) {
             target.add(IndexingConstants.getFieldType(i));
         }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java b/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
index 51e1612..714e05c 100644
--- a/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
+++ b/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
@@ -63,6 +63,8 @@ import org.apache.asterix.common.feeds.api.IFeedLifecycleEventSubscriber.FeedLif
 import org.apache.asterix.common.feeds.api.IFeedLifecycleListener.ConnectionLocation;
 import org.apache.asterix.common.functions.FunctionSignature;
 import org.apache.asterix.compiler.provider.ILangCompilationProvider;
+import org.apache.asterix.external.adapter.factory.IFeedAdapterFactory;
+import org.apache.asterix.external.indexing.ExternalFile;
 import org.apache.asterix.feeds.CentralFeedManager;
 import org.apache.asterix.feeds.FeedJoint;
 import org.apache.asterix.feeds.FeedLifecycleListener;
@@ -127,7 +129,6 @@ import org.apache.asterix.metadata.entities.Dataset;
 import org.apache.asterix.metadata.entities.Datatype;
 import org.apache.asterix.metadata.entities.Dataverse;
 import org.apache.asterix.metadata.entities.ExternalDatasetDetails;
-import org.apache.asterix.metadata.entities.ExternalFile;
 import org.apache.asterix.metadata.entities.Feed;
 import org.apache.asterix.metadata.entities.Feed.FeedType;
 import org.apache.asterix.metadata.entities.FeedPolicy;
@@ -139,7 +140,6 @@ import org.apache.asterix.metadata.entities.PrimaryFeed;
 import org.apache.asterix.metadata.entities.SecondaryFeed;
 import org.apache.asterix.metadata.feeds.FeedLifecycleEventSubscriber;
 import org.apache.asterix.metadata.feeds.FeedUtil;
-import org.apache.asterix.metadata.feeds.IFeedAdapterFactory;
 import org.apache.asterix.metadata.utils.DatasetUtils;
 import org.apache.asterix.metadata.utils.ExternalDatasetsRegistry;
 import org.apache.asterix.metadata.utils.MetadataLockManager;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-app/src/main/java/org/apache/asterix/file/ExternalIndexingOperations.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/file/ExternalIndexingOperations.java b/asterix-app/src/main/java/org/apache/asterix/file/ExternalIndexingOperations.java
index 1b7bd1e..d3fade5 100644
--- a/asterix-app/src/main/java/org/apache/asterix/file/ExternalIndexingOperations.java
+++ b/asterix-app/src/main/java/org/apache/asterix/file/ExternalIndexingOperations.java
@@ -48,6 +48,9 @@ import org.apache.asterix.dataflow.data.nontagged.valueproviders.AqlPrimitiveVal
 import org.apache.asterix.external.adapter.factory.HDFSAdapterFactory;
 import org.apache.asterix.external.adapter.factory.HDFSIndexingAdapterFactory;
 import org.apache.asterix.external.adapter.factory.HiveAdapterFactory;
+import org.apache.asterix.external.indexing.ExternalFile;
+import org.apache.asterix.external.indexing.FilesIndexDescription;
+import org.apache.asterix.external.indexing.IndexingConstants;
 import org.apache.asterix.external.indexing.operators.ExternalDatasetIndexesAbortOperatorDescriptor;
 import org.apache.asterix.external.indexing.operators.ExternalDatasetIndexesCommitOperatorDescriptor;
 import org.apache.asterix.external.indexing.operators.ExternalDatasetIndexesRecoverOperatorDescriptor;
@@ -60,10 +63,7 @@ import org.apache.asterix.metadata.MetadataManager;
 import org.apache.asterix.metadata.declared.AqlMetadataProvider;
 import org.apache.asterix.metadata.entities.Dataset;
 import org.apache.asterix.metadata.entities.ExternalDatasetDetails;
-import org.apache.asterix.metadata.entities.ExternalFile;
 import org.apache.asterix.metadata.entities.Index;
-import org.apache.asterix.metadata.external.FilesIndexDescription;
-import org.apache.asterix.metadata.external.IndexingConstants;
 import org.apache.asterix.metadata.feeds.ExternalDataScanOperatorDescriptor;
 import org.apache.asterix.metadata.utils.DatasetUtils;
 import org.apache.asterix.metadata.utils.ExternalDatasetsRegistry;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-app/src/main/java/org/apache/asterix/file/FeedOperations.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/file/FeedOperations.java b/asterix-app/src/main/java/org/apache/asterix/file/FeedOperations.java
index cd2f304..c7bb0e3 100644
--- a/asterix-app/src/main/java/org/apache/asterix/file/FeedOperations.java
+++ b/asterix-app/src/main/java/org/apache/asterix/file/FeedOperations.java
@@ -33,11 +33,11 @@ import org.apache.asterix.common.feeds.api.IFeedMessage;
 import org.apache.asterix.common.feeds.api.IFeedRuntime.FeedRuntimeType;
 import org.apache.asterix.common.feeds.message.EndFeedMessage;
 import org.apache.asterix.common.feeds.message.ThrottlingEnabledFeedMessage;
+import org.apache.asterix.external.adapter.factory.IFeedAdapterFactory;
 import org.apache.asterix.feeds.FeedLifecycleListener;
 import org.apache.asterix.metadata.declared.AqlMetadataProvider;
 import org.apache.asterix.metadata.entities.PrimaryFeed;
 import org.apache.asterix.metadata.feeds.FeedMessageOperatorDescriptor;
-import org.apache.asterix.metadata.feeds.IFeedAdapterFactory;
 import org.apache.asterix.metadata.feeds.PrepareStallMessage;
 import org.apache.asterix.metadata.feeds.TerminateDataFlowMessage;
 import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-app/src/main/java/org/apache/asterix/file/IndexOperations.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/file/IndexOperations.java b/asterix-app/src/main/java/org/apache/asterix/file/IndexOperations.java
index c5870a6..4cb8a0f 100644
--- a/asterix-app/src/main/java/org/apache/asterix/file/IndexOperations.java
+++ b/asterix-app/src/main/java/org/apache/asterix/file/IndexOperations.java
@@ -26,10 +26,10 @@ import org.apache.asterix.common.config.OptimizationConfUtil;
 import org.apache.asterix.common.context.AsterixVirtualBufferCacheProvider;
 import org.apache.asterix.common.exceptions.AsterixException;
 import org.apache.asterix.common.ioopcallbacks.LSMBTreeIOOperationCallbackFactory;
+import org.apache.asterix.external.indexing.ExternalFile;
 import org.apache.asterix.metadata.MetadataException;
 import org.apache.asterix.metadata.declared.AqlMetadataProvider;
 import org.apache.asterix.metadata.entities.Dataset;
-import org.apache.asterix.metadata.entities.ExternalFile;
 import org.apache.asterix.metadata.utils.DatasetUtils;
 import org.apache.asterix.om.types.ARecordType;
 import org.apache.asterix.om.util.AsterixAppContextInfo;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-app/src/main/java/org/apache/asterix/file/SecondaryBTreeOperationsHelper.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/file/SecondaryBTreeOperationsHelper.java b/asterix-app/src/main/java/org/apache/asterix/file/SecondaryBTreeOperationsHelper.java
index 7c14e5d..a579d2c 100644
--- a/asterix-app/src/main/java/org/apache/asterix/file/SecondaryBTreeOperationsHelper.java
+++ b/asterix-app/src/main/java/org/apache/asterix/file/SecondaryBTreeOperationsHelper.java
@@ -30,9 +30,9 @@ import org.apache.asterix.common.context.AsterixVirtualBufferCacheProvider;
 import org.apache.asterix.common.exceptions.AsterixException;
 import org.apache.asterix.common.ioopcallbacks.LSMBTreeIOOperationCallbackFactory;
 import org.apache.asterix.common.ioopcallbacks.LSMBTreeWithBuddyIOOperationCallbackFactory;
+import org.apache.asterix.external.indexing.IndexingConstants;
 import org.apache.asterix.metadata.declared.AqlMetadataProvider;
 import org.apache.asterix.metadata.entities.Index;
-import org.apache.asterix.metadata.external.IndexingConstants;
 import org.apache.asterix.metadata.feeds.ExternalDataScanOperatorDescriptor;
 import org.apache.asterix.metadata.utils.ExternalDatasetsRegistry;
 import org.apache.asterix.om.types.IAType;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-app/src/main/java/org/apache/asterix/file/SecondaryIndexOperationsHelper.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/file/SecondaryIndexOperationsHelper.java b/asterix-app/src/main/java/org/apache/asterix/file/SecondaryIndexOperationsHelper.java
index 0cc137b..4eb6944 100644
--- a/asterix-app/src/main/java/org/apache/asterix/file/SecondaryIndexOperationsHelper.java
+++ b/asterix-app/src/main/java/org/apache/asterix/file/SecondaryIndexOperationsHelper.java
@@ -36,6 +36,8 @@ import org.apache.asterix.common.exceptions.AsterixException;
 import org.apache.asterix.common.ioopcallbacks.LSMBTreeIOOperationCallbackFactory;
 import org.apache.asterix.common.transactions.IRecoveryManager.ResourceType;
 import org.apache.asterix.common.transactions.JobId;
+import org.apache.asterix.external.indexing.ExternalFile;
+import org.apache.asterix.external.indexing.IndexingConstants;
 import org.apache.asterix.external.indexing.operators.ExternalIndexBulkModifyOperatorDescriptor;
 import org.apache.asterix.formats.nontagged.AqlBinaryBooleanInspectorImpl;
 import org.apache.asterix.formats.nontagged.AqlBinaryComparatorFactoryProvider;
@@ -44,8 +46,6 @@ import org.apache.asterix.formats.nontagged.AqlTypeTraitProvider;
 import org.apache.asterix.metadata.MetadataException;
 import org.apache.asterix.metadata.declared.AqlMetadataProvider;
 import org.apache.asterix.metadata.entities.Dataset;
-import org.apache.asterix.metadata.entities.ExternalFile;
-import org.apache.asterix.metadata.external.IndexingConstants;
 import org.apache.asterix.metadata.feeds.ExternalDataScanOperatorDescriptor;
 import org.apache.asterix.metadata.utils.DatasetUtils;
 import org.apache.asterix.om.types.ARecordType;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-app/src/main/java/org/apache/asterix/file/SecondaryRTreeOperationsHelper.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/file/SecondaryRTreeOperationsHelper.java b/asterix-app/src/main/java/org/apache/asterix/file/SecondaryRTreeOperationsHelper.java
index 6777d82..4d887dc 100644
--- a/asterix-app/src/main/java/org/apache/asterix/file/SecondaryRTreeOperationsHelper.java
+++ b/asterix-app/src/main/java/org/apache/asterix/file/SecondaryRTreeOperationsHelper.java
@@ -30,12 +30,12 @@ import org.apache.asterix.common.context.AsterixVirtualBufferCacheProvider;
 import org.apache.asterix.common.exceptions.AsterixException;
 import org.apache.asterix.common.ioopcallbacks.LSMRTreeIOOperationCallbackFactory;
 import org.apache.asterix.dataflow.data.nontagged.valueproviders.AqlPrimitiveValueProviderFactory;
+import org.apache.asterix.external.indexing.IndexingConstants;
 import org.apache.asterix.formats.nontagged.AqlBinaryComparatorFactoryProvider;
 import org.apache.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
 import org.apache.asterix.formats.nontagged.AqlTypeTraitProvider;
 import org.apache.asterix.metadata.declared.AqlMetadataProvider;
 import org.apache.asterix.metadata.entities.Index;
-import org.apache.asterix.metadata.external.IndexingConstants;
 import org.apache.asterix.metadata.feeds.ExternalDataScanOperatorDescriptor;
 import org.apache.asterix.metadata.utils.ExternalDatasetsRegistry;
 import org.apache.asterix.om.types.ATypeTag;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/AsterixGlobalRecoveryManager.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/AsterixGlobalRecoveryManager.java b/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/AsterixGlobalRecoveryManager.java
index 7cb2fd7..c6e04df 100644
--- a/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/AsterixGlobalRecoveryManager.java
+++ b/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/AsterixGlobalRecoveryManager.java
@@ -29,6 +29,7 @@ import org.apache.asterix.common.api.IClusterManagementWorkResponse;
 import org.apache.asterix.common.config.DatasetConfig.DatasetType;
 import org.apache.asterix.common.config.DatasetConfig.ExternalDatasetTransactionState;
 import org.apache.asterix.common.config.DatasetConfig.ExternalFilePendingOp;
+import org.apache.asterix.external.indexing.ExternalFile;
 import org.apache.asterix.feeds.CentralFeedManager;
 import org.apache.asterix.file.ExternalIndexingOperations;
 import org.apache.asterix.metadata.MetadataManager;
@@ -38,7 +39,6 @@ import org.apache.asterix.metadata.declared.AqlMetadataProvider;
 import org.apache.asterix.metadata.entities.Dataset;
 import org.apache.asterix.metadata.entities.Dataverse;
 import org.apache.asterix.metadata.entities.ExternalDatasetDetails;
-import org.apache.asterix.metadata.entities.ExternalFile;
 import org.apache.asterix.metadata.entities.Index;
 import org.apache.asterix.om.util.AsterixClusterProperties;
 import org.apache.asterix.common.api.IClusterManagementWork.ClusterState;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/ExternalLibraryBootstrap.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/ExternalLibraryBootstrap.java b/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/ExternalLibraryBootstrap.java
index 51d857a..267be3d 100755
--- a/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/ExternalLibraryBootstrap.java
+++ b/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/ExternalLibraryBootstrap.java
@@ -37,6 +37,7 @@ import org.apache.asterix.common.exceptions.ACIDException;
 import org.apache.asterix.common.exceptions.AsterixException;
 import org.apache.asterix.common.functions.FunctionSignature;
 import org.apache.asterix.external.library.ExternalLibrary;
+import org.apache.asterix.external.library.ExternalLibraryManager;
 import org.apache.asterix.external.library.LibraryAdapter;
 import org.apache.asterix.external.library.LibraryFunction;
 import org.apache.asterix.metadata.MetadataManager;
@@ -46,7 +47,6 @@ import org.apache.asterix.metadata.entities.DatasourceAdapter;
 import org.apache.asterix.metadata.entities.DatasourceAdapter.AdapterType;
 import org.apache.asterix.metadata.entities.Dataverse;
 import org.apache.asterix.metadata.feeds.AdapterIdentifier;
-import org.apache.asterix.metadata.functions.ExternalLibraryManager;
 import org.apache.asterix.runtime.formats.NonTaggedDataFormat;
 
 public class ExternalLibraryBootstrap {

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-external-data/pom.xml
----------------------------------------------------------------------
diff --git a/asterix-external-data/pom.xml b/asterix-external-data/pom.xml
index 86e466e..4062b23 100644
--- a/asterix-external-data/pom.xml
+++ b/asterix-external-data/pom.xml
@@ -17,184 +17,174 @@
  ! under the License.
  !-->
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-	<modelVersion>4.0.0</modelVersion>
-	<parent>
-		<artifactId>asterix</artifactId>
-		<groupId>org.apache.asterix</groupId>
-		<version>0.8.8-SNAPSHOT</version>
-	</parent>
-
-  <licenses>
-    <license>
-      <name>Apache License, Version 2.0</name>
-      <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
-      <distribution>repo</distribution>
-      <comments>A business-friendly OSS license</comments>
-    </license>
-  </licenses>
-
-	<artifactId>asterix-external-data</artifactId>
-
-	<build>
-		<plugins>
-			<plugin>
-				<groupId>org.jvnet.jaxb2.maven2</groupId>
-				<artifactId>maven-jaxb2-plugin</artifactId>
-				<version>0.9.0</version>
-				<executions>
-					<execution>
-						<id>configuration</id>
-						<goals>
-							<goal>generate</goal>
-						</goals>
-						<configuration>
-							<schemaDirectory>src/main/resources/schema</schemaDirectory>
-							<schemaIncludes>
-								<include>library.xsd</include>
-							</schemaIncludes>
-							<generatePackage>org.apache.asterix.external.library</generatePackage>
-							<generateDirectory>${project.build.directory}/generated-sources/configuration</generateDirectory>
-						</configuration>
-					</execution>
-				</executions>
-			</plugin>
-			<plugin>
-				<groupId>org.apache.maven.plugins</groupId>
-				<artifactId>maven-jar-plugin</artifactId>
-				<version>2.4</version>
-				<configuration>
-					<includes>
-						<include>**/*.class</include>
-						<include>**/*.txt</include>
-					</includes>
-				</configuration>
-				<executions>
-					<execution>
-						<goals>
-							<goal>test-jar</goal>
-						</goals>
-						<phase>package</phase>
-					</execution>
-				</executions>
-			</plugin>
-			<plugin>
-				<artifactId>maven-assembly-plugin</artifactId>
-				<version>2.2-beta-5</version>
-				<executions>
-					<execution>
-						<configuration>
-							<descriptor>src/main/assembly/binary-assembly-libzip.xml</descriptor>
-							<finalName>testlib-zip</finalName>
-						</configuration>
-						<phase>package</phase>
-						<goals>
-							<goal>attached</goal>
-						</goals>
-					</execution>
-				</executions>
-			</plugin>
-		</plugins>
-	</build>
-	<dependencies>
-		<dependency>
-			<groupId>javax.servlet</groupId>
-			<artifactId>servlet-api</artifactId>
-			<type>jar</type>
-		</dependency>
-		<dependency>
-			<groupId>org.apache.asterix</groupId>
-			<artifactId>asterix-om</artifactId>
-			<version>0.8.8-SNAPSHOT</version>
-			<type>jar</type>
-			<scope>compile</scope>
-		</dependency>
-		<dependency>
-			<groupId>org.apache.asterix</groupId>
-			<artifactId>asterix-runtime</artifactId>
-			<version>0.8.8-SNAPSHOT</version>
-			<type>jar</type>
-			<scope>compile</scope>
-		</dependency>
-		<dependency>
-			<groupId>org.apache.asterix</groupId>
-			<artifactId>asterix-metadata</artifactId>
-			<version>0.8.8-SNAPSHOT</version>
-			<type>jar</type>
-			<scope>compile</scope>
-		</dependency>
-		<dependency>
-			<groupId>org.apache.hyracks</groupId>
-			<artifactId>algebricks-compiler</artifactId>
-		</dependency>
-		<dependency>
-			<groupId>com.kenai.nbpwr</groupId>
-			<artifactId>org-apache-commons-io</artifactId>
-			<version>1.3.1-201002241208</version>
-			<scope>test</scope>
-		</dependency>
-		<dependency>
-			<groupId>org.twitter4j</groupId>
-			<artifactId>twitter4j-core</artifactId>
-			<version>[4.0,)</version>
-		</dependency>
-		<dependency>
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <artifactId>asterix</artifactId>
+        <groupId>org.apache.asterix</groupId>
+        <version>0.8.8-SNAPSHOT</version>
+    </parent>
+    <licenses>
+        <license>
+            <name>Apache License, Version 2.0</name>
+            <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+            <distribution>repo</distribution>
+            <comments>A business-friendly OSS license</comments>
+        </license>
+    </licenses>
+    <artifactId>asterix-external-data</artifactId>
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.jvnet.jaxb2.maven2</groupId>
+                <artifactId>maven-jaxb2-plugin</artifactId>
+                <version>0.9.0</version>
+                <executions>
+                    <execution>
+                        <id>configuration</id>
+                        <goals>
+                            <goal>generate</goal>
+                        </goals>
+                        <configuration>
+                            <schemaDirectory>src/main/resources/schema</schemaDirectory>
+                            <schemaIncludes>
+                                <include>library.xsd</include>
+                            </schemaIncludes>
+                            <generatePackage>org.apache.asterix.external.library</generatePackage>
+                            <generateDirectory>${project.build.directory}/generated-sources/configuration</generateDirectory>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-jar-plugin</artifactId>
+                <version>2.4</version>
+                <configuration>
+                    <includes>
+                        <include>**/*.class</include>
+                        <include>**/*.txt</include>
+                    </includes>
+                </configuration>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>test-jar</goal>
+                        </goals>
+                        <phase>package</phase>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <version>2.2-beta-5</version>
+                <executions>
+                    <execution>
+                        <configuration>
+                            <descriptor>src/main/assembly/binary-assembly-libzip.xml</descriptor>
+                            <finalName>testlib-zip</finalName>
+                        </configuration>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>attached</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+    <dependencies>
+        <dependency>
+            <groupId>javax.servlet</groupId>
+            <artifactId>servlet-api</artifactId>
+            <type>jar</type>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.asterix</groupId>
+            <artifactId>asterix-om</artifactId>
+            <version>0.8.8-SNAPSHOT</version>
+            <type>jar</type>
+            <scope>compile</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.asterix</groupId>
+            <artifactId>asterix-runtime</artifactId>
+            <version>0.8.8-SNAPSHOT</version>
+            <type>jar</type>
+            <scope>compile</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.hyracks</groupId>
+            <artifactId>algebricks-compiler</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.kenai.nbpwr</groupId>
+            <artifactId>org-apache-commons-io</artifactId>
+            <version>1.3.1-201002241208</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.twitter4j</groupId>
+            <artifactId>twitter4j-core</artifactId>
+            <version>[4.0,)</version>
+        </dependency>
+        <dependency>
             <groupId>org.twitter4j</groupId>
             <artifactId>twitter4j-stream</artifactId>
             <version>[4.0,)</version>
         </dependency>
-		<dependency>
-			<groupId>org.apache.hadoop</groupId>
-			<artifactId>hadoop-client</artifactId>
-			<type>jar</type>
-			<scope>compile</scope>
-		</dependency>
-		<dependency>
-			<groupId>net.java.dev.rome</groupId>
-			<artifactId>rome-fetcher</artifactId>
-			<version>1.0.0</version>
-			<type>jar</type>
-			<scope>compile</scope>
-			<exclusions>
-				<exclusion>
-					<artifactId>rome</artifactId>
-					<groupId>net.java.dev.rome</groupId>
-				</exclusion>
-			</exclusions>
-		</dependency>
-		<dependency>
-			<groupId>rome</groupId>
-			<artifactId>rome</artifactId>
-			<version>1.0.1-modified-01</version>
-		</dependency>
-		<dependency>
-			<groupId>org.apache.hyracks</groupId>
-			<artifactId>hyracks-hdfs-core</artifactId>
-			<version>${hyracks.version}</version>
-		</dependency>
-		<dependency>
-			<groupId>jdom</groupId>
-			<artifactId>jdom</artifactId>
-			<version>1.0</version>
-		</dependency>
-		<dependency>
-			<groupId>org.apache.asterix</groupId>
-			<artifactId>asterix-common</artifactId>
-			<version>0.8.8-SNAPSHOT</version>
-		</dependency>
-		<dependency>
-			<groupId>com.microsoft.windowsazure</groupId>
-			<artifactId>microsoft-windowsazure-api</artifactId>
-			<version>0.4.4</version>
-		</dependency>
-		<dependency>
-			<groupId>org.apache.hive</groupId>
-			<artifactId>hive-exec</artifactId>
-			<version>0.13.0</version>
-		</dependency>
-		<dependency>
-			<groupId>javax.jdo</groupId>
-			<artifactId>jdo2-api</artifactId>
-			<version>2.3-20090302111651</version>
-		</dependency>
-	</dependencies>
-</project>
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-client</artifactId>
+            <type>jar</type>
+            <scope>compile</scope>
+        </dependency>
+        <dependency>
+            <groupId>net.java.dev.rome</groupId>
+            <artifactId>rome-fetcher</artifactId>
+            <version>1.0.0</version>
+            <type>jar</type>
+            <scope>compile</scope>
+            <exclusions>
+                <exclusion>
+                    <artifactId>rome</artifactId>
+                    <groupId>net.java.dev.rome</groupId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>rome</groupId>
+            <artifactId>rome</artifactId>
+            <version>1.0.1-modified-01</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.hyracks</groupId>
+            <artifactId>hyracks-hdfs-core</artifactId>
+            <version>${hyracks.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>jdom</groupId>
+            <artifactId>jdom</artifactId>
+            <version>1.0</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.asterix</groupId>
+            <artifactId>asterix-common</artifactId>
+            <version>0.8.8-SNAPSHOT</version>
+        </dependency>
+        <dependency>
+            <groupId>com.microsoft.windowsazure</groupId>
+            <artifactId>microsoft-windowsazure-api</artifactId>
+            <version>0.4.4</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.hive</groupId>
+            <artifactId>hive-exec</artifactId>
+            <version>0.13.0</version>
+        </dependency>
+        <dependency>
+            <groupId>javax.jdo</groupId>
+            <artifactId>jdo2-api</artifactId>
+            <version>2.3-20090302111651</version>
+        </dependency>
+    </dependencies>
+</project>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/CNNFeedAdapterFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/CNNFeedAdapterFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/CNNFeedAdapterFactory.java
index a74ae4d..8b7b6d5 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/CNNFeedAdapterFactory.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/CNNFeedAdapterFactory.java
@@ -26,7 +26,6 @@ import java.util.Map;
 import org.apache.asterix.common.feeds.api.IDatasourceAdapter;
 import org.apache.asterix.common.feeds.api.IIntakeProgressTracker;
 import org.apache.asterix.external.dataset.adapter.RSSFeedAdapter;
-import org.apache.asterix.metadata.feeds.IFeedAdapterFactory;
 import org.apache.asterix.om.types.ARecordType;
 import org.apache.hyracks.algebricks.common.constraints.AlgebricksCountPartitionConstraint;
 import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/HDFSAdapterFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/HDFSAdapterFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/HDFSAdapterFactory.java
index cc4e2ef..ebf41cc 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/HDFSAdapterFactory.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/HDFSAdapterFactory.java
@@ -26,26 +26,24 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-import org.apache.hadoop.fs.BlockLocation;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapred.FileSplit;
-import org.apache.hadoop.mapred.InputSplit;
-import org.apache.hadoop.mapred.JobConf;
-
 import org.apache.asterix.common.config.DatasetConfig.ExternalFilePendingOp;
 import org.apache.asterix.common.feeds.api.IDatasourceAdapter;
 import org.apache.asterix.external.dataset.adapter.HDFSAdapter;
+import org.apache.asterix.external.indexing.ExternalFile;
 import org.apache.asterix.external.indexing.dataflow.HDFSObjectTupleParserFactory;
-import org.apache.asterix.metadata.entities.ExternalFile;
-import org.apache.asterix.metadata.external.IAdapterFactory;
 import org.apache.asterix.om.types.ARecordType;
 import org.apache.asterix.om.types.IAType;
 import org.apache.asterix.om.util.AsterixAppContextInfo;
 import org.apache.asterix.om.util.AsterixClusterProperties;
 import org.apache.asterix.runtime.operators.file.AsterixTupleParserFactory;
 import org.apache.asterix.runtime.operators.file.AsterixTupleParserFactory.InputDataFormat;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.FileSplit;
+import org.apache.hadoop.mapred.InputSplit;
+import org.apache.hadoop.mapred.JobConf;
 import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
 import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
 import org.apache.hyracks.api.context.ICCContext;
@@ -106,8 +104,8 @@ public class HDFSAdapterFactory extends StreamBasedAdapterFactory implements IAd
         ICCContext ccContext = AsterixAppContextInfo.getInstance().getCCApplicationContext().getCCContext();
         Scheduler scheduler = null;
         try {
-            scheduler = new Scheduler(ccContext.getClusterControllerInfo().getClientNetAddress(), ccContext
-                    .getClusterControllerInfo().getClientNetPort());
+            scheduler = new Scheduler(ccContext.getClusterControllerInfo().getClientNetAddress(),
+                    ccContext.getClusterControllerInfo().getClientNetPort());
         } catch (HyracksException e) {
             throw new IllegalStateException("Cannot obtain hdfs scheduler");
         }
@@ -145,15 +143,15 @@ public class HDFSAdapterFactory extends StreamBasedAdapterFactory implements IAd
 
     public static JobConf configureJobConf(Map<String, String> configuration) throws Exception {
         JobConf conf = new JobConf();
-        String formatClassName = (String) formatClassNames.get(((String) configuration.get(KEY_INPUT_FORMAT)).trim());
-        String localShortCircuitSocketPath = (String) configuration.get(KEY_LOCAL_SOCKET_PATH);
+        String formatClassName = formatClassNames.get(configuration.get(KEY_INPUT_FORMAT).trim());
+        String localShortCircuitSocketPath = configuration.get(KEY_LOCAL_SOCKET_PATH);
         if (formatClassName == null) {
-            formatClassName = ((String) configuration.get(KEY_INPUT_FORMAT)).trim();
+            formatClassName = configuration.get(KEY_INPUT_FORMAT).trim();
         }
-        conf.set(KEY_HADOOP_FILESYSTEM_URI, ((String) configuration.get(KEY_HDFS_URL)).trim());
+        conf.set(KEY_HADOOP_FILESYSTEM_URI, configuration.get(KEY_HDFS_URL).trim());
         conf.set(KEY_HADOOP_FILESYSTEM_CLASS, CLASS_NAME_HDFS_FILESYSTEM);
         conf.setClassLoader(HDFSAdapter.class.getClassLoader());
-        conf.set(KEY_HADOOP_INPUT_DIR, ((String) configuration.get(KEY_PATH)).trim());
+        conf.set(KEY_HADOOP_INPUT_DIR, configuration.get(KEY_PATH).trim());
         conf.set(KEY_HADOOP_INPUT_FORMAT, formatClassName);
 
         // Enable local short circuit reads if user supplied the parameters
@@ -169,7 +167,7 @@ public class HDFSAdapterFactory extends StreamBasedAdapterFactory implements IAd
         if (!configured) {
             throw new IllegalStateException("Adapter factory has not been configured yet");
         }
-        return (AlgebricksPartitionConstraint) clusterLocations;
+        return clusterLocations;
     }
 
     @Override
@@ -199,7 +197,7 @@ public class HDFSAdapterFactory extends StreamBasedAdapterFactory implements IAd
         Arrays.fill(executed, false);
         configured = true;
 
-        atype = (IAType) outputType;
+        atype = outputType;
         configureFormat(atype);
     }
 
@@ -241,26 +239,26 @@ public class HDFSAdapterFactory extends StreamBasedAdapterFactory implements IAd
      * if data is text data (adm or delimited text), it will use a text tuple parser,
      * otherwise it will use hdfs record object parser
      */
+    @Override
     protected void configureFormat(IAType sourceDatatype) throws Exception {
-         String specifiedFormat = (String) configuration.get(AsterixTupleParserFactory.KEY_FORMAT);
-         if (specifiedFormat == null) {
-                 throw new IllegalArgumentException(" Unspecified data format");
-         } 
-         
-         if(AsterixTupleParserFactory.FORMAT_BINARY.equalsIgnoreCase(specifiedFormat)){
-             parserFactory = new HDFSObjectTupleParserFactory((ARecordType) atype, this, configuration);
-         } else {
-             InputDataFormat inputFormat = InputDataFormat.UNKNOWN;
-             if (AsterixTupleParserFactory.FORMAT_DELIMITED_TEXT.equalsIgnoreCase(specifiedFormat)) {
-                 inputFormat = InputDataFormat.DELIMITED;
-             }   else if (AsterixTupleParserFactory.FORMAT_ADM.equalsIgnoreCase(specifiedFormat)) {
-                 inputFormat = InputDataFormat.ADM;
-             }    
-             parserFactory = new AsterixTupleParserFactory(configuration, (ARecordType) sourceDatatype
-                     , inputFormat);
-         }  
-        
-     }
+        String specifiedFormat = configuration.get(AsterixTupleParserFactory.KEY_FORMAT);
+        if (specifiedFormat == null) {
+            throw new IllegalArgumentException(" Unspecified data format");
+        }
+
+        if (AsterixTupleParserFactory.FORMAT_BINARY.equalsIgnoreCase(specifiedFormat)) {
+            parserFactory = new HDFSObjectTupleParserFactory((ARecordType) atype, this, configuration);
+        } else {
+            InputDataFormat inputFormat = InputDataFormat.UNKNOWN;
+            if (AsterixTupleParserFactory.FORMAT_DELIMITED_TEXT.equalsIgnoreCase(specifiedFormat)) {
+                inputFormat = InputDataFormat.DELIMITED;
+            } else if (AsterixTupleParserFactory.FORMAT_ADM.equalsIgnoreCase(specifiedFormat)) {
+                inputFormat = InputDataFormat.ADM;
+            }
+            parserFactory = new AsterixTupleParserFactory(configuration, (ARecordType) sourceDatatype, inputFormat);
+        }
+
+    }
 
     /**
      * Instead of creating the split using the input format, we do it manually
@@ -296,9 +294,11 @@ public class HDFSAdapterFactory extends StreamBasedAdapterFactory implements IAd
                 // Create a split per block
                 for (BlockLocation block : fileBlocks) {
                     if (block.getOffset() < file.getSize()) {
-                        fileSplits.add(new FileSplit(filePath, block.getOffset(), (block.getLength() + block
-                                .getOffset()) < file.getSize() ? block.getLength() : (file.getSize() - block
-                                .getOffset()), block.getHosts()));
+                        fileSplits
+                                .add(new FileSplit(filePath,
+                                        block.getOffset(), (block.getLength() + block.getOffset()) < file.getSize()
+                                                ? block.getLength() : (file.getSize() - block.getOffset()),
+                                block.getHosts()));
                         orderedExternalFiles.add(file);
                     }
                 }
@@ -322,11 +322,11 @@ public class HDFSAdapterFactory extends StreamBasedAdapterFactory implements IAd
                         if (block.getOffset() < newSize) {
                             // Block interact with delta -> Create a split
                             long startCut = (block.getOffset() > oldSize) ? 0L : oldSize - block.getOffset();
-                            long endCut = (block.getOffset() + block.getLength() < newSize) ? 0L : block.getOffset()
-                                    + block.getLength() - newSize;
+                            long endCut = (block.getOffset() + block.getLength() < newSize) ? 0L
+                                    : block.getOffset() + block.getLength() - newSize;
                             long splitLength = block.getLength() - startCut - endCut;
-                            fileSplits.add(new FileSplit(filePath, block.getOffset() + startCut, splitLength, block
-                                    .getHosts()));
+                            fileSplits.add(new FileSplit(filePath, block.getOffset() + startCut, splitLength,
+                                    block.getHosts()));
                             orderedExternalFiles.add(file);
                         }
                     }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/HDFSIndexingAdapterFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/HDFSIndexingAdapterFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/HDFSIndexingAdapterFactory.java
index 0033e5c..11e2b96 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/HDFSIndexingAdapterFactory.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/HDFSIndexingAdapterFactory.java
@@ -23,9 +23,6 @@ import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
 
-import org.apache.hadoop.mapred.InputSplit;
-import org.apache.hadoop.mapred.JobConf;
-
 import org.apache.asterix.common.feeds.api.IDatasourceAdapter;
 import org.apache.asterix.external.dataset.adapter.HDFSIndexingAdapter;
 import org.apache.asterix.external.indexing.dataflow.HDFSIndexingParserFactory;
@@ -39,6 +36,8 @@ import org.apache.asterix.om.util.AsterixClusterProperties;
 import org.apache.asterix.om.util.NonTaggedFormatUtil;
 import org.apache.asterix.runtime.operators.file.AsterixTupleParserFactory;
 import org.apache.asterix.runtime.operators.file.DelimitedDataParser;
+import org.apache.hadoop.mapred.InputSplit;
+import org.apache.hadoop.mapred.JobConf;
 import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
 import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
 import org.apache.hyracks.algebricks.common.exceptions.NotImplementedException;
@@ -75,8 +74,8 @@ public class HDFSIndexingAdapterFactory extends HDFSAdapterFactory {
         ICCContext ccContext = AsterixAppContextInfo.getInstance().getCCApplicationContext().getCCContext();
         IndexingScheduler scheduler = null;
         try {
-            scheduler = new IndexingScheduler(ccContext.getClusterControllerInfo().getClientNetAddress(), ccContext
-                    .getClusterControllerInfo().getClientNetPort());
+            scheduler = new IndexingScheduler(ccContext.getClusterControllerInfo().getClientNetAddress(),
+                    ccContext.getClusterControllerInfo().getClientNetPort());
         } catch (HyracksException e) {
             throw new IllegalStateException("Cannot obtain hdfs scheduler");
         }
@@ -98,7 +97,7 @@ public class HDFSIndexingAdapterFactory extends HDFSAdapterFactory {
         if (!configured) {
             throw new IllegalStateException("Adapter factory has not been configured yet");
         }
-        return (AlgebricksPartitionConstraint) clusterLocations;
+        return clusterLocations;
     }
 
     @Override
@@ -131,11 +130,12 @@ public class HDFSIndexingAdapterFactory extends HDFSAdapterFactory {
         executed = new boolean[readSchedule.length];
         Arrays.fill(executed, false);
         configured = true;
-        atype = (IAType) outputType;
+        atype = outputType;
         // The function below is overwritten to create indexing adapter factory instead of regular adapter factory
         configureFormat(atype);
     }
 
+    @Override
     protected void configureFormat(IAType sourceDatatype) throws Exception {
 
         char delimiter = AsterixTupleParserFactory.getDelimiter(configuration);

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/HiveAdapterFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/HiveAdapterFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/HiveAdapterFactory.java
index b61f5f3..553682e 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/HiveAdapterFactory.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/HiveAdapterFactory.java
@@ -24,8 +24,7 @@ import java.util.Map;
 import org.apache.asterix.common.feeds.api.IDatasourceAdapter;
 import org.apache.asterix.external.dataset.adapter.HDFSAdapter;
 import org.apache.asterix.external.dataset.adapter.HiveAdapter;
-import org.apache.asterix.metadata.entities.ExternalFile;
-import org.apache.asterix.metadata.external.IAdapterFactory;
+import org.apache.asterix.external.indexing.ExternalFile;
 import org.apache.asterix.om.types.ARecordType;
 import org.apache.asterix.om.types.IAType;
 import org.apache.asterix.runtime.operators.file.AsterixTupleParserFactory;
@@ -67,7 +66,6 @@ public class HiveAdapterFactory extends StreamBasedAdapterFactory implements IAd
         return "hive";
     }
 
-  
     @Override
     public SupportedOperation getSupportedOperations() {
         return SupportedOperation.READ;
@@ -78,13 +76,13 @@ public class HiveAdapterFactory extends StreamBasedAdapterFactory implements IAd
         if (!configured) {
             populateConfiguration(configuration);
             hdfsAdapterFactory.configure(configuration, outputType);
-            this.atype = (IAType) outputType;
+            this.atype = outputType;
         }
     }
 
     public static void populateConfiguration(Map<String, String> configuration) throws Exception {
         /** configure hive */
-        String database = (String) configuration.get(HIVE_DATABASE);
+        String database = configuration.get(HIVE_DATABASE);
         String tablePath = null;
         if (database == null) {
             tablePath = configuration.get(HIVE_WAREHOUSE_DIR) + "/" + configuration.get(HIVE_TABLE);
@@ -93,14 +91,17 @@ public class HiveAdapterFactory extends StreamBasedAdapterFactory implements IAd
                     + configuration.get(HIVE_TABLE);
         }
         configuration.put(HDFSAdapterFactory.KEY_PATH, tablePath);
-        if (!configuration.get(AsterixTupleParserFactory.KEY_FORMAT).equals(AsterixTupleParserFactory.FORMAT_DELIMITED_TEXT)) {
-            throw new IllegalArgumentException("format" + configuration.get(AsterixTupleParserFactory.KEY_FORMAT) + " is not supported");
+        if (!configuration.get(AsterixTupleParserFactory.KEY_FORMAT)
+                .equals(AsterixTupleParserFactory.FORMAT_DELIMITED_TEXT)) {
+            throw new IllegalArgumentException(
+                    "format" + configuration.get(AsterixTupleParserFactory.KEY_FORMAT) + " is not supported");
         }
 
-        if (!(configuration.get(HDFSAdapterFactory.KEY_INPUT_FORMAT).equals(HDFSAdapterFactory.INPUT_FORMAT_TEXT) || configuration
-                .get(HDFSAdapterFactory.KEY_INPUT_FORMAT).equals(HDFSAdapterFactory.INPUT_FORMAT_SEQUENCE))) {
-            throw new IllegalArgumentException("file input format"
-                    + configuration.get(HDFSAdapterFactory.KEY_INPUT_FORMAT) + " is not supported");
+        if (!(configuration.get(HDFSAdapterFactory.KEY_INPUT_FORMAT).equals(HDFSAdapterFactory.INPUT_FORMAT_TEXT)
+                || configuration.get(HDFSAdapterFactory.KEY_INPUT_FORMAT)
+                        .equals(HDFSAdapterFactory.INPUT_FORMAT_SEQUENCE))) {
+            throw new IllegalArgumentException(
+                    "file input format" + configuration.get(HDFSAdapterFactory.KEY_INPUT_FORMAT) + " is not supported");
         }
     }
 
@@ -109,7 +110,6 @@ public class HiveAdapterFactory extends StreamBasedAdapterFactory implements IAd
         return hdfsAdapterFactory.getPartitionConstraint();
     }
 
-    
     @Override
     public ARecordType getAdapterOutputType() {
         return (ARecordType) atype;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/IAdapterFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/IAdapterFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/IAdapterFactory.java
new file mode 100644
index 0000000..b8005cd
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/IAdapterFactory.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.adapter.factory;
+
+import java.io.Serializable;
+import java.util.Map;
+
+import org.apache.asterix.common.feeds.api.IDatasourceAdapter;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+
+/**
+ * Base interface for IGenericDatasetAdapterFactory and ITypedDatasetAdapterFactory.
+ * Acts as a marker interface indicating that the implementation provides functionality
+ * for creating an adapter.
+ */
+public interface IAdapterFactory extends Serializable {
+
+    public static final String KEY_TYPE_NAME = "type-name";
+
+    public enum SupportedOperation {
+        READ,
+        WRITE,
+        READ_WRITE
+    }
+
+    /**
+     * Returns the type of adapter indicating if the adapter can be used for
+     * reading from an external data source or writing to an external data
+     * source or can be used for both purposes.
+     * 
+     * @see SupportedOperation
+     * @return
+     */
+    public SupportedOperation getSupportedOperations();
+
+    /**
+     * Returns the display name corresponding to the Adapter type that is created by the factory.
+     * 
+     * @return the display name
+     */
+    public String getName();
+
+    /**
+     * Gets a list of partition constraints. A partition constraint can be a
+     * requirement to execute at a particular location or could be cardinality
+     * constraints indicating the number of instances that need to run in
+     * parallel. example, a IDatasourceAdapter implementation written for data
+     * residing on the local file system of a node cannot run on any other node
+     * and thus has a location partition constraint. The location partition
+     * constraint can be expressed as a node IP address or a node controller id.
+     * In the former case, the IP address is translated to a node controller id
+     * running on the node with the given IP address.
+     */
+    public AlgebricksPartitionConstraint getPartitionConstraint() throws Exception;
+
+    /**
+     * Creates an instance of IDatasourceAdapter.
+     * 
+     * @param HyracksTaskContext
+     * @param partition
+     * @return An instance of IDatasourceAdapter.
+     * @throws Exception
+     */
+    public IDatasourceAdapter createAdapter(IHyracksTaskContext ctx, int partition) throws Exception;
+
+    /**
+     * @param configuration
+     * @param outputType
+     * @throws Exception
+     */
+    public void configure(Map<String, String> configuration, ARecordType outputType) throws Exception;
+
+    /**
+     * Gets the record type associated with the output of the adapter
+     * 
+     * @return
+     */
+    public ARecordType getAdapterOutputType();
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/IControlledAdapterFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/IControlledAdapterFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/IControlledAdapterFactory.java
new file mode 100644
index 0000000..0de6fad
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/IControlledAdapterFactory.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.adapter.factory;
+
+import java.io.Serializable;
+import java.util.Map;
+
+import org.apache.asterix.external.dataset.adapter.IControlledAdapter;
+import org.apache.asterix.external.indexing.ExternalFileIndexAccessor;
+import org.apache.asterix.om.types.IAType;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+
+public interface IControlledAdapterFactory extends Serializable {
+    public IControlledAdapter createAdapter(IHyracksTaskContext ctx, ExternalFileIndexAccessor fileIndexAccessor,
+            RecordDescriptor inRecDesc);
+
+    public void configure(IAType atype, boolean propagateInput, int[] ridFields,
+            Map<String, String> adapterConfiguration, boolean retainNull);
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/IFeedAdapterFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/IFeedAdapterFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/IFeedAdapterFactory.java
new file mode 100644
index 0000000..9358a52
--- /dev/null
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/IFeedAdapterFactory.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.adapter.factory;
+
+import org.apache.asterix.common.feeds.api.IIntakeProgressTracker;
+
+public interface IFeedAdapterFactory extends IAdapterFactory {
+
+    public boolean isRecordTrackingEnabled();
+
+    public IIntakeProgressTracker createIntakeProgressTracker();
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/NCFileSystemAdapterFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/NCFileSystemAdapterFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/NCFileSystemAdapterFactory.java
index 61bf4a3..251d69a 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/NCFileSystemAdapterFactory.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/NCFileSystemAdapterFactory.java
@@ -26,11 +26,10 @@ import java.util.logging.Level;
 import org.apache.asterix.common.exceptions.AsterixException;
 import org.apache.asterix.common.feeds.api.IDatasourceAdapter;
 import org.apache.asterix.external.dataset.adapter.NCFileSystemAdapter;
+import org.apache.asterix.external.indexing.ExternalFile;
 import org.apache.asterix.external.util.DNSResolverFactory;
 import org.apache.asterix.external.util.INodeResolver;
 import org.apache.asterix.external.util.INodeResolverFactory;
-import org.apache.asterix.metadata.entities.ExternalFile;
-import org.apache.asterix.metadata.external.IAdapterFactory;
 import org.apache.asterix.om.types.ARecordType;
 import org.apache.asterix.om.types.IAType;
 import org.apache.asterix.runtime.operators.file.AsterixTupleParserFactory;
@@ -58,7 +57,6 @@ public class NCFileSystemAdapterFactory extends StreamBasedAdapterFactory implem
     private FileSplit[] fileSplits;
     private ARecordType outputType;
 
-
     @Override
     public IDatasourceAdapter createAdapter(IHyracksTaskContext ctx, int partition) throws Exception {
         NCFileSystemAdapter fsAdapter = new NCFileSystemAdapter(fileSplits, parserFactory, sourceDatatype, ctx);
@@ -70,7 +68,6 @@ public class NCFileSystemAdapterFactory extends StreamBasedAdapterFactory implem
         return NC_FILE_SYSTEM_ADAPTER_NAME;
     }
 
-
     @Override
     public SupportedOperation getSupportedOperations() {
         return SupportedOperation.READ;
@@ -80,8 +77,8 @@ public class NCFileSystemAdapterFactory extends StreamBasedAdapterFactory implem
     public void configure(Map<String, String> configuration, ARecordType outputType) throws Exception {
         this.configuration = configuration;
         this.outputType = outputType;
-        String[] splits = ((String) configuration.get(AsterixTupleParserFactory.KEY_PATH)).split(",");
-        IAType sourceDatatype = (IAType) outputType;
+        String[] splits = configuration.get(AsterixTupleParserFactory.KEY_PATH).split(",");
+        IAType sourceDatatype = outputType;
         configureFileSplits(splits);
         configureFormat(sourceDatatype);
 
@@ -102,8 +99,8 @@ public class NCFileSystemAdapterFactory extends StreamBasedAdapterFactory implem
             for (String splitPath : splits) {
                 trimmedValue = splitPath.trim();
                 if (!trimmedValue.contains("://")) {
-                    throw new AsterixException("Invalid path: " + splitPath
-                            + "\nUsage- path=\"Host://Absolute File Path\"");
+                    throw new AsterixException(
+                            "Invalid path: " + splitPath + "\nUsage- path=\"Host://Absolute File Path\"");
                 }
                 nodeName = trimmedValue.split(":")[0];
                 nodeLocalPath = trimmedValue.split("://")[1];
@@ -132,7 +129,8 @@ public class NCFileSystemAdapterFactory extends StreamBasedAdapterFactory implem
 
     private static INodeResolver initializeNodeResolver() {
         INodeResolver nodeResolver = null;
-        String configuredNodeResolverFactory = System.getProperty(AsterixTupleParserFactory.NODE_RESOLVER_FACTORY_PROPERTY);
+        String configuredNodeResolverFactory = System
+                .getProperty(AsterixTupleParserFactory.NODE_RESOLVER_FACTORY_PROPERTY);
         if (configuredNodeResolverFactory != null) {
             try {
                 nodeResolver = ((INodeResolverFactory) (Class.forName(configuredNodeResolverFactory).newInstance()))
@@ -150,12 +148,12 @@ public class NCFileSystemAdapterFactory extends StreamBasedAdapterFactory implem
         }
         return nodeResolver;
     }
-    
+
     @Override
     public ARecordType getAdapterOutputType() {
         return outputType;
     }
-    
+
     @Override
     public InputDataFormat getInputDataFormat() {
         return InputDataFormat.UNKNOWN;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/PullBasedAzureTwitterAdapterFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/PullBasedAzureTwitterAdapterFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/PullBasedAzureTwitterAdapterFactory.java
index bf92480..fbde1b4 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/PullBasedAzureTwitterAdapterFactory.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/PullBasedAzureTwitterAdapterFactory.java
@@ -25,13 +25,7 @@ import org.apache.asterix.common.feeds.FeedPolicyAccessor;
 import org.apache.asterix.common.feeds.api.IDatasourceAdapter;
 import org.apache.asterix.common.feeds.api.IIntakeProgressTracker;
 import org.apache.asterix.external.dataset.adapter.PullBasedAzureTwitterAdapter;
-import org.apache.asterix.metadata.MetadataManager;
-import org.apache.asterix.metadata.MetadataTransactionContext;
-import org.apache.asterix.metadata.entities.Datatype;
-import org.apache.asterix.metadata.feeds.IFeedAdapterFactory;
 import org.apache.asterix.om.types.ARecordType;
-import org.apache.asterix.om.types.ATypeTag;
-import org.apache.asterix.om.types.IAType;
 import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
 import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
 import org.apache.hyracks.api.context.IHyracksTaskContext;
@@ -42,7 +36,6 @@ public class PullBasedAzureTwitterAdapterFactory implements IFeedAdapterFactory
 
     private static final String INGESTOR_LOCATIONS_KEY = "ingestor-locations";
     private static final String PARTITIONS_KEY = "partitions";
-    private static final String OUTPUT_TYPE_KEY = "output-type";
     private static final String TABLE_NAME_KEY = "table-name";
     private static final String ACCOUNT_NAME_KEY = "account-name";
     private static final String ACCOUNT_KEY_KEY = "account-key";
@@ -78,8 +71,8 @@ public class PullBasedAzureTwitterAdapterFactory implements IFeedAdapterFactory
 
     @Override
     public IDatasourceAdapter createAdapter(IHyracksTaskContext ctx, int partition) throws Exception {
-        return new PullBasedAzureTwitterAdapter(azureAccountName, azureAccountKey, tableName, partitions,
-                configuration, ctx, outputType);
+        return new PullBasedAzureTwitterAdapter(azureAccountName, azureAccountKey, tableName, partitions, configuration,
+                ctx, outputType);
     }
 
     @Override
@@ -120,38 +113,6 @@ public class PullBasedAzureTwitterAdapterFactory implements IFeedAdapterFactory
             throw new AsterixException("Invalid adapter configuration: number of ingestion-locations ("
                     + nIngestLocations + ") must be the same as the number of partitions (" + nPartitions + ")");
         }
-        configureType();
-    }
-
-    private void configureType() throws Exception {
-        String fqOutputType = configuration.get(OUTPUT_TYPE_KEY);
-
-        if (fqOutputType == null) {
-            throw new IllegalArgumentException("No output type specified");
-        }
-        String[] dataverseAndType = fqOutputType.split("[.]");
-        String dataverseName = dataverseAndType[0];
-        String datatypeName = dataverseAndType[1];
-
-        MetadataTransactionContext ctx = null;
-        MetadataManager.INSTANCE.acquireReadLatch();
-        try {
-            ctx = MetadataManager.INSTANCE.beginTransaction();
-            Datatype t = MetadataManager.INSTANCE.getDatatype(ctx, dataverseName, datatypeName);
-            IAType type = t.getDatatype();
-            if (type.getTypeTag() != ATypeTag.RECORD) {
-                throw new IllegalStateException();
-            }
-            outputType = (ARecordType) t.getDatatype();
-            MetadataManager.INSTANCE.commitTransaction(ctx);
-        } catch (Exception e) {
-            if (ctx != null) {
-                MetadataManager.INSTANCE.abortTransaction(ctx);
-            }
-            throw e;
-        } finally {
-            MetadataManager.INSTANCE.releaseReadLatch();
-        }
     }
 
     @Override
@@ -167,7 +128,5 @@ public class PullBasedAzureTwitterAdapterFactory implements IFeedAdapterFactory
     public FeedPolicyAccessor getIngestionPolicy() {
         return ingestionPolicy;
     }
-    
-    
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/PullBasedTwitterAdapterFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/PullBasedTwitterAdapterFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/PullBasedTwitterAdapterFactory.java
index c05df33..7d2dd73 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/PullBasedTwitterAdapterFactory.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/PullBasedTwitterAdapterFactory.java
@@ -28,7 +28,6 @@ import org.apache.asterix.common.feeds.api.IIntakeProgressTracker;
 import org.apache.asterix.external.dataset.adapter.PullBasedTwitterAdapter;
 import org.apache.asterix.external.util.TwitterUtil;
 import org.apache.asterix.external.util.TwitterUtil.SearchAPIConstants;
-import org.apache.asterix.metadata.feeds.IFeedAdapterFactory;
 import org.apache.asterix.om.types.ARecordType;
 import org.apache.hyracks.algebricks.common.constraints.AlgebricksCountPartitionConstraint;
 import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/PushBasedTwitterAdapterFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/PushBasedTwitterAdapterFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/PushBasedTwitterAdapterFactory.java
index 2606063..5bfdbcf 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/PushBasedTwitterAdapterFactory.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/PushBasedTwitterAdapterFactory.java
@@ -25,7 +25,6 @@ import org.apache.asterix.common.feeds.api.IIntakeProgressTracker;
 import org.apache.asterix.external.dataset.adapter.PushBasedTwitterAdapter;
 import org.apache.asterix.external.util.TwitterUtil;
 import org.apache.asterix.external.util.TwitterUtil.AuthenticationConstants;
-import org.apache.asterix.metadata.feeds.IFeedAdapterFactory;
 import org.apache.asterix.om.types.ARecordType;
 import org.apache.hyracks.algebricks.common.constraints.AlgebricksCountPartitionConstraint;
 import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/RSSFeedAdapterFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/RSSFeedAdapterFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/RSSFeedAdapterFactory.java
index 8f47aa2..4d893fe 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/RSSFeedAdapterFactory.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/RSSFeedAdapterFactory.java
@@ -26,7 +26,6 @@ import org.apache.asterix.common.feeds.FeedPolicyAccessor;
 import org.apache.asterix.common.feeds.api.IDatasourceAdapter;
 import org.apache.asterix.common.feeds.api.IIntakeProgressTracker;
 import org.apache.asterix.external.dataset.adapter.RSSFeedAdapter;
-import org.apache.asterix.metadata.feeds.IFeedAdapterFactory;
 import org.apache.asterix.om.types.ARecordType;
 import org.apache.hyracks.algebricks.common.constraints.AlgebricksCountPartitionConstraint;
 import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
@@ -68,7 +67,7 @@ public class RSSFeedAdapterFactory implements IFeedAdapterFactory {
     public void configure(Map<String, String> configuration, ARecordType outputType) throws Exception {
         this.configuration = configuration;
         this.outputType = outputType;
-        String rssURLProperty = (String) configuration.get(KEY_RSS_URL);
+        String rssURLProperty = configuration.get(KEY_RSS_URL);
         if (rssURLProperty == null) {
             throw new IllegalArgumentException("no rss url provided");
         }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/StreamBasedAdapterFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/StreamBasedAdapterFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/StreamBasedAdapterFactory.java
index e37cf3e..c7e582f 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/StreamBasedAdapterFactory.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/StreamBasedAdapterFactory.java
@@ -22,7 +22,6 @@ import java.util.Map;
 import java.util.logging.Logger;
 
 import org.apache.asterix.external.util.INodeResolver;
-import org.apache.asterix.metadata.external.IAdapterFactory;
 import org.apache.asterix.om.types.ARecordType;
 import org.apache.asterix.om.types.IAType;
 import org.apache.asterix.runtime.operators.file.AsterixTupleParserFactory;
@@ -42,7 +41,8 @@ public abstract class StreamBasedAdapterFactory implements IAdapterFactory {
     public abstract InputDataFormat getInputDataFormat();
 
     protected void configureFormat(IAType sourceDatatype) throws Exception {
-        parserFactory = new AsterixTupleParserFactory(configuration, (ARecordType) sourceDatatype, getInputDataFormat());
+        parserFactory = new AsterixTupleParserFactory(configuration, (ARecordType) sourceDatatype,
+                getInputDataFormat());
 
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/ClientBasedFeedAdapter.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/ClientBasedFeedAdapter.java b/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/ClientBasedFeedAdapter.java
index 26d4d8f..a197368 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/ClientBasedFeedAdapter.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/ClientBasedFeedAdapter.java
@@ -25,7 +25,7 @@ import java.util.logging.Logger;
 import org.apache.asterix.common.feeds.api.IFeedAdapter;
 import org.apache.asterix.common.parse.ITupleForwardPolicy;
 import org.apache.asterix.external.dataset.adapter.IFeedClient.InflowState;
-import org.apache.asterix.metadata.feeds.FeedPolicyEnforcer;
+import org.apache.asterix.external.feeds.FeedPolicyEnforcer;
 import org.apache.asterix.om.types.ARecordType;
 import org.apache.hyracks.api.comm.IFrame;
 import org.apache.hyracks.api.comm.IFrameWriter;
@@ -138,6 +138,7 @@ public abstract class ClientBasedFeedAdapter implements IFeedAdapter {
      * 
      * @throws Exception
      */
+    @Override
     public void stop() throws Exception {
         continueIngestion = false;
     }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/HDFSAdapter.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/HDFSAdapter.java b/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/HDFSAdapter.java
index df0926b..5f1b1ae 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/HDFSAdapter.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/HDFSAdapter.java
@@ -23,11 +23,11 @@ import java.io.InputStream;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.asterix.external.indexing.ExternalFile;
 import org.apache.asterix.external.indexing.input.GenericFileAwareRecordReader;
 import org.apache.asterix.external.indexing.input.GenericRecordReader;
 import org.apache.asterix.external.indexing.input.TextualDataReader;
 import org.apache.asterix.external.indexing.input.TextualFullScanDataReader;
-import org.apache.asterix.metadata.entities.ExternalFile;
 import org.apache.asterix.om.types.IAType;
 import org.apache.asterix.runtime.operators.file.AsterixTupleParserFactory;
 import org.apache.hadoop.mapred.InputSplit;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/486e47ad/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/HDFSIndexingAdapter.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/HDFSIndexingAdapter.java b/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/HDFSIndexingAdapter.java
index 7da182a..92a049d0 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/HDFSIndexingAdapter.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/dataset/adapter/HDFSIndexingAdapter.java
@@ -22,16 +22,15 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.util.List;
 
-import org.apache.hadoop.mapred.InputSplit;
-import org.apache.hadoop.mapred.JobConf;
-
 import org.apache.asterix.external.adapter.factory.HDFSAdapterFactory;
+import org.apache.asterix.external.indexing.ExternalFile;
 import org.apache.asterix.external.indexing.input.GenericFileAwareRecordReader;
 import org.apache.asterix.external.indexing.input.RCFileDataReader;
 import org.apache.asterix.external.indexing.input.TextualDataReader;
-import org.apache.asterix.metadata.entities.ExternalFile;
 import org.apache.asterix.om.types.IAType;
 import org.apache.asterix.runtime.operators.file.AsterixTupleParserFactory;
+import org.apache.hadoop.mapred.InputSplit;
+import org.apache.hadoop.mapred.JobConf;
 import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
 import org.apache.hyracks.api.context.IHyracksTaskContext;
 import org.apache.hyracks.dataflow.std.file.ITupleParserFactory;



Mime
View raw message