asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amo...@apache.org
Subject [26/26] incubator-asterixdb git commit: Feed Fixes and Cleanup
Date Thu, 14 Jan 2016 20:32:20 GMT
Feed Fixes and Cleanup

1. Introduce filesystem feed data source.
2. Fix the order of closing feed stages on disconnection.
3. Added Twitter feed to the compatibility utility to allow
   using it with its alias.
4. First part of the feed log space.
5. Fixed the handling of duplicate key exception.

Change-Id: I4e8db26a810efd1fbaa52ceeb3efd0c8328ab070
Reviewed-on: https://asterix-gerrit.ics.uci.edu/574
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <hubailmor@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/commit/e800e6d5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/tree/e800e6d5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/diff/e800e6d5

Branch: refs/heads/master
Commit: e800e6d5f87358519c5c90e296edadb4ad383554
Parents: dace5f2
Author: Abdullah Alamoudi <bamousaa@gmail.com>
Authored: Thu Jan 14 11:20:55 2016 +0300
Committer: abdullah alamoudi <bamousaa@gmail.com>
Committed: Thu Jan 14 12:26:31 2016 -0800

----------------------------------------------------------------------
 ...ceRandomPartitioningFeedComputationRule.java |    3 +-
 .../optimizer/rules/UnnestToDataScanRule.java   |   10 +-
 .../am/AbstractIntroduceAccessMethodRule.java   |    1 +
 .../translator/AbstractLangTranslator.java      |    2 +-
 .../asterix/translator/CompiledStatements.java  |    2 +-
 asterix-algebra/src/main/javacc/AQLPlus.jj      |    2 +-
 .../api/common/AsterixAppRuntimeContext.java    |    4 +-
 .../asterix/api/common/FeedWorkCollection.java  |  205 ---
 .../api/http/servlet/ConnectorAPIServlet.java   |    2 +-
 .../asterix/api/http/servlet/FeedServlet.java   |   14 +-
 .../api/http/servlet/FeedServletUtil.java       |    6 +-
 .../asterix/aql/translator/QueryTranslator.java |   92 +-
 .../apache/asterix/feed/CentralFeedManager.java |  110 ++
 .../feed/FeedJobNotificationHandler.java        |  742 +++++++++++
 .../java/org/apache/asterix/feed/FeedJoint.java |  190 +++
 .../asterix/feed/FeedLifecycleListener.java     |  499 ++++++++
 .../apache/asterix/feed/FeedLoadManager.java    |  302 +++++
 .../asterix/feed/FeedMessageReceiver.java       |   91 ++
 .../asterix/feed/FeedTrackingManager.java       |  188 +++
 .../apache/asterix/feed/FeedWorkCollection.java |  206 +++
 .../feed/FeedWorkRequestResponseHandler.java    |  269 ++++
 .../org/apache/asterix/feed/FeedsActivator.java |  118 ++
 .../asterix/feeds/CentralFeedManager.java       |  110 --
 .../apache/asterix/feeds/FeedCollectInfo.java   |   54 -
 .../java/org/apache/asterix/feeds/FeedInfo.java |   53 -
 .../feeds/FeedJobNotificationHandler.java       |  743 -----------
 .../org/apache/asterix/feeds/FeedJoint.java     |  190 ---
 .../asterix/feeds/FeedLifecycleListener.java    |  497 --------
 .../apache/asterix/feeds/FeedLoadManager.java   |  302 -----
 .../org/apache/asterix/feeds/FeedManager.java   |  144 ---
 .../asterix/feeds/FeedMessageReceiver.java      |   92 --
 .../asterix/feeds/FeedMetadataManager.java      |  113 --
 .../asterix/feeds/FeedTrackingManager.java      |  188 ---
 .../feeds/FeedWorkRequestResponseHandler.java   |  269 ----
 .../apache/asterix/feeds/FeedsActivator.java    |  117 --
 .../file/ExternalIndexingOperations.java        |    2 +-
 .../org/apache/asterix/file/FeedOperations.java |   34 +-
 .../file/SecondaryBTreeOperationsHelper.java    |    2 +-
 .../file/SecondaryIndexOperationsHelper.java    |    2 +-
 .../file/SecondaryRTreeOperationsHelper.java    |    2 +-
 .../bootstrap/AsterixGlobalRecoveryManager.java |    4 +-
 .../bootstrap/CCApplicationEntryPoint.java      |    6 +-
 .../bootstrap/ExternalLibraryBootstrap.java     |    7 +-
 .../hyracks/bootstrap/FeedBootstrap.java        |   28 +-
 .../bootstrap/NCApplicationEntryPoint.java      |   40 +-
 .../http/servlet/ConnectorAPIServletTest.java   |    2 +-
 .../metadata/results/basic/meta15/meta15.1.adm  |    4 +-
 .../results/feeds/feeds_01/feeds_01.1.adm       |    2 +-
 .../src/test/resources/runtimets/testsuite.xml  |  153 ++-
 .../common/api/IAsterixAppRuntimeContext.java   |    3 +-
 .../config/AsterixPropertiesAccessor.java       |    2 +-
 .../common/config/MetadataConstants.java        |   33 +
 ...erixLSMInsertDeleteOperatorNodePushable.java |    8 +-
 .../common/exceptions/FrameDataException.java   |    2 +-
 .../common/feeds/BasicMonitoredBuffer.java      |   76 --
 .../asterix/common/feeds/CollectionRuntime.java |   93 --
 .../feeds/ComputeSideMonitoredBuffer.java       |   75 --
 .../apache/asterix/common/feeds/DataBucket.java |   88 --
 .../asterix/common/feeds/DataBucketPool.java    |  110 --
 .../common/feeds/DistributeFeedFrameWriter.java |  143 ---
 .../asterix/common/feeds/FeedActivity.java      |  119 --
 .../feeds/FeedCollectRuntimeInputHandler.java   |   61 -
 .../common/feeds/FeedConnectJobInfo.java        |   93 --
 .../asterix/common/feeds/FeedConnectionId.java  |   74 --
 .../common/feeds/FeedConnectionRequest.java     |  126 --
 .../asterix/common/feeds/FeedConstants.java     |   71 --
 .../common/feeds/FeedExceptionHandler.java      |  108 --
 .../asterix/common/feeds/FeedFrameCache.java    |  171 ---
 .../common/feeds/FeedFrameCollector.java        |  158 ---
 .../common/feeds/FeedFrameDiscarder.java        |   63 -
 .../asterix/common/feeds/FeedFrameHandlers.java |  303 -----
 .../asterix/common/feeds/FeedFrameSpiller.java  |  176 ---
 .../common/feeds/FeedFrameTupleAccessor.java    |  110 --
 .../asterix/common/feeds/FeedFrameUtil.java     |  102 --
 .../org/apache/asterix/common/feeds/FeedId.java |   66 -
 .../asterix/common/feeds/FeedIntakeInfo.java    |   62 -
 .../asterix/common/feeds/FeedJobInfo.java       |   86 --
 .../asterix/common/feeds/FeedJointKey.java      |   79 --
 .../asterix/common/feeds/FeedMemoryManager.java |  112 --
 .../common/feeds/FeedMessageService.java        |  144 ---
 .../common/feeds/FeedMetricCollector.java       |  187 ---
 .../common/feeds/FeedPolicyAccessor.java        |  168 ---
 .../asterix/common/feeds/FeedRuntime.java       |   74 --
 .../asterix/common/feeds/FeedRuntimeId.java     |   80 --
 .../common/feeds/FeedRuntimeInputHandler.java   |  440 -------
 .../common/feeds/FeedRuntimeManager.java        |   81 --
 .../asterix/common/feeds/FeedRuntimeReport.java |   23 -
 .../common/feeds/FeedTupleCommitAckMessage.java |   97 --
 .../feeds/FeedTupleCommitResponseMessage.java   |   80 --
 .../asterix/common/feeds/FrameCollection.java   |  101 --
 .../asterix/common/feeds/FrameDistributor.java  |  360 ------
 .../common/feeds/FrameEventCallback.java        |  102 --
 .../asterix/common/feeds/IngestionRuntime.java  |   75 --
 .../common/feeds/IntakePartitionStatistics.java |   41 -
 .../common/feeds/IntakeSideMonitoredBuffer.java |   76 --
 .../asterix/common/feeds/MessageListener.java   |  126 --
 .../asterix/common/feeds/MessageReceiver.java   |  111 --
 .../asterix/common/feeds/MonitoredBuffer.java   |  388 ------
 .../common/feeds/MonitoredBufferTimerTasks.java |  294 -----
 .../apache/asterix/common/feeds/NodeLoad.java   |   62 -
 .../asterix/common/feeds/NodeLoadReport.java    |   99 --
 .../common/feeds/NodeLoadReportService.java     |  106 --
 .../org/apache/asterix/common/feeds/Series.java |   44 -
 .../apache/asterix/common/feeds/SeriesAvg.java  |   47 -
 .../apache/asterix/common/feeds/SeriesRate.java |   92 --
 .../common/feeds/StorageFrameHandler.java       |  118 --
 .../feeds/StorageSideMonitoredBuffer.java       |  206 ---
 .../common/feeds/SubscribableFeedRuntimeId.java |   52 -
 .../common/feeds/SubscribableRuntime.java       |   90 --
 .../feeds/api/IAdapterRuntimeManager.java       |   82 --
 .../common/feeds/api/ICentralFeedManager.java   |   34 -
 .../common/feeds/api/IDataSourceAdapter.java    |   60 -
 .../common/feeds/api/IExceptionHandler.java     |   43 -
 .../feeds/api/IFeedConnectionManager.java       |   75 --
 .../common/feeds/api/IFeedFrameHandler.java     |   39 -
 .../asterix/common/feeds/api/IFeedJoint.java    |  121 --
 .../api/IFeedLifecycleEventSubscriber.java      |   36 -
 .../IFeedLifecycleIntakeEventSubscriber.java    |   28 -
 .../feeds/api/IFeedLifecycleListener.java       |   56 -
 .../common/feeds/api/IFeedLoadManager.java      |   60 -
 .../asterix/common/feeds/api/IFeedManager.java  |   97 --
 .../common/feeds/api/IFeedMemoryComponent.java  |   58 -
 .../common/feeds/api/IFeedMemoryManager.java    |   58 -
 .../asterix/common/feeds/api/IFeedMessage.java  |   52 -
 .../common/feeds/api/IFeedMessageService.java   |   34 -
 .../common/feeds/api/IFeedMetadataManager.java  |   39 -
 .../common/feeds/api/IFeedMetricCollector.java  |   50 -
 .../api/IFeedOperatorOutputSideHandler.java     |   36 -
 .../asterix/common/feeds/api/IFeedProvider.java |   26 -
 .../asterix/common/feeds/api/IFeedRuntime.java  |   62 -
 .../asterix/common/feeds/api/IFeedService.java  |   26 -
 .../feeds/api/IFeedSubscriptionManager.java     |   41 -
 .../common/feeds/api/IFeedTrackingManager.java  |   29 -
 .../asterix/common/feeds/api/IFeedWork.java     |   28 -
 .../feeds/api/IFeedWorkEventListener.java       |   41 -
 .../common/feeds/api/IFeedWorkManager.java      |   25 -
 .../common/feeds/api/IFrameEventCallback.java   |   32 -
 .../common/feeds/api/IFramePostProcessor.java   |   28 -
 .../common/feeds/api/IFramePreprocessor.java    |   26 -
 .../feeds/api/IIntakeProgressTracker.java       |   29 -
 .../common/feeds/api/IMessageReceiver.java      |   28 -
 .../common/feeds/api/ISubscribableRuntime.java  |   61 -
 .../common/feeds/api/ISubscriberRuntime.java    |   30 -
 .../common/feeds/api/ISubscriptionProvider.java |   29 -
 .../feeds/api/ITupleTrackingFeedAdapter.java    |   24 -
 .../common/feeds/message/EndFeedMessage.java    |   97 --
 .../feeds/message/FeedCongestionMessage.java    |  103 --
 .../common/feeds/message/FeedMessage.java       |   42 -
 .../common/feeds/message/FeedReportMessage.java |  100 --
 .../common/feeds/message/NodeReportMessage.java |   69 -
 .../feeds/message/ScaleInReportMessage.java     |  114 --
 .../feeds/message/StorageReportFeedMessage.java |  129 --
 .../message/ThrottlingEnabledFeedMessage.java   |   86 --
 .../asterix/common/parse/ITupleForwarder.java   |    5 +-
 .../asterix/common/utils/StoragePathUtil.java   |   61 +
 .../adapter/factory/GenericAdapterFactory.java  |    2 +-
 .../asterix/external/api/IAdapterFactory.java   |    1 -
 .../external/api/IAdapterRuntimeManager.java    |   83 ++
 .../external/api/IDataFlowController.java       |   15 +-
 .../external/api/IDataSourceAdapter.java        |   51 +
 .../asterix/external/api/IFeedAdapter.java      |   50 +
 .../asterix/external/api/IStreamDataParser.java |   12 +-
 .../dataflow/AbstractDataFlowController.java    |    3 +-
 .../AbstractFeedDataFlowController.java         |   68 +
 .../dataflow/FeedRecordDataFlowController.java  |  114 ++
 .../dataflow/FeedStreamDataFlowController.java  |   87 ++
 .../external/dataflow/FeedTupleForwarder.java   |   83 ++
 .../dataflow/RecordDataFlowController.java      |   12 +-
 .../dataflow/StreamDataFlowController.java      |   10 +
 .../dataset/adapter/AdapterIdentifier.java      |   66 +
 .../dataset/adapter/GenericAdapter.java         |   19 +-
 .../dataset/adapter/StreamBasedAdapter.java     |   65 -
 .../external/feed/api/ICentralFeedManager.java  |   34 +
 .../external/feed/api/IExceptionHandler.java    |   43 +
 .../apache/asterix/external/feed/api/IFeed.java |   55 +
 .../feed/api/IFeedConnectionManager.java        |   75 ++
 .../external/feed/api/IFeedFrameHandler.java    |   39 +
 .../asterix/external/feed/api/IFeedJoint.java   |  121 ++
 .../feed/api/IFeedLifecycleEventSubscriber.java |   36 +
 .../IFeedLifecycleIntakeEventSubscriber.java    |   28 +
 .../feed/api/IFeedLifecycleListener.java        |   56 +
 .../external/feed/api/IFeedLoadManager.java     |   60 +
 .../asterix/external/feed/api/IFeedManager.java |   72 ++
 .../external/feed/api/IFeedMemoryComponent.java |   58 +
 .../external/feed/api/IFeedMemoryManager.java   |   58 +
 .../asterix/external/feed/api/IFeedMessage.java |   52 +
 .../external/feed/api/IFeedMessageService.java  |   34 +
 .../external/feed/api/IFeedMetadataManager.java |   39 +
 .../external/feed/api/IFeedMetricCollector.java |   50 +
 .../api/IFeedOperatorOutputSideHandler.java     |   49 +
 .../external/feed/api/IFeedProvider.java        |   26 +
 .../asterix/external/feed/api/IFeedRuntime.java |   62 +
 .../asterix/external/feed/api/IFeedService.java |   26 +
 .../feed/api/IFeedSubscriptionManager.java      |   41 +
 .../external/feed/api/IFeedTrackingManager.java |   29 +
 .../asterix/external/feed/api/IFeedWork.java    |   28 +
 .../feed/api/IFeedWorkEventListener.java        |   41 +
 .../external/feed/api/IFeedWorkManager.java     |   25 +
 .../external/feed/api/IFrameEventCallback.java  |   32 +
 .../external/feed/api/IFramePostProcessor.java  |   28 +
 .../external/feed/api/IFramePreprocessor.java   |   26 +
 .../feed/api/IIntakeProgressTracker.java        |   29 +
 .../external/feed/api/IMessageReceiver.java     |   28 +
 .../external/feed/api/ISubscribableRuntime.java |   61 +
 .../external/feed/api/ISubscriberRuntime.java   |   30 +
 .../feed/api/ISubscriptionProvider.java         |   29 +
 .../CollectTransformFeedFrameWriter.java        |  119 ++
 .../external/feed/dataflow/DataBucket.java      |   89 ++
 .../external/feed/dataflow/DataBucketPool.java  |  110 ++
 .../dataflow/DistributeFeedFrameWriter.java     |  159 +++
 .../FeedCollectRuntimeInputHandler.java         |   64 +
 .../feed/dataflow/FeedExceptionHandler.java     |   80 ++
 .../external/feed/dataflow/FeedFrameCache.java  |  172 +++
 .../feed/dataflow/FeedFrameCollector.java       |  160 +++
 .../feed/dataflow/FeedFrameDiscarder.java       |   67 +
 .../feed/dataflow/FeedFrameHandlers.java        |  305 +++++
 .../feed/dataflow/FeedFrameSpiller.java         |  188 +++
 .../feed/dataflow/FeedFrameTupleAccessor.java   |  110 ++
 .../feed/dataflow/FeedFrameTupleDecorator.java  |  108 ++
 .../feed/dataflow/FeedRuntimeInputHandler.java  |  468 +++++++
 .../external/feed/dataflow/FrameCollection.java |  101 ++
 .../feed/dataflow/FrameDistributor.java         |  361 ++++++
 .../feed/dataflow/FrameEventCallback.java       |  103 ++
 .../feed/dataflow/StorageFrameHandler.java      |  119 ++
 .../feed/management/FeedCollectInfo.java        |   52 +
 .../feed/management/FeedConnectionId.java       |   74 ++
 .../feed/management/FeedConnectionManager.java  |  107 ++
 .../feed/management/FeedConnectionRequest.java  |  125 ++
 .../external/feed/management/FeedId.java        |   66 +
 .../external/feed/management/FeedInfo.java      |   53 +
 .../external/feed/management/FeedJointKey.java  |   79 ++
 .../FeedLifecycleEventSubscriber.java           |   66 +
 .../external/feed/management/FeedManager.java   |  137 ++
 .../feed/management/FeedMemoryManager.java      |  114 ++
 .../feed/management/FeedMetadataManager.java    |  112 ++
 .../feed/management/FeedRuntimeManager.java     |   83 ++
 .../management/FeedSubscriptionManager.java     |   76 ++
 .../feed/management/FeedWorkManager.java        |   50 +
 .../external/feed/message/EndFeedMessage.java   |   96 ++
 .../feed/message/FeedCongestionMessage.java     |  102 ++
 .../external/feed/message/FeedMessage.java      |   42 +
 .../feed/message/FeedMessageService.java        |  145 +++
 .../feed/message/FeedReportMessage.java         |   99 ++
 .../feed/message/FeedTupleCommitAckMessage.java |   98 ++
 .../message/FeedTupleCommitResponseMessage.java |   81 ++
 .../external/feed/message/MessageListener.java  |  126 ++
 .../external/feed/message/MessageReceiver.java  |  111 ++
 .../feed/message/NodeReportMessage.java         |   68 +
 .../feed/message/PrepareStallMessage.java       |   68 +
 .../message/RemoteSocketMessageListener.java    |  134 ++
 .../feed/message/ScaleInReportMessage.java      |  113 ++
 .../feed/message/SocketMessageListener.java     |  160 +++
 .../feed/message/StorageReportFeedMessage.java  |  128 ++
 .../feed/message/TerminateDataFlowMessage.java  |   52 +
 .../message/ThrottlingEnabledFeedMessage.java   |   85 ++
 .../external/feed/message/XAQLFeedMessage.java  |   66 +
 .../external/feed/policy/FeedPolicy.java        |   80 ++
 .../feed/policy/FeedPolicyAccessor.java         |  185 +++
 .../feed/policy/FeedPolicyEnforcer.java         |   49 +
 .../external/feed/runtime/AdapterExecutor.java  |   81 ++
 .../feed/runtime/AdapterRuntimeManager.java     |  146 +++
 .../feed/runtime/CollectionRuntime.java         |   96 ++
 .../external/feed/runtime/FeedRuntime.java      |   75 ++
 .../external/feed/runtime/FeedRuntimeId.java    |   80 ++
 .../external/feed/runtime/IngestionRuntime.java |   83 ++
 .../feed/runtime/SubscribableFeedRuntimeId.java |   53 +
 .../feed/runtime/SubscribableRuntime.java       |   95 ++
 .../feed/watch/BasicMonitoredBuffer.java        |   80 ++
 .../feed/watch/ComputeSideMonitoredBuffer.java  |   79 ++
 .../external/feed/watch/FeedActivity.java       |  116 ++
 .../external/feed/watch/FeedConnectJobInfo.java |   94 ++
 .../external/feed/watch/FeedIntakeInfo.java     |   63 +
 .../external/feed/watch/FeedJobInfo.java        |   86 ++
 .../feed/watch/FeedMetricCollector.java         |  189 +++
 .../feed/watch/IntakePartitionStatistics.java   |   41 +
 .../feed/watch/IntakeSideMonitoredBuffer.java   |   80 ++
 .../external/feed/watch/MonitoredBuffer.java    |  396 ++++++
 .../feed/watch/MonitoredBufferTimerTasks.java   |  299 +++++
 .../asterix/external/feed/watch/NodeLoad.java   |   62 +
 .../external/feed/watch/NodeLoadReport.java     |  100 ++
 .../feed/watch/NodeLoadReportService.java       |  107 ++
 .../asterix/external/feed/watch/Series.java     |   44 +
 .../asterix/external/feed/watch/SeriesAvg.java  |   47 +
 .../asterix/external/feed/watch/SeriesRate.java |   92 ++
 .../feed/watch/StorageSideMonitoredBuffer.java  |  211 ++++
 .../external/feeds/FeedPolicyEnforcer.java      |   50 -
 .../reader/AbstractStreamRecordReader.java      |   17 +-
 .../input/record/reader/LineRecordReader.java   |   10 +-
 .../record/reader/QuotedLineRecordReader.java   |    4 +
 .../reader/SemiStructuredRecordReader.java      |   12 +-
 .../record/reader/TwitterPushRecordReader.java  |   18 +-
 .../factory/TwitterRecordReaderFactory.java     |    2 +-
 .../input/stream/AInputStreamReader.java        |    9 +
 .../input/stream/HDFSInputStreamProvider.java   |    2 +-
 .../stream/LocalFSInputStreamProvider.java      |   42 +-
 .../stream/LocalFileSystemInputStream.java      |  125 ++
 .../TwitterFirehoseInputStreamProvider.java     |    2 +-
 .../LocalFSInputStreamProviderFactory.java      |   41 +-
 .../external/library/ExternalFunction.java      |    2 +-
 .../external/library/ResultCollector.java       |    1 +
 .../external/library/java/JObjectUtil.java      |   44 +-
 .../ExternalDataScanOperatorDescriptor.java     |   74 ++
 ...DatasetIndexesRecoverOperatorDescriptor.java |    1 -
 .../FeedCollectOperatorDescriptor.java          |  171 +++
 .../FeedCollectOperatorNodePushable.java        |  207 +++
 .../operators/FeedIntakeOperatorDescriptor.java |  138 ++
 .../FeedIntakeOperatorNodePushable.java         |  216 ++++
 .../FeedMessageOperatorDescriptor.java          |   53 +
 .../FeedMessageOperatorNodePushable.java        |  303 +++++
 .../operators/FeedMetaComputeNodePushable.java  |  224 ++++
 .../operators/FeedMetaNodePushable.java         |  184 +++
 .../operators/FeedMetaOperatorDescriptor.java   |  132 ++
 .../operators/FeedMetaStoreNodePushable.java    |  220 ++++
 .../asterix/external/parser/ADMDataParser.java  |    7 +
 .../external/parser/DelimitedDataParser.java    |    8 +-
 .../provider/AdapterFactoryProvider.java        |   14 +-
 .../provider/DataflowControllerProvider.java    |   16 +-
 .../provider/DatasourceFactoryProvider.java     |    7 +-
 .../asterix/external/runtime/DataGenerator.java | 1188 ------------------
 .../runtime/GenericSocketFeedAdapter.java       |  118 --
 .../GenericSocketFeedAdapterFactory.java        |  154 ---
 .../external/runtime/SocketClientAdapter.java   |  106 --
 .../runtime/SocketClientAdapterFactory.java     |   77 --
 .../external/runtime/TweetGenerator.java        |  156 ---
 .../external/util/DNSResolverFactory.java       |   36 -
 .../asterix/external/util/DataGenerator.java    | 1186 +++++++++++++++++
 .../asterix/external/util/DataflowUtils.java    |   11 +-
 .../apache/asterix/external/util/Datatypes.java |    9 +-
 .../util/ExternalDataCompatibilityUtils.java    |   35 +-
 .../external/util/ExternalDataConstants.java    |   18 +-
 .../util/ExternalDataExceptionUtils.java        |   22 +
 .../external/util/ExternalDataUtils.java        |   37 +-
 .../asterix/external/util/FeedConstants.java    |   76 ++
 .../asterix/external/util/FeedFrameUtil.java    |  102 ++
 .../asterix/external/util/FeedLogManager.java   |  152 +++
 .../apache/asterix/external/util/FeedUtils.java |   69 +
 .../external/util/FileSystemWatcher.java        |  244 ++++
 .../external/util/LocalFileSystemUtils.java     |   75 ++
 .../external/util/NodeResolverFactory.java      |   36 +
 .../asterix/external/util/TweetGenerator.java   |  156 +++
 .../asterix/external/util/TweetProcessor.java   |  121 --
 .../external/library/UpperCaseFunction.java     |   10 +-
 .../library/adapter/TestTypedAdapter.java       |   43 +-
 .../adapter/TestTypedAdapterFactory.java        |    2 +-
 .../aql/statement/SubscribeFeedStatement.java   |   21 +-
 asterix-lang-aql/src/main/javacc/AQL.jj         |    2 +-
 .../lang/common/statement/DatasetDecl.java      |    2 +-
 .../lang/common/visitor/FormatPrintVisitor.java |    2 +-
 .../visitor/VariableCheckAndRewriteVisitor.java |    2 +-
 asterix-lang-sqlpp/src/main/javacc/SQLPP.jj     |    2 +-
 .../apache/asterix/metadata/MetadataCache.java  |  138 +-
 .../asterix/metadata/MetadataManager.java       |   15 +-
 .../apache/asterix/metadata/MetadataNode.java   |   26 +-
 .../metadata/MetadataTransactionContext.java    |   23 +-
 .../asterix/metadata/api/IMetadataEntity.java   |    6 +-
 .../asterix/metadata/api/IMetadataManager.java  |   41 +-
 .../asterix/metadata/api/IMetadataNode.java     |    8 +-
 .../metadata/bootstrap/MetadataBootstrap.java   |   42 +-
 .../metadata/bootstrap/MetadataConstants.java   |   33 -
 .../metadata/bootstrap/MetadataIndex.java       |    1 +
 .../MetadataIndexImmutableProperties.java       |    2 +
 .../metadata/declared/AqlMetadataProvider.java  |   48 +-
 .../metadata/declared/FeedDataSource.java       |   17 +-
 .../metadata/entities/CompactionPolicy.java     |    6 +-
 .../asterix/metadata/entities/Dataset.java      |    6 +-
 .../metadata/entities/DatasourceAdapter.java    |   14 +-
 .../asterix/metadata/entities/Datatype.java     |    6 +-
 .../asterix/metadata/entities/Dataverse.java    |    6 +-
 .../apache/asterix/metadata/entities/Feed.java  |   72 +-
 .../asterix/metadata/entities/FeedPolicy.java   |   99 --
 .../metadata/entities/FeedPolicyEntity.java     |   49 +
 .../asterix/metadata/entities/Function.java     |    6 +-
 .../apache/asterix/metadata/entities/Index.java |    6 +-
 .../asterix/metadata/entities/Library.java      |    6 +-
 .../asterix/metadata/entities/NodeGroup.java    |    6 +-
 .../asterix/metadata/entities/PrimaryFeed.java  |   80 --
 .../metadata/entities/SecondaryFeed.java        |   64 -
 .../DatasourceAdapterTupleTranslator.java       |    7 +-
 .../FeedPolicyTupleTranslator.java              |   14 +-
 .../FeedTupleTranslator.java                    |   37 +-
 .../feeds/AbstractDatasourceAdapter.java        |   43 -
 .../feeds/AbstractFeedDatasourceAdapter.java    |   39 -
 .../asterix/metadata/feeds/AdapterExecutor.java |   74 --
 .../metadata/feeds/AdapterIdentifier.java       |   66 -
 .../metadata/feeds/AdapterRuntimeManager.java   |  131 --
 .../metadata/feeds/BuiltinFeedPolicies.java     |   64 +-
 .../feeds/CollectTransformFeedFrameWriter.java  |  120 --
 .../ExternalDataScanOperatorDescriptor.java     |   74 --
 .../metadata/feeds/FeedActivityIdFactory.java   |   40 -
 .../feeds/FeedCollectOperatorDescriptor.java    |  170 ---
 .../feeds/FeedCollectOperatorNodePushable.java  |  208 ---
 .../metadata/feeds/FeedConnectionManager.java   |  109 --
 .../metadata/feeds/FeedFrameTupleDecorator.java |  108 --
 .../feeds/FeedIntakeOperatorDescriptor.java     |  136 --
 .../feeds/FeedIntakeOperatorNodePushable.java   |  215 ----
 .../feeds/FeedLifecycleEventSubscriber.java     |   66 -
 .../feeds/FeedMessageOperatorDescriptor.java    |   53 -
 .../feeds/FeedMessageOperatorNodePushable.java  |  302 -----
 .../feeds/FeedMetaComputeNodePushable.java      |  227 ----
 .../metadata/feeds/FeedMetaNodePushable.java    |  189 ---
 .../feeds/FeedMetaOperatorDescriptor.java       |  132 --
 .../feeds/FeedMetaStoreNodePushable.java        |  224 ----
 .../metadata/feeds/FeedMetadataUtil.java        |  583 +++++++++
 .../metadata/feeds/FeedSubscriptionManager.java |   76 --
 .../apache/asterix/metadata/feeds/FeedUtil.java |  590 ---------
 .../asterix/metadata/feeds/FeedWorkManager.java |   50 -
 .../metadata/feeds/IAdapterExecutor.java        |   40 -
 .../asterix/metadata/feeds/IFeedMessage.java    |   32 -
 .../metadata/feeds/ITypedAdapterFactory.java    |   31 -
 .../asterix/metadata/feeds/MessageListener.java |  138 --
 .../metadata/feeds/PrepareStallMessage.java     |   70 --
 .../feeds/RemoteSocketMessageListener.java      |  134 --
 .../metadata/feeds/SocketMessageListener.java   |  160 ---
 .../feeds/TerminateDataFlowMessage.java         |   54 -
 .../asterix/metadata/feeds/XAQLFeedMessage.java |   68 -
 .../asterix/metadata/utils/DatasetUtils.java    |    2 +-
 .../utils/SplitsAndConstraintsUtil.java         |   79 +-
 .../om/util/AsterixClusterProperties.java       |   24 +-
 418 files changed, 19689 insertions(+), 19427 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/e800e6d5/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceRandomPartitioningFeedComputationRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceRandomPartitioningFeedComputationRule.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceRandomPartitioningFeedComputationRule.java
index 35bc3fd..fb81885 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceRandomPartitioningFeedComputationRule.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceRandomPartitioningFeedComputationRule.java
@@ -20,11 +20,10 @@ package org.apache.asterix.optimizer.rules;
 
 import org.apache.commons.lang3.mutable.Mutable;
 import org.apache.commons.lang3.mutable.MutableObject;
-
 import org.apache.asterix.metadata.declared.AqlDataSource;
 import org.apache.asterix.metadata.declared.AqlDataSource.AqlDataSourceType;
-import org.apache.asterix.metadata.declared.FeedDataSource;
 import org.apache.asterix.metadata.entities.Feed;
+import org.apache.asterix.metadata.declared.FeedDataSource;
 import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
 import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/e800e6d5/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java
index dab6193..1945be3 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java
@@ -22,8 +22,8 @@ import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.asterix.common.config.DatasetConfig.DatasetType;
-import org.apache.asterix.common.feeds.FeedActivity.FeedActivityDetails;
-import org.apache.asterix.common.feeds.api.IFeedLifecycleListener.ConnectionLocation;
+import org.apache.asterix.external.feed.api.IFeedLifecycleListener.ConnectionLocation;
+import org.apache.asterix.external.feed.watch.FeedActivity.FeedActivityDetails;
 import org.apache.asterix.metadata.declared.AqlDataSource;
 import org.apache.asterix.metadata.declared.AqlMetadataProvider;
 import org.apache.asterix.metadata.declared.AqlSourceId;
@@ -31,7 +31,7 @@ import org.apache.asterix.metadata.declared.FeedDataSource;
 import org.apache.asterix.metadata.entities.Dataset;
 import org.apache.asterix.metadata.entities.Dataverse;
 import org.apache.asterix.metadata.entities.Feed;
-import org.apache.asterix.metadata.entities.FeedPolicy;
+import org.apache.asterix.metadata.entities.FeedPolicyEntity;
 import org.apache.asterix.metadata.feeds.BuiltinFeedPolicies;
 import org.apache.asterix.metadata.utils.DatasetUtils;
 import org.apache.asterix.om.base.AString;
@@ -154,7 +154,7 @@ public class UnnestToDataScanRule implements IAlgebraicRewriteRule {
 
                 AqlSourceId asid = new AqlSourceId(dataverse, getTargetFeed);
                 String policyName = metadataProvider.getConfig().get(FeedActivityDetails.FEED_POLICY_NAME);
-                FeedPolicy policy = metadataProvider.findFeedPolicy(dataverse, policyName);
+                FeedPolicyEntity policy = metadataProvider.findFeedPolicy(dataverse, policyName);
                 if (policy == null) {
                     policy = BuiltinFeedPolicies.getFeedPolicy(policyName);
                     if (policy == null) {
@@ -193,7 +193,7 @@ public class UnnestToDataScanRule implements IAlgebraicRewriteRule {
     }
 
     private AqlDataSource createFeedDataSource(AqlSourceId aqlId, String targetDataset, String sourceFeedName,
-            String subscriptionLocation, AqlMetadataProvider metadataProvider, FeedPolicy feedPolicy,
+            String subscriptionLocation, AqlMetadataProvider metadataProvider, FeedPolicyEntity feedPolicy,
             String outputType, String locations) throws AlgebricksException {
         if (!aqlId.getDataverseName().equals(
                 metadataProvider.getDefaultDataverse() == null ? null : metadataProvider.getDefaultDataverse()

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/e800e6d5/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java
index 1e24ea8..0c2463c 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java
@@ -25,6 +25,7 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.asterix.common.config.MetadataConstants;
 import org.apache.asterix.common.config.DatasetConfig.IndexType;
 import org.apache.asterix.dataflow.data.common.AqlExpressionTypeComputer;
 import org.apache.asterix.metadata.api.IMetadataEntity;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/e800e6d5/asterix-algebra/src/main/java/org/apache/asterix/translator/AbstractLangTranslator.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/translator/AbstractLangTranslator.java b/asterix-algebra/src/main/java/org/apache/asterix/translator/AbstractLangTranslator.java
index 87d4f1c..a161717 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/translator/AbstractLangTranslator.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/translator/AbstractLangTranslator.java
@@ -24,6 +24,7 @@ import java.util.logging.Level;
 import java.util.logging.Logger;
 
 import org.apache.asterix.common.api.IClusterManagementWork.ClusterState;
+import org.apache.asterix.common.config.MetadataConstants;
 import org.apache.asterix.common.exceptions.AsterixException;
 import org.apache.asterix.lang.common.base.Statement;
 import org.apache.asterix.lang.common.statement.DatasetDecl;
@@ -32,7 +33,6 @@ import org.apache.asterix.lang.common.statement.DeleteStatement;
 import org.apache.asterix.lang.common.statement.DropStatement;
 import org.apache.asterix.lang.common.statement.InsertStatement;
 import org.apache.asterix.lang.common.statement.NodeGroupDropStatement;
-import org.apache.asterix.metadata.bootstrap.MetadataConstants;
 import org.apache.asterix.metadata.dataset.hints.DatasetHints;
 import org.apache.asterix.metadata.entities.AsterixBuiltinTypeMap;
 import org.apache.asterix.metadata.entities.Dataverse;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/e800e6d5/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java b/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java
index d4825c4..7221972 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java
@@ -22,7 +22,7 @@ import java.util.List;
 import java.util.Map;
 
 import org.apache.asterix.common.config.DatasetConfig.IndexType;
-import org.apache.asterix.common.feeds.FeedConnectionRequest;
+import org.apache.asterix.external.feed.management.FeedConnectionRequest;
 import org.apache.asterix.lang.common.base.Expression;
 import org.apache.asterix.lang.common.base.Statement.Kind;
 import org.apache.asterix.lang.common.expression.VariableExpr;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/e800e6d5/asterix-algebra/src/main/javacc/AQLPlus.jj
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/javacc/AQLPlus.jj b/asterix-algebra/src/main/javacc/AQLPlus.jj
index 853631c..b10d837 100644
--- a/asterix-algebra/src/main/javacc/AQLPlus.jj
+++ b/asterix-algebra/src/main/javacc/AQLPlus.jj
@@ -41,6 +41,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.asterix.common.config.MetadataConstants;
 import org.apache.asterix.common.exceptions.AsterixException;
 import org.apache.asterix.common.functions.FunctionSignature;
 import org.apache.asterix.lang.aql.clause.DistinctClause;
@@ -100,7 +101,6 @@ import org.apache.asterix.lang.common.statement.WriteStatement;
 import org.apache.asterix.lang.common.struct.Identifier;
 import org.apache.asterix.lang.common.struct.QuantifiedPair;
 import org.apache.asterix.lang.common.struct.VarIdentifier;
-import org.apache.asterix.metadata.bootstrap.MetadataConstants;
 import org.apache.hyracks.algebricks.core.algebra.expressions.IndexedNLJoinExpressionAnnotation;
 
 

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/e800e6d5/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java b/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java
index 4a8a323..6d0b321 100644
--- a/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java
+++ b/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java
@@ -39,7 +39,6 @@ import org.apache.asterix.common.context.AsterixFileMapManager;
 import org.apache.asterix.common.context.DatasetLifecycleManager;
 import org.apache.asterix.common.exceptions.ACIDException;
 import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.common.feeds.api.IFeedManager;
 import org.apache.asterix.common.replication.IRemoteRecoveryManager;
 import org.apache.asterix.common.replication.IReplicaResourcesManager;
 import org.apache.asterix.common.replication.IReplicationChannel;
@@ -47,8 +46,9 @@ import org.apache.asterix.common.replication.IReplicationManager;
 import org.apache.asterix.common.transactions.IAsterixAppRuntimeContextProvider;
 import org.apache.asterix.common.transactions.IRecoveryManager;
 import org.apache.asterix.common.transactions.IRecoveryManager.SystemState;
+import org.apache.asterix.external.feed.api.IFeedManager;
+import org.apache.asterix.external.feed.management.FeedManager;
 import org.apache.asterix.common.transactions.ITransactionSubsystem;
-import org.apache.asterix.feeds.FeedManager;
 import org.apache.asterix.metadata.bootstrap.MetadataIndexImmutableProperties;
 import org.apache.asterix.om.util.AsterixClusterProperties;
 import org.apache.asterix.replication.management.ReplicationChannel;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/e800e6d5/asterix-app/src/main/java/org/apache/asterix/api/common/FeedWorkCollection.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/api/common/FeedWorkCollection.java b/asterix-app/src/main/java/org/apache/asterix/api/common/FeedWorkCollection.java
deleted file mode 100644
index 6aad64b..0000000
--- a/asterix-app/src/main/java/org/apache/asterix/api/common/FeedWorkCollection.java
+++ /dev/null
@@ -1,205 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.api.common;
-
-import java.io.PrintWriter;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import org.apache.asterix.api.common.SessionConfig.OutputFormat;
-import org.apache.asterix.aql.translator.QueryTranslator;
-import org.apache.asterix.common.feeds.FeedConnectionRequest;
-import org.apache.asterix.common.feeds.FeedConnectionRequest.ConnectionStatus;
-import org.apache.asterix.common.feeds.api.IFeedWork;
-import org.apache.asterix.common.feeds.api.IFeedWorkEventListener;
-import org.apache.asterix.compiler.provider.AqlCompilationProvider;
-import org.apache.asterix.compiler.provider.ILangCompilationProvider;
-import org.apache.asterix.feeds.FeedCollectInfo;
-import org.apache.asterix.lang.aql.statement.SubscribeFeedStatement;
-import org.apache.asterix.lang.common.base.Statement;
-import org.apache.asterix.lang.common.statement.DataverseDecl;
-import org.apache.asterix.lang.common.struct.Identifier;
-import org.apache.asterix.om.util.AsterixAppContextInfo;
-import org.apache.hyracks.api.job.JobId;
-
-/**
- * A collection of feed management related task, each represented as an implementation of {@code IFeedWork}.
- */
-public class FeedWorkCollection {
-
-    private static Logger LOGGER = Logger.getLogger(FeedWorkCollection.class.getName());
-    private static final ILangCompilationProvider compilationProvider = new AqlCompilationProvider();
-
-    /**
-     * The task of subscribing to a feed to obtain data.
-     */
-    public static class SubscribeFeedWork implements IFeedWork {
-
-        private final Runnable runnable;
-
-        private final FeedConnectionRequest request;
-
-        @Override
-        public Runnable getRunnable() {
-            return runnable;
-        }
-
-        public SubscribeFeedWork(String[] locations, FeedConnectionRequest request) {
-            this.runnable = new SubscribeFeedWorkRunnable(locations, request);
-            this.request = request;
-        }
-
-        private static class SubscribeFeedWorkRunnable implements Runnable {
-
-            private final FeedConnectionRequest request;
-            private final String[] locations;
-
-            public SubscribeFeedWorkRunnable(String[] locations, FeedConnectionRequest request) {
-                this.request = request;
-                this.locations = locations;
-            }
-
-            @Override
-            public void run() {
-                try {
-                    PrintWriter writer = new PrintWriter(System.out, true);
-                    SessionConfig pc = new SessionConfig(writer, OutputFormat.ADM);
-                    DataverseDecl dataverseDecl = new DataverseDecl(
-                            new Identifier(request.getReceivingFeedId().getDataverse()));
-                    SubscribeFeedStatement subscribeStmt = new SubscribeFeedStatement(locations, request);
-                    List<Statement> statements = new ArrayList<Statement>();
-                    statements.add(dataverseDecl);
-                    statements.add(subscribeStmt);
-                    QueryTranslator translator = new QueryTranslator(statements, pc, compilationProvider);
-                    translator.compileAndExecute(AsterixAppContextInfo.getInstance().getHcc(), null,
-                            QueryTranslator.ResultDelivery.SYNC);
-                    if (LOGGER.isLoggable(Level.INFO)) {
-                        LOGGER.info("Submitted connection requests for execution: " + request);
-                    }
-                } catch (Exception e) {
-                    if (LOGGER.isLoggable(Level.SEVERE)) {
-                        LOGGER.severe("Exception in executing " + request);
-                    }
-                    throw new RuntimeException(e);
-                }
-            }
-        }
-
-        public static class FeedSubscribeWorkEventListener implements IFeedWorkEventListener {
-
-            @Override
-            public void workFailed(IFeedWork work, Exception e) {
-                if (LOGGER.isLoggable(Level.WARNING)) {
-                    LOGGER.warning(" Feed subscription request " + ((SubscribeFeedWork) work).request
-                            + " failed with exception " + e);
-                }
-            }
-
-            @Override
-            public void workCompleted(IFeedWork work) {
-                ((SubscribeFeedWork) work).request.setSubscriptionStatus(ConnectionStatus.ACTIVE);
-                if (LOGGER.isLoggable(Level.INFO)) {
-                    LOGGER.warning(" Feed subscription request " + ((SubscribeFeedWork) work).request + " completed ");
-                }
-            }
-
-        }
-
-        public FeedConnectionRequest getRequest() {
-            return request;
-        }
-
-        @Override
-        public String toString() {
-            return "SubscribeFeedWork for [" + request + "]";
-        }
-
-    }
-
-    /**
-     * The task of activating a set of feeds.
-     */
-    public static class ActivateFeedWork implements IFeedWork {
-
-        private final Runnable runnable;
-
-        @Override
-        public Runnable getRunnable() {
-            return runnable;
-        }
-
-        public ActivateFeedWork(List<FeedCollectInfo> feedsToRevive) {
-            this.runnable = new FeedsActivateRunnable(feedsToRevive);
-        }
-
-        public ActivateFeedWork() {
-            this.runnable = new FeedsActivateRunnable();
-        }
-
-        private static class FeedsActivateRunnable implements Runnable {
-
-            private List<FeedCollectInfo> feedsToRevive;
-            private Mode mode;
-
-            public enum Mode {
-                REVIVAL_POST_NODE_REJOIN
-            }
-
-            public FeedsActivateRunnable(List<FeedCollectInfo> feedsToRevive) {
-                this.feedsToRevive = feedsToRevive;
-            }
-
-            public FeedsActivateRunnable() {
-            }
-
-            @Override
-            public void run() {
-                switch (mode) {
-                    case REVIVAL_POST_NODE_REJOIN:
-                        try {
-                            Thread.sleep(10000);
-                        } catch (InterruptedException e1) {
-                            if (LOGGER.isLoggable(Level.INFO)) {
-                                LOGGER.info("Attempt to resume feed interrupted");
-                            }
-                            throw new IllegalStateException(e1.getMessage());
-                        }
-                        for (FeedCollectInfo finfo : feedsToRevive) {
-                            try {
-                                JobId jobId = AsterixAppContextInfo.getInstance().getHcc().startJob(finfo.jobSpec);
-                                if (LOGGER.isLoggable(Level.INFO)) {
-                                    LOGGER.info("Resumed feed :" + finfo.feedConnectionId + " job id " + jobId);
-                                    LOGGER.info("Job:" + finfo.jobSpec);
-                                }
-                            } catch (Exception e) {
-                                if (LOGGER.isLoggable(Level.WARNING)) {
-                                    LOGGER.warning(
-                                            "Unable to resume feed " + finfo.feedConnectionId + " " + e.getMessage());
-                                }
-                            }
-                        }
-                }
-            }
-
-        }
-
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/e800e6d5/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ConnectorAPIServlet.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ConnectorAPIServlet.java b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ConnectorAPIServlet.java
index 1e6cc66..4df461b 100644
--- a/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ConnectorAPIServlet.java
+++ b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ConnectorAPIServlet.java
@@ -29,7 +29,7 @@ import javax.servlet.http.HttpServlet;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
-import org.apache.asterix.feeds.CentralFeedManager;
+import org.apache.asterix.feed.CentralFeedManager;
 import org.apache.asterix.metadata.MetadataManager;
 import org.apache.asterix.metadata.MetadataTransactionContext;
 import org.apache.asterix.metadata.declared.AqlMetadataProvider;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/e800e6d5/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/FeedServlet.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/FeedServlet.java b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/FeedServlet.java
index 35efcb1..6957926 100644
--- a/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/FeedServlet.java
+++ b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/FeedServlet.java
@@ -32,13 +32,13 @@ import javax.servlet.http.HttpServlet;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
-import org.apache.asterix.common.feeds.FeedActivity;
-import org.apache.asterix.common.feeds.FeedActivity.FeedActivityDetails;
-import org.apache.asterix.common.feeds.FeedConnectionId;
-import org.apache.asterix.common.feeds.FeedId;
-import org.apache.asterix.common.feeds.api.IFeedLoadManager;
-import org.apache.asterix.common.feeds.api.IFeedRuntime.FeedRuntimeType;
-import org.apache.asterix.feeds.CentralFeedManager;
+import org.apache.asterix.external.feed.api.IFeedLoadManager;
+import org.apache.asterix.external.feed.api.IFeedRuntime.FeedRuntimeType;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.feed.management.FeedId;
+import org.apache.asterix.external.feed.watch.FeedActivity;
+import org.apache.asterix.external.feed.watch.FeedActivity.FeedActivityDetails;
+import org.apache.asterix.feed.CentralFeedManager;
 
 public class FeedServlet extends HttpServlet {
     private static final long serialVersionUID = 1L;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/e800e6d5/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/FeedServletUtil.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/FeedServletUtil.java b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/FeedServletUtil.java
index f1473f4..d459775 100644
--- a/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/FeedServletUtil.java
+++ b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/FeedServletUtil.java
@@ -26,9 +26,9 @@ import java.util.concurrent.LinkedBlockingQueue;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
-import org.apache.asterix.common.feeds.FeedConnectionId;
-import org.apache.asterix.feeds.FeedLifecycleListener;
-import org.apache.asterix.metadata.feeds.RemoteSocketMessageListener;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.feed.message.RemoteSocketMessageListener;
+import org.apache.asterix.feed.FeedLifecycleListener;
 
 public class FeedServletUtil {
 

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/e800e6d5/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java b/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
index 545cb67..3b6af6d 100644
--- a/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
+++ b/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
@@ -48,26 +48,30 @@ import org.apache.asterix.common.config.DatasetConfig.ExternalDatasetTransaction
 import org.apache.asterix.common.config.DatasetConfig.ExternalFilePendingOp;
 import org.apache.asterix.common.config.DatasetConfig.IndexType;
 import org.apache.asterix.common.config.GlobalConfig;
+import org.apache.asterix.common.config.MetadataConstants;
 import org.apache.asterix.common.exceptions.ACIDException;
 import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.common.feeds.FeedActivity.FeedActivityDetails;
-import org.apache.asterix.common.feeds.FeedConnectionId;
-import org.apache.asterix.common.feeds.FeedConnectionRequest;
-import org.apache.asterix.common.feeds.FeedId;
-import org.apache.asterix.common.feeds.FeedJointKey;
-import org.apache.asterix.common.feeds.FeedPolicyAccessor;
-import org.apache.asterix.common.feeds.api.IFeedJoint;
-import org.apache.asterix.common.feeds.api.IFeedJoint.FeedJointType;
-import org.apache.asterix.common.feeds.api.IFeedLifecycleEventSubscriber;
-import org.apache.asterix.common.feeds.api.IFeedLifecycleEventSubscriber.FeedLifecycleEvent;
-import org.apache.asterix.common.feeds.api.IFeedLifecycleListener.ConnectionLocation;
 import org.apache.asterix.common.functions.FunctionSignature;
 import org.apache.asterix.compiler.provider.ILangCompilationProvider;
 import org.apache.asterix.external.api.IAdapterFactory;
+import org.apache.asterix.external.feed.api.IFeed;
+import org.apache.asterix.external.feed.api.IFeed.FeedType;
+import org.apache.asterix.external.feed.api.IFeedJoint;
+import org.apache.asterix.external.feed.api.IFeedJoint.FeedJointType;
+import org.apache.asterix.external.feed.api.IFeedLifecycleEventSubscriber;
+import org.apache.asterix.external.feed.api.IFeedLifecycleEventSubscriber.FeedLifecycleEvent;
+import org.apache.asterix.external.feed.api.IFeedLifecycleListener.ConnectionLocation;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.feed.management.FeedConnectionRequest;
+import org.apache.asterix.external.feed.management.FeedId;
+import org.apache.asterix.external.feed.management.FeedJointKey;
+import org.apache.asterix.external.feed.management.FeedLifecycleEventSubscriber;
+import org.apache.asterix.external.feed.policy.FeedPolicyAccessor;
+import org.apache.asterix.external.feed.watch.FeedActivity.FeedActivityDetails;
 import org.apache.asterix.external.indexing.ExternalFile;
-import org.apache.asterix.feeds.CentralFeedManager;
-import org.apache.asterix.feeds.FeedJoint;
-import org.apache.asterix.feeds.FeedLifecycleListener;
+import org.apache.asterix.feed.CentralFeedManager;
+import org.apache.asterix.feed.FeedJoint;
+import org.apache.asterix.feed.FeedLifecycleListener;
 import org.apache.asterix.file.DatasetOperations;
 import org.apache.asterix.file.DataverseOperations;
 import org.apache.asterix.file.ExternalIndexingOperations;
@@ -120,7 +124,6 @@ import org.apache.asterix.metadata.MetadataException;
 import org.apache.asterix.metadata.MetadataManager;
 import org.apache.asterix.metadata.MetadataTransactionContext;
 import org.apache.asterix.metadata.api.IMetadataEntity;
-import org.apache.asterix.metadata.bootstrap.MetadataConstants;
 import org.apache.asterix.metadata.dataset.hints.DatasetHints;
 import org.apache.asterix.metadata.dataset.hints.DatasetHints.DatasetNodegroupCardinalityHint;
 import org.apache.asterix.metadata.declared.AqlMetadataProvider;
@@ -130,16 +133,12 @@ import org.apache.asterix.metadata.entities.Datatype;
 import org.apache.asterix.metadata.entities.Dataverse;
 import org.apache.asterix.metadata.entities.ExternalDatasetDetails;
 import org.apache.asterix.metadata.entities.Feed;
-import org.apache.asterix.metadata.entities.Feed.FeedType;
-import org.apache.asterix.metadata.entities.FeedPolicy;
+import org.apache.asterix.metadata.entities.FeedPolicyEntity;
 import org.apache.asterix.metadata.entities.Function;
 import org.apache.asterix.metadata.entities.Index;
 import org.apache.asterix.metadata.entities.InternalDatasetDetails;
 import org.apache.asterix.metadata.entities.NodeGroup;
-import org.apache.asterix.metadata.entities.PrimaryFeed;
-import org.apache.asterix.metadata.entities.SecondaryFeed;
-import org.apache.asterix.metadata.feeds.FeedLifecycleEventSubscriber;
-import org.apache.asterix.metadata.feeds.FeedUtil;
+import org.apache.asterix.metadata.feeds.FeedMetadataUtil;
 import org.apache.asterix.metadata.utils.DatasetUtils;
 import org.apache.asterix.metadata.utils.ExternalDatasetsRegistry;
 import org.apache.asterix.metadata.utils.MetadataLockManager;
@@ -239,11 +238,11 @@ public class QueryTranslator extends AbstractLangTranslator {
     /**
      * Compiles and submits for execution a list of AQL statements.
      * @param hcc
-     *        A Hyracks client connection that is used to submit a jobspec to Hyracks.
+     *            A Hyracks client connection that is used to submit a jobspec to Hyracks.
      * @param hdc
-     *        A Hyracks dataset client object that is used to read the results.
+     *            A Hyracks dataset client object that is used to read the results.
      * @param resultDelivery
-     *        True if the results should be read asynchronously or false if we should wait for results to be read.
+     *            True if the results should be read asynchronously or false if we should wait for results to be read.
      * @return A List<QueryResult> containing a QueryResult instance corresponding to each submitted query.
      * @throws Exception
      */
@@ -1933,13 +1932,13 @@ public class QueryTranslator extends AbstractLangTranslator {
                 case CREATE_PRIMARY_FEED:
                     CreatePrimaryFeedStatement cpfs = (CreatePrimaryFeedStatement) stmt;
                     String adaptorName = cpfs.getAdaptorName();
-                    feed = new PrimaryFeed(dataverseName, feedName, adaptorName, cpfs.getAdaptorConfiguration(),
-                            cfs.getAppliedFunction());
+                    feed = new Feed(dataverseName, feedName, cfs.getAppliedFunction(), FeedType.PRIMARY, feedName,
+                            adaptorName, cpfs.getAdaptorConfiguration());
                     break;
                 case CREATE_SECONDARY_FEED:
                     CreateSecondaryFeedStatement csfs = (CreateSecondaryFeedStatement) stmt;
-                    feed = new SecondaryFeed(dataverseName, feedName, csfs.getSourceFeedName(),
-                            csfs.getAppliedFunction());
+                    feed = new Feed(dataverseName, feedName, csfs.getAppliedFunction(), FeedType.SECONDARY,
+                            csfs.getSourceFeedName(), null, null);
                     break;
                 default:
                     throw new IllegalStateException();
@@ -1961,14 +1960,14 @@ public class QueryTranslator extends AbstractLangTranslator {
         metadataProvider.setMetadataTxnContext(mdTxnCtx);
         String dataverse;
         String policy;
-        FeedPolicy newPolicy = null;
+        FeedPolicyEntity newPolicy = null;
         CreateFeedPolicyStatement cfps = (CreateFeedPolicyStatement) stmt;
         dataverse = getActiveDataverse(null);
         policy = cfps.getPolicyName();
         MetadataLockManager.INSTANCE.createFeedPolicyBegin(dataverse, dataverse + "." + policy);
         try {
-            FeedPolicy feedPolicy = MetadataManager.INSTANCE.getFeedPolicy(metadataProvider.getMetadataTxnContext(),
-                    dataverse, policy);
+            FeedPolicyEntity feedPolicy = MetadataManager.INSTANCE
+                    .getFeedPolicy(metadataProvider.getMetadataTxnContext(), dataverse, policy);
             if (feedPolicy != null) {
                 if (cfps.getIfNotExists()) {
                     MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
@@ -1980,7 +1979,7 @@ public class QueryTranslator extends AbstractLangTranslator {
             boolean extendingExisting = cfps.getSourcePolicyName() != null;
             String description = cfps.getDescription() == null ? "" : cfps.getDescription();
             if (extendingExisting) {
-                FeedPolicy sourceFeedPolicy = MetadataManager.INSTANCE
+                FeedPolicyEntity sourceFeedPolicy = MetadataManager.INSTANCE
                         .getFeedPolicy(metadataProvider.getMetadataTxnContext(), dataverse, cfps.getSourcePolicyName());
                 if (sourceFeedPolicy == null) {
                     sourceFeedPolicy = MetadataManager.INSTANCE.getFeedPolicy(metadataProvider.getMetadataTxnContext(),
@@ -1991,7 +1990,7 @@ public class QueryTranslator extends AbstractLangTranslator {
                 }
                 Map<String, String> policyProperties = sourceFeedPolicy.getProperties();
                 policyProperties.putAll(cfps.getProperties());
-                newPolicy = new FeedPolicy(dataverse, policy, description, policyProperties);
+                newPolicy = new FeedPolicyEntity(dataverse, policy, description, policyProperties);
             } else {
                 Properties prop = new Properties();
                 try {
@@ -2004,7 +2003,7 @@ public class QueryTranslator extends AbstractLangTranslator {
                 for (Entry<Object, Object> entry : prop.entrySet()) {
                     policyProperties.put((String) entry.getKey(), (String) entry.getValue());
                 }
-                newPolicy = new FeedPolicy(dataverse, policy, description, policyProperties);
+                newPolicy = new FeedPolicyEntity(dataverse, policy, description, policyProperties);
             }
             MetadataManager.INSTANCE.addFeedPolicy(mdTxnCtx, newPolicy);
             MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
@@ -2071,7 +2070,7 @@ public class QueryTranslator extends AbstractLangTranslator {
         MetadataLockManager.INSTANCE.dropFeedPolicyBegin(dataverseName, dataverseName + "." + policyName);
 
         try {
-            FeedPolicy feedPolicy = MetadataManager.INSTANCE.getFeedPolicy(mdTxnCtx, dataverseName, policyName);
+            FeedPolicyEntity feedPolicy = MetadataManager.INSTANCE.getFeedPolicy(mdTxnCtx, dataverseName, policyName);
             if (feedPolicy == null) {
                 if (!stmtFeedPolicyDrop.getIfExists()) {
                     throw new AlgebricksException("Unknown policy " + policyName + " in dataverse " + dataverseName);
@@ -2112,10 +2111,10 @@ public class QueryTranslator extends AbstractLangTranslator {
             CompiledConnectFeedStatement cbfs = new CompiledConnectFeedStatement(dataverseName, cfs.getFeedName(),
                     cfs.getDatasetName().getValue(), cfs.getPolicy(), cfs.getQuery(), cfs.getVarCounter());
 
-            FeedUtil.validateIfDatasetExists(dataverseName, cfs.getDatasetName().getValue(),
+            FeedMetadataUtil.validateIfDatasetExists(dataverseName, cfs.getDatasetName().getValue(),
                     metadataProvider.getMetadataTxnContext());
 
-            Feed feed = FeedUtil.validateIfFeedExists(dataverseName, cfs.getFeedName(),
+            Feed feed = FeedMetadataUtil.validateIfFeedExists(dataverseName, cfs.getFeedName(),
                     metadataProvider.getMetadataTxnContext());
 
             feedConnId = new FeedConnectionId(dataverseName, cfs.getFeedName(), cfs.getDatasetName().getValue());
@@ -2125,7 +2124,8 @@ public class QueryTranslator extends AbstractLangTranslator {
                         + cfs.getDatasetName().getValue());
             }
 
-            FeedPolicy feedPolicy = FeedUtil.validateIfPolicyExists(dataverseName, cbfs.getPolicyName(), mdTxnCtx);
+            FeedPolicyEntity feedPolicy = FeedMetadataUtil.validateIfPolicyExists(dataverseName, cbfs.getPolicyName(),
+                    mdTxnCtx);
 
             // All Metadata checks have passed. Feed connect request is valid. //
 
@@ -2139,8 +2139,8 @@ public class QueryTranslator extends AbstractLangTranslator {
             subscriberRegistered = true;
             if (createFeedIntakeJob) {
                 FeedId feedId = connectionRequest.getFeedJointKey().getFeedId();
-                PrimaryFeed primaryFeed = (PrimaryFeed) MetadataManager.INSTANCE.getFeed(mdTxnCtx,
-                        feedId.getDataverse(), feedId.getFeedName());
+                Feed primaryFeed = MetadataManager.INSTANCE.getFeed(mdTxnCtx, feedId.getDataverse(),
+                        feedId.getFeedName());
                 Pair<JobSpecification, IAdapterFactory> pair = FeedOperations.buildFeedIntakeJobSpec(primaryFeed,
                         metadataProvider, policyAccessor);
                 // adapter configuration are valid at this stage
@@ -2205,7 +2205,7 @@ public class QueryTranslator extends AbstractLangTranslator {
      * @throws MetadataException
      */
     private Triple<FeedConnectionRequest, Boolean, List<IFeedJoint>> getFeedConnectionRequest(String dataverse,
-            Feed feed, String dataset, FeedPolicy feedPolicy, MetadataTransactionContext mdTxnCtx)
+            Feed feed, String dataset, FeedPolicyEntity feedPolicy, MetadataTransactionContext mdTxnCtx)
                     throws MetadataException {
         IFeedJoint sourceFeedJoint = null;
         FeedConnectionRequest request = null;
@@ -2269,12 +2269,12 @@ public class QueryTranslator extends AbstractLangTranslator {
     private FeedJointKey getFeedJointKey(Feed feed, MetadataTransactionContext ctx) throws MetadataException {
         Feed sourceFeed = feed;
         List<String> appliedFunctions = new ArrayList<String>();
-        while (sourceFeed.getFeedType().equals(FeedType.SECONDARY)) {
+        while (sourceFeed.getFeedType().equals(IFeed.FeedType.SECONDARY)) {
             if (sourceFeed.getAppliedFunction() != null) {
                 appliedFunctions.add(0, sourceFeed.getAppliedFunction().getName());
             }
             Feed parentFeed = MetadataManager.INSTANCE.getFeed(ctx, feed.getDataverseName(),
-                    ((SecondaryFeed) sourceFeed).getSourceFeedName());
+                    sourceFeed.getSourceFeedName());
             sourceFeed = parentFeed;
         }
 
@@ -2295,8 +2295,8 @@ public class QueryTranslator extends AbstractLangTranslator {
         boolean bActiveTxn = true;
         metadataProvider.setMetadataTxnContext(mdTxnCtx);
 
-        FeedUtil.validateIfDatasetExists(dataverseName, cfs.getDatasetName().getValue(), mdTxnCtx);
-        Feed feed = FeedUtil.validateIfFeedExists(dataverseName, cfs.getFeedName().getValue(), mdTxnCtx);
+        FeedMetadataUtil.validateIfDatasetExists(dataverseName, cfs.getDatasetName().getValue(), mdTxnCtx);
+        Feed feed = FeedMetadataUtil.validateIfFeedExists(dataverseName, cfs.getFeedName().getValue(), mdTxnCtx);
 
         FeedConnectionId connectionId = new FeedConnectionId(feed.getFeedId(), cfs.getDatasetName().getValue());
         boolean isFeedConnectionActive = FeedLifecycleListener.INSTANCE.isFeedConnectionActive(connectionId);
@@ -2369,7 +2369,7 @@ public class QueryTranslator extends AbstractLangTranslator {
 
         try {
 
-            JobSpecification alteredJobSpec = FeedUtil.alterJobSpecificationForFeed(compiled, feedConnectionId,
+            JobSpecification alteredJobSpec = FeedMetadataUtil.alterJobSpecificationForFeed(compiled, feedConnectionId,
                     bfs.getSubscriptionRequest().getPolicyParameters());
             MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
             bActiveTxn = false;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/e800e6d5/asterix-app/src/main/java/org/apache/asterix/feed/CentralFeedManager.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/feed/CentralFeedManager.java b/asterix-app/src/main/java/org/apache/asterix/feed/CentralFeedManager.java
new file mode 100644
index 0000000..4020bde
--- /dev/null
+++ b/asterix-app/src/main/java/org/apache/asterix/feed/CentralFeedManager.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.feed;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringReader;
+import java.util.List;
+
+import org.apache.asterix.api.common.SessionConfig;
+import org.apache.asterix.api.common.SessionConfig.OutputFormat;
+import org.apache.asterix.aql.translator.QueryTranslator;
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.compiler.provider.AqlCompilationProvider;
+import org.apache.asterix.compiler.provider.ILangCompilationProvider;
+import org.apache.asterix.external.feed.api.ICentralFeedManager;
+import org.apache.asterix.external.feed.api.IFeedLoadManager;
+import org.apache.asterix.external.feed.api.IFeedTrackingManager;
+import org.apache.asterix.external.feed.message.SocketMessageListener;
+import org.apache.asterix.lang.aql.parser.AQLParserFactory;
+import org.apache.asterix.lang.common.base.IParser;
+import org.apache.asterix.lang.common.base.IParserFactory;
+import org.apache.asterix.lang.common.base.Statement;
+import org.apache.asterix.om.util.AsterixAppContextInfo;
+import org.apache.hyracks.api.client.IHyracksClientConnection;
+import org.apache.hyracks.api.job.JobId;
+import org.apache.hyracks.api.job.JobSpecification;
+
+public class CentralFeedManager implements ICentralFeedManager {
+
+    private static final ICentralFeedManager centralFeedManager = new CentralFeedManager();
+    private static final ILangCompilationProvider compilationProvider = new AqlCompilationProvider();
+
+    public static ICentralFeedManager getInstance() {
+        return centralFeedManager;
+    }
+
+    private final int port;
+    private final IFeedLoadManager feedLoadManager;
+    private final IFeedTrackingManager feedTrackingManager;
+    private final SocketMessageListener messageListener;
+
+    private CentralFeedManager() {
+        this.port = AsterixAppContextInfo.getInstance().getFeedProperties().getFeedCentralManagerPort();
+        this.feedLoadManager = new FeedLoadManager();
+        this.feedTrackingManager = new FeedTrackingManager();
+        this.messageListener = new SocketMessageListener(port, new FeedMessageReceiver(this));
+    }
+
+    @Override
+    public void start() throws AsterixException {
+        messageListener.start();
+    }
+
+    @Override
+    public void stop() throws AsterixException, IOException {
+        messageListener.stop();
+    }
+
+    public static JobId runJob(JobSpecification spec, boolean waitForCompletion) throws Exception {
+        IHyracksClientConnection hcc = AsterixAppContextInfo.getInstance().getHcc();
+        JobId jobId = hcc.startJob(spec);
+        if (waitForCompletion) {
+            hcc.waitForCompletion(jobId);
+        }
+        return jobId;
+    }
+
+    @Override
+    public IFeedLoadManager getFeedLoadManager() {
+        return feedLoadManager;
+    }
+
+    @Override
+    public IFeedTrackingManager getFeedTrackingManager() {
+        return feedTrackingManager;
+    }
+
+    public static class AQLExecutor {
+
+        private static final PrintWriter out = new PrintWriter(System.out, true);
+        private static final IParserFactory parserFactory = new AQLParserFactory();
+
+        public static void executeAQL(String aql) throws Exception {
+            IParser parser = parserFactory.createParser(new StringReader(aql));
+            List<Statement> statements = parser.parse();
+            SessionConfig pc = new SessionConfig(out, OutputFormat.ADM);
+            QueryTranslator translator = new QueryTranslator(statements, pc, compilationProvider);
+            translator.compileAndExecute(AsterixAppContextInfo.getInstance().getHcc(), null,
+                    QueryTranslator.ResultDelivery.SYNC);
+        }
+    }
+
+}


Mime
View raw message