asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amo...@apache.org
Subject incubator-asterixdb git commit: Fix Asterix Installer Test Cases
Date Tue, 22 Mar 2016 17:34:37 GMT
Repository: incubator-asterixdb
Updated Branches:
  refs/heads/master 36fbc4160 -> 30904b839


Fix Asterix Installer Test Cases

Many tests started to fail in installer after being
disabled for a long time. This change fixes the following:
1. asterix-installer: external library tests.
2. asterix-installer: recovery: query-after-restart.

Change-Id: Iefad5ac3ff754693025312f1e2fa01b08832514c
Reviewed-on: https://asterix-gerrit.ics.uci.edu/737
Reviewed-by: Ian Maxon <imaxon@apache.org>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>


Project: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/commit/30904b83
Tree: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/tree/30904b83
Diff: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/diff/30904b83

Branch: refs/heads/master
Commit: 30904b839e0da98dd7342e451618b4a67a4d9881
Parents: 36fbc41
Author: Abdullah Alamoudi <bamousaa@gmail.com>
Authored: Tue Mar 22 19:25:10 2016 +0300
Committer: abdullah alamoudi <bamousaa@gmail.com>
Committed: Tue Mar 22 10:29:10 2016 -0700

----------------------------------------------------------------------
 .gitignore                                      |  1 +
 asterix-app/pom.xml                             | 84 ++++++++++----------
 .../feed-push-socket.4.server.aql               |  2 +-
 .../feed-with-external-function.1.ddl.aql       | 58 ++++++++++++++
 .../feed-with-external-function.2.lib.aql       | 20 +++++
 .../feed-with-external-function.3.update.aql    | 32 ++++++++
 .../feed-with-external-function.4.sleep.aql     | 19 +++++
 .../feed-with-external-function.5.query.aql     | 32 ++++++++
 .../feed-with-external-function.6.ddl.aql       | 19 +++++
 .../feed-with-external-function.7.lib.aql       | 19 +++++
 .../feed-with-meta-pk-in-meta.1.ddl.aql         |  2 +-
 ..._dataset_with_meta_primary_index-1.1.ddl.aql |  2 +-
 .../feed-with-external-function.1.adm           | 12 +++
 .../src/test/resources/runtimets/testsuite.xml  |  5 ++
 .../dataflow/RecordDataFlowController.java      |  2 +
 .../external/feed/watch/MonitoredBuffer.java    | 12 +--
 .../record/reader/kv/KVTestReaderFactory.java   |  4 +-
 .../operators/FeedMetaComputeNodePushable.java  | 11 ++-
 .../operators/FeedMetaOperatorDescriptor.java   |  4 +-
 .../apache/asterix/external/util/FeedUtils.java | 13 ++-
 .../adapter/TestTypedAdapterFactory.java        | 18 ++++-
 asterix-installer/pom.xml                       | 51 +++++++++++-
 .../installer/test/AbstractExecutionIT.java     |  6 +-
 .../test/AsterixExternalLibraryIT.java          |  8 +-
 .../installer/test/ManagixExecutionIT.java      |  3 -
 .../feed_ingest/feed_ingest.1.ddl.aql           |  3 +-
 .../feed_ingest/feed_ingest.3.query.aql         | 32 --------
 .../feed_ingest/feed_ingest.3.sleep.aql         | 26 ++++++
 .../feed_ingest/feed_ingest.4.query.aql         | 32 ++++++++
 .../record-parser/record-parser.1.ddl.aql       |  8 +-
 .../record-parser/record-parser.2.query.aql     |  1 +
 .../dataverseDataset/dataverseDataset.1.adm     |  4 +-
 .../external_index/external_index.1.adm         |  6 +-
 .../metadata/feeds/FeedMetadataUtil.java        |  9 ++-
 .../ExternalBTreeLocalResourceMetadata.java     |  3 +-
 35 files changed, 443 insertions(+), 120 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index d802def..1b129d6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -18,6 +18,7 @@ asterix-app/opt_parserts/
 asterix-app/runtime_parserts/
 asterix-installer/ittest/
 asterix-installer/repliationtest/
+asterix-installer/src/test/resources/clusterts/asterix-installer-0.8.9-SNAPSHOT-binary-assembly/
 build
 asterix_logs
 bin/

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-app/pom.xml
----------------------------------------------------------------------
diff --git a/asterix-app/pom.xml b/asterix-app/pom.xml
index 00e924f..d8e7aec 100644
--- a/asterix-app/pom.xml
+++ b/asterix-app/pom.xml
@@ -64,48 +64,48 @@
                 </executions>
             </plugin>
             <plugin>
-    <artifactId>maven-resources-plugin</artifactId>
-    <executions>
-        <execution>
-            <id>copy-external-library</id>
-            <phase>generate-resources</phase>
-            <goals>
-                <goal>copy-resources</goal>
-            </goals>
-            <configuration>
-                <outputDirectory>src/test/resources/externallib</outputDirectory>
-                <overwrite>true</overwrite>
-                <resources>
-                    <resource>
-                        <directory>../asterix-external-data/target</directory>
-                        <includes>
-                            <include>testlib-zip-binary-assembly.zip</include>
-                        </includes>
-                    </resource>
-                </resources>
-            </configuration>
-        </execution>
-        <execution>
-            <id>copy-beer-csv</id>
-            <phase>generate-resources</phase>
-            <goals>
-                <goal>copy-resources</goal>
-            </goals>
-            <configuration>
-                <outputDirectory>data/csv</outputDirectory>
-                <overwrite>true</overwrite>
-                <resources>
-                    <resource>
-                        <directory>../asterix-external-data/src/test/resources</directory>
-                        <includes>
-                            <include>beer.csv</include>
-                        </includes>
-                    </resource>
-                </resources>
-            </configuration>
-       </execution>
-    </executions>
-</plugin>
+                <artifactId>maven-resources-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>copy-external-library</id>
+                        <phase>generate-resources</phase>
+                        <goals>
+                            <goal>copy-resources</goal>
+                        </goals>
+                        <configuration>
+                            <outputDirectory>src/test/resources/externallib</outputDirectory>
+                            <overwrite>true</overwrite>
+                            <resources>
+                                <resource>
+                                    <directory>../asterix-external-data/target</directory>
+                                    <includes>
+                                        <include>testlib-zip-binary-assembly.zip</include>
+                                    </includes>
+                                </resource>
+                            </resources>
+                        </configuration>
+                    </execution>
+                    <execution>
+                        <id>copy-beer-csv</id>
+                        <phase>generate-resources</phase>
+                        <goals>
+                            <goal>copy-resources</goal>
+                        </goals>
+                        <configuration>
+                            <outputDirectory>target/data/csv</outputDirectory>
+                            <overwrite>true</overwrite>
+                            <resources>
+                                <resource>
+                                    <directory>../asterix-external-data/src/test/resources</directory>
+                                    <includes>
+                                        <include>beer.csv</include>
+                                    </includes>
+                                </resource>
+                            </resources>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
         </plugins>
     </build>
     <dependencies>

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-app/src/test/resources/runtimets/queries/feeds/feed-push-socket/feed-push-socket.4.server.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/feeds/feed-push-socket/feed-push-socket.4.server.aql b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-push-socket/feed-push-socket.4.server.aql
index 578d458..5b610de 100644
--- a/asterix-app/src/test/resources/runtimets/queries/feeds/feed-push-socket/feed-push-socket.4.server.aql
+++ b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-push-socket/feed-push-socket.4.server.aql
@@ -23,4 +23,4 @@
  * Expected Res : Success
  * Date         : 24th Feb 2016
  */
-start client 10001 file-client localhost data/twitter/tw_messages.adm 500 50 1000
\ No newline at end of file
+start client 10001 file-client localhost ../asterix-app/data/twitter/tw_messages.adm 500 50 1000
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.1.ddl.aql
new file mode 100644
index 0000000..ce1d7fc
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.1.ddl.aql
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description  : Create a feed dataset that uses the feed simulator adapter.
+                  The feed simulator simulates feed from a file in the local fs.
+                  Associate with the feed an external user-defined function. The UDF
+                  finds topics in each tweet. A topic is identified by a #.
+                  Begin ingestion and apply external user defined function
+ * Expected Res : Success
+ * Date         : 23rd Apr 2013
+ */
+
+drop dataverse externallibtest if exists;
+create dataverse externallibtest;
+use dataverse externallibtest;
+
+create type TweetInputType as closed {
+  id: string,
+  username : string,
+  location : string,
+  text : string,
+  timestamp : string
+}
+
+create type TweetOutputType as closed {
+  id: string,
+  username : string,
+  location : string,
+  text : string,
+  timestamp : string,
+  topics : {{string}}
+}
+
+create feed TweetFeed
+using localfs
+(("type-name"="TweetInputType"),
+("path"="asterix_nc1://data/twitter/obamatweets.adm"),
+("format"="adm"))
+apply function testlib#parseTweet;
+
+create dataset TweetsFeedIngest(TweetOutputType)
+primary key id;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.2.lib.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.2.lib.aql b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.2.lib.aql
new file mode 100644
index 0000000..124038a
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.2.lib.aql
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+install externallibtest testlib src/test/resources/externallib/testlib-zip-binary-assembly.zip
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.3.update.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.3.update.aql b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.3.update.aql
new file mode 100644
index 0000000..ea1a379
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.3.update.aql
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description  : Create a feed dataset that uses the feed simulator adapter.
+                  The feed simulator simulates feed from a file in the local fs.
+                  Associate with the feed an external user-defined function. The UDF
+                  finds topics in each tweet. A topic is identified by a #.
+                  Begin ingestion and apply external user defined function
+ * Expected Res : Success
+ * Date         : 23rd Apr 2013
+ */
+use dataverse externallibtest;
+
+set wait-for-completion-feed "true";
+
+connect feed TweetFeed to dataset TweetsFeedIngest;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.4.sleep.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.4.sleep.aql b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.4.sleep.aql
new file mode 100644
index 0000000..af2f691
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.4.sleep.aql
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+10000
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.5.query.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.5.query.aql b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.5.query.aql
new file mode 100644
index 0000000..8879fa8
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.5.query.aql
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description  : Create a feed dataset that uses the feed simulator adapter.
+                  The feed simulator simulates feed from a file in the local fs.
+                  Associate with the feed an external user-defined function. The UDF
+                  finds topics in each tweet. A topic is identified by a #.
+                  Begin ingestion and apply external user defined function
+ * Expected Res : Success
+ * Date         : 23rd Apr 2013
+ */
+use dataverse externallibtest;
+
+for $x in dataset TweetsFeedIngest
+order by $x.id
+return $x

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.6.ddl.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.6.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.6.ddl.aql
new file mode 100644
index 0000000..2a7acef
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.6.ddl.aql
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+drop dataverse externallibtest if exists;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.7.lib.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.7.lib.aql b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.7.lib.aql
new file mode 100644
index 0000000..86af80f
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-external-function/feed-with-external-function.7.lib.aql
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+uninstall externallibtest testlib
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-meta-pk-in-meta/feed-with-meta-pk-in-meta.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-meta-pk-in-meta/feed-with-meta-pk-in-meta.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-meta-pk-in-meta/feed-with-meta-pk-in-meta.1.ddl.aql
index ec4a712..de9446a 100644
--- a/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-meta-pk-in-meta/feed-with-meta-pk-in-meta.1.ddl.aql
+++ b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-meta-pk-in-meta/feed-with-meta-pk-in-meta.1.ddl.aql
@@ -46,7 +46,7 @@ create feed KVStream using adapter(
  ("parser"="record-with-metadata"),
  ("type-name"="DocumentType"),
  ("meta-type-name"="KVMetaType"),
- ("path"="asterix_nc1://data/csv/beer.csv"),
+ ("path"="asterix_nc1://target/data/csv/beer.csv"),
  ("format"="csv"),
  ("delimiter"=","),
  ("record-format"="adm"),

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.1.ddl.aql
index 718d2a6..26d4b09 100644
--- a/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.1.ddl.aql
+++ b/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.1.ddl.aql
@@ -41,7 +41,7 @@ create feed KVStream using adapter(
  ("parser"="record-with-metadata"),
  ("type-name"="DocumentType"),
  ("meta-type-name"="KVMetaType"),
- ("path"="asterix_nc1://data/csv/beer.csv"),
+ ("path"="asterix_nc1://target/data/csv/beer.csv"),
  ("format"="csv"),
  ("delimiter"=","),
  ("record-format"="adm"),

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-app/src/test/resources/runtimets/results/feeds/feed-with-external-function/feed-with-external-function.1.adm
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/results/feeds/feed-with-external-function/feed-with-external-function.1.adm b/asterix-app/src/test/resources/runtimets/results/feeds/feed-with-external-function/feed-with-external-function.1.adm
new file mode 100644
index 0000000..1291213
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/feeds/feed-with-external-function/feed-with-external-function.1.adm
@@ -0,0 +1,12 @@
+{ "id": "nc1:1", "username": "BronsonMike", "location": "", "text": "@GottaLaff @reutersus Christie and obama just foul weather friends", "timestamp": "Thu Dec 06 16:53:06 PST 2012", "topics": {{  }} }
+{ "id": "nc1:100", "username": "KidrauhlProuds", "location": "", "text": "RT @01Direclieber: A filha do Michael Jackson  uma Belieber,a filha do Eminem e uma Belieber,as filhas de Obama sao Beliebers, e a filha do meu pai e Belieber", "timestamp": "Thu Dec 06 16:53:16 PST 2012", "topics": {{  }} }
+{ "id": "nc1:102", "username": "jaysauce82", "location": "", "text": "Not voting for President Obama #BadDecision", "timestamp": "Thu Dec 06 16:53:16 PST 2012", "topics": {{ "#BadDecision" }} }
+{ "id": "nc1:104", "username": "princeofsupras", "location": "", "text": "RT @01Direclieber: A filha do Michael Jackson e uma Belieber,a filha do Eminem e uma Belieber,as filhas de Obama sao Beliebers, e a filha do meu pai e Belieber", "timestamp": "Thu Dec 06 16:53:15 PST 2012", "topics": {{  }} }
+{ "id": "nc1:106", "username": "GulfDogs", "location": "", "text": "Obama Admin Knew Libyan Terrorists Had US-Provided Weaponsteaparty #tcot #ccot #NewGuards #BreitbartArmy #patriotwttp://t.co/vJxzrQUE", "timestamp": "Thu Dec 06 16:53:14 PST 2012", "topics": {{ "#tcot", "#ccot", "#NewGuards", "#BreitbartArmy", "#patriotwttp://t.co/vJxzrQUE" }} }
+{ "id": "nc1:108", "username": "Laugzpz", "location": "", "text": "@AlfredoJalife Maestro Obama se hace de la vista gorda, es un acuerdo de siempre creo yo.", "timestamp": "Thu Dec 06 16:53:14 PST 2012", "topics": {{  }} }
+{ "id": "nc1:11", "username": "magarika", "location": "", "text": "RT @ken24xavier: Obama tells SOROS - our plan is ALMOST finished http://t.co/WvzK0GtU", "timestamp": "Thu Dec 06 16:53:05 PST 2012", "topics": {{  }} }
+{ "id": "nc1:111", "username": "ToucanMall", "location": "", "text": "RT @WorldWar3Watch: Michelle Obama Gets More Grammy Nominations Than Justin ...  #Obama #WW3 http://t.co/0Wv2GKij", "timestamp": "Thu Dec 06 16:53:13 PST 2012", "topics": {{ "#Obama", "#WW3" }} }
+{ "id": "nc1:113", "username": "ToucanMall", "location": "", "text": "RT @ObamaPalooza: Tiffany Shared What $2,000 Meant to Her ... and the President Stopped by to Talk About It http://t.co/sgT7lsNV #Obama", "timestamp": "Thu Dec 06 16:53:12 PST 2012", "topics": {{ "#Obama" }} }
+{ "id": "nc1:115", "username": "thewildpitch", "location": "", "text": "RT @RevkahJC: Dennis Miller: Obama Should Just Say He Wants To Tax Successful People http://t.co/Ihlemy9Y", "timestamp": "Thu Dec 06 16:53:11 PST 2012", "topics": {{  }} }
+{ "id": "nc1:117", "username": "Rnugent24", "location": "", "text": "RT @ConservativeQuo: unemployment is above 8% again. I wonder how long it will take for Obama to start blaming Bush? 3-2-1 #tcot #antiobama", "timestamp": "Thu Dec 06 16:53:10 PST 2012", "topics": {{ "#tcot", "#antiobama" }} }
+{ "id": "nc1:119", "username": "ToucanMall", "location": "", "text": "RT @Newitrsdotcom: I hope #Obama will win re-election... Other four years without meaningless #wars", "timestamp": "Thu Dec 06 16:53:09 PST 2012", "topics": {{ "#Obama", "#wars" }} }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-app/src/test/resources/runtimets/testsuite.xml
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/testsuite.xml b/asterix-app/src/test/resources/runtimets/testsuite.xml
index 115cf92..fbc821a 100644
--- a/asterix-app/src/test/resources/runtimets/testsuite.xml
+++ b/asterix-app/src/test/resources/runtimets/testsuite.xml
@@ -30,6 +30,11 @@
         QueryFileExtension=".aql">
     <test-group name="feeds">
         <test-case FilePath="feeds">
+            <compilation-unit name="feed-with-external-function">
+                <output-dir compare="Text">feed-with-external-function</output-dir>
+            </compilation-unit>
+        </test-case>
+        <test-case FilePath="feeds">
             <compilation-unit name="change-feed-with-meta-pk-in-meta">
                 <output-dir compare="Text">change-feed-with-meta-pk-in-meta</output-dir>
             </compilation-unit>

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/RecordDataFlowController.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/RecordDataFlowController.java b/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/RecordDataFlowController.java
index 99654d0..915c343 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/RecordDataFlowController.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/RecordDataFlowController.java
@@ -28,9 +28,11 @@ import org.apache.hyracks.api.comm.IFrameWriter;
 import org.apache.hyracks.api.context.IHyracksTaskContext;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import org.apache.log4j.Logger;
 
 public class RecordDataFlowController<T> extends AbstractDataFlowController {
 
+    private static final Logger LOGGER = Logger.getLogger(RecordDataFlowController.class.getName());
     protected final IRecordDataParser<T> dataParser;
     protected final IRecordReader<? extends T> recordReader;
     protected final int numOfTupleFields;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/MonitoredBuffer.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/MonitoredBuffer.java b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/MonitoredBuffer.java
index b36fb6a..67e6295 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/MonitoredBuffer.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/feed/watch/MonitoredBuffer.java
@@ -26,13 +26,13 @@ import java.util.logging.Level;
 
 import org.apache.asterix.external.feed.api.IExceptionHandler;
 import org.apache.asterix.external.feed.api.IFeedMetricCollector;
-import org.apache.asterix.external.feed.api.IFrameEventCallback;
-import org.apache.asterix.external.feed.api.IFramePostProcessor;
-import org.apache.asterix.external.feed.api.IFramePreprocessor;
 import org.apache.asterix.external.feed.api.IFeedMetricCollector.MetricType;
 import org.apache.asterix.external.feed.api.IFeedMetricCollector.ValueType;
 import org.apache.asterix.external.feed.api.IFeedRuntime.Mode;
+import org.apache.asterix.external.feed.api.IFrameEventCallback;
 import org.apache.asterix.external.feed.api.IFrameEventCallback.FrameEvent;
+import org.apache.asterix.external.feed.api.IFramePostProcessor;
+import org.apache.asterix.external.feed.api.IFramePreprocessor;
 import org.apache.asterix.external.feed.dataflow.DataBucket;
 import org.apache.asterix.external.feed.dataflow.FeedRuntimeInputHandler;
 import org.apache.asterix.external.feed.dataflow.StorageFrameHandler;
@@ -170,7 +170,7 @@ public abstract class MonitoredBuffer extends MessageReceiver<DataBucket> {
             this.timer.scheduleAtFixedRate(monitorInputQueueLengthTask, 0, INPUT_QUEUE_MEASURE_FREQUENCY);
         }
 
-        if (logInflowOutflowRate || reportInflowRate || reportOutflowRate) {
+        if (reportInflowRate || reportOutflowRate) {
             this.logInflowOutflowRateTask = new LogInputOutputRateTask(this, logInflowOutflowRate, reportInflowRate,
                     reportOutflowRate);
             this.timer.scheduleAtFixedRate(logInflowOutflowRateTask, 0, LOG_INPUT_OUTPUT_RATE_FREQUENCY);
@@ -188,7 +188,7 @@ public abstract class MonitoredBuffer extends MessageReceiver<DataBucket> {
         if (processingRateTask != null) {
             processingRateTask.cancel();
         }
-        if (logInflowOutflowRate || reportInflowRate || reportOutflowRate) {
+        if (reportInflowRate || reportOutflowRate) {
             metricCollector.removeReportSender(inflowReportSenderId);
             metricCollector.removeReportSender(outflowReportSenderId);
             logInflowOutflowRateTask.cancel();
@@ -244,7 +244,7 @@ public abstract class MonitoredBuffer extends MessageReceiver<DataBucket> {
     }
 
     public void sendReport(ByteBuffer frame) {
-        if ((logInflowOutflowRate || reportInflowRate) && !(inputHandler.getMode().equals(Mode.PROCESS_BACKLOG)
+        if ((reportInflowRate) && !(inputHandler.getMode().equals(Mode.PROCESS_BACKLOG)
                 || inputHandler.getMode().equals(Mode.PROCESS_SPILL))) {
             inflowFta.reset(frame);
             metricCollector.sendReport(inflowReportSenderId, inflowFta.getTupleCount());

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVTestReaderFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVTestReaderFactory.java b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVTestReaderFactory.java
index adaa525..f5afef6 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVTestReaderFactory.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/kv/KVTestReaderFactory.java
@@ -69,8 +69,8 @@ public class KVTestReaderFactory implements IRecordReaderFactory<DCPRequest> {
     @Override
     public IRecordReader<? extends DCPRequest> createRecordReader(final IHyracksTaskContext ctx, final int partition) {
         return new KVTestReader(partition, bucket, schedule,
-                (int) Math.ceil((double) numOfRecords / (double) getPartitionConstraint().getLocations().length),
-                deleteCycle, upsertCycle, (numOfRecords / numOfReaders) * partition);
+                (int) Math.ceil((double) numOfRecords / (double) numOfReaders), deleteCycle, upsertCycle,
+                (numOfRecords / numOfReaders) * partition);
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaComputeNodePushable.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaComputeNodePushable.java b/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaComputeNodePushable.java
index 66b520f..210a94d 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaComputeNodePushable.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaComputeNodePushable.java
@@ -95,9 +95,14 @@ public class FeedMetaComputeNodePushable extends AbstractUnaryInputUnaryOutputOp
 
     private ByteBuffer message = ByteBuffer.allocate(MessagingFrameTupleAppender.MAX_MESSAGE_SIZE);
 
+    private final FeedMetaOperatorDescriptor opDesc;
+
+    private final IRecordDescriptorProvider recordDescProvider;
+
     public FeedMetaComputeNodePushable(IHyracksTaskContext ctx, IRecordDescriptorProvider recordDescProvider,
             int partition, int nPartitions, IOperatorDescriptor coreOperator, FeedConnectionId feedConnectionId,
-            Map<String, String> feedPolicyProperties, String operationId) throws HyracksDataException {
+            Map<String, String> feedPolicyProperties, String operationId,
+            FeedMetaOperatorDescriptor feedMetaOperatorDescriptor) throws HyracksDataException {
         this.ctx = ctx;
         this.coreOperator = (AbstractUnaryInputUnaryOutputOperatorNodePushable) ((IActivity) coreOperator)
                 .createPushRuntime(ctx, recordDescProvider, partition, nPartitions);
@@ -108,6 +113,8 @@ public class FeedMetaComputeNodePushable extends AbstractUnaryInputUnaryOutputOp
         this.feedManager = (IFeedManager) ((IAsterixAppRuntimeContext) ctx.getJobletContext().getApplicationContext()
                 .getApplicationObject()).getFeedManager();
         ctx.setSharedObject(message);
+        this.opDesc = feedMetaOperatorDescriptor;
+        this.recordDescProvider = recordDescProvider;
     }
 
     @Override
@@ -129,7 +136,7 @@ public class FeedMetaComputeNodePushable extends AbstractUnaryInputUnaryOutputOp
     }
 
     private void initializeNewFeedRuntime(FeedRuntimeId runtimeId) throws Exception {
-        this.fta = new FrameTupleAccessor(recordDesc);
+        this.fta = new FrameTupleAccessor(recordDescProvider.getInputRecordDescriptor(opDesc.getActivityId(), 0));
         this.inputSideHandler = new FeedRuntimeInputHandler(ctx, connectionId, runtimeId, coreOperator,
                 policyEnforcer.getFeedPolicyAccessor(), policyEnforcer.getFeedPolicyAccessor().bufferingEnabled(), fta,
                 recordDesc, feedManager, nPartitions);

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaOperatorDescriptor.java b/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaOperatorDescriptor.java
index 354636d..6591795 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaOperatorDescriptor.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaOperatorDescriptor.java
@@ -90,12 +90,12 @@ public class FeedMetaOperatorDescriptor extends AbstractSingleActivityOperatorDe
     @Override
     public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
             final IRecordDescriptorProvider recordDescProvider, final int partition, final int nPartitions)
-                    throws HyracksDataException {
+            throws HyracksDataException {
         IOperatorNodePushable nodePushable = null;
         switch (runtimeType) {
             case COMPUTE:
                 nodePushable = new FeedMetaComputeNodePushable(ctx, recordDescProvider, partition, nPartitions,
-                        coreOperator, feedConnectionId, feedPolicyProperties, operandId);
+                        coreOperator, feedConnectionId, feedPolicyProperties, operandId, this);
                 break;
             case STORE:
                 nodePushable = new FeedMetaStoreNodePushable(ctx, recordDescProvider, partition, nPartitions,

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedUtils.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedUtils.java b/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedUtils.java
index c9fc5fc..5ab41af 100644
--- a/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedUtils.java
+++ b/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedUtils.java
@@ -45,16 +45,17 @@ public class FeedUtils {
         return dataverseName + File.separator + feedName;
     }
 
-    public static FileSplit[] splitsForAdapter(String dataverseName, String feedName, String nodeName, int partition) {
+    public static FileSplit splitsForAdapter(String dataverseName, String feedName, int partition,
+            ClusterPartition[] nodePartitions) {
         File relPathFile = new File(prepareDataverseFeedName(dataverseName, feedName));
         String storageDirName = AsterixClusterProperties.INSTANCE.getStorageDirectoryName();
-        ClusterPartition nodePartition = AsterixClusterProperties.INSTANCE.getNodePartitions(nodeName)[0];
+        ClusterPartition nodePartition = nodePartitions[0];
         String storagePartitionPath = StoragePathUtil.prepareStoragePartitionPath(storageDirName,
                 nodePartition.getPartitionId());
         // format: 'storage dir name'/partition_#/dataverse/feed/adapter_#
         File f = new File(storagePartitionPath + File.separator + relPathFile + File.separator
                 + StoragePathUtil.ADAPTER_INSTANCE_PREFIX + partition);
-        return new FileSplit[] { StoragePathUtil.getFileSplitForClusterPartition(nodePartition, f) };
+        return StoragePathUtil.getFileSplitForClusterPartition(nodePartition, f);
     }
 
     public static FileSplit[] splitsForAdapter(String dataverseName, String feedName,
@@ -93,6 +94,12 @@ public class FeedUtils {
                         feedLogFileSplits[partition].getIODeviceId(), ctx.getIOManager()).getFile());
     }
 
+    public static FeedLogManager getFeedLogManager(IHyracksTaskContext ctx, FileSplit feedLogFileSplit)
+            throws HyracksDataException {
+        return new FeedLogManager(FeedUtils.getAbsoluteFileRef(feedLogFileSplit.getLocalFile().getFile().getPath(),
+                feedLogFileSplit.getIODeviceId(), ctx.getIOManager()).getFile());
+    }
+
     public static void processFeedMessage(ByteBuffer input, ByteBuffer message, FrameTupleAccessor fta) {
         // read the message and reduce the number of tuples
         fta.reset(input);

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-external-data/src/test/java/org/apache/asterix/external/library/adapter/TestTypedAdapterFactory.java
----------------------------------------------------------------------
diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/library/adapter/TestTypedAdapterFactory.java b/asterix-external-data/src/test/java/org/apache/asterix/external/library/adapter/TestTypedAdapterFactory.java
index 9082a72..7da6389 100644
--- a/asterix-external-data/src/test/java/org/apache/asterix/external/library/adapter/TestTypedAdapterFactory.java
+++ b/asterix-external-data/src/test/java/org/apache/asterix/external/library/adapter/TestTypedAdapterFactory.java
@@ -22,6 +22,8 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.util.Map;
 
+import org.apache.asterix.common.cluster.ClusterPartition;
+import org.apache.asterix.common.config.IAsterixPropertiesProvider;
 import org.apache.asterix.common.exceptions.AsterixException;
 import org.apache.asterix.external.api.IAdapterFactory;
 import org.apache.asterix.external.api.IDataSourceAdapter;
@@ -36,6 +38,7 @@ import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartit
 import org.apache.hyracks.api.comm.IFrameWriter;
 import org.apache.hyracks.api.context.IHyracksTaskContext;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.control.nc.NodeControllerService;
 import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
 import org.apache.hyracks.dataflow.std.file.ITupleParser;
 import org.apache.hyracks.dataflow.std.file.ITupleParserFactory;
@@ -74,12 +77,19 @@ public class TestTypedAdapterFactory implements IAdapterFactory {
                 ADMDataParser parser;
                 ITupleForwarder forwarder;
                 ArrayTupleBuilder tb;
+                IAsterixPropertiesProvider propertiesProvider = (IAsterixPropertiesProvider) ((NodeControllerService) ctx
+                        .getJobletContext().getApplicationContext().getControllerService()).getApplicationContext()
+                                .getApplicationObject();
+                ClusterPartition[] nodePartitions = propertiesProvider.getMetadataProperties().getNodePartitions()
+                        .get(nodeId);
                 try {
                     parser = new ADMDataParser(outputType, true);
-                    forwarder = DataflowUtils.getTupleForwarder(configuration,
-                            FeedUtils.getFeedLogManager(ctx, partition,
-                                    FeedUtils.splitsForAdapter(ExternalDataUtils.getDataverse(configuration),
-                                            ExternalDataUtils.getFeedName(configuration), nodeId, partition)));
+                    forwarder = DataflowUtils
+                            .getTupleForwarder(configuration,
+                                    FeedUtils.getFeedLogManager(ctx,
+                                            FeedUtils.splitsForAdapter(ExternalDataUtils.getDataverse(configuration),
+                                                    ExternalDataUtils.getFeedName(configuration), partition,
+                                                    nodePartitions)));
                     tb = new ArrayTupleBuilder(1);
                 } catch (Exception e) {
                     throw new HyracksDataException(e);

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-installer/pom.xml
----------------------------------------------------------------------
diff --git a/asterix-installer/pom.xml b/asterix-installer/pom.xml
index 96e8fa1..cdda6b7 100644
--- a/asterix-installer/pom.xml
+++ b/asterix-installer/pom.xml
@@ -138,8 +138,57 @@
 					</execution>
 				</executions>
 			</plugin>
+            <plugin>
+                <artifactId>maven-resources-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>copy-external-library</id>
+                        <phase>generate-resources</phase>
+                        <goals>
+                            <goal>copy-resources</goal>
+                        </goals>
+                        <configuration>
+                            <outputDirectory>target/resources/externallib</outputDirectory>
+                            <overwrite>true</overwrite>
+                            <resources>
+                                <resource>
+                                    <directory>../asterix-external-data/target</directory>
+                                    <includes>
+                                        <include>testlib-zip-binary-assembly.zip</include>
+                                    </includes>
+                                </resource>
+                            </resources>
+                        </configuration>
+                    </execution>
+                    <execution>
+                        <id>copy-test-data</id>
+                        <phase>generate-resources</phase>
+                        <goals>
+                            <goal>copy-resources</goal>
+                        </goals>
+                        <configuration>
+                            <outputDirectory>target/data</outputDirectory>
+                            <overwrite>true</overwrite>
+                            <resources>
+                                <resource>
+                                    <directory>../asterix-external-data/src/test/resources</directory>
+                                    <includes>
+                                        <include>beer.csv</include>
+                                    </includes>
+                                </resource>
+                                <resource>
+                                    <directory>../asterix-app/data/external-parser</directory>
+                                    <includes>
+                                        <include>jobads.new</include>
+                                    </includes>
+                                </resource>
+                            </resources>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
 		</plugins>
-	</build>
+    </build>
 
 
     <profiles>

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-installer/src/test/java/org/apache/asterix/installer/test/AbstractExecutionIT.java
----------------------------------------------------------------------
diff --git a/asterix-installer/src/test/java/org/apache/asterix/installer/test/AbstractExecutionIT.java b/asterix-installer/src/test/java/org/apache/asterix/installer/test/AbstractExecutionIT.java
index a6d544e..ba2d7f1 100644
--- a/asterix-installer/src/test/java/org/apache/asterix/installer/test/AbstractExecutionIT.java
+++ b/asterix-installer/src/test/java/org/apache/asterix/installer/test/AbstractExecutionIT.java
@@ -72,7 +72,7 @@ public abstract class AbstractExecutionIT {
                 .copyFile(
                         new File(StringUtils.join(new String[] { "src", "test", "resources", "integrationts",
                                 "asterix-configuration.xml" }, File.separator)),
-                new File(AsterixInstallerIntegrationUtil.getManagixHome() + "/conf/asterix-configuration.xml"));
+                        new File(AsterixInstallerIntegrationUtil.getManagixHome() + "/conf/asterix-configuration.xml"));
 
         AsterixLifecycleIT.setUp();
 
@@ -80,6 +80,10 @@ public abstract class AbstractExecutionIT {
                 new File(StringUtils.join(new String[] { "..", "asterix-app", "data" }, File.separator)),
                 new File(AsterixInstallerIntegrationUtil.getManagixHome() + "/clusters/local/working_dir/data"));
 
+        FileUtils.copyDirectoryStructure(new File(StringUtils.join(new String[] { "target", "data" }, File.separator)),
+                new File(AsterixInstallerIntegrationUtil.getManagixHome()
+                        + "/clusters/local/working_dir/target/data/csv"));
+
         // Set the node resolver to be the identity resolver that expects node names
         // to be node controller ids; a valid assumption in test environment.
         System.setProperty(ExternalDataConstants.NODE_RESOLVER_FACTORY_PROPERTY,

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-installer/src/test/java/org/apache/asterix/installer/test/AsterixExternalLibraryIT.java
----------------------------------------------------------------------
diff --git a/asterix-installer/src/test/java/org/apache/asterix/installer/test/AsterixExternalLibraryIT.java b/asterix-installer/src/test/java/org/apache/asterix/installer/test/AsterixExternalLibraryIT.java
index 1da01c3..506e31f 100644
--- a/asterix-installer/src/test/java/org/apache/asterix/installer/test/AsterixExternalLibraryIT.java
+++ b/asterix-installer/src/test/java/org/apache/asterix/installer/test/AsterixExternalLibraryIT.java
@@ -35,8 +35,8 @@ public class AsterixExternalLibraryIT {
     private static final String LIBRARY_DATAVERSE = "externallibtest";
     private static final String PATH_BASE = "src/test/resources/integrationts/library";
     private static final String PATH_ACTUAL = "ittest/";
-    private static final String LIBRARY_PATH = "asterix-external-data" + File.separator + "target" + File.separator
-            + "testlib-zip-binary-assembly.zip";
+    private static final String LIBRARY_PATH = "target" + File.separator + "resources" + File.separator + "externallib"
+            + File.separator + "testlib-zip-binary-assembly.zip";
     private static final Logger LOGGER = Logger.getLogger(AsterixExternalLibraryIT.class.getName());
     private static List<TestCaseContext> testCaseCollection;
     private final TestExecutor testExecutor = new TestExecutor();
@@ -46,8 +46,8 @@ public class AsterixExternalLibraryIT {
         try {
             AsterixInstallerIntegrationUtil.init();
             File asterixInstallerProjectDir = new File(System.getProperty("user.dir"));
-            String asterixExternalLibraryPath = asterixInstallerProjectDir.getParentFile().getAbsolutePath()
-                    + File.separator + LIBRARY_PATH;
+            String asterixExternalLibraryPath = asterixInstallerProjectDir.getAbsolutePath() + File.separator
+                    + LIBRARY_PATH;
             LOGGER.info("Installing library :" + LIBRARY_NAME + " located at " + asterixExternalLibraryPath
                     + " in dataverse " + LIBRARY_DATAVERSE);
             AsterixInstallerIntegrationUtil.installLibrary(LIBRARY_NAME, LIBRARY_DATAVERSE, asterixExternalLibraryPath);

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-installer/src/test/java/org/apache/asterix/installer/test/ManagixExecutionIT.java
----------------------------------------------------------------------
diff --git a/asterix-installer/src/test/java/org/apache/asterix/installer/test/ManagixExecutionIT.java b/asterix-installer/src/test/java/org/apache/asterix/installer/test/ManagixExecutionIT.java
index 14a1a58..6df34ce 100644
--- a/asterix-installer/src/test/java/org/apache/asterix/installer/test/ManagixExecutionIT.java
+++ b/asterix-installer/src/test/java/org/apache/asterix/installer/test/ManagixExecutionIT.java
@@ -24,10 +24,7 @@ import org.junit.runners.Parameterized;
 @RunWith(Parameterized.class)
 public class ManagixExecutionIT extends AbstractExecutionIT {
 
-    private TestCaseContext tcCtx;
-
     public ManagixExecutionIT(TestCaseContext tcCtx) {
         super(tcCtx);
-        this.tcCtx = tcCtx;
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-installer/src/test/resources/integrationts/library/queries/library-feeds/feed_ingest/feed_ingest.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterix-installer/src/test/resources/integrationts/library/queries/library-feeds/feed_ingest/feed_ingest.1.ddl.aql b/asterix-installer/src/test/resources/integrationts/library/queries/library-feeds/feed_ingest/feed_ingest.1.ddl.aql
index f9a6eda..dbe3cfa 100644
--- a/asterix-installer/src/test/resources/integrationts/library/queries/library-feeds/feed_ingest/feed_ingest.1.ddl.aql
+++ b/asterix-installer/src/test/resources/integrationts/library/queries/library-feeds/feed_ingest/feed_ingest.1.ddl.aql
@@ -48,8 +48,7 @@ create feed TweetFeed
 using localfs
 (("type-name"="TweetInputType"),
 ("path"="asterix_nc1://../../../../../../asterix-app/data/twitter/obamatweets.adm"),
-("format"="adm"),
-("tuple-interval"="10"))
+("format"="adm"))
 apply function testlib#parseTweet;
 
 create dataset TweetsFeedIngest(TweetOutputType)

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-installer/src/test/resources/integrationts/library/queries/library-feeds/feed_ingest/feed_ingest.3.query.aql
----------------------------------------------------------------------
diff --git a/asterix-installer/src/test/resources/integrationts/library/queries/library-feeds/feed_ingest/feed_ingest.3.query.aql b/asterix-installer/src/test/resources/integrationts/library/queries/library-feeds/feed_ingest/feed_ingest.3.query.aql
deleted file mode 100644
index e4e1b45..0000000
--- a/asterix-installer/src/test/resources/integrationts/library/queries/library-feeds/feed_ingest/feed_ingest.3.query.aql
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Description  : Create a feed dataset that uses the feed simulator adapter.
-                  The feed simulator simulates feed from a file in the local fs.
-                  Associate with the feed an external user-defined function. The UDF 
-                  finds topics in each tweet. A topic is identified by a #. 
-                  Begin ingestion and apply external user defined function
- * Expected Res : Success
- * Date         : 23rd Apr 2013
- */
-use dataverse externallibtest;
-
-for $x in dataset TweetsFeedIngest
-order by $x.id
-return $x

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-installer/src/test/resources/integrationts/library/queries/library-feeds/feed_ingest/feed_ingest.3.sleep.aql
----------------------------------------------------------------------
diff --git a/asterix-installer/src/test/resources/integrationts/library/queries/library-feeds/feed_ingest/feed_ingest.3.sleep.aql b/asterix-installer/src/test/resources/integrationts/library/queries/library-feeds/feed_ingest/feed_ingest.3.sleep.aql
new file mode 100644
index 0000000..18bbbbc
--- /dev/null
+++ b/asterix-installer/src/test/resources/integrationts/library/queries/library-feeds/feed_ingest/feed_ingest.3.sleep.aql
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description  : Create a socket feed with a client that pushes
+ * 10 records. The feed is connected to a dataset that is then
+ * queried for the data.
+ * Expected Res : Success
+ * Date         : 24th Feb 2016
+ */
+10000
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-installer/src/test/resources/integrationts/library/queries/library-feeds/feed_ingest/feed_ingest.4.query.aql
----------------------------------------------------------------------
diff --git a/asterix-installer/src/test/resources/integrationts/library/queries/library-feeds/feed_ingest/feed_ingest.4.query.aql b/asterix-installer/src/test/resources/integrationts/library/queries/library-feeds/feed_ingest/feed_ingest.4.query.aql
new file mode 100644
index 0000000..e4e1b45
--- /dev/null
+++ b/asterix-installer/src/test/resources/integrationts/library/queries/library-feeds/feed_ingest/feed_ingest.4.query.aql
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description  : Create a feed dataset that uses the feed simulator adapter.
+                  The feed simulator simulates feed from a file in the local fs.
+                  Associate with the feed an external user-defined function. The UDF 
+                  finds topics in each tweet. A topic is identified by a #. 
+                  Begin ingestion and apply external user defined function
+ * Expected Res : Success
+ * Date         : 23rd Apr 2013
+ */
+use dataverse externallibtest;
+
+for $x in dataset TweetsFeedIngest
+order by $x.id
+return $x

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-installer/src/test/resources/integrationts/library/queries/library-parsers/record-parser/record-parser.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterix-installer/src/test/resources/integrationts/library/queries/library-parsers/record-parser/record-parser.1.ddl.aql b/asterix-installer/src/test/resources/integrationts/library/queries/library-parsers/record-parser/record-parser.1.ddl.aql
index d523247..09db37c 100644
--- a/asterix-installer/src/test/resources/integrationts/library/queries/library-parsers/record-parser/record-parser.1.ddl.aql
+++ b/asterix-installer/src/test/resources/integrationts/library/queries/library-parsers/record-parser/record-parser.1.ddl.aql
@@ -24,10 +24,12 @@
 use dataverse externallibtest;
 
 create type Classad as open {
-  GlobalJobId: int64
+  GlobalJobId: string
 }
 
 create external dataset Condor(Classad) using localfs(
-("path"="asterix_nc1://data/external-parser/jobads.new"),
-("format"="adm"),
+("path"="asterix_nc1://../../../../data/jobads.new"),
+("format"="semi-structured"),
+("record-start"="["),
+("record-end"="]"),
 ("parser"="testlib#org.apache.asterix.external.library.ClassAdParserFactory"));

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-installer/src/test/resources/integrationts/library/queries/library-parsers/record-parser/record-parser.2.query.aql
----------------------------------------------------------------------
diff --git a/asterix-installer/src/test/resources/integrationts/library/queries/library-parsers/record-parser/record-parser.2.query.aql b/asterix-installer/src/test/resources/integrationts/library/queries/library-parsers/record-parser/record-parser.2.query.aql
index ca2d832..26f980f 100644
--- a/asterix-installer/src/test/resources/integrationts/library/queries/library-parsers/record-parser/record-parser.2.query.aql
+++ b/asterix-installer/src/test/resources/integrationts/library/queries/library-parsers/record-parser/record-parser.2.query.aql
@@ -24,4 +24,5 @@
 use dataverse externallibtest;
 
 for $x in dataset Condor
+order by $x.GlobalJobId
 return $x;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-installer/src/test/resources/integrationts/library/results/library-metadata/dataverseDataset/dataverseDataset.1.adm
----------------------------------------------------------------------
diff --git a/asterix-installer/src/test/resources/integrationts/library/results/library-metadata/dataverseDataset/dataverseDataset.1.adm b/asterix-installer/src/test/resources/integrationts/library/results/library-metadata/dataverseDataset/dataverseDataset.1.adm
index d4e5619..2c4af14 100644
--- a/asterix-installer/src/test/resources/integrationts/library/results/library-metadata/dataverseDataset/dataverseDataset.1.adm
+++ b/asterix-installer/src/test/resources/integrationts/library/results/library-metadata/dataverseDataset/dataverseDataset.1.adm
@@ -1,2 +1,2 @@
-{ "DataverseName": "Metadata", "DataFormat": "org.apache.asterix.runtime.formats.NonTaggedDataFormat", "Timestamp": "Thu Apr 25 11:17:56 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "externallibtest", "DataFormat": "org.apache.asterix.runtime.formats.NonTaggedDataFormat", "Timestamp": "Thu Apr 25 11:18:12 PDT 2013", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DataFormat": "org.apache.asterix.runtime.formats.NonTaggedDataFormat", "Timestamp": "Thu Apr 25 11:17:56 PDT 2013", "PendingOp": 0i32 }
+{ "DataverseName": "externallibtest", "DataFormat": "org.apache.asterix.runtime.formats.NonTaggedDataFormat", "Timestamp": "Thu Apr 25 11:18:12 PDT 2013", "PendingOp": 0i32 }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-installer/src/test/resources/transactionts/results/query_after_restart/external_index/external_index.1.adm
----------------------------------------------------------------------
diff --git a/asterix-installer/src/test/resources/transactionts/results/query_after_restart/external_index/external_index.1.adm b/asterix-installer/src/test/resources/transactionts/results/query_after_restart/external_index/external_index.1.adm
index 36b836c..3ee4a1e 100644
--- a/asterix-installer/src/test/resources/transactionts/results/query_after_restart/external_index/external_index.1.adm
+++ b/asterix-installer/src/test/resources/transactionts/results/query_after_restart/external_index/external_index.1.adm
@@ -1,4 +1,2 @@
-[ { "id": 3, "name": "Samuel", "age": 22 }
-, { "id": 10, "name": "David", "age": 22 }
- ]
-
+{ "id": 3, "name": "Samuel", "age": 22 }
+{ "id": 10, "name": "David", "age": 22 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java
----------------------------------------------------------------------
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java
index 78c6587..8561bf5 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java
@@ -214,12 +214,15 @@ public class FeedMetadataUtil {
         Map<ConnectorDescriptorId, ConnectorDescriptorId> connectorMapping = new HashMap<ConnectorDescriptorId, ConnectorDescriptorId>();
         for (Entry<ConnectorDescriptorId, IConnectorDescriptor> entry : spec.getConnectorMap().entrySet()) {
             IConnectorDescriptor connDesc = entry.getValue();
+            ConnectorDescriptorId newConnId;
             if (connDesc instanceof MToNPartitioningConnectorDescriptor) {
                 MToNPartitioningConnectorDescriptor m2nConn = (MToNPartitioningConnectorDescriptor) connDesc;
                 connDesc = new MToNPartitioningWithMessageConnectorDescriptor(altered,
                         m2nConn.getTuplePartitionComputerFactory());
+                newConnId = connDesc.getConnectorId();
+            } else {
+                newConnId = altered.createConnectorDescriptor(connDesc);
             }
-            ConnectorDescriptorId newConnId = altered.createConnectorDescriptor(connDesc);
             connectorMapping.put(entry.getKey(), newConnId);
         }
 
@@ -466,7 +469,7 @@ public class FeedMetadataUtil {
     @SuppressWarnings("rawtypes")
     public static Triple<IAdapterFactory, RecordDescriptor, IDataSourceAdapter.AdapterType> getPrimaryFeedFactoryAndOutput(
             Feed feed, FeedPolicyAccessor policyAccessor, MetadataTransactionContext mdTxnCtx)
-                    throws AlgebricksException {
+            throws AlgebricksException {
         // This method needs to be re-visited
         String adapterName = null;
         DatasourceAdapter adapterEntity = null;
@@ -606,7 +609,7 @@ public class FeedMetadataUtil {
 
     public static String getSecondaryFeedOutput(Feed feed, FeedPolicyAccessor policyAccessor,
             MetadataTransactionContext mdTxnCtx)
-                    throws AlgebricksException, MetadataException, RemoteException, ACIDException {
+            throws AlgebricksException, MetadataException, RemoteException, ACIDException {
         String outputType = null;
         String primaryFeedName = feed.getSourceFeedName();
         Feed primaryFeed = MetadataManager.INSTANCE.getFeed(mdTxnCtx, feed.getDataverseName(), primaryFeedName);

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/30904b83/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java
----------------------------------------------------------------------
diff --git a/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java b/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java
index 33b7f30..b76e829 100644
--- a/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java
+++ b/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalBTreeLocalResourceMetadata.java
@@ -45,8 +45,9 @@ public class ExternalBTreeLocalResourceMetadata extends LSMBTreeLocalResourceMet
 
     @Override
     public ILSMIndex createIndexInstance(IAsterixAppRuntimeContextProvider runtimeContextProvider, String filePath,
-                                         int partition, int ioDeviceNum) {
+            int partition, int ioDeviceNum) {
         FileReference file = new FileReference(new File(filePath));
+
         LSMBTree lsmBTree = LSMBTreeUtils.createExternalBTree(file, runtimeContextProvider.getBufferCache(),
                 runtimeContextProvider.getFileMapManager(), typeTraits, cmpFactories, bloomFilterKeyFields,
                 runtimeContextProvider.getBloomFilterFalsePositiveRate(),


Mime
View raw message