asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amo...@apache.org
Subject [3/6] asterixdb git commit: Introduce Strategy Based Replication and Fault-Tolerance
Date Sun, 19 Feb 2017 20:30:29 GMT
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef173f34/asterixdb/asterix-installer/src/main/resources/clusters/local/local_chained_declustering_rep.xml
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-installer/src/main/resources/clusters/local/local_chained_declustering_rep.xml b/asterixdb/asterix-installer/src/main/resources/clusters/local/local_chained_declustering_rep.xml
new file mode 100644
index 0000000..954a311
--- /dev/null
+++ b/asterixdb/asterix-installer/src/main/resources/clusters/local/local_chained_declustering_rep.xml
@@ -0,0 +1,81 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+ ! Licensed to the Apache Software Foundation (ASF) under one
+ ! or more contributor license agreements.  See the NOTICE file
+ ! distributed with this work for additional information
+ ! regarding copyright ownership.  The ASF licenses this file
+ ! to you under the Apache License, Version 2.0 (the
+ ! "License"); you may not use this file except in compliance
+ ! with the License.  You may obtain a copy of the License at
+ !
+ !   http://www.apache.org/licenses/LICENSE-2.0
+ !
+ ! Unless required by applicable law or agreed to in writing,
+ ! software distributed under the License is distributed on an
+ ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ! KIND, either express or implied.  See the License for the
+ ! specific language governing permissions and limitations
+ ! under the License.
+ !-->
+<cluster xmlns="cluster">
+  <java_home>/usr/lib/jvm/jdk-8-oracle-x64/jre</java_home>
+  <log_dir>/tmp/asterix/logs</log_dir>
+  <txn_log_dir>/tmp/asterix/txnLogs</txn_log_dir>
+  <store>storage</store>
+  <working_dir>
+    <dir>/tmp/asterix-installer</dir>
+    <NFS>true</NFS>
+  </working_dir>
+  <!-- Sets the time duration between two heartbeats from each node controller in milliseconds (default: 10000) -->
+  <heartbeat_period>1000</heartbeat_period>
+  <!-- Sets the maximum number of missed heartbeats before a node is marked as dead (default: 5) -->
+  <max_heartbeat_lapse_periods>5</max_heartbeat_lapse_periods>
+  <!-- Sets the time duration between two profile dumps from each node controller in milliseconds. 0 to disable. (default: 0) -->
+  <profile_dump_period>0</profile_dump_period>
+  <!-- Sets the default number of job attempts allowed if not specified in the job specification. (default: 5) -->
+  <default_max_job_attempts>5</default_max_job_attempts>
+  <!-- Limits the number of historical jobs remembered by the system to the specified value. (default: 10) -->
+  <job_history_size>10</job_history_size>
+  <!-- Limits the amount of time results for asynchronous jobs should be retained by the system in milliseconds. (default: 24 hours) -->
+  <result_time_to_live>86400000</result_time_to_live>
+  <!-- The duration within which an instance of the result cleanup should be invoked in milliseconds. (default: 1 minute) -->
+  <result_sweep_threshold>60000</result_sweep_threshold>
+
+  <metadata_node>nc1</metadata_node>
+
+  <high_availability>
+    <enabled>true</enabled>
+    <data_replication>
+      <strategy>chained_declustering</strategy>
+      <replication_port>2000</replication_port>
+      <replication_factor>2</replication_factor>
+      <replication_time_out>30</replication_time_out>
+    </data_replication>
+    <fault_tolerance>
+       <strategy>auto</strategy>
+    </fault_tolerance>
+  </high_availability>
+
+  <master_node>
+    <id>master</id>
+    <client_ip>127.0.0.1</client_ip>
+    <cluster_ip>127.0.0.1</cluster_ip>
+    <client_port>1098</client_port>
+    <cluster_port>1099</cluster_port>
+    <http_port>8888</http_port>
+  </master_node>
+  <node>
+    <id>nc1</id>
+    <cluster_ip>127.0.0.1</cluster_ip>
+    <txn_log_dir>/tmp/asterix/nc1/txnLogs</txn_log_dir>
+    <iodevices>/tmp/asterix/nc1/p1,/tmp/asterix/nc1/p2</iodevices>
+    <replication_port>2000</replication_port>
+  </node>
+  <node>
+    <id>nc2</id>
+    <cluster_ip>127.0.0.1</cluster_ip>
+    <txn_log_dir>/tmp/asterix/nc2/txnLogs</txn_log_dir>
+    <iodevices>/tmp/asterix/nc2/p1,/tmp/asterix/nc2/p2</iodevices>
+    <replication_port>2001</replication_port>
+  </node>
+</cluster>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef173f34/asterixdb/asterix-installer/src/main/resources/clusters/local/local_metadata_only_rep.xml
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-installer/src/main/resources/clusters/local/local_metadata_only_rep.xml b/asterixdb/asterix-installer/src/main/resources/clusters/local/local_metadata_only_rep.xml
new file mode 100644
index 0000000..7a435b7
--- /dev/null
+++ b/asterixdb/asterix-installer/src/main/resources/clusters/local/local_metadata_only_rep.xml
@@ -0,0 +1,83 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+ ! Licensed to the Apache Software Foundation (ASF) under one
+ ! or more contributor license agreements.  See the NOTICE file
+ ! distributed with this work for additional information
+ ! regarding copyright ownership.  The ASF licenses this file
+ ! to you under the Apache License, Version 2.0 (the
+ ! "License"); you may not use this file except in compliance
+ ! with the License.  You may obtain a copy of the License at
+ !
+ !   http://www.apache.org/licenses/LICENSE-2.0
+ !
+ ! Unless required by applicable law or agreed to in writing,
+ ! software distributed under the License is distributed on an
+ ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ! KIND, either express or implied.  See the License for the
+ ! specific language governing permissions and limitations
+ ! under the License.
+ !-->
+<cluster xmlns="cluster">
+  <java_home>/usr/lib/jvm/jdk-8-oracle-x64/jre</java_home>
+  <log_dir>/tmp/asterix/logs</log_dir>
+  <txn_log_dir>/tmp/asterix/txnLogs</txn_log_dir>
+  <store>storage</store>
+  <working_dir>
+    <dir>/tmp/asterix-installer</dir>
+    <NFS>true</NFS>
+  </working_dir>
+  <!-- Sets the time duration between two heartbeats from each node controller in milliseconds (default: 10000) -->
+  <heartbeat_period>1000</heartbeat_period>
+  <!-- Sets the maximum number of missed heartbeats before a node is marked as dead (default: 5) -->
+  <max_heartbeat_lapse_periods>5</max_heartbeat_lapse_periods>
+  <!-- Sets the time duration between two profile dumps from each node controller in milliseconds. 0 to disable. (default: 0) -->
+  <profile_dump_period>0</profile_dump_period>
+  <!-- Sets the default number of job attempts allowed if not specified in the job specification. (default: 5) -->
+  <default_max_job_attempts>5</default_max_job_attempts>
+  <!-- Limits the number of historical jobs remembered by the system to the specified value. (default: 10) -->
+  <job_history_size>10</job_history_size>
+  <!-- Limits the amount of time results for asynchronous jobs should be retained by the system in milliseconds. (default: 24 hours) -->
+  <result_time_to_live>86400000</result_time_to_live>
+  <!-- The duration within which an instance of the result cleanup should be invoked in milliseconds. (default: 1 minute) -->
+  <result_sweep_threshold>60000</result_sweep_threshold>
+
+  <metadata_node>nc1</metadata_node>
+
+  <high_availability>
+    <enabled>true</enabled>
+    <data_replication>
+      <strategy>metadata_only</strategy>
+      <replication_port>2000</replication_port>
+      <replication_time_out>30</replication_time_out>
+    </data_replication>
+    <fault_tolerance>
+      <strategy>metadata_node</strategy>
+      <replica>
+        <node_id>nc2</node_id>
+      </replica>
+    </fault_tolerance>
+  </high_availability>
+
+  <master_node>
+    <id>master</id>
+    <client_ip>127.0.0.1</client_ip>
+    <cluster_ip>127.0.0.1</cluster_ip>
+    <client_port>1098</client_port>
+    <cluster_port>1099</cluster_port>
+    <http_port>8888</http_port>
+  </master_node>
+  <node>
+    <id>nc1</id>
+    <cluster_ip>127.0.0.1</cluster_ip>
+    <txn_log_dir>/tmp/asterix/nc1/txnLogs</txn_log_dir>
+    <iodevices>/tmp/asterix/nc1/p1,/tmp/asterix/nc1/p2</iodevices>
+    <replication_port>2000</replication_port>
+  </node>
+  <node>
+    <id>nc2</id>
+    <cluster_ip>127.0.0.1</cluster_ip>
+    <txn_log_dir>/tmp/asterix/nc2/txnLogs</txn_log_dir>
+    <iodevices>/tmp/asterix/nc2/p1,/tmp/asterix/nc2/p2</iodevices>
+    <replication_port>2001</replication_port>
+  </node>
+</cluster>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef173f34/asterixdb/asterix-installer/src/main/resources/clusters/local/local_with_replication.xml
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-installer/src/main/resources/clusters/local/local_with_replication.xml b/asterixdb/asterix-installer/src/main/resources/clusters/local/local_with_replication.xml
deleted file mode 100644
index e6a3547..0000000
--- a/asterixdb/asterix-installer/src/main/resources/clusters/local/local_with_replication.xml
+++ /dev/null
@@ -1,76 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<!--
- ! Licensed to the Apache Software Foundation (ASF) under one
- ! or more contributor license agreements.  See the NOTICE file
- ! distributed with this work for additional information
- ! regarding copyright ownership.  The ASF licenses this file
- ! to you under the Apache License, Version 2.0 (the
- ! "License"); you may not use this file except in compliance
- ! with the License.  You may obtain a copy of the License at
- !
- !   http://www.apache.org/licenses/LICENSE-2.0
- !
- ! Unless required by applicable law or agreed to in writing,
- ! software distributed under the License is distributed on an
- ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ! KIND, either express or implied.  See the License for the
- ! specific language governing permissions and limitations
- ! under the License.
- !-->
-<cluster xmlns="cluster">
-  <java_home>/usr/lib/jvm/jdk-8-oracle-x64/jre</java_home>
-  <log_dir>/tmp/asterix/logs</log_dir>
-  <txn_log_dir>/tmp/asterix/txnLogs</txn_log_dir>
-  <store>storage</store>
-  <working_dir>
-    <dir>/tmp/asterix-installer</dir>
-    <NFS>true</NFS>
-  </working_dir>
-  <!-- Sets the time duration between two heartbeats from each node controller in milliseconds (default: 10000) -->
-  <heartbeat_period>1000</heartbeat_period>
-  <!-- Sets the maximum number of missed heartbeats before a node is marked as dead (default: 5) -->
-  <max_heartbeat_lapse_periods>5</max_heartbeat_lapse_periods>
-  <!-- Sets the time duration between two profile dumps from each node controller in milliseconds. 0 to disable. (default: 0) -->
-  <profile_dump_period>0</profile_dump_period>
-  <!-- Sets the default number of job attempts allowed if not specified in the job specification. (default: 5) -->
-  <default_max_job_attempts>5</default_max_job_attempts>
-  <!-- Limits the number of historical jobs remembered by the system to the specified value. (default: 10) -->
-  <job_history_size>10</job_history_size>
-  <!-- Limits the amount of time results for asynchronous jobs should be retained by the system in milliseconds. (default: 24 hours) -->
-  <result_time_to_live>86400000</result_time_to_live>
-  <!-- The duration within which an instance of the result cleanup should be invoked in milliseconds. (default: 1 minute) -->
-  <result_sweep_threshold>60000</result_sweep_threshold>
-
-  <metadata_node>nc1</metadata_node>
-
-  <data_replication>
-    <enabled>true</enabled>
-    <replication_port>2000</replication_port>
-    <replication_factor>2</replication_factor>
-    <auto_failover>true</auto_failover>
-    <replication_time_out>10</replication_time_out>
-  </data_replication>
-
-  <master_node>
-    <id>master</id>
-    <client_ip>127.0.0.1</client_ip>
-    <cluster_ip>127.0.0.1</cluster_ip>
-    <client_port>1098</client_port>
-    <cluster_port>1099</cluster_port>
-    <http_port>8888</http_port>
-  </master_node>
-  <node>
-    <id>nc1</id>
-    <cluster_ip>127.0.0.1</cluster_ip>
-    <txn_log_dir>/tmp/asterix/nc1/txnLogs</txn_log_dir>
-    <iodevices>/tmp/asterix/nc1/p1,/tmp/asterix/nc1/p2</iodevices>
-    <replication_port>2000</replication_port>
-  </node>
-  <node>
-    <id>nc2</id>
-    <cluster_ip>127.0.0.1</cluster_ip>
-    <txn_log_dir>/tmp/asterix/nc2/txnLogs</txn_log_dir>
-    <iodevices>/tmp/asterix/nc2/p1,/tmp/asterix/nc2/p2</iodevices>
-    <replication_port>2001</replication_port>
-  </node>
-</cluster>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef173f34/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/AsterixInstallerIntegrationUtil.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/AsterixInstallerIntegrationUtil.java b/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/AsterixInstallerIntegrationUtil.java
index e840796..bfe63be 100644
--- a/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/AsterixInstallerIntegrationUtil.java
+++ b/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/AsterixInstallerIntegrationUtil.java
@@ -55,8 +55,10 @@ public class AsterixInstallerIntegrationUtil {
     private static IHyracksClientConnection hcc;
     private static final String CLUSTERS_BASE_PATH = "clusters" + File.separator + "local" + File.separator;
     public static final String LOCAL_CLUSTER_PATH = CLUSTERS_BASE_PATH + File.separator + "local.xml";
-    public static final String LOCAL_CLUSTER_WITH_REPLICATION_PATH = CLUSTERS_BASE_PATH + File.separator
-            + "local_with_replication.xml";
+    public static final String LOCAL_CLUSTER_CHAINED_DECLUSTERING_REP_PATH = CLUSTERS_BASE_PATH + File.separator
+            + "local_chained_declustering_rep.xml";
+    public static final String LOCAL_CLUSTER_METADATA_ONLY_REP_PATH = CLUSTERS_BASE_PATH + File.separator
+            + "local_metadata_only_rep.xml";
 
     public static void deinit() throws Exception {
         deleteInstance();

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef173f34/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/MetadataReplicationIT.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/MetadataReplicationIT.java b/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/MetadataReplicationIT.java
new file mode 100644
index 0000000..56ac759
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/MetadataReplicationIT.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.installer.test;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Map;
+import java.util.logging.Logger;
+
+import org.apache.asterix.event.model.AsterixInstance.State;
+import org.apache.asterix.test.aql.TestExecutor;
+import org.apache.asterix.test.base.RetainLogsRule;
+import org.apache.asterix.testframework.context.TestCaseContext;
+import org.apache.commons.lang3.StringUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestRule;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+@RunWith(Parameterized.class)
+public class MetadataReplicationIT {
+
+    private static final String PATH_BASE = "src/test/resources/integrationts/metadata_only_replication/";
+    private static final String PATH_ACTUAL = "target" + File.separator + "ittest" + File.separator;
+    private static final Logger LOGGER = Logger.getLogger(MetadataReplicationIT.class.getName());
+    private static String reportPath = new File(
+            StringUtils.join(new String[] { "target", "failsafe-reports" }, File.separator)).getAbsolutePath();
+
+    private final TestExecutor testExecutor = new TestExecutor();
+    private TestCaseContext tcCtx;
+    private static String scriptHomePath;
+    private static File asterixInstallerPath;
+    private static ProcessBuilder pb;
+    private static Map<String, String> env;
+
+    public MetadataReplicationIT(TestCaseContext tcCtx) {
+        this.tcCtx = tcCtx;
+    }
+
+    @Rule
+    public TestRule retainLogs = new RetainLogsRule(AsterixInstallerIntegrationUtil.getManagixHome(), reportPath);
+
+    @BeforeClass
+    public static void setUp() throws Exception {
+        try {
+            pb = new ProcessBuilder();
+            env = pb.environment();
+            asterixInstallerPath = new File(System.getProperty("user.dir"));
+            scriptHomePath = asterixInstallerPath + File.separator + "src" + File.separator + "test" + File.separator
+                    + "resources" + File.separator + "integrationts" + File.separator + "metadata_only_replication" + File.separator
+                    + "scripts";
+            env.put("SCRIPT_HOME", scriptHomePath);
+        } catch (Throwable th) {
+            th.printStackTrace();
+            throw th;
+        }
+    }
+
+    @Before
+    public void before() throws Exception {
+        LOGGER.info("Creating new instance...");
+        AsterixInstallerIntegrationUtil.init(AsterixInstallerIntegrationUtil.LOCAL_CLUSTER_METADATA_ONLY_REP_PATH);
+        LOGGER.info("Instacne created.");
+        AsterixInstallerIntegrationUtil.transformIntoRequiredState(State.ACTIVE);
+        LOGGER.info("Instance is in ACTIVE state.");
+    }
+
+    @After
+    public void after() throws Exception {
+        LOGGER.info("Destroying instance...");
+        AsterixInstallerIntegrationUtil.deinit();
+        LOGGER.info("Instance destroyed.");
+    }
+
+    @Test
+    public void test() throws Exception {
+        testExecutor.executeTest(PATH_ACTUAL, tcCtx, pb, false);
+    }
+
+    @Parameterized.Parameters(name = "MetadataReplicationIT {index}: {0}")
+    public static Collection<Object[]> tests() throws Exception {
+        Collection<Object[]> testArgs = buildTestsInXml(TestCaseContext.DEFAULT_TESTSUITE_XML_NAME);
+        if (testArgs.size() == 0) {
+            testArgs = buildTestsInXml(TestCaseContext.DEFAULT_TESTSUITE_XML_NAME);
+        }
+        return testArgs;
+    }
+
+    protected static Collection<Object[]> buildTestsInXml(String xmlfile) throws Exception {
+        Collection<Object[]> testArgs = new ArrayList<>();
+        TestCaseContext.Builder b = new TestCaseContext.Builder();
+        for (TestCaseContext ctx : b.build(new File(PATH_BASE), xmlfile)) {
+            testArgs.add(new Object[] { ctx });
+        }
+        return testArgs;
+    }
+}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef173f34/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/ReplicationIT.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/ReplicationIT.java b/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/ReplicationIT.java
index 5f47849..5c8a68a 100644
--- a/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/ReplicationIT.java
+++ b/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/ReplicationIT.java
@@ -80,7 +80,7 @@ public class ReplicationIT {
     @Before
     public void before() throws Exception {
         LOGGER.info("Creating new instance...");
-        AsterixInstallerIntegrationUtil.init(AsterixInstallerIntegrationUtil.LOCAL_CLUSTER_WITH_REPLICATION_PATH);
+        AsterixInstallerIntegrationUtil.init(AsterixInstallerIntegrationUtil.LOCAL_CLUSTER_CHAINED_DECLUSTERING_REP_PATH);
         LOGGER.info("Instacne created.");
         AsterixInstallerIntegrationUtil.transformIntoRequiredState(State.ACTIVE);
         LOGGER.info("Instance is in ACTIVE state.");

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef173f34/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.1.ddl.aql b/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.1.ddl.aql
new file mode 100644
index 0000000..725ed61
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.1.ddl.aql
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name  : metadata_node_recovery.aql
+ * Description     : Make sure metadata only replication completes as expected.
+                     The test goes as follows:
+                     start metadata node and a replica, update metadata, kill metadata node,
+                     start metadata node again, validate metadata, update metadata, kill replica,
+                     start replica, update metadata, kill metadata node, start metadata node,
+                     validate metadata.
+ * Expected Result : Success
+ * Date            : January 10 2017
+ */
+drop dataverse Social if exists;
+create dataverse Social;
+use dataverse Social;
+
+create type EmploymentType as open {
+    organization-name: string,
+    start-date: date,
+    end-date: date?
+}
+
+create type UserType as closed {
+    id: int,
+    alias: string,
+    name: string,
+    user-since: datetime,
+    friend-ids: {{ int32 }},
+    employment: [EmploymentType]
+}
+
+create dataset Users(UserType)
+primary key id;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef173f34/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.10.node.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.10.node.aql b/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.10.node.aql
new file mode 100644
index 0000000..f4685df
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.10.node.aql
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name  : metadata_node_recovery.aql
+ * Description     : Make sure metadata only replication completes as expected.
+                     The test goes as follows:
+                     start metadata node and a replica, update metadata, kill metadata node,
+                     start metadata node again, validate metadata, update metadata, kill replica,
+                     start replica, update metadata, kill metadata node, start metadata node,
+                     validate metadata.
+ * Expected Result : Success
+ * Date            : January 10 2017
+ */
+kill asterix_nc2
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef173f34/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.11.sleep.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.11.sleep.aql b/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.11.sleep.aql
new file mode 100644
index 0000000..51b0e76
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.11.sleep.aql
@@ -0,0 +1 @@
+60000
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef173f34/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.12.mgx.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.12.mgx.aql b/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.12.mgx.aql
new file mode 100644
index 0000000..0e7bef0
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.12.mgx.aql
@@ -0,0 +1 @@
+startnode -n asterix -nodes nc2
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef173f34/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.13.sleep.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.13.sleep.aql b/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.13.sleep.aql
new file mode 100644
index 0000000..d0000aa
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.13.sleep.aql
@@ -0,0 +1 @@
+20000
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef173f34/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.14.ddl.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.14.ddl.aql b/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.14.ddl.aql
new file mode 100644
index 0000000..eee7423
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.14.ddl.aql
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name  : metadata_node_recovery.aql
+ * Description     : Make sure metadata only replication completes as expected.
+                     The test goes as follows:
+                     start metadata node and a replica, update metadata, kill metadata node,
+                     start metadata node again, validate metadata, update metadata, kill replica,
+                     start replica, update metadata, kill metadata node, start metadata node,
+                     validate metadata.
+ * Expected Result : Success
+ * Date            : January 10 2017
+ */
+use dataverse Social;
+
+create dataset UsersAfterReplicaRecovery(UserType)
+primary key id;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef173f34/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.15.node.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.15.node.aql b/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.15.node.aql
new file mode 100644
index 0000000..720f33e
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.15.node.aql
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name  : metadata_node_recovery.aql
+ * Description     : Make sure metadata only replication completes as expected.
+                     The test goes as follows:
+                     start metadata node and a replica, update metadata, kill metadata node,
+                     start metadata node again, validate metadata, update metadata, kill replica,
+                     start replica, update metadata, kill metadata node, start metadata node,
+                     validate metadata.
+ * Expected Result : Success
+ * Date            : January 10 2017
+ */
+kill asterix_nc1
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef173f34/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.16.sleep.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.16.sleep.aql b/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.16.sleep.aql
new file mode 100644
index 0000000..51b0e76
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.16.sleep.aql
@@ -0,0 +1 @@
+60000
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef173f34/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.17.mgx.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.17.mgx.aql b/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.17.mgx.aql
new file mode 100644
index 0000000..67b492c
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.17.mgx.aql
@@ -0,0 +1 @@
+startnode -n asterix -nodes nc1
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef173f34/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.18.sleep.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.18.sleep.aql b/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.18.sleep.aql
new file mode 100644
index 0000000..d0000aa
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.18.sleep.aql
@@ -0,0 +1 @@
+20000
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef173f34/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.19.query.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.19.query.aql b/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.19.query.aql
new file mode 100644
index 0000000..5da6c59
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.19.query.aql
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name  : metadata_node_recovery.aql
+ * Description     : Make sure metadata only replication completes as expected.
+                     The test goes as follows:
+                     start metadata node and a replica, update metadata, kill metadata node,
+                     start metadata node again, validate metadata, update metadata, kill replica,
+                     start replica, update metadata, kill metadata node, start metadata node,
+                     validate metadata.
+ * Expected Result : Success
+ * Date            : January 10 2017
+ */
+count(
+for $x in dataset Metadata.Dataset
+where $x.DatasetName ='UsersAfterRecovery'
+or $x.DatasetName ='UsersAfterReplicaRecovery'
+return $x.DatasetName
+);
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef173f34/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.2.node.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.2.node.aql b/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.2.node.aql
new file mode 100644
index 0000000..720f33e
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.2.node.aql
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name  : metadata_node_recovery.aql
+ * Description     : Make sure metadata only replication completes as expected.
+                     The test goes as follows:
+                     start metadata node and a replica, update metadata, kill metadata node,
+                     start metadata node again, validate metadata, update metadata, kill replica,
+                     start replica, update metadata, kill metadata node, start metadata node,
+                     validate metadata.
+ * Expected Result : Success
+ * Date            : January 10 2017
+ */
+kill asterix_nc1
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef173f34/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.3.sleep.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.3.sleep.aql b/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.3.sleep.aql
new file mode 100644
index 0000000..51b0e76
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.3.sleep.aql
@@ -0,0 +1 @@
+60000
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef173f34/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.4.get.http
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.4.get.http b/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.4.get.http
new file mode 100644
index 0000000..fdacfd0
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.4.get.http
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name  : metadata_node_recovery.aql
+ * Description     : Make sure metadata only replication completes as expected.
+                     The test goes as follows:
+                     start metadata node and a replica, update metadata, kill metadata node,
+                     start metadata node again, validate metadata, update metadata, kill replica,
+                     start replica, update metadata, kill metadata node, start metadata node,
+                     validate metadata.
+ * Expected Result : Success
+ * Date            : January 10 2017
+ */
+/admin/cluster/summary
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef173f34/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.5.mgx.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.5.mgx.aql b/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.5.mgx.aql
new file mode 100644
index 0000000..67b492c
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.5.mgx.aql
@@ -0,0 +1 @@
+startnode -n asterix -nodes nc1
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef173f34/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.6.sleep.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.6.sleep.aql b/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.6.sleep.aql
new file mode 100644
index 0000000..d0000aa
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.6.sleep.aql
@@ -0,0 +1 @@
+20000
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef173f34/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.7.get.http
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.7.get.http b/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.7.get.http
new file mode 100644
index 0000000..fdacfd0
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.7.get.http
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name  : metadata_node_recovery.aql
+ * Description     : Make sure metadata only replication completes as expected.
+                     The test goes as follows:
+                     start metadata node and a replica, update metadata, kill metadata node,
+                     start metadata node again, validate metadata, update metadata, kill replica,
+                     start replica, update metadata, kill metadata node, start metadata node,
+                     validate metadata.
+ * Expected Result : Success
+ * Date            : January 10 2017
+ */
+/admin/cluster/summary
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef173f34/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.8.query.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.8.query.aql b/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.8.query.aql
new file mode 100644
index 0000000..778e9f3
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.8.query.aql
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name  : metadata_node_recovery.aql
+ * Description     : Make sure metadata only replication completes as expected.
+                     The test goes as follows:
+                     start metadata node and a replica, update metadata, kill metadata node,
+                     start metadata node again, validate metadata, update metadata, kill replica,
+                     start replica, update metadata, kill metadata node, start metadata node,
+                     validate metadata.
+ * Expected Result : Success
+ * Date            : January 10 2017
+ */
+count(
+for $x in dataset Metadata.Dataset
+where $x.DatasetName ='Users'
+return $x.DatasetName
+);
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef173f34/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.9.ddl.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.9.ddl.aql b/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.9.ddl.aql
new file mode 100644
index 0000000..376df70
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/queries/metadata_recovery/metadata_node_recovery/metadata_node_recovery.9.ddl.aql
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name  : metadata_node_recovery.aql
+ * Description     : Make sure metadata only replication completes as expected.
+                     The test goes as follows:
+                     start metadata node and a replica, update metadata, kill metadata node,
+                     start metadata node again, validate metadata, update metadata, kill replica,
+                     start replica, update metadata, kill metadata node, start metadata node,
+                     validate metadata.
+ * Expected Result : Success
+ * Date            : January 10 2017
+ */
+use dataverse Social;
+
+create dataset UsersAfterRecovery(UserType)
+primary key id;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef173f34/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/results/metadata_recovery/metadata_node_recovery/metadata_node_recovery.cluster_state.4.adm
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/results/metadata_recovery/metadata_node_recovery/metadata_node_recovery.cluster_state.4.adm b/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/results/metadata_recovery/metadata_node_recovery/metadata_node_recovery.cluster_state.4.adm
new file mode 100644
index 0000000..d6ea4b7
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/results/metadata_recovery/metadata_node_recovery/metadata_node_recovery.cluster_state.4.adm
@@ -0,0 +1,34 @@
+{
+  "metadata_node" : "asterix_nc1",
+  "partitions" : {
+    "0" : {
+      "partitionId" : 0,
+      "nodeId" : "asterix_nc1",
+      "activeNodeId" : "asterix_nc1",
+      "active" : false,
+      "iodeviceNum" : 0
+    },
+    "1" : {
+      "partitionId" : 1,
+      "nodeId" : "asterix_nc1",
+      "activeNodeId" : "asterix_nc1",
+      "active" : false,
+      "iodeviceNum" : 1
+    },
+    "2" : {
+      "partitionId" : 2,
+      "nodeId" : "asterix_nc2",
+      "activeNodeId" : "asterix_nc2",
+      "active" : true,
+      "iodeviceNum" : 0
+    },
+    "3" : {
+      "partitionId" : 3,
+      "nodeId" : "asterix_nc2",
+      "activeNodeId" : "asterix_nc2",
+      "active" : true,
+      "iodeviceNum" : 1
+    }
+  },
+  "state" : "UNUSABLE"
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef173f34/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/results/metadata_recovery/metadata_node_recovery/metadata_node_recovery.cluster_state.7.adm
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/results/metadata_recovery/metadata_node_recovery/metadata_node_recovery.cluster_state.7.adm b/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/results/metadata_recovery/metadata_node_recovery/metadata_node_recovery.cluster_state.7.adm
new file mode 100644
index 0000000..579caac
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/results/metadata_recovery/metadata_node_recovery/metadata_node_recovery.cluster_state.7.adm
@@ -0,0 +1,34 @@
+{
+  "metadata_node" : "asterix_nc1",
+  "partitions" : {
+    "0" : {
+      "partitionId" : 0,
+      "nodeId" : "asterix_nc1",
+      "activeNodeId" : "asterix_nc1",
+      "active" : true,
+      "iodeviceNum" : 0
+    },
+    "1" : {
+      "partitionId" : 1,
+      "nodeId" : "asterix_nc1",
+      "activeNodeId" : "asterix_nc1",
+      "active" : true,
+      "iodeviceNum" : 1
+    },
+    "2" : {
+      "partitionId" : 2,
+      "nodeId" : "asterix_nc2",
+      "activeNodeId" : "asterix_nc2",
+      "active" : true,
+      "iodeviceNum" : 0
+    },
+    "3" : {
+      "partitionId" : 3,
+      "nodeId" : "asterix_nc2",
+      "activeNodeId" : "asterix_nc2",
+      "active" : true,
+      "iodeviceNum" : 1
+    }
+  },
+  "state" : "ACTIVE"
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef173f34/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/results/metadata_recovery/metadata_node_recovery/metadata_node_recovery.query.19.adm
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/results/metadata_recovery/metadata_node_recovery/metadata_node_recovery.query.19.adm b/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/results/metadata_recovery/metadata_node_recovery/metadata_node_recovery.query.19.adm
new file mode 100644
index 0000000..d8263ee
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/results/metadata_recovery/metadata_node_recovery/metadata_node_recovery.query.19.adm
@@ -0,0 +1 @@
+2
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef173f34/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/results/metadata_recovery/metadata_node_recovery/metadata_node_recovery.query.8.adm
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/results/metadata_recovery/metadata_node_recovery/metadata_node_recovery.query.8.adm b/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/results/metadata_recovery/metadata_node_recovery/metadata_node_recovery.query.8.adm
new file mode 100644
index 0000000..56a6051
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/results/metadata_recovery/metadata_node_recovery/metadata_node_recovery.query.8.adm
@@ -0,0 +1 @@
+1
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef173f34/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/testsuite.xml
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/testsuite.xml b/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/testsuite.xml
new file mode 100644
index 0000000..d0ac325
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/integrationts/metadata_only_replication/testsuite.xml
@@ -0,0 +1,27 @@
+<!--
+ ! Licensed to the Apache Software Foundation (ASF) under one
+ ! or more contributor license agreements.  See the NOTICE file
+ ! distributed with this work for additional information
+ ! regarding copyright ownership.  The ASF licenses this file
+ ! to you under the Apache License, Version 2.0 (the
+ ! "License"); you may not use this file except in compliance
+ ! with the License.  You may obtain a copy of the License at
+ !
+ !   http://www.apache.org/licenses/LICENSE-2.0
+ !
+ ! Unless required by applicable law or agreed to in writing,
+ ! software distributed under the License is distributed on an
+ ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ! KIND, either express or implied.  See the License for the
+ ! specific language governing permissions and limitations
+ ! under the License.
+ !-->
+<test-suite xmlns="urn:xml.testframework.asterix.apache.org" ResultOffsetPath="results" QueryOffsetPath="queries" QueryFileExtension=".aql">
+  <test-group name="metadata_recovery">
+    <test-case FilePath="metadata_recovery">
+      <compilation-unit name="metadata_node_recovery">
+        <output-dir compare="Text">metadata_node_recovery</output-dir>
+      </compilation-unit>
+    </test-case>
+  </test-group>
+</test-suite>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef173f34/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/functions/ReplicaFilesRequest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/functions/ReplicaFilesRequest.java b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/functions/ReplicaFilesRequest.java
index 7502737..f5b2378 100644
--- a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/functions/ReplicaFilesRequest.java
+++ b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/functions/ReplicaFilesRequest.java
@@ -26,19 +26,19 @@ import java.util.HashSet;
 import java.util.Set;
 
 public class ReplicaFilesRequest {
-    private final Set<String> replicaIds;
+    private final Set<Integer> partitionIds;
     private final Set<String> existingFiles;
 
-    public ReplicaFilesRequest(Set<String> replicaIds, Set<String> existingFiles) {
-        this.replicaIds = replicaIds;
+    public ReplicaFilesRequest(Set<Integer> partitionIds, Set<String> existingFiles) {
+        this.partitionIds = partitionIds;
         this.existingFiles = existingFiles;
     }
 
     public void serialize(OutputStream out) throws IOException {
         DataOutputStream dos = new DataOutputStream(out);
-        dos.writeInt(replicaIds.size());
-        for (String replicaId : replicaIds) {
-            dos.writeUTF(replicaId);
+        dos.writeInt(partitionIds.size());
+        for (Integer partitionId : partitionIds) {
+            dos.writeInt(partitionId);
         }
         dos.writeInt(existingFiles.size());
         for (String fileName : existingFiles) {
@@ -48,20 +48,20 @@ public class ReplicaFilesRequest {
 
     public static ReplicaFilesRequest create(DataInput input) throws IOException {
         int size = input.readInt();
-        Set<String> replicaIds = new HashSet<String>(size);
+        Set<Integer> partitionIds = new HashSet<>(size);
         for (int i = 0; i < size; i++) {
-            replicaIds.add(input.readUTF());
+            partitionIds.add(input.readInt());
         }
         int filesCount = input.readInt();
-        Set<String> existingFiles = new HashSet<String>(filesCount);
+        Set<String> existingFiles = new HashSet<>(filesCount);
         for (int i = 0; i < filesCount; i++) {
             existingFiles.add(input.readUTF());
         }
-        return new ReplicaFilesRequest(replicaIds, existingFiles);
+        return new ReplicaFilesRequest(partitionIds, existingFiles);
     }
 
-    public Set<String> getReplicaIds() {
-        return replicaIds;
+    public Set<Integer> getPartitionIds() {
+        return partitionIds;
     }
 
     public Set<String> getExistingFiles() {

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef173f34/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/ReplicationChannel.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/ReplicationChannel.java b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/ReplicationChannel.java
index 8050e8d..044707a 100644
--- a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/ReplicationChannel.java
+++ b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/ReplicationChannel.java
@@ -52,13 +52,16 @@ import org.apache.asterix.common.ioopcallbacks.AbstractLSMIOOperationCallback;
 import org.apache.asterix.common.replication.IReplicaResourcesManager;
 import org.apache.asterix.common.replication.IReplicationChannel;
 import org.apache.asterix.common.replication.IReplicationManager;
+import org.apache.asterix.common.replication.IReplicationStrategy;
 import org.apache.asterix.common.replication.IReplicationThread;
 import org.apache.asterix.common.replication.ReplicaEvent;
+import org.apache.asterix.common.storage.IndexFileProperties;
 import org.apache.asterix.common.transactions.IAppRuntimeContextProvider;
 import org.apache.asterix.common.transactions.ILogManager;
 import org.apache.asterix.common.transactions.LogRecord;
 import org.apache.asterix.common.transactions.LogSource;
 import org.apache.asterix.common.transactions.LogType;
+import org.apache.asterix.common.utils.StoragePathUtil;
 import org.apache.asterix.common.utils.TransactionUtil;
 import org.apache.asterix.replication.functions.ReplicaFilesRequest;
 import org.apache.asterix.replication.functions.ReplicaIndexFlushRequest;
@@ -101,6 +104,8 @@ public class ReplicationChannel extends Thread implements IReplicationChannel {
     private final Set<Integer> nodeHostedPartitions;
     private final ReplicationNotifier replicationNotifier;
     private final Object flushLogslock = new Object();
+    private final IDatasetLifecycleManager dsLifecycleManager;
+    private final PersistentLocalResourceRepository localResourceRep;
 
     public ReplicationChannel(String nodeId, ReplicationProperties replicationProperties, ILogManager logManager,
             IReplicaResourcesManager replicaResoucesManager, IReplicationManager replicationManager,
@@ -111,6 +116,9 @@ public class ReplicationChannel extends Thread implements IReplicationChannel {
         this.replicationManager = replicationManager;
         this.replicationProperties = replicationProperties;
         this.appContextProvider = asterixAppRuntimeContextProvider;
+        this.dsLifecycleManager = asterixAppRuntimeContextProvider.getDatasetLifecycleManager();
+        this.localResourceRep = (PersistentLocalResourceRepository) asterixAppRuntimeContextProvider
+                .getLocalResourceRepository();
         lsmComponentRemoteLSN2LocalLSNMappingTaskQ = new LinkedBlockingQueue<>();
         pendingNotificationRemoteLogsQ = new LinkedBlockingQueue<>();
         lsmComponentId2PropertiesMap = new ConcurrentHashMap<>();
@@ -118,10 +126,9 @@ public class ReplicationChannel extends Thread implements IReplicationChannel {
         lsmComponentLSNMappingService = new LSMComponentsSyncService();
         replicationNotifier = new ReplicationNotifier();
         replicationThreads = Executors.newCachedThreadPool(appContext.getThreadFactory());
-        Map<String, ClusterPartition[]> nodePartitions =
-                ((IPropertiesProvider) asterixAppRuntimeContextProvider.getAppContext()).getMetadataProperties()
-                        .getNodePartitions();
-        Set<String> nodeReplicationClients = replicationProperties.getNodeReplicationClients(nodeId);
+        Map<String, ClusterPartition[]> nodePartitions = ((IPropertiesProvider) asterixAppRuntimeContextProvider
+                .getAppContext()).getMetadataProperties().getNodePartitions();
+        Set<String> nodeReplicationClients = replicationProperties.getRemotePrimaryReplicasIds(nodeId);
         List<Integer> clientsPartitions = new ArrayList<>();
         for (String clientId : nodeReplicationClients) {
             for (ClusterPartition clusterPartition : nodePartitions.get(clientId)) {
@@ -141,8 +148,8 @@ public class ReplicationChannel extends Thread implements IReplicationChannel {
         try {
             serverSocketChannel = ServerSocketChannel.open();
             serverSocketChannel.configureBlocking(true);
-            InetSocketAddress replicationChannelAddress =
-                    new InetSocketAddress(InetAddress.getByName(nodeIP), dataPort);
+            InetSocketAddress replicationChannelAddress = new InetSocketAddress(InetAddress.getByName(nodeIP),
+                    dataPort);
             serverSocketChannel.socket().bind(replicationChannelAddress);
             lsmComponentLSNMappingService.start();
             replicationNotifier.start();
@@ -169,9 +176,8 @@ public class ReplicationChannel extends Thread implements IReplicationChannel {
         if (remainingFile == 0) {
             if (lsmCompProp.getOpType() == LSMOperationType.FLUSH && lsmCompProp.getReplicaLSN() != null
                     && replicaUniqueLSN2RemoteMapping.containsKey(lsmCompProp.getNodeUniqueLSN())) {
-                int remainingIndexes =
-                        replicaUniqueLSN2RemoteMapping.get(lsmCompProp.getNodeUniqueLSN()).numOfFlushedIndexes
-                                .decrementAndGet();
+                int remainingIndexes = replicaUniqueLSN2RemoteMapping
+                        .get(lsmCompProp.getNodeUniqueLSN()).numOfFlushedIndexes.decrementAndGet();
                 if (remainingIndexes == 0) {
                     /**
                      * Note: there is a chance that this will never be removed because some
@@ -217,8 +223,8 @@ public class ReplicationChannel extends Thread implements IReplicationChannel {
         public void run() {
             Thread.currentThread().setName("Replication Thread");
             try {
-                ReplicationRequestType replicationFunction =
-                        ReplicationProtocol.getRequestType(socketChannel, inBuffer);
+                ReplicationRequestType replicationFunction = ReplicationProtocol.getRequestType(socketChannel,
+                        inBuffer);
                 while (replicationFunction != ReplicationRequestType.GOODBYE) {
                     switch (replicationFunction) {
                         case REPLICATE_LOG:
@@ -282,8 +288,8 @@ public class ReplicationChannel extends Thread implements IReplicationChannel {
             Set<Integer> datasetsToForceFlush = new HashSet<>();
             for (IndexInfo iInfo : openIndexesInfo) {
                 if (requestedIndexesToBeFlushed.contains(iInfo.getResourceId())) {
-                    AbstractLSMIOOperationCallback ioCallback =
-                            (AbstractLSMIOOperationCallback) iInfo.getIndex().getIOOperationCallback();
+                    AbstractLSMIOOperationCallback ioCallback = (AbstractLSMIOOperationCallback) iInfo.getIndex()
+                            .getIOOperationCallback();
                     //if an index has a pending flush, then the request to flush it will succeed.
                     if (ioCallback.hasPendingFlush()) {
                         //remove index to indicate that it will be flushed
@@ -372,34 +378,40 @@ public class ReplicationChannel extends Thread implements IReplicationChannel {
             LSMIndexFileProperties fileProperties = new LSMIndexFileProperties();
 
             List<String> filesList;
-            Set<String> replicaIds = request.getReplicaIds();
+            Set<Integer> partitionIds = request.getPartitionIds();
             Set<String> requesterExistingFiles = request.getExistingFiles();
-            Map<String, ClusterPartition[]> nodePartitions =
-                    ((IPropertiesProvider) appContextProvider.getAppContext()).getMetadataProperties()
-                            .getNodePartitions();
-            for (String replicaId : replicaIds) {
-                //get replica partitions
-                ClusterPartition[] replicaPatitions = nodePartitions.get(replicaId);
-                for (ClusterPartition partition : replicaPatitions) {
-                    filesList = replicaResourcesManager.getPartitionIndexesFiles(partition.getPartitionId(), false);
-                    //start sending files
-                    for (String filePath : filesList) {
-                        String relativeFilePath = PersistentLocalResourceRepository.getResourceRelativePath(filePath);
-                        //if the file already exists on the requester, skip it
-                        if (!requesterExistingFiles.contains(relativeFilePath)) {
-                            try (RandomAccessFile fromFile = new RandomAccessFile(filePath, "r");
-                                    FileChannel fileChannel = fromFile.getChannel();) {
-                                long fileSize = fileChannel.size();
-                                fileProperties.initialize(filePath, fileSize, replicaId, false,
-                                        AbstractLSMIOOperationCallback.INVALID, false);
-                                outBuffer = ReplicationProtocol.writeFileReplicationRequest(outBuffer, fileProperties,
-                                        ReplicationRequestType.REPLICATE_FILE);
-
-                                //send file info
-                                NetworkingUtil.transferBufferToChannel(socketChannel, outBuffer);
-                                //transfer file
-                                NetworkingUtil.sendFile(fileChannel, socketChannel);
-                            }
+            Map<Integer, ClusterPartition> clusterPartitions = ((IPropertiesProvider) appContextProvider
+                    .getAppContext()).getMetadataProperties().getClusterPartitions();
+
+            final IReplicationStrategy repStrategy = replicationProperties.getReplicationStrategy();
+            // Flush replicated datasets to generate the latest LSM components
+            dsLifecycleManager.flushDataset(repStrategy);
+            for (Integer partitionId : partitionIds) {
+                ClusterPartition partition = clusterPartitions.get(partitionId);
+                filesList = replicaResourcesManager.getPartitionIndexesFiles(partition.getPartitionId(), false);
+                //start sending files
+                for (String filePath : filesList) {
+                    // Send only files of datasets that are replciated.
+                    IndexFileProperties indexFileRef = localResourceRep.getIndexFileRef(filePath);
+                    if (!repStrategy.isMatch(indexFileRef.getDatasetId())) {
+                        continue;
+                    }
+                    String relativeFilePath = StoragePathUtil.getIndexFileRelativePath(filePath);
+                    //if the file already exists on the requester, skip it
+                    if (!requesterExistingFiles.contains(relativeFilePath)) {
+                        try (RandomAccessFile fromFile = new RandomAccessFile(filePath, "r");
+                                FileChannel fileChannel = fromFile.getChannel();) {
+                            long fileSize = fileChannel.size();
+                            fileProperties.initialize(filePath, fileSize, partition.getNodeId(), false,
+                                    AbstractLSMIOOperationCallback.INVALID, false);
+                            outBuffer = ReplicationProtocol.writeFileReplicationRequest(outBuffer, fileProperties,
+                                    ReplicationRequestType.REPLICATE_FILE);
+
+                            //send file info
+                            NetworkingUtil.transferBufferToChannel(socketChannel, outBuffer);
+
+                            //transfer file
+                            NetworkingUtil.sendFile(fileChannel, socketChannel);
                         }
                     }
                 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef173f34/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/ReplicationManager.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/ReplicationManager.java b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/ReplicationManager.java
index 936f5a0..72cc7d1 100644
--- a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/ReplicationManager.java
+++ b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/management/ReplicationManager.java
@@ -50,18 +50,20 @@ import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.logging.Level;
 import java.util.logging.Logger;
+import java.util.stream.Collectors;
 
 import org.apache.asterix.common.cluster.ClusterPartition;
-import org.apache.asterix.common.config.ClusterProperties;
 import org.apache.asterix.common.config.IPropertiesProvider;
 import org.apache.asterix.common.config.ReplicationProperties;
 import org.apache.asterix.common.dataflow.LSMIndexUtil;
 import org.apache.asterix.common.replication.IReplicaResourcesManager;
 import org.apache.asterix.common.replication.IReplicationManager;
+import org.apache.asterix.common.replication.IReplicationStrategy;
 import org.apache.asterix.common.replication.Replica;
 import org.apache.asterix.common.replication.Replica.ReplicaState;
 import org.apache.asterix.common.replication.ReplicaEvent;
 import org.apache.asterix.common.replication.ReplicationJob;
+import org.apache.asterix.common.storage.IndexFileProperties;
 import org.apache.asterix.common.transactions.IAppRuntimeContextProvider;
 import org.apache.asterix.common.transactions.ILogManager;
 import org.apache.asterix.common.transactions.ILogRecord;
@@ -101,7 +103,6 @@ public class ReplicationManager implements IReplicationManager {
     private final Map<Integer, Set<String>> jobCommitAcks;
     private final Map<Integer, ILogRecord> replicationJobsPendingAcks;
     private ByteBuffer dataBuffer;
-
     private final LinkedBlockingQueue<IReplicationJob> replicationJobsQ;
     private final LinkedBlockingQueue<ReplicaEvent> replicaEventsQ;
 
@@ -134,6 +135,8 @@ public class ReplicationManager implements IReplicationManager {
     private Future<? extends Object> txnLogReplicatorTask;
     private SocketChannel[] logsRepSockets;
     private final ByteBuffer txnLogsBatchSizeBuffer = ByteBuffer.allocate(Integer.BYTES);
+    private final IReplicationStrategy replicationStrategy;
+    private final PersistentLocalResourceRepository localResourceRepo;
 
     //TODO this class needs to be refactored by moving its private classes to separate files
     //and possibly using MessageBroker to send/receive remote replicas events.
@@ -142,36 +145,38 @@ public class ReplicationManager implements IReplicationManager {
             IAppRuntimeContextProvider asterixAppRuntimeContextProvider) {
         this.nodeId = nodeId;
         this.replicationProperties = replicationProperties;
+        replicationStrategy = replicationProperties.getReplicationStrategy();
         this.replicaResourcesManager = (ReplicaResourcesManager) remoteResoucesManager;
         this.asterixAppRuntimeContextProvider = asterixAppRuntimeContextProvider;
-        this.hostIPAddressFirstOctet = replicationProperties.getReplicaIPAddress(nodeId).substring(0, 3);
         this.logManager = logManager;
+        localResourceRepo = (PersistentLocalResourceRepository) asterixAppRuntimeContextProvider
+                .getLocalResourceRepository();
+        this.hostIPAddressFirstOctet = replicationProperties.getReplicaIPAddress(nodeId).substring(0, 3);
+        replicas = new HashMap<>();
         replicationJobsQ = new LinkedBlockingQueue<>();
         replicaEventsQ = new LinkedBlockingQueue<>();
         terminateJobsReplication = new AtomicBoolean(false);
         jobsReplicationSuspended = new AtomicBoolean(true);
         replicationSuspended = new AtomicBoolean(true);
-        replicas = new HashMap<>();
         jobCommitAcks = new ConcurrentHashMap<>();
         replicationJobsPendingAcks = new ConcurrentHashMap<>();
         shuttingDownReplicaIds = new HashSet<>();
         dataBuffer = ByteBuffer.allocate(INITIAL_BUFFER_SIZE);
+        replicationMonitor = new ReplicasEventsMonitor();
+        //add list of replicas from configurations (To be read from another source e.g. Zookeeper)
+        Set<Replica> replicaNodes = replicationProperties.getReplicationStrategy().getRemoteReplicas(nodeId);
 
         //Used as async listeners from replicas
         replicationListenerThreads = Executors.newCachedThreadPool();
         replicationJobsProcessor = new ReplicationJobsProccessor();
-        replicationMonitor = new ReplicasEventsMonitor();
 
-        Map<String, ClusterPartition[]> nodePartitions =
-                ((IPropertiesProvider) asterixAppRuntimeContextProvider.getAppContext()).getMetadataProperties()
-                        .getNodePartitions();
-        //add list of replicas from configurations (To be read from another source e.g. Zookeeper)
-        Set<Replica> replicaNodes = replicationProperties.getRemoteReplicas(nodeId);
+        Map<String, ClusterPartition[]> nodePartitions = ((IPropertiesProvider) asterixAppRuntimeContextProvider
+                .getAppContext()).getMetadataProperties().getNodePartitions();
         replica2PartitionsMap = new HashMap<>(replicaNodes.size());
         for (Replica replica : replicaNodes) {
-            replicas.put(replica.getNode().getId(), replica);
+            replicas.put(replica.getId(), replica);
             //for each remote replica, get the list of replication clients
-            Set<String> nodeReplicationClients = replicationProperties.getNodeReplicationClients(replica.getId());
+            Set<String> nodeReplicationClients = replicationProperties.getRemotePrimaryReplicasIds(replica.getId());
             //get the partitions of each client
             List<Integer> clientPartitions = new ArrayList<>();
             for (String clientId : nodeReplicationClients) {
@@ -255,7 +260,6 @@ public class ReplicationManager implements IReplicationManager {
                 getAndInitNewPage();
             }
         }
-
         currentTxnLogBuffer.append(logRecord);
     }
 
@@ -277,7 +281,12 @@ public class ReplicationManager implements IReplicationManager {
             //all of the job's files belong to a single storage partition.
             //get any of them to determine the partition from the file path.
             String jobFile = job.getJobFiles().iterator().next();
-            int jobPartitionId = PersistentLocalResourceRepository.getResourcePartition(jobFile);
+            IndexFileProperties indexFileRef = localResourceRepo.getIndexFileRef(jobFile);
+            if (!replicationStrategy.isMatch(indexFileRef.getDatasetId())) {
+                return;
+            }
+
+            int jobPartitionId = indexFileRef.getPartitionId();
 
             ByteBuffer responseBuffer = null;
             LSMIndexFileProperties asterixFileProperties = new LSMIndexFileProperties();
@@ -442,7 +451,7 @@ public class ReplicationManager implements IReplicationManager {
 
     @Override
     public boolean isReplicationEnabled() {
-        return ClusterProperties.INSTANCE.isReplicationEnabled();
+        return replicationProperties.isParticipant(nodeId);
     }
 
     @Override
@@ -822,7 +831,6 @@ public class ReplicationManager implements IReplicationManager {
                 }
             }
         }
-
         //presume replicated
         return true;
     }
@@ -908,18 +916,37 @@ public class ReplicationManager implements IReplicationManager {
     public void stop(boolean dumpState, OutputStream ouputStream) throws IOException {
         //stop replication thread afters all jobs/logs have been processed
         suspendReplication(false);
-        //send shutdown event to remote replicas
-        sendShutdownNotifiction();
-        //wait until all shutdown events come from all remote replicas
-        synchronized (shuttingDownReplicaIds) {
-            while (!shuttingDownReplicaIds.containsAll(getActiveReplicasIds())) {
-                try {
-                    shuttingDownReplicaIds.wait(1000);
-                } catch (InterruptedException e) {
-                    Thread.currentThread().interrupt();
+
+        /**
+         * If this node has any remote replicas, it needs to inform them
+         * that it is shutting down.
+         */
+        if (!replicationStrategy.getRemoteReplicas(nodeId).isEmpty()) {
+            //send shutdown event to remote replicas
+            sendShutdownNotifiction();
+        }
+
+        /**
+         * If this node has any remote primary replicas, then it needs to wait
+         * until all of them send the shutdown notification.
+         */
+        // find active remote primary replicas
+        Set<String> activeRemotePrimaryReplicas = replicationStrategy.getRemotePrimaryReplicas(nodeId).stream()
+                .map(Replica::getId).filter(getActiveReplicasIds()::contains).collect(Collectors.toSet());
+
+        if (!activeRemotePrimaryReplicas.isEmpty()) {
+            //wait until all shutdown events come from all remote primary replicas
+            synchronized (shuttingDownReplicaIds) {
+                while (!shuttingDownReplicaIds.containsAll(activeRemotePrimaryReplicas)) {
+                    try {
+                        shuttingDownReplicaIds.wait();
+                    } catch (InterruptedException e) {
+                        Thread.currentThread().interrupt();
+                    }
                 }
             }
         }
+
         LOGGER.log(Level.INFO, "Got shutdown notification from all remote replicas");
         //close replication channel
         asterixAppRuntimeContextProvider.getAppContext().getReplicationChannel().close();
@@ -996,6 +1023,9 @@ public class ReplicationManager implements IReplicationManager {
     public void requestFlushLaggingReplicaIndexes(long nonSharpCheckpointTargetLSN) throws IOException {
         long startLSN = logManager.getAppendLSN();
         Set<String> replicaIds = getActiveReplicasIds();
+        if (replicaIds.isEmpty()) {
+            return;
+        }
         ByteBuffer requestBuffer = ByteBuffer.allocate(INITIAL_BUFFER_SIZE);
         for (String replicaId : replicaIds) {
             //1. identify replica indexes with LSN less than nonSharpCheckpointTargetLSN.
@@ -1079,9 +1109,9 @@ public class ReplicationManager implements IReplicationManager {
 
     //Recovery Method
     @Override
-    public void requestReplicaFiles(String selectedReplicaId, Set<String> replicasDataToRecover,
+    public void requestReplicaFiles(String selectedReplicaId, Set<Integer> partitionsToRecover,
             Set<String> existingFiles) throws IOException {
-        ReplicaFilesRequest request = new ReplicaFilesRequest(replicasDataToRecover, existingFiles);
+        ReplicaFilesRequest request = new ReplicaFilesRequest(partitionsToRecover, existingFiles);
         dataBuffer = ReplicationProtocol.writeGetReplicaFilesRequest(dataBuffer, request);
 
         try (SocketChannel socketChannel = getReplicaSocket(selectedReplicaId)) {


Mime
View raw message