asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mb...@apache.org
Subject asterixdb git commit: Fix AsterixHyracksIntegrationUtil regressions
Date Sat, 20 May 2017 03:37:10 GMT
Repository: asterixdb
Updated Branches:
  refs/heads/master 9d63f629a -> a7f8a359a


Fix AsterixHyracksIntegrationUtil regressions

- indicate NCs are virtual on CC's ConfigManager instance
- properly fixup IO devices, after ConfigManager processes the config

Change-Id: I69ad013fcb383e671e879d6c7c3de1b79e52e070
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1764
Sonar-Qube: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
BAD: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: abdullah alamoudi <bamousaa@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/a7f8a359
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/a7f8a359
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/a7f8a359

Branch: refs/heads/master
Commit: a7f8a359afd100f6ff6c0e8c6edb33910623ac7e
Parents: 9d63f62
Author: Michael Blow <mblow@apache.org>
Authored: Fri May 19 20:24:19 2017 -0400
Committer: Michael Blow <mblow@apache.org>
Committed: Fri May 19 20:36:52 2017 -0700

----------------------------------------------------------------------
 .../common/AsterixHyracksIntegrationUtil.java   | 50 ++++++++------------
 .../common/config/PropertiesAccessor.java       |  2 +
 .../recovery/AbstractCheckpointManager.java     |  2 +-
 3 files changed, 24 insertions(+), 30 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a7f8a359/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
index 28a42f5..8d60ec6 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
@@ -19,6 +19,7 @@
 package org.apache.asterix.api.common;
 
 import static org.apache.asterix.api.common.AsterixHyracksIntegrationUtil.LoggerHolder.LOGGER;
+import static org.apache.hyracks.util.file.FileUtil.joinPath;
 
 import java.io.File;
 import java.io.IOException;
@@ -47,8 +48,10 @@ import org.apache.hyracks.api.job.JobSpecification;
 import org.apache.hyracks.control.cc.ClusterControllerService;
 import org.apache.hyracks.control.common.config.ConfigManager;
 import org.apache.hyracks.control.common.controllers.CCConfig;
+import org.apache.hyracks.control.common.controllers.ControllerConfig;
 import org.apache.hyracks.control.common.controllers.NCConfig;
 import org.apache.hyracks.control.nc.NodeControllerService;
+import org.kohsuke.args4j.CmdLineException;
 
 public class AsterixHyracksIntegrationUtil {
     static class LoggerHolder {
@@ -58,7 +61,7 @@ public class AsterixHyracksIntegrationUtil {
         }
     }
 
-    private static final String IO_DIR_KEY = "java.io.tmpdir";
+    protected static final String IO_DIR_KEY = "java.io.tmpdir";
     public static final int DEFAULT_HYRACKS_CC_CLIENT_PORT = 1098;
     public static final int DEFAULT_HYRACKS_CC_CLUSTER_PORT = 1099;
 
@@ -76,13 +79,16 @@ public class AsterixHyracksIntegrationUtil {
         final CCConfig ccConfig = createCCConfig(configManager);
         cc = new ClusterControllerService(ccConfig, ccApplication);
 
-        nodeNames = ccConfig.getConfigManager().getNodeNames();
+
+            nodeNames = ccConfig.getConfigManager().getNodeNames();
         if (deleteOldInstanceData) {
             deleteTransactionLogs();
             removeTestStorageFiles();
         }
         final List<NodeControllerService> nodeControllers = new ArrayList<>();
         for (String nodeId : nodeNames) {
+            // mark this NC as virtual in the CC's config manager, so he doesn't try to contact
NCService...
+            configManager.set(nodeId, NCConfig.Option.VIRTUAL_NC, true);
             final INCApplication ncApplication = createNCApplication();
             ConfigManager ncConfigManager = new ConfigManager();
             ncApplication.registerConfig(ncConfigManager);
@@ -113,13 +119,6 @@ public class AsterixHyracksIntegrationUtil {
         for (Thread thread : startupThreads) {
             thread.join();
         }
-        for (NodeControllerService nc : nodeControllers) {
-            for (String ioDevice : nc.getConfiguration().getIODevices()) {
-                if (!new File(ioDevice).isAbsolute()) {
-                    throw new IllegalStateException("iodevice not absolute: " + ioDevice);
-                }
-            }
-        }
         // Wait until cluster becomes active
         ClusterStateManager.INSTANCE.waitForState(ClusterState.ACTIVE);
         hcc = new HyracksConnection(cc.getConfig().getClientListenAddress(), cc.getConfig().getClientListenPort());
@@ -134,6 +133,7 @@ public class AsterixHyracksIntegrationUtil {
         ccConfig.setClusterListenPort(DEFAULT_HYRACKS_CC_CLUSTER_PORT);
         ccConfig.setResultTTL(120000L);
         ccConfig.setResultSweepThreshold(1000L);
+        configManager.set(ControllerConfig.Option.DEFAULT_DIR, joinPath(System.getProperty(IO_DIR_KEY),
"asterixdb"));
         return ccConfig;
     }
 
@@ -152,6 +152,8 @@ public class AsterixHyracksIntegrationUtil {
         ncConfig.setResultTTL(120000L);
         ncConfig.setResultSweepThreshold(1000L);
         ncConfig.setVirtualNC(true);
+        configManager.set(ControllerConfig.Option.DEFAULT_DIR,
+                joinPath(System.getProperty(IO_DIR_KEY), "asterixdb", ncName));
         return ncConfig;
     }
 
@@ -159,32 +161,22 @@ public class AsterixHyracksIntegrationUtil {
         return new NCApplication();
     }
 
-    private NCConfig fixupIODevices(NCConfig ncConfig) throws IOException, AsterixException
{
-        PropertiesAccessor accessor = PropertiesAccessor.getInstance(ncConfig.getAppConfig());
-        String tempPath = System.getProperty(IO_DIR_KEY);
-        if (tempPath.endsWith(File.separator)) {
-            tempPath = tempPath.substring(0, tempPath.length() - 1);
-        }
-        LOGGER.info("Using the temp path: " + tempPath);
-        // get initial partitions from properties
-        String[] nodeStores = accessor.getStores().get(ncConfig.getNodeId());
+    private NCConfig fixupIODevices(NCConfig ncConfig) throws IOException, AsterixException,
CmdLineException {
+        // we have to first process the config
+        ncConfig.getConfigManager().processConfig();
+
+        // get initial partitions from config
+        String[] nodeStores = ncConfig.getAppConfig().getStringArray(NCConfig.Option.IODEVICES);
         if (nodeStores == null) {
             throw new IllegalStateException("Couldn't find stores for NC: " + ncConfig.getNodeId());
         }
         String tempDirPath = System.getProperty(IO_DIR_KEY);
-        if (!tempDirPath.endsWith(File.separator)) {
-            tempDirPath += File.separator;
-        }
-        List<String> ioDevices = new ArrayList<>();
-        for (String nodeStore : nodeStores) {
+        LOGGER.info("Using the temp path: " + tempDirPath);
+        for (int i = 0; i < nodeStores.length; i++) {
             // create IO devices based on stores
-            String iodevicePath = tempDirPath + ncConfig.getNodeId() + File.separator + nodeStore;
-            File ioDeviceDir = new File(iodevicePath);
-            ioDeviceDir.mkdirs();
-            ioDevices.add(iodevicePath);
+            nodeStores[i] = joinPath(tempDirPath, ncConfig.getNodeId(), nodeStores[i]);
         }
-        ncConfig.getConfigManager().set(ncConfig.getNodeId(), NCConfig.Option.IODEVICES,
-                ioDevices.toArray(new String[0]));
+        ncConfig.getConfigManager().set(ncConfig.getNodeId(), NCConfig.Option.IODEVICES,
nodeStores);
         return ncConfig;
     }
 

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a7f8a359/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/PropertiesAccessor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/PropertiesAccessor.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/PropertiesAccessor.java
index 233daa7..1ea512a 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/PropertiesAccessor.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/PropertiesAccessor.java
@@ -134,6 +134,8 @@ public class PropertiesAccessor implements IApplicationConfig {
                 stores.put(store.getNcId(), nodeStores);
                 nodePartitionsMap.put(store.getNcId(), nodePartitions);
                 configManager.registerVirtualNode(store.getNcId());
+                // push the store info to the config manager
+                configManager.set(store.getNcId(), NCConfig.Option.IODEVICES, nodeStores);
                 // marking node as virtual, as we're not using NCServices with old-style
config
                 configManager.set(store.getNcId(), NCConfig.Option.VIRTUAL_NC, true);
             }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a7f8a359/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/recovery/AbstractCheckpointManager.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/recovery/AbstractCheckpointManager.java
b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/recovery/AbstractCheckpointManager.java
index 585e8ec..8d3e0a7 100644
--- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/recovery/AbstractCheckpointManager.java
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/recovery/AbstractCheckpointManager.java
@@ -70,7 +70,7 @@ public abstract class AbstractCheckpointManager implements ICheckpointManager
{
         checkpointDir = new File(checkpointDirPath);
         // Create the checkpoint directory if missing
         if (!checkpointDir.exists()) {
-            (new File(checkpointDirPath)).mkdir();
+            checkpointDir.mkdirs();
         }
         lsnThreshold = checkpointProperties.getLsnThreshold();
         pollFrequency = checkpointProperties.getPollFrequency();


Mime
View raw message