asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amo...@apache.org
Subject asterixdb git commit: Change IO directory for integration util to be a directory in target
Date Wed, 31 May 2017 16:15:46 GMT
Repository: asterixdb
Updated Branches:
  refs/heads/master ffd6e4ac5 -> 146417247


Change IO directory for integration util to be a directory in target

Change-Id: I3dfb74dd4228725fb624eb1d21f621b7855f3d37
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1789
Sonar-Qube: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
BAD: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mblow@apache.org>
Reviewed-by: Till Westmann <tillw@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/14641724
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/14641724
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/14641724

Branch: refs/heads/master
Commit: 1464172470d9425a222e7a0f13d8d0f7179098bb
Parents: ffd6e4a
Author: Abdullah Alamoudi <bamousaa@gmail.com>
Authored: Tue May 30 23:40:37 2017 -0700
Committer: abdullah alamoudi <bamousaa@gmail.com>
Committed: Wed May 31 09:15:29 2017 -0700

----------------------------------------------------------------------
 .../common/AsterixHyracksIntegrationUtil.java   | 20 ++++-----
 .../bootstrap/ApplicationConfigurator.java      |  7 +--
 .../asterix/test/runtime/LangExecutionUtil.java | 46 +++++++++-----------
 .../cluster_state_1/cluster_state_1.1.regexadm  |  2 +-
 .../cluster_state_1_full.1.regexadm             |  2 +-
 .../cluster_state_1_less.1.regexadm             |  2 +-
 .../common/config/ClusterProperties.java        |  4 +-
 .../common/controllers/ControllerConfig.java    |  9 ++--
 .../control/common/utils/ConfigurationUtil.java | 28 ++++++++++++
 9 files changed, 73 insertions(+), 47 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/14641724/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
index 8d60ec6..2a0631c 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
@@ -61,7 +61,6 @@ public class AsterixHyracksIntegrationUtil {
         }
     }
 
-    protected static final String IO_DIR_KEY = "java.io.tmpdir";
     public static final int DEFAULT_HYRACKS_CC_CLIENT_PORT = 1098;
     public static final int DEFAULT_HYRACKS_CC_CLUSTER_PORT = 1099;
 
@@ -79,8 +78,7 @@ public class AsterixHyracksIntegrationUtil {
         final CCConfig ccConfig = createCCConfig(configManager);
         cc = new ClusterControllerService(ccConfig, ccApplication);
 
-
-            nodeNames = ccConfig.getConfigManager().getNodeNames();
+        nodeNames = ccConfig.getConfigManager().getNodeNames();
         if (deleteOldInstanceData) {
             deleteTransactionLogs();
             removeTestStorageFiles();
@@ -133,7 +131,7 @@ public class AsterixHyracksIntegrationUtil {
         ccConfig.setClusterListenPort(DEFAULT_HYRACKS_CC_CLUSTER_PORT);
         ccConfig.setResultTTL(120000L);
         ccConfig.setResultSweepThreshold(1000L);
-        configManager.set(ControllerConfig.Option.DEFAULT_DIR, joinPath(System.getProperty(IO_DIR_KEY),
"asterixdb"));
+        configManager.set(ControllerConfig.Option.DEFAULT_DIR, joinPath(getDefaultStoragePath(),
"asterixdb"));
         return ccConfig;
     }
 
@@ -152,8 +150,7 @@ public class AsterixHyracksIntegrationUtil {
         ncConfig.setResultTTL(120000L);
         ncConfig.setResultSweepThreshold(1000L);
         ncConfig.setVirtualNC(true);
-        configManager.set(ControllerConfig.Option.DEFAULT_DIR,
-                joinPath(System.getProperty(IO_DIR_KEY), "asterixdb", ncName));
+        configManager.set(ControllerConfig.Option.DEFAULT_DIR, joinPath(getDefaultStoragePath(),
"asterixdb", ncName));
         return ncConfig;
     }
 
@@ -170,11 +167,10 @@ public class AsterixHyracksIntegrationUtil {
         if (nodeStores == null) {
             throw new IllegalStateException("Couldn't find stores for NC: " + ncConfig.getNodeId());
         }
-        String tempDirPath = System.getProperty(IO_DIR_KEY);
-        LOGGER.info("Using the temp path: " + tempDirPath);
+        LOGGER.info("Using the path: " + getDefaultStoragePath());
         for (int i = 0; i < nodeStores.length; i++) {
             // create IO devices based on stores
-            nodeStores[i] = joinPath(tempDirPath, ncConfig.getNodeId(), nodeStores[i]);
+            nodeStores[i] = joinPath(getDefaultStoragePath(), ncConfig.getNodeId(), nodeStores[i]);
         }
         ncConfig.getConfigManager().set(ncConfig.getNodeId(), NCConfig.Option.IODEVICES,
nodeStores);
         return ncConfig;
@@ -226,8 +222,12 @@ public class AsterixHyracksIntegrationUtil {
         hcc.waitForCompletion(jobId);
     }
 
+    protected String getDefaultStoragePath() {
+        return joinPath("target", "io", "dir");
+    }
+
     public void removeTestStorageFiles() {
-        File dir = new File(System.getProperty(IO_DIR_KEY));
+        File dir = new File(getDefaultStoragePath());
         for (String ncName : nodeNames) {
             File ncDir = new File(dir, ncName);
             FileUtils.deleteQuietly(ncDir);

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/14641724/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/ApplicationConfigurator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/ApplicationConfigurator.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/ApplicationConfigurator.java
index 9678e09..adb3c18 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/ApplicationConfigurator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/ApplicationConfigurator.java
@@ -28,6 +28,7 @@ import org.apache.hyracks.api.config.IConfigManager;
 import org.apache.hyracks.control.common.controllers.CCConfig;
 import org.apache.hyracks.control.common.controllers.ControllerConfig;
 import org.apache.hyracks.control.common.controllers.NCConfig;
+import org.apache.hyracks.control.common.utils.ConfigurationUtil;
 import org.apache.hyracks.util.file.FileUtil;
 
 class ApplicationConfigurator {
@@ -37,12 +38,12 @@ class ApplicationConfigurator {
     static void registerConfigOptions(IConfigManager configManager) {
         AsterixProperties.registerConfigOptions(configManager);
         ControllerConfig.Option.DEFAULT_DIR
-                .setDefaultValue(FileUtil.joinPath(System.getProperty("java.io.tmpdir"),
"asterixdb"));
+                .setDefaultValue(FileUtil.joinPath(System.getProperty(ConfigurationUtil.JAVA_IO_TMPDIR),
"asterixdb"));
         NCConfig.Option.APP_CLASS.setDefaultValue(NCApplication.class.getName());
         CCConfig.Option.APP_CLASS.setDefaultValue(CCApplication.class.getName());
         try {
-            InputStream propertyStream = ApplicationConfigurator.class.getClassLoader()
-                    .getResourceAsStream("git.properties");
+            InputStream propertyStream =
+                    ApplicationConfigurator.class.getClassLoader().getResourceAsStream("git.properties");
             if (propertyStream != null) {
                 Properties gitProperties = new Properties();
                 gitProperties.load(propertyStream);

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/14641724/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java
index 64877ea..ec8333a 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java
@@ -32,12 +32,15 @@ import java.util.Collection;
 import java.util.List;
 
 import org.apache.asterix.app.external.TestLibrarian;
+import org.apache.asterix.common.config.ClusterProperties;
 import org.apache.asterix.common.library.ILibraryManager;
 import org.apache.asterix.test.common.TestExecutor;
 import org.apache.asterix.testframework.context.TestCaseContext;
 import org.apache.commons.lang.SystemUtils;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.hyracks.api.io.IODeviceHandle;
 import org.apache.hyracks.control.common.utils.ThreadDumpHelper;
+import org.apache.hyracks.control.nc.NodeControllerService;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
@@ -48,8 +51,8 @@ import org.junit.runners.Parameterized;
 public class LangExecutionUtil {
 
     private static final String PATH_ACTUAL = "target" + File.separator + "rttest" + File.separator;
-    private static final String PATH_BASE = StringUtils.join(new String[] { "src", "test",
"resources", "runtimets" },
-            File.separator);
+    private static final String PATH_BASE =
+            StringUtils.join(new String[] { "src", "test", "resources", "runtimets" }, File.separator);
 
     private static final boolean cleanupOnStart = true;
     private static final boolean cleanupOnStop = true;
@@ -137,31 +140,22 @@ public class LangExecutionUtil {
 
     // Checks whether data files are uniformly distributed among io devices.
     private static void checkStorageFiles() throws Exception {
-        String tempDirPath = System.getProperty("java.io.tmpdir");
-        File dir = new File(tempDirPath);
-        File[] subDirs = dir.listFiles();
-        List<File> ncStores = new ArrayList<>();
-        // Finds nc stores.
-        for (File file : subDirs) {
-            if (file.getName().startsWith("asterix_nc")) {
-                ncStores.add(file);
-            }
-        }
-        // Checks that dataset files are uniformly distributed across each nc store.
-        for (File ncStore : ncStores) {
-            checkNcStore(ncStore);
+        NodeControllerService[] ncs = ExecutionTestUtil.integrationUtil.ncs;
+        // Checks that dataset files are uniformly distributed across each io device.
+        for (NodeControllerService nc : ncs) {
+            checkNcStore(nc);
         }
     }
 
     // For each NC, check whether data files are uniformly distributed among io devices.
-    private static void checkNcStore(File ncStore) throws Exception {
-        File[] ioDevices = ncStore.listFiles();
+    private static void checkNcStore(NodeControllerService nc) throws Exception {
+        List<IODeviceHandle> ioDevices = nc.getIoManager().getIODevices();
         int expectedPartitionNum = -1;
-        for (File ioDevice : ioDevices) {
-            File[] dataDirs = ioDevice.listFiles();
+        for (IODeviceHandle ioDevice : ioDevices) {
+            File[] dataDirs = ioDevice.getMount().listFiles();
             for (File dataDir : dataDirs) {
                 String dirName = dataDir.getName();
-                if (!dirName.equals("storage")) {
+                if (!dirName.equals(ClusterProperties.DEFAULT_STORAGE_DIR_NAME)) {
                     // Skips non-storage directories.
                     continue;
                 }
@@ -202,8 +196,8 @@ public class LangExecutionUtil {
         String threadDump = ThreadDumpHelper.takeDumpJSON(ManagementFactory.getThreadMXBean());
         // Currently we only do sanity check for threads used in the execution engine.
         // Later we should check if there are leaked storage threads as well.
-        if (threadDump.contains("Operator") || threadDump.contains("SuperActivity") || threadDump
-                .contains("PipelinedPartition")) {
+        if (threadDump.contains("Operator") || threadDump.contains("SuperActivity")
+                || threadDump.contains("PipelinedPartition")) {
             System.out.print(threadDump);
             throw new AssertionError("There are leaked threads in the execution engine.");
         }
@@ -219,8 +213,8 @@ public class LangExecutionUtil {
         String processId = processName.split("@")[0];
 
         // Checks whether there are leaked run files from operators.
-        Process process = Runtime.getRuntime()
-                .exec(new String[] { "bash", "-c", "lsof -p " + processId + "|grep waf|wc
-l" });
+        Process process =
+                Runtime.getRuntime().exec(new String[] { "bash", "-c", "lsof -p " + processId
+ "|grep waf|wc -l" });
         try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream())))
{
             int runFileCount = Integer.parseInt(reader.readLine().trim());
             if (runFileCount != 0) {
@@ -232,8 +226,8 @@ public class LangExecutionUtil {
     }
 
     private static void outputLeakedOpenFiles(String processId) throws IOException {
-        Process process = Runtime.getRuntime()
-                .exec(new String[] { "bash", "-c", "lsof -p " + processId + "|grep waf" });
+        Process process =
+                Runtime.getRuntime().exec(new String[] { "bash", "-c", "lsof -p " + processId
+ "|grep waf" });
         try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream())))
{
             String line;
             while ((line = reader.readLine()) != null) {

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/14641724/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
index 5ec13a4..e8f018e 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
@@ -11,7 +11,7 @@
     "compiler\.joinmemory" : 262144,
     "compiler\.parallelism" : 0,
     "compiler\.sortmemory" : 327680,
-    "default\.dir" : "/.*/asterixdb",
+    "default\.dir" : "target/io/dir/asterixdb",
     "instance\.name" : "DEFAULT_INSTANCE",
     "log\.level" : "INFO",
     "max\.wait\.active\.cluster" : 60,

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/14641724/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
index d845b2a..e099835 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
@@ -11,7 +11,7 @@
     "compiler\.joinmemory" : 262144,
     "compiler\.parallelism" : -1,
     "compiler\.sortmemory" : 327680,
-    "default\.dir" : "/.*/asterixdb",
+    "default\.dir" : "target/io/dir/asterixdb",
     "instance\.name" : "DEFAULT_INSTANCE",
     "log\.level" : "WARNING",
     "max\.wait\.active\.cluster" : 60,

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/14641724/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
index 1315220..b3fe5cc 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
@@ -11,7 +11,7 @@
     "compiler\.joinmemory" : 262144,
     "compiler\.parallelism" : 3,
     "compiler\.sortmemory" : 327680,
-    "default\.dir" : "/.*/asterixdb",
+    "default\.dir" : "target/io/dir/asterixdb",
     "instance\.name" : "DEFAULT_INSTANCE",
     "log\.level" : "WARNING",
     "max\.wait\.active\.cluster" : 60,

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/14641724/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ClusterProperties.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ClusterProperties.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ClusterProperties.java
index 1ba9471..0abb92f 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ClusterProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ClusterProperties.java
@@ -37,8 +37,8 @@ import org.apache.hyracks.api.exceptions.HyracksDataException;
 public class ClusterProperties {
 
     public static final ClusterProperties INSTANCE = new ClusterProperties();
-    private static final String CLUSTER_CONFIGURATION_FILE = "cluster.xml";
-    private static final String DEFAULT_STORAGE_DIR_NAME = "storage";
+    public static final String CLUSTER_CONFIGURATION_FILE = "cluster.xml";
+    public static final String DEFAULT_STORAGE_DIR_NAME = "storage";
     private String nodeNamePrefix = StringUtils.EMPTY;
     private Cluster cluster;
 

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/14641724/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/ControllerConfig.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/ControllerConfig.java
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/ControllerConfig.java
index a9b3f97..1745e2a 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/ControllerConfig.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/ControllerConfig.java
@@ -27,16 +27,19 @@ import org.apache.hyracks.api.config.IOptionType;
 import org.apache.hyracks.api.config.Section;
 import org.apache.hyracks.control.common.config.ConfigManager;
 import org.apache.hyracks.control.common.config.OptionTypes;
+import org.apache.hyracks.control.common.utils.ConfigurationUtil;
 import org.apache.hyracks.util.file.FileUtil;
 
 public class ControllerConfig implements Serializable {
 
+    private static final long serialVersionUID = 1L;
     public enum Option implements IOption {
         CONFIG_FILE(OptionTypes.STRING, "Specify path to master configuration file", null),
         CONFIG_FILE_URL(OptionTypes.URL, "Specify URL to master configuration file", null),
-        DEFAULT_DIR(OptionTypes.STRING, "Directory where files are written to by default",
-                FileUtil.joinPath(System.getProperty("java.io.tmpdir"), "hyracks")),
-        ;
+        DEFAULT_DIR(
+                OptionTypes.STRING,
+                "Directory where files are written to by default",
+                FileUtil.joinPath(System.getProperty(ConfigurationUtil.JAVA_IO_TMPDIR), "hyracks")),;
 
         private final IOptionType type;
         private final String description;

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/14641724/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/utils/ConfigurationUtil.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/utils/ConfigurationUtil.java
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/utils/ConfigurationUtil.java
new file mode 100644
index 0000000..d2a455f
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/utils/ConfigurationUtil.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.control.common.utils;
+
+public class ConfigurationUtil {
+    public static final String JAVA_IO_TMPDIR = "java.io.tmpdir";
+
+    private ConfigurationUtil() {
+
+    }
+
+}


Mime
View raw message