cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aweisb...@apache.org
Subject [1/6] cassandra git commit: Set test.runners based on cores and memory size
Date Tue, 11 Jul 2017 18:29:24 GMT
Repository: cassandra
Updated Branches:
  refs/heads/cassandra-3.0 97fb4d102 -> 2400d07bf
  refs/heads/cassandra-3.11 48ffad89c -> e406700cf
  refs/heads/trunk ebd0aaefe -> 19914dc1d


Set test.runners based on cores and memory size

patch by Jay Zhuang; reviewed by Ariel Weisberg for CASSANDRA-13078


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/2400d07b
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/2400d07b
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/2400d07b

Branch: refs/heads/cassandra-3.0
Commit: 2400d07bf52829b25a7c03c19b22ddd3301899be
Parents: 97fb4d1
Author: Jay Zhuang <jay.zhuang@yahoo.com>
Authored: Thu Jul 6 18:01:38 2017 -0700
Committer: Ariel Weisberg <aweisberg@apple.com>
Committed: Tue Jul 11 14:08:14 2017 -0400

----------------------------------------------------------------------
 CHANGES.txt |  1 +
 build.xml   | 58 +++++++++++++++++++++++++++++++++++++++++++++++++++-----
 circle.yml  |  2 +-
 3 files changed, 55 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/2400d07b/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 8095e25..ce2324d 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
 3.0.15
+ * Set test.runners based on cores and memory size (CASSANDRA-13078)
  * Allow different NUMACTL_ARGS to be passed in (CASSANDRA-13557)
  * Allow native function calls in CQLSSTableWriter (CASSANDRA-12606)
  * Fix secondary index queries on COMPACT tables (CASSANDRA-13627)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/2400d07b/build.xml
----------------------------------------------------------------------
diff --git a/build.xml b/build.xml
index 5eb6572..53c2cea 100644
--- a/build.xml
+++ b/build.xml
@@ -59,7 +59,6 @@
     <property name="test.classlistfile" value="testlist.txt"/>
     <property name="benchmark.name" value=""/>
     <property name="test.methods" value=""/>
-    <property name="test.runners" value="1"/>
     <property name="test.unit.src" value="${test.dir}/unit"/>
     <property name="test.long.src" value="${test.dir}/long"/>
     <property name="test.burn.src" value="${test.dir}/burn"/>
@@ -1625,12 +1624,25 @@
   <scriptdef name="testparallelhelper" language="javascript">
     <attribute name="testdelegate"/>
     <![CDATA[
-        var Integer = java.lang.Integer;
         sep = project.getProperty("path.separator");
         all = project.getProperty("all-test-classes").split(sep);
-        dir = project.getProperty("test.unit.src");
+        runners = project.getProperty("test.runners")
+        cores = project.getProperty("cores.count")
+        mem = project.getProperty("mem.size")
 
-        numRunners = parseInt(project.getProperty("test.runners"));
+        numRunners = 1
+        if (runners != null) // there's test.runners override
+            numRunners = parseInt(runners) || 1;
+        else if (cores != null && mem != null) // only if cores and memory size is
set
+            numRunners = Math.min(Math.floor(Math.sqrt(parseInt(cores) || 1)),
+                                  Math.floor((parseInt(mem) || 1)/(4*1024*1024*1024)));
+
+        if (numRunners < 1)
+            numRunners = 1
+
+        var echo = project.createTask("echo");
+        echo.setMessage("Number of test runners: " + numRunners);
+        echo.perform();
 
         var p = project.createTask('parallel');
         p.setThreadCount(numRunners);
@@ -1652,7 +1664,42 @@
     ]]>
   </scriptdef>
 
-  <target name="test" depends="build-test" description="Parallel Test Runner">
+  <target name="get-cores">
+    <property environment="env"/>
+    <!-- support for Windows -->
+    <condition property="cores.count" value="${env.NUMBER_OF_PROCESSORS}">
+      <os family="windows" />
+    </condition>
+    <!-- support for Linux and Solaris (package SUNWgnu-coreutils is required) -->
+    <exec executable="nproc" outputproperty="cores.count" os="Linux,SunOS,Solaris" failifexecutionfails="false">
+      <arg value="--all"/>
+    </exec>
+    <!-- support for Mac OS X -->
+    <exec executable="sysctl" outputproperty="cores.count" os="Mac,Mac OS X,Darwin" failifexecutionfails="false">
+      <arg value="-n"/>
+      <arg value="hw.ncpu"/>
+    </exec>
+    <echo message="Number of cores: ${cores.count}"/>
+  </target>
+
+  <target name="get-mem">
+    <condition property="mem.size" value="unknown">
+      <os family="windows" />
+    </condition>
+    <!-- support for Linux and Solaris (package SUNWgnu-coreutils is required) -->
+    <exec executable="bash" outputproperty="mem.size" os="Linux,SunOS,Solaris" failifexecutionfails="false">
+      <arg value="-c"/>
+      <arg value="free -b | grep Mem: | awk '{print $2}'"/>
+    </exec>
+    <!-- support for Mac OS X -->
+    <exec executable="sysctl" outputproperty="mem.size" os="Mac,Mac OS X,Darwin" failifexecutionfails="false">
+      <arg value="-n"/>
+      <arg value="hw.memsize"/>
+    </exec>
+    <echo message="Mem size : ${mem.size}"/>
+  </target>
+
+  <target name="test" depends="build-test,get-cores,get-mem" description="Parallel Test
Runner">
     <path id="all-test-classes-path">
       <fileset dir="${test.unit.src}" includes="**/${test.name}.java" />
     </path>
@@ -1933,4 +1980,5 @@
             file="${build.dir}/${final.name}-javadoc.jar"
             classifier="javadoc"/>
   </target>
+
 </project>

http://git-wip-us.apache.org/repos/asf/cassandra/blob/2400d07b/circle.yml
----------------------------------------------------------------------
diff --git a/circle.yml b/circle.yml
index 9d31277..f4801b7 100644
--- a/circle.yml
+++ b/circle.yml
@@ -7,7 +7,7 @@ test:
     - sudo apt-get update; sudo apt-get install wamerican:
         parallel: true
   override:
-    - case $CIRCLE_NODE_INDEX in 0) ant eclipse-warnings; ant test ;; 1) ant long-test ;;
2) ant test-compression ;; 3) ant stress-test ;;esac:
+    - case $CIRCLE_NODE_INDEX in 0) ant eclipse-warnings; ant test -Dtest.runners=1;; 1)
ant long-test ;; 2) ant test-compression ;; 3) ant stress-test ;;esac:
         parallel: true
 
   post:


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org


Mime
View raw message