hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From j..@apache.org
Subject hadoop git commit: HDFS-9448. Enable valgrind for libhdfspp unit tests. Contributed by Bob Hansen.
Date Tue, 15 Dec 2015 23:38:07 GMT
Repository: hadoop
Updated Branches:
  refs/heads/HDFS-8707 01087b0df -> 0cf5e66ab


HDFS-9448. Enable valgrind for libhdfspp unit tests.  Contributed by Bob Hansen.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0cf5e66a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0cf5e66a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0cf5e66a

Branch: refs/heads/HDFS-8707
Commit: 0cf5e66ab3a2d7540b06366af5035e2f0ab2f022
Parents: 01087b0
Author: James <jhc@apache.org>
Authored: Tue Dec 15 18:37:34 2015 -0500
Committer: James <jhc@apache.org>
Committed: Tue Dec 15 18:37:34 2015 -0500

----------------------------------------------------------------------
 dev-support/docker/Dockerfile                   |  2 +-
 .../hadoop-hdfs-native-client/pom.xml           | 14 ++++++++--
 .../src/main/native/libhdfspp/CMakeLists.txt    | 12 +++++++++
 .../main/native/libhdfspp/tests/CMakeLists.txt  | 27 +++++++++++++++-----
 .../native/libhdfspp/tests/bad_datanode_test.cc |  6 ++++-
 .../libhdfspp/tests/remote_block_reader_test.cc |  6 ++++-
 .../native/libhdfspp/tests/rpc_engine_test.cc   |  6 ++++-
 7 files changed, 60 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0cf5e66a/dev-support/docker/Dockerfile
----------------------------------------------------------------------
diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile
index e80df29..c934653 100644
--- a/dev-support/docker/Dockerfile
+++ b/dev-support/docker/Dockerfile
@@ -39,7 +39,7 @@ RUN apt-get update && apt-get install --no-install-recommends -y
\
     fuse libfuse-dev \
     libcurl4-openssl-dev \
     python python2.7 pylint \
-    openjdk-7-jdk doxygen
+    openjdk-7-jdk doxygen valgrind
 
 # Fixing the Apache commons / Maven dependency problem under Ubuntu:
 # See http://wiki.apache.org/commons/VfsProblems

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0cf5e66a/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml
index d554f1a..8ad6313 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml
@@ -33,6 +33,10 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <properties>
     <require.fuse>false</require.fuse>
     <require.libwebhdfs>false</require.libwebhdfs>
+    <require.valgrind>false</require.valgrind>
+    <native_ctest_args></native_ctest_args>
+    <native_cmake_args></native_cmake_args>
+    <native_make_args></native_make_args>
   </properties>
 
   <dependencies>
@@ -141,11 +145,13 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
                     <mkdir dir="${project.build.directory}/native"/>
                     <exec executable="cmake" dir="${project.build.directory}/native"
                           failonerror="true">
-                      <arg line="${basedir}/src/ -DGENERATED_JAVAH=${project.build.directory}/native/javah
-DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DREQUIRE_LIBWEBHDFS=${require.libwebhdfs} -DREQUIRE_FUSE=${require.fuse}
-G '${generator}'"/>
+                      <arg line="${basedir}/src/ -DGENERATED_JAVAH=${project.build.directory}/native/javah
-DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DREQUIRE_LIBWEBHDFS=${require.libwebhdfs} -DREQUIRE_FUSE=${require.fuse}
-DREQUIRE_VALGRIND=${require.valgrind} -G '${generator}'"/>
+                      <arg line="${native_cmake_args}"/>
                     </exec>
                     <exec executable="msbuild" dir="${project.build.directory}/native"
                           failonerror="true">
                       <arg line="ALL_BUILD.vcxproj /nologo /p:Configuration=RelWithDebInfo
/p:LinkIncremental=false"/>
+                      <arg line="${native_make_args}"/>
                     </exec>
                     <!-- Copy for inclusion in distribution. -->
                     <copy todir="${project.build.directory}/bin">
@@ -167,6 +173,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
                       <property name="compile_classpath" refid="maven.compile.classpath"/>
                       <property name="test_classpath" refid="maven.test.classpath"/>
                       <exec executable="ctest" failonerror="true" dir="${project.build.directory}/">
+                        <arg line="${native_ctest_args}"/>
                         <env key="CLASSPATH" value="${test_classpath}:${compile_classpath}"/>
                         <!-- Make sure libhadoop.so is on LD_LIBRARY_PATH. -->
                         <env key="LD_LIBRARY_PATH" value="${env.LD_LIBRARY_PATH}:${project.build.directory}/target/usr/local/lib:${hadoop.common.build.dir}/native/target/usr/local/lib"/>
@@ -202,9 +209,11 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
                   <target>
                     <mkdir dir="${project.build.directory}"/>
                     <exec executable="cmake" dir="${project.build.directory}" failonerror="true">
-                      <arg line="${basedir}/src/ -DGENERATED_JAVAH=${project.build.directory}/native/javah
-DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DREQUIRE_LIBWEBHDFS=${require.libwebhdfs} -DREQUIRE_FUSE=${require.fuse}"/>
+                      <arg line="${basedir}/src/ -DGENERATED_JAVAH=${project.build.directory}/native/javah
-DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DREQUIRE_LIBWEBHDFS=${require.libwebhdfs} -DREQUIRE_FUSE=${require.fuse}
-DREQUIRE_VALGRIND=${require.valgrind} "/>
+                      <arg line="${native_cmake_args}"/>
                     </exec>
                     <exec executable="make" dir="${project.build.directory}" failonerror="true">
+                      <arg line="${native_make_args}"/>
                     </exec>
                   </target>
                 </configuration>
@@ -219,6 +228,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
                     <property name="compile_classpath" refid="maven.compile.classpath"/>
                     <property name="test_classpath" refid="maven.test.classpath"/>
                     <exec executable="ctest" failonerror="true" dir="${project.build.directory}/">
+                      <arg line="${native_ctest_args}"/>
                       <env key="CLASSPATH" value="${test_classpath}:${compile_classpath}"/>
                       <!-- Make sure libhadoop.so is on LD_LIBRARY_PATH. -->
                       <env key="LD_LIBRARY_PATH" value="${env.LD_LIBRARY_PATH}:${project.build.directory}/target/usr/local/lib:${hadoop.common.build.dir}/native/target/usr/local/lib"/>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0cf5e66a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
index 128527a..6338d5d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
@@ -19,14 +19,26 @@
 project (libhdfspp)
 
 enable_testing()
+include (CTest)
 
 find_package(Doxygen)
 find_package(OpenSSL REQUIRED)
 find_package(Protobuf REQUIRED)
 find_package(Threads)
 
+find_program(MEMORYCHECK_COMMAND valgrind HINTS ${VALGRIND_DIR} )
+set(MEMORYCHECK_COMMAND_OPTIONS "--trace-children=yes --leak-check=full --error-exitcode=1")
+message(STATUS "valgrind location: ${MEMORYCHECK_COMMAND}")
+
+if (REQUIRE_VALGRIND AND MEMORYCHECK_COMMAND MATCHES "MEMORYCHECK_COMMAND-NOTFOUND" )
+  message(FATAL_ERROR "valgrind was required but not found.  "
+                      "The path can be included via a -DVALGRIND_DIR=... flag passed to CMake.")
+endif (REQUIRE_VALGRIND AND MEMORYCHECK_COMMAND MATCHES "MEMORYCHECK_COMMAND-NOTFOUND" )
+
+
 add_definitions(-DASIO_STANDALONE -DASIO_CPP11_DATE_TIME)
 
+
 if(UNIX)
 set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -pedantic -std=c++11 -g -fPIC -fno-strict-aliasing")
 endif()

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0cf5e66a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt
b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt
index 60e71d6..6490484 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt
@@ -43,34 +43,47 @@ protobuf_generate_cpp(PROTO_TEST_SRCS PROTO_TEST_HDRS
   ${PROTO_HADOOP_TEST_DIR}/test_rpc_service.proto
 )
 
+# Shamelessly stolen from
+#    http://stackoverflow.com/questions/9303711/how-do-i-make-ctest-run-a-program-with-valgrind-without-dart
+function(add_memcheck_test name binary)
+  add_test(${name} ${binary} ${ARGN})
+
+  if(NOT MEMORYCHECK_COMMAND MATCHES "MEMORYCHECK_COMMAND-NOTFOUND" AND NOT SKIP_VALGRIND)
+    set(memcheck_command "${MEMORYCHECK_COMMAND} ${MEMORYCHECK_COMMAND_OPTIONS}")
+    separate_arguments(memcheck_command)
+    add_test(memcheck_${name} ${memcheck_command} ./${binary} ${ARGN})
+  endif()
+endfunction(add_memcheck_test)
+
+
 add_executable(remote_block_reader_test remote_block_reader_test.cc $<TARGET_OBJECTS:test_common>)
 target_link_libraries(remote_block_reader_test reader proto common connection ${PROTOBUF_LIBRARIES}
${OPENSSL_LIBRARIES} gmock_main ${CMAKE_THREAD_LIBS_INIT})
-add_test(remote_block_reader remote_block_reader_test)
+add_memcheck_test(remote_block_reader remote_block_reader_test)
 
 add_executable(sasl_digest_md5_test sasl_digest_md5_test.cc)
 target_link_libraries(sasl_digest_md5_test common ${OPENSSL_LIBRARIES} gmock_main ${CMAKE_THREAD_LIBS_INIT})
-add_test(sasl_digest_md5 sasl_digest_md5_test)
+add_memcheck_test(sasl_digest_md5 sasl_digest_md5_test)
 
 add_executable(retry_policy_test retry_policy_test.cc)
 target_link_libraries(retry_policy_test common gmock_main ${CMAKE_THREAD_LIBS_INIT})
-add_test(retry_policy retry_policy_test)
+add_memcheck_test(retry_policy retry_policy_test)
 
 include_directories(${CMAKE_CURRENT_BINARY_DIR})
 add_executable(rpc_engine_test rpc_engine_test.cc ${PROTO_TEST_SRCS} ${PROTO_TEST_HDRS} $<TARGET_OBJECTS:test_common>)
 target_link_libraries(rpc_engine_test rpc proto common ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES}
gmock_main ${CMAKE_THREAD_LIBS_INIT})
-add_test(rpc_engine rpc_engine_test)
+add_memcheck_test(rpc_engine rpc_engine_test)
 
 add_executable(bad_datanode_test bad_datanode_test.cc)
 target_link_libraries(bad_datanode_test rpc reader proto fs bindings_c rpc proto common reader
connection ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} gmock_main ${CMAKE_THREAD_LIBS_INIT})
-add_test(bad_datanode bad_datanode_test)
+add_memcheck_test(bad_datanode bad_datanode_test)
 
 add_executable(node_exclusion_test node_exclusion_test.cc)
 target_link_libraries(node_exclusion_test fs gmock_main common ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES}
${CMAKE_THREAD_LIBS_INIT})
-add_test(node_exclusion node_exclusion_test)
+add_memcheck_test(node_exclusion node_exclusion_test)
 
 add_executable(configuration_test configuration_test.cc)
 target_link_libraries(configuration_test common gmock_main ${CMAKE_THREAD_LIBS_INIT})
-add_test(configuration configuration_test)
+add_memcheck_test(configuration configuration_test)
 
 add_executable(hdfs_configuration_test hdfs_configuration_test.cc)
 target_link_libraries(hdfs_configuration_test common gmock_main ${CMAKE_THREAD_LIBS_INIT})

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0cf5e66a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/bad_datanode_test.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/bad_datanode_test.cc
b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/bad_datanode_test.cc
index 0f69195..771d85f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/bad_datanode_test.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/bad_datanode_test.cc
@@ -242,5 +242,9 @@ int main(int argc, char *argv[]) {
   // The following line must be executed to initialize Google Mock
   // (and Google Test) before running the tests.
   ::testing::InitGoogleMock(&argc, argv);
-  return RUN_ALL_TESTS();
+  int exit_code = RUN_ALL_TESTS();
+
+  // Clean up static data and prevent valgrind memory leaks
+  google::protobuf::ShutdownProtobufLibrary();
+  return exit_code;
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0cf5e66a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/remote_block_reader_test.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/remote_block_reader_test.cc
b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/remote_block_reader_test.cc
index 6f3122e..7d926ba 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/remote_block_reader_test.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/remote_block_reader_test.cc
@@ -410,5 +410,9 @@ int main(int argc, char *argv[]) {
   // The following line must be executed to initialize Google Mock
   // (and Google Test) before running the tests.
   ::testing::InitGoogleMock(&argc, argv);
-  return RUN_ALL_TESTS();
+  int exit_code = RUN_ALL_TESTS();
+
+  // Clean up static data and prevent valgrind memory leaks
+  google::protobuf::ShutdownProtobufLibrary();
+  return exit_code;
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0cf5e66a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc
b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc
index 063bcee..71e3978 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc
@@ -395,5 +395,9 @@ int main(int argc, char *argv[]) {
   // The following line must be executed to initialize Google Mock
   // (and Google Test) before running the tests.
   ::testing::InitGoogleMock(&argc, argv);
-  return RUN_ALL_TESTS();
+  int exit_code = RUN_ALL_TESTS();
+
+  // Clean up static data and prevent valgrind memory leaks
+  google::protobuf::ShutdownProtobufLibrary();
+  return exit_code;
 }


Mime
View raw message