nifi-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sza...@apache.org
Subject [nifi-minifi-cpp] 03/06: MINIFICPP-1580 Fix Tensorflow extension build
Date Wed, 23 Jun 2021 16:55:51 GMT
This is an automated email from the ASF dual-hosted git repository.

szaszm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git

commit e6ecb2e182008e3d6f7bdebc591409fafd2cd6c9
Author: Gabor Gyimesi <gamezbird@gmail.com>
AuthorDate: Wed Jun 23 18:14:24 2021 +0200

    MINIFICPP-1580 Fix Tensorflow extension build
    
    Closes #1096
    
    Signed-off-by: Marton Szasz <szaszm@apache.org>
---
 cmake/FindTensorFlow.cmake                         | 55 ++++++++++++++++------
 extensions/tensorflow/CMakeLists.txt               |  3 +-
 extensions/tensorflow/TFApplyGraph.cpp             | 11 +++--
 extensions/tensorflow/TFApplyGraph.h               |  2 +-
 extensions/tensorflow/TFConvertImageToTensor.cpp   |  6 +--
 extensions/tensorflow/TFConvertImageToTensor.h     |  2 +-
 extensions/tensorflow/TFExtractTopLabels.cpp       | 11 +++--
 extensions/tensorflow/TFExtractTopLabels.h         |  2 +-
 libminifi/test/tensorflow-tests/CMakeLists.txt     |  2 +-
 .../test/tensorflow-tests/TensorFlowTests.cpp      | 10 ++--
 10 files changed, 67 insertions(+), 37 deletions(-)

diff --git a/cmake/FindTensorFlow.cmake b/cmake/FindTensorFlow.cmake
index 732e3ae..2996048 100644
--- a/cmake/FindTensorFlow.cmake
+++ b/cmake/FindTensorFlow.cmake
@@ -18,8 +18,20 @@
 include(FindPackageHandleStandardArgs)
 unset(TENSORFLOW_FOUND)
 
-if (TENSORFLOW_PATH)
-  message("-- Checking for TensorFlow in provided TENSORFLOW_PATH: ${TENSORFLOW_PATH}")
+if (TENSORFLOW_INCLUDE_PATH)
+  message("-- Checking for TensorFlow includes in provided TENSORFLOW_INCLUDE_PATH: ${TENSORFLOW_INCLUDE_PATH}")
+endif()
+
+if (TENSORFLOW_LIB_PATH)
+  message("-- Checking for TensorFlow libs in provided TENSORFLOW_LIB_PATH: ${TENSORFLOW_LIB_PATH}")
+endif()
+
+if (GOOGLE_PROTOBUF_INCLUDE_PATH)
+  message("-- Checking for Google Protobuf includes in provided GOOGLE_PROTOBUF_INCLUDE_PATH:
${GOOGLE_PROTOBUF_INCLUDE_PATH}")
+endif()
+
+if (GOOGLE_PROTOBUF_LIB_PATH)
+  message("-- Checking for Google Protobuf libs in provided GOOGLE_PROTOBUF_LIB_PATH: ${GOOGLE_PROTOBUF_LIB_PATH}")
 endif()
 
 find_path(TENSORFLOW_INCLUDE_DIR
@@ -28,34 +40,46 @@ find_path(TENSORFLOW_INCLUDE_DIR
           tensorflow/cc
           third_party
           HINTS
-          ${TENSORFLOW_PATH}
+          ${TENSORFLOW_INCLUDE_PATH}
           /usr/include/tensorflow
           /usr/local/include/google/tensorflow
           /usr/local/include/tensorflow
+          /usr/local/include/tensorflow/bazel-bin/tensorflow/include
           /usr/include/google/tensorflow)
 
 find_library(TENSORFLOW_CC_LIBRARY NAMES tensorflow_cc
              HINTS
-             ${TENSORFLOW_PATH}
-             ${TENSORFLOW_PATH}/bazel-bin/tensorflow
+             ${TENSORFLOW_LIB_PATH}
+             ${TENSORFLOW_INCLUDE_PATH}/bazel-bin/tensorflow
              /usr/lib
              /usr/local/lib
              /usr/local/lib/tensorflow_cc)
 
-find_library(TENSORFLOW_FRAMEWORK_LIBRARY NAMES tensorflow_framework
+find_path(GOOGLE_PROTOBUF_INCLUDE NAMES google/protobuf
+          HINTS
+          ${GOOGLE_PROTOBUF_INCLUDE_PATH}
+          ${TENSORFLOW_INCLUDE_PATH}/src
+          /usr/include/tensorflow/src
+          /usr/local/include/google/tensorflow/src
+          /usr/local/include/tensorflow/src
+          /usr/local/include/tensorflow/bazel-bin/tensorflow/include/src
+          /usr/include/google/tensorflow/src)
+
+find_library(GOOGLE_PROTOBUF_LIBRARY NAMES protobuf
              HINTS
-             ${TENSORFLOW_PATH}
-             ${TENSORFLOW_PATH}/bazel-bin/tensorflow
+             ${GOOGLE_PROTOBUF_LIB_PATH}
              /usr/lib
              /usr/local/lib
-             /usr/local/lib/tensorflow_cc)
+             /usr/lib/x86_64-linux-gnu)
 
-find_package_handle_standard_args(TENSORFLOW DEFAULT_MSG TENSORFLOW_INCLUDE_DIR TENSORFLOW_CC_LIBRARY
TENSORFLOW_FRAMEWORK_LIBRARY)
+find_package_handle_standard_args(TENSORFLOW DEFAULT_MSG TENSORFLOW_INCLUDE_DIR TENSORFLOW_CC_LIBRARY
GOOGLE_PROTOBUF_INCLUDE GOOGLE_PROTOBUF_LIBRARY)
 
 if(TENSORFLOW_FOUND)
     message("-- Found TensorFlow includes: ${TENSORFLOW_INCLUDE_DIR}")
-    message("-- Found TensorFlow libraries: ${TENSORFLOW_CC_LIBRARY} ${TENSORFLOW_FRAMEWORK_LIBRARY}")
-    set(TENSORFLOW_LIBRARIES ${TENSORFLOW_CC_LIBRARY} ${TENSORFLOW_FRAMEWORK_LIBRARY})
+    message("-- Found TensorFlow libs: ${TENSORFLOW_CC_LIBRARY}")
+    message("-- Found Google Protobuf includes: ${GOOGLE_PROTOBUF_INCLUDE}")
+    message("-- Found Google Protobuf libs: ${GOOGLE_PROTOBUF_LIBRARY}")
+    set(TENSORFLOW_LIBRARIES ${TENSORFLOW_CC_LIBRARY} ${GOOGLE_PROTOBUF_LIBRARY})
     set(TENSORFLOW_INCLUDE_DIRS
         ${TENSORFLOW_INCLUDE_DIR}
         ${TENSORFLOW_INCLUDE_DIR}/bazel-genfiles
@@ -63,9 +87,10 @@ if(TENSORFLOW_FOUND)
         ${TENSORFLOW_INCLUDE_DIR}/tensorflow/contrib/makefile/downloads/eigen
         ${TENSORFLOW_INCLUDE_DIR}/tensorflow/contrib/makefile/downloads/gemmlowp
         ${TENSORFLOW_INCLUDE_DIR}/tensorflow/contrib/makefile/downloads/nsync/public
-        ${TENSORFLOW_INCLUDE_DIR}/tensorflow/contrib/makefile/gen/protobuf-host/include)
+        ${TENSORFLOW_INCLUDE_DIR}/tensorflow/contrib/makefile/gen/protobuf-host/include
+        ${GOOGLE_PROTOBUF_INCLUDE})
 else()
-  message(FATAL_ERROR "TensorFlow was not found. Check or set TENSORFLOW_PATH to TensorFlow
build, Install libtensorflow_cc.so and headers into the system, or disable the TensorFlow
extension.")
+  message(FATAL_ERROR "TensorFlow or Google Protobuf dependency was not found. Check or set
TENSORFLOW_INCLUDE_PATH, TENSORFLOW_LIB_PATH, GOOGLE_PROTOBUF_INCLUDE, GOOGLE_PROTOBUF_LIBRARY
to TensorFlow build, Install libtensorflow_cc.so, libprotobuf.so and headers into the system,
or disable the TensorFlow extension.")
 endif()
 
-mark_as_advanced(TENSORFLOW_INCLUDE_DIR TENSORFLOW_CC_LIBRARY TENSORFLOW_FRAMEWORK_LIBRARY)
+mark_as_advanced(TENSORFLOW_INCLUDE_DIR TENSORFLOW_CC_LIBRARY GOOGLE_PROTOBUF_INCLUDE GOOGLE_PROTOBUF_LIBRARY)
diff --git a/extensions/tensorflow/CMakeLists.txt b/extensions/tensorflow/CMakeLists.txt
index 3c792ee..b6f74c2 100644
--- a/extensions/tensorflow/CMakeLists.txt
+++ b/extensions/tensorflow/CMakeLists.txt
@@ -26,13 +26,12 @@ find_package(TensorFlow REQUIRED)
 
 message("-- Found TensorFlow: ${TENSORFLOW_INCLUDE_DIRS}")
 
-include_directories(${TENSORFLOW_INCLUDE_DIRS})
-
 file(GLOB SOURCES  "*.cpp")
 
 add_library(minifi-tensorflow-extensions STATIC ${SOURCES})
 set_property(TARGET minifi-tensorflow-extensions PROPERTY POSITION_INDEPENDENT_CODE ON)
 
+target_include_directories(minifi-tensorflow-extensions SYSTEM PRIVATE ${TENSORFLOW_INCLUDE_DIRS})
 target_link_libraries(minifi-tensorflow-extensions ${LIBMINIFI} Threads::Threads)
 target_link_libraries(minifi-tensorflow-extensions ${TENSORFLOW_LIBRARIES})
 
diff --git a/extensions/tensorflow/TFApplyGraph.cpp b/extensions/tensorflow/TFApplyGraph.cpp
index 14d349e..b14c2e3 100644
--- a/extensions/tensorflow/TFApplyGraph.cpp
+++ b/extensions/tensorflow/TFApplyGraph.cpp
@@ -65,7 +65,7 @@ void TFApplyGraph::initialize() {
   setSupportedRelationships(std::move(relationships));
 }
 
-void TFApplyGraph::onSchedule(core::ProcessContext *context, core::ProcessSessionFactory
*sessionFactory) {
+void TFApplyGraph::onSchedule(core::ProcessContext *context, core::ProcessSessionFactory*
/*sessionFactory*/) {
   context->getProperty(InputNode.getName(), input_node_);
 
   if (input_node_.empty()) {
@@ -79,8 +79,8 @@ void TFApplyGraph::onSchedule(core::ProcessContext *context, core::ProcessSessio
   }
 }
 
-void TFApplyGraph::onTrigger(const std::shared_ptr<core::ProcessContext> &context,
-                             const std::shared_ptr<core::ProcessSession> &session)
{
+void TFApplyGraph::onTrigger(const std::shared_ptr<core::ProcessContext>& /*context*/,
+                             const std::shared_ptr<core::ProcessSession>& session)
{
   auto flow_file = session->get();
 
   if (!flow_file) {
@@ -152,7 +152,10 @@ void TFApplyGraph::onTrigger(const std::shared_ptr<core::ProcessContext>
&contex
     TensorReadCallback tensor_cb(input_tensor_proto);
     session->read(flow_file, &tensor_cb);
     tensorflow::Tensor input;
-    input.FromProto(*input_tensor_proto);
+    if (!input.FromProto(*input_tensor_proto)) {
+      // failure deliberately ignored at this time
+      // added to avoid warn_unused_result build errors
+    }
     std::vector<tensorflow::Tensor> outputs;
     auto status = ctx->tf_session->Run({{input_node_, input}}, {output_node_}, {},
&outputs);
 
diff --git a/extensions/tensorflow/TFApplyGraph.h b/extensions/tensorflow/TFApplyGraph.h
index 3f26872..e00d528 100644
--- a/extensions/tensorflow/TFApplyGraph.h
+++ b/extensions/tensorflow/TFApplyGraph.h
@@ -47,7 +47,7 @@ class TFApplyGraph : public core::Processor {
 
   void initialize() override;
   void onSchedule(core::ProcessContext *context, core::ProcessSessionFactory *sessionFactory)
override;
-  void onTrigger(core::ProcessContext *context, core::ProcessSession *session) override {
+  void onTrigger(core::ProcessContext* /*context*/, core::ProcessSession* /*session*/) override
{
     logger_->log_error("onTrigger invocation with raw pointers is not implemented");
   }
   void onTrigger(const std::shared_ptr<core::ProcessContext> &context,
diff --git a/extensions/tensorflow/TFConvertImageToTensor.cpp b/extensions/tensorflow/TFConvertImageToTensor.cpp
index aea09f8..efa4716 100644
--- a/extensions/tensorflow/TFConvertImageToTensor.cpp
+++ b/extensions/tensorflow/TFConvertImageToTensor.cpp
@@ -115,7 +115,7 @@ void TFConvertImageToTensor::initialize() {
   setSupportedRelationships(std::move(relationships));
 }
 
-void TFConvertImageToTensor::onSchedule(core::ProcessContext *context, core::ProcessSessionFactory
*sessionFactory) {
+void TFConvertImageToTensor::onSchedule(core::ProcessContext* context, core::ProcessSessionFactory*
/*sessionFactory*/) {
   context->getProperty(ImageFormat.getName(), input_format_);
 
   if (input_format_.empty()) {
@@ -189,8 +189,8 @@ void TFConvertImageToTensor::onSchedule(core::ProcessContext *context,
core::Pro
   }
 }
 
-void TFConvertImageToTensor::onTrigger(const std::shared_ptr<core::ProcessContext>
&context,
-                                       const std::shared_ptr<core::ProcessSession>
&session) {
+void TFConvertImageToTensor::onTrigger(const std::shared_ptr<core::ProcessContext>&
/*context*/,
+                                       const std::shared_ptr<core::ProcessSession>&
session) {
   auto flow_file = session->get();
 
   if (!flow_file) {
diff --git a/extensions/tensorflow/TFConvertImageToTensor.h b/extensions/tensorflow/TFConvertImageToTensor.h
index da3a11c..93c5b38 100644
--- a/extensions/tensorflow/TFConvertImageToTensor.h
+++ b/extensions/tensorflow/TFConvertImageToTensor.h
@@ -54,7 +54,7 @@ class TFConvertImageToTensor : public core::Processor {
 
   void initialize() override;
   void onSchedule(core::ProcessContext *context, core::ProcessSessionFactory *sessionFactory)
override;
-  void onTrigger(core::ProcessContext *context, core::ProcessSession *session) override {
+  void onTrigger(core::ProcessContext* /*context*/, core::ProcessSession* /*session*/) override
{
     logger_->log_error("onTrigger invocation with raw pointers is not implemented");
   }
   void onTrigger(const std::shared_ptr<core::ProcessContext> &context,
diff --git a/extensions/tensorflow/TFExtractTopLabels.cpp b/extensions/tensorflow/TFExtractTopLabels.cpp
index 2f44f84..cfdaa9d 100644
--- a/extensions/tensorflow/TFExtractTopLabels.cpp
+++ b/extensions/tensorflow/TFExtractTopLabels.cpp
@@ -48,10 +48,10 @@ void TFExtractTopLabels::initialize() {
   setSupportedRelationships(std::move(relationships));
 }
 
-void TFExtractTopLabels::onSchedule(core::ProcessContext *context, core::ProcessSessionFactory
*sessionFactory) {
+void TFExtractTopLabels::onSchedule(core::ProcessContext* /*context*/, core::ProcessSessionFactory*
/*sessionFactory*/) {
 }
 
-void TFExtractTopLabels::onTrigger(const std::shared_ptr<core::ProcessContext> &context,
+void TFExtractTopLabels::onTrigger(const std::shared_ptr<core::ProcessContext>&
/*context*/,
                                    const std::shared_ptr<core::ProcessSession> &session)
{
   auto flow_file = session->get();
 
@@ -88,7 +88,10 @@ void TFExtractTopLabels::onTrigger(const std::shared_ptr<core::ProcessContext>
&
     session->read(flow_file, &tensor_cb);
 
     tensorflow::Tensor input;
-    input.FromProto(*input_tensor_proto);
+    if (!input.FromProto(*input_tensor_proto)) {
+      // failure deliberately ignored at this time
+      // added to avoid warn_unused_result build errors
+    }
     auto input_flat = input.flat<float>();
 
     std::vector<std::pair<uint64_t, float>> scores;
@@ -102,7 +105,7 @@ void TFExtractTopLabels::onTrigger(const std::shared_ptr<core::ProcessContext>
&
       return a.second > b.second;
     });
 
-    for (int i = 0; i < 5 && i < scores.size(); i++) {
+    for (std::size_t i = 0; i < 5 && i < scores.size(); i++) {
       if (!labels || scores[i].first > labels->size()) {
         logger_->log_error("Label index is out of range (are the correct labels loaded?);
routing to retry...");
         session->transfer(flow_file, Retry);
diff --git a/extensions/tensorflow/TFExtractTopLabels.h b/extensions/tensorflow/TFExtractTopLabels.h
index a1b2211..03ca8ca 100644
--- a/extensions/tensorflow/TFExtractTopLabels.h
+++ b/extensions/tensorflow/TFExtractTopLabels.h
@@ -44,7 +44,7 @@ class TFExtractTopLabels : public core::Processor {
 
   void initialize() override;
   void onSchedule(core::ProcessContext *context, core::ProcessSessionFactory *sessionFactory)
override;
-  void onTrigger(core::ProcessContext *context, core::ProcessSession *session) override {
+  void onTrigger(core::ProcessContext* /*context*/, core::ProcessSession* /*session*/) override
{
     logger_->log_error("onTrigger invocation with raw pointers is not implemented");
   }
   void onTrigger(const std::shared_ptr<core::ProcessContext> &context,
diff --git a/libminifi/test/tensorflow-tests/CMakeLists.txt b/libminifi/test/tensorflow-tests/CMakeLists.txt
index c7e68eb..97b45ec 100644
--- a/libminifi/test/tensorflow-tests/CMakeLists.txt
+++ b/libminifi/test/tensorflow-tests/CMakeLists.txt
@@ -27,7 +27,7 @@ FOREACH(testfile ${TENSORFLOW_INTEGRATION_TESTS})
 	add_executable("${testfilename}" "${testfile}" "${TEST_DIR}/TestBase.cpp")
 	target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/tensorflow")
 	target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/standard-processors")
-	target_include_directories(${testfilename} PRIVATE BEFORE ${TENSORFLOW_INCLUDE_DIRS})
+	target_include_directories(${testfilename} SYSTEM PRIVATE BEFORE ${TENSORFLOW_INCLUDE_DIRS})
 	createTests("${testfilename}")
 	target_wholearchive_library(${testfilename} minifi-tensorflow-extensions)
 	target_wholearchive_library(${testfilename} minifi-standard-processors)
diff --git a/libminifi/test/tensorflow-tests/TensorFlowTests.cpp b/libminifi/test/tensorflow-tests/TensorFlowTests.cpp
index d4dc27c..6959cfb 100644
--- a/libminifi/test/tensorflow-tests/TensorFlowTests.cpp
+++ b/libminifi/test/tensorflow-tests/TensorFlowTests.cpp
@@ -120,13 +120,13 @@ TEST_CASE("TensorFlow: Apply Graph", "[tfApplyGraph]") { // NOLINT
     tensorflow::GraphDef graph;
 
     // Write test TensorFlow graph
-    root.ToGraphDef(&graph);
+    REQUIRE(root.ToGraphDef(&graph).ok());
     std::ofstream in_file_stream(in_graph_file);
     graph.SerializeToOstream(&in_file_stream);
   }
 
   // Read test TensorFlow graph into TFApplyGraph
-  plan->runNextProcessor([&get_file, &in_graph_file, &plan](const std::shared_ptr<core::ProcessContext>
context,
+  plan->runNextProcessor([&in_graph_file](const std::shared_ptr<core::ProcessContext>
/*context*/,
                                                             const std::shared_ptr<core::ProcessSession>
session) {
     // Intercept the call so that we can add an attr (won't be required when we have UpdateAttribute
processor)
     auto flow_file = session->create();
@@ -163,7 +163,7 @@ TEST_CASE("TensorFlow: Apply Graph", "[tfApplyGraph]") { // NOLINT
     tensorflow::TensorProto tensor_proto;
     tensor_proto.ParseFromIstream(&out_file_stream);
     tensorflow::Tensor tensor;
-    tensor.FromProto(tensor_proto);
+    REQUIRE(tensor.FromProto(tensor_proto));
 
     // Verify output tensor
     float tensor_val = tensor.flat<float>().data()[0];
@@ -286,7 +286,7 @@ TEST_CASE("TensorFlow: ConvertImageToTensor", "[tfConvertImageToTensor]")
{ // N
     tensorflow::TensorProto tensor_proto;
     tensor_proto.ParseFromIstream(&out_file_stream);
     tensorflow::Tensor tensor;
-    tensor.FromProto(tensor_proto);
+    REQUIRE(tensor.FromProto(tensor_proto));
 
     // Verify output tensor
     auto shape = tensor.shape();
@@ -358,7 +358,7 @@ TEST_CASE("TensorFlow: Extract Top Labels", "[tfExtractTopLabels]") {
// NOLINT
   }
 
   // Read labels
-  plan->runNextProcessor([&get_file, &in_labels_file, &plan](const std::shared_ptr<core::ProcessContext>
context,
+  plan->runNextProcessor([&in_labels_file](const std::shared_ptr<core::ProcessContext>
/*context*/,
                                                              const std::shared_ptr<core::ProcessSession>
session) {
     // Intercept the call so that we can add an attr (won't be required when we have UpdateAttribute
processor)
     auto flow_file = session->create();

Mime
View raw message