From commits-return-44816-archive-asf-public=cust-asf.ponee.io@nifi.apache.org Wed Jun 23 16:57:05 2021 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mxout1-ec2-va.apache.org (mxout1-ec2-va.apache.org [3.227.148.255]) by mx-eu-01.ponee.io (Postfix) with ESMTPS id 0F2A0180636 for ; Wed, 23 Jun 2021 18:57:05 +0200 (CEST) Received: from mail.apache.org (mailroute1-lw-us.apache.org [207.244.88.153]) by mxout1-ec2-va.apache.org (ASF Mail Server at mxout1-ec2-va.apache.org) with SMTP id 0A6364317E for ; Wed, 23 Jun 2021 16:55:58 +0000 (UTC) Received: (qmail 29749 invoked by uid 500); 23 Jun 2021 16:55:49 -0000 Mailing-List: contact commits-help@nifi.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@nifi.apache.org Delivered-To: mailing list commits@nifi.apache.org Received: (qmail 29691 invoked by uid 99); 23 Jun 2021 16:55:49 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 23 Jun 2021 16:55:49 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id 4DC2F821EC; Wed, 23 Jun 2021 16:55:49 +0000 (UTC) Date: Wed, 23 Jun 2021 16:55:51 +0000 To: "commits@nifi.apache.org" Subject: [nifi-minifi-cpp] 03/06: MINIFICPP-1580 Fix Tensorflow extension build MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit From: szaszm@apache.org In-Reply-To: <162446734879.17647.16161136135857412236@gitbox.apache.org> References: <162446734879.17647.16161136135857412236@gitbox.apache.org> X-Git-Host: gitbox.apache.org X-Git-Repo: nifi-minifi-cpp X-Git-Refname: refs/heads/main X-Git-Reftype: branch X-Git-Rev: e6ecb2e182008e3d6f7bdebc591409fafd2cd6c9 X-Git-NotificationType: diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated Message-Id: <20210623165549.4DC2F821EC@gitbox.apache.org> This is an automated email from the ASF dual-hosted git repository. szaszm pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git commit e6ecb2e182008e3d6f7bdebc591409fafd2cd6c9 Author: Gabor Gyimesi AuthorDate: Wed Jun 23 18:14:24 2021 +0200 MINIFICPP-1580 Fix Tensorflow extension build Closes #1096 Signed-off-by: Marton Szasz --- cmake/FindTensorFlow.cmake | 55 ++++++++++++++++------ extensions/tensorflow/CMakeLists.txt | 3 +- extensions/tensorflow/TFApplyGraph.cpp | 11 +++-- extensions/tensorflow/TFApplyGraph.h | 2 +- extensions/tensorflow/TFConvertImageToTensor.cpp | 6 +-- extensions/tensorflow/TFConvertImageToTensor.h | 2 +- extensions/tensorflow/TFExtractTopLabels.cpp | 11 +++-- extensions/tensorflow/TFExtractTopLabels.h | 2 +- libminifi/test/tensorflow-tests/CMakeLists.txt | 2 +- .../test/tensorflow-tests/TensorFlowTests.cpp | 10 ++-- 10 files changed, 67 insertions(+), 37 deletions(-) diff --git a/cmake/FindTensorFlow.cmake b/cmake/FindTensorFlow.cmake index 732e3ae..2996048 100644 --- a/cmake/FindTensorFlow.cmake +++ b/cmake/FindTensorFlow.cmake @@ -18,8 +18,20 @@ include(FindPackageHandleStandardArgs) unset(TENSORFLOW_FOUND) -if (TENSORFLOW_PATH) - message("-- Checking for TensorFlow in provided TENSORFLOW_PATH: ${TENSORFLOW_PATH}") +if (TENSORFLOW_INCLUDE_PATH) + message("-- Checking for TensorFlow includes in provided TENSORFLOW_INCLUDE_PATH: ${TENSORFLOW_INCLUDE_PATH}") +endif() + +if (TENSORFLOW_LIB_PATH) + message("-- Checking for TensorFlow libs in provided TENSORFLOW_LIB_PATH: ${TENSORFLOW_LIB_PATH}") +endif() + +if (GOOGLE_PROTOBUF_INCLUDE_PATH) + message("-- Checking for Google Protobuf includes in provided GOOGLE_PROTOBUF_INCLUDE_PATH: ${GOOGLE_PROTOBUF_INCLUDE_PATH}") +endif() + +if (GOOGLE_PROTOBUF_LIB_PATH) + message("-- Checking for Google Protobuf libs in provided GOOGLE_PROTOBUF_LIB_PATH: ${GOOGLE_PROTOBUF_LIB_PATH}") endif() find_path(TENSORFLOW_INCLUDE_DIR @@ -28,34 +40,46 @@ find_path(TENSORFLOW_INCLUDE_DIR tensorflow/cc third_party HINTS - ${TENSORFLOW_PATH} + ${TENSORFLOW_INCLUDE_PATH} /usr/include/tensorflow /usr/local/include/google/tensorflow /usr/local/include/tensorflow + /usr/local/include/tensorflow/bazel-bin/tensorflow/include /usr/include/google/tensorflow) find_library(TENSORFLOW_CC_LIBRARY NAMES tensorflow_cc HINTS - ${TENSORFLOW_PATH} - ${TENSORFLOW_PATH}/bazel-bin/tensorflow + ${TENSORFLOW_LIB_PATH} + ${TENSORFLOW_INCLUDE_PATH}/bazel-bin/tensorflow /usr/lib /usr/local/lib /usr/local/lib/tensorflow_cc) -find_library(TENSORFLOW_FRAMEWORK_LIBRARY NAMES tensorflow_framework +find_path(GOOGLE_PROTOBUF_INCLUDE NAMES google/protobuf + HINTS + ${GOOGLE_PROTOBUF_INCLUDE_PATH} + ${TENSORFLOW_INCLUDE_PATH}/src + /usr/include/tensorflow/src + /usr/local/include/google/tensorflow/src + /usr/local/include/tensorflow/src + /usr/local/include/tensorflow/bazel-bin/tensorflow/include/src + /usr/include/google/tensorflow/src) + +find_library(GOOGLE_PROTOBUF_LIBRARY NAMES protobuf HINTS - ${TENSORFLOW_PATH} - ${TENSORFLOW_PATH}/bazel-bin/tensorflow + ${GOOGLE_PROTOBUF_LIB_PATH} /usr/lib /usr/local/lib - /usr/local/lib/tensorflow_cc) + /usr/lib/x86_64-linux-gnu) -find_package_handle_standard_args(TENSORFLOW DEFAULT_MSG TENSORFLOW_INCLUDE_DIR TENSORFLOW_CC_LIBRARY TENSORFLOW_FRAMEWORK_LIBRARY) +find_package_handle_standard_args(TENSORFLOW DEFAULT_MSG TENSORFLOW_INCLUDE_DIR TENSORFLOW_CC_LIBRARY GOOGLE_PROTOBUF_INCLUDE GOOGLE_PROTOBUF_LIBRARY) if(TENSORFLOW_FOUND) message("-- Found TensorFlow includes: ${TENSORFLOW_INCLUDE_DIR}") - message("-- Found TensorFlow libraries: ${TENSORFLOW_CC_LIBRARY} ${TENSORFLOW_FRAMEWORK_LIBRARY}") - set(TENSORFLOW_LIBRARIES ${TENSORFLOW_CC_LIBRARY} ${TENSORFLOW_FRAMEWORK_LIBRARY}) + message("-- Found TensorFlow libs: ${TENSORFLOW_CC_LIBRARY}") + message("-- Found Google Protobuf includes: ${GOOGLE_PROTOBUF_INCLUDE}") + message("-- Found Google Protobuf libs: ${GOOGLE_PROTOBUF_LIBRARY}") + set(TENSORFLOW_LIBRARIES ${TENSORFLOW_CC_LIBRARY} ${GOOGLE_PROTOBUF_LIBRARY}) set(TENSORFLOW_INCLUDE_DIRS ${TENSORFLOW_INCLUDE_DIR} ${TENSORFLOW_INCLUDE_DIR}/bazel-genfiles @@ -63,9 +87,10 @@ if(TENSORFLOW_FOUND) ${TENSORFLOW_INCLUDE_DIR}/tensorflow/contrib/makefile/downloads/eigen ${TENSORFLOW_INCLUDE_DIR}/tensorflow/contrib/makefile/downloads/gemmlowp ${TENSORFLOW_INCLUDE_DIR}/tensorflow/contrib/makefile/downloads/nsync/public - ${TENSORFLOW_INCLUDE_DIR}/tensorflow/contrib/makefile/gen/protobuf-host/include) + ${TENSORFLOW_INCLUDE_DIR}/tensorflow/contrib/makefile/gen/protobuf-host/include + ${GOOGLE_PROTOBUF_INCLUDE}) else() - message(FATAL_ERROR "TensorFlow was not found. Check or set TENSORFLOW_PATH to TensorFlow build, Install libtensorflow_cc.so and headers into the system, or disable the TensorFlow extension.") + message(FATAL_ERROR "TensorFlow or Google Protobuf dependency was not found. Check or set TENSORFLOW_INCLUDE_PATH, TENSORFLOW_LIB_PATH, GOOGLE_PROTOBUF_INCLUDE, GOOGLE_PROTOBUF_LIBRARY to TensorFlow build, Install libtensorflow_cc.so, libprotobuf.so and headers into the system, or disable the TensorFlow extension.") endif() -mark_as_advanced(TENSORFLOW_INCLUDE_DIR TENSORFLOW_CC_LIBRARY TENSORFLOW_FRAMEWORK_LIBRARY) +mark_as_advanced(TENSORFLOW_INCLUDE_DIR TENSORFLOW_CC_LIBRARY GOOGLE_PROTOBUF_INCLUDE GOOGLE_PROTOBUF_LIBRARY) diff --git a/extensions/tensorflow/CMakeLists.txt b/extensions/tensorflow/CMakeLists.txt index 3c792ee..b6f74c2 100644 --- a/extensions/tensorflow/CMakeLists.txt +++ b/extensions/tensorflow/CMakeLists.txt @@ -26,13 +26,12 @@ find_package(TensorFlow REQUIRED) message("-- Found TensorFlow: ${TENSORFLOW_INCLUDE_DIRS}") -include_directories(${TENSORFLOW_INCLUDE_DIRS}) - file(GLOB SOURCES "*.cpp") add_library(minifi-tensorflow-extensions STATIC ${SOURCES}) set_property(TARGET minifi-tensorflow-extensions PROPERTY POSITION_INDEPENDENT_CODE ON) +target_include_directories(minifi-tensorflow-extensions SYSTEM PRIVATE ${TENSORFLOW_INCLUDE_DIRS}) target_link_libraries(minifi-tensorflow-extensions ${LIBMINIFI} Threads::Threads) target_link_libraries(minifi-tensorflow-extensions ${TENSORFLOW_LIBRARIES}) diff --git a/extensions/tensorflow/TFApplyGraph.cpp b/extensions/tensorflow/TFApplyGraph.cpp index 14d349e..b14c2e3 100644 --- a/extensions/tensorflow/TFApplyGraph.cpp +++ b/extensions/tensorflow/TFApplyGraph.cpp @@ -65,7 +65,7 @@ void TFApplyGraph::initialize() { setSupportedRelationships(std::move(relationships)); } -void TFApplyGraph::onSchedule(core::ProcessContext *context, core::ProcessSessionFactory *sessionFactory) { +void TFApplyGraph::onSchedule(core::ProcessContext *context, core::ProcessSessionFactory* /*sessionFactory*/) { context->getProperty(InputNode.getName(), input_node_); if (input_node_.empty()) { @@ -79,8 +79,8 @@ void TFApplyGraph::onSchedule(core::ProcessContext *context, core::ProcessSessio } } -void TFApplyGraph::onTrigger(const std::shared_ptr &context, - const std::shared_ptr &session) { +void TFApplyGraph::onTrigger(const std::shared_ptr& /*context*/, + const std::shared_ptr& session) { auto flow_file = session->get(); if (!flow_file) { @@ -152,7 +152,10 @@ void TFApplyGraph::onTrigger(const std::shared_ptr &contex TensorReadCallback tensor_cb(input_tensor_proto); session->read(flow_file, &tensor_cb); tensorflow::Tensor input; - input.FromProto(*input_tensor_proto); + if (!input.FromProto(*input_tensor_proto)) { + // failure deliberately ignored at this time + // added to avoid warn_unused_result build errors + } std::vector outputs; auto status = ctx->tf_session->Run({{input_node_, input}}, {output_node_}, {}, &outputs); diff --git a/extensions/tensorflow/TFApplyGraph.h b/extensions/tensorflow/TFApplyGraph.h index 3f26872..e00d528 100644 --- a/extensions/tensorflow/TFApplyGraph.h +++ b/extensions/tensorflow/TFApplyGraph.h @@ -47,7 +47,7 @@ class TFApplyGraph : public core::Processor { void initialize() override; void onSchedule(core::ProcessContext *context, core::ProcessSessionFactory *sessionFactory) override; - void onTrigger(core::ProcessContext *context, core::ProcessSession *session) override { + void onTrigger(core::ProcessContext* /*context*/, core::ProcessSession* /*session*/) override { logger_->log_error("onTrigger invocation with raw pointers is not implemented"); } void onTrigger(const std::shared_ptr &context, diff --git a/extensions/tensorflow/TFConvertImageToTensor.cpp b/extensions/tensorflow/TFConvertImageToTensor.cpp index aea09f8..efa4716 100644 --- a/extensions/tensorflow/TFConvertImageToTensor.cpp +++ b/extensions/tensorflow/TFConvertImageToTensor.cpp @@ -115,7 +115,7 @@ void TFConvertImageToTensor::initialize() { setSupportedRelationships(std::move(relationships)); } -void TFConvertImageToTensor::onSchedule(core::ProcessContext *context, core::ProcessSessionFactory *sessionFactory) { +void TFConvertImageToTensor::onSchedule(core::ProcessContext* context, core::ProcessSessionFactory* /*sessionFactory*/) { context->getProperty(ImageFormat.getName(), input_format_); if (input_format_.empty()) { @@ -189,8 +189,8 @@ void TFConvertImageToTensor::onSchedule(core::ProcessContext *context, core::Pro } } -void TFConvertImageToTensor::onTrigger(const std::shared_ptr &context, - const std::shared_ptr &session) { +void TFConvertImageToTensor::onTrigger(const std::shared_ptr& /*context*/, + const std::shared_ptr& session) { auto flow_file = session->get(); if (!flow_file) { diff --git a/extensions/tensorflow/TFConvertImageToTensor.h b/extensions/tensorflow/TFConvertImageToTensor.h index da3a11c..93c5b38 100644 --- a/extensions/tensorflow/TFConvertImageToTensor.h +++ b/extensions/tensorflow/TFConvertImageToTensor.h @@ -54,7 +54,7 @@ class TFConvertImageToTensor : public core::Processor { void initialize() override; void onSchedule(core::ProcessContext *context, core::ProcessSessionFactory *sessionFactory) override; - void onTrigger(core::ProcessContext *context, core::ProcessSession *session) override { + void onTrigger(core::ProcessContext* /*context*/, core::ProcessSession* /*session*/) override { logger_->log_error("onTrigger invocation with raw pointers is not implemented"); } void onTrigger(const std::shared_ptr &context, diff --git a/extensions/tensorflow/TFExtractTopLabels.cpp b/extensions/tensorflow/TFExtractTopLabels.cpp index 2f44f84..cfdaa9d 100644 --- a/extensions/tensorflow/TFExtractTopLabels.cpp +++ b/extensions/tensorflow/TFExtractTopLabels.cpp @@ -48,10 +48,10 @@ void TFExtractTopLabels::initialize() { setSupportedRelationships(std::move(relationships)); } -void TFExtractTopLabels::onSchedule(core::ProcessContext *context, core::ProcessSessionFactory *sessionFactory) { +void TFExtractTopLabels::onSchedule(core::ProcessContext* /*context*/, core::ProcessSessionFactory* /*sessionFactory*/) { } -void TFExtractTopLabels::onTrigger(const std::shared_ptr &context, +void TFExtractTopLabels::onTrigger(const std::shared_ptr& /*context*/, const std::shared_ptr &session) { auto flow_file = session->get(); @@ -88,7 +88,10 @@ void TFExtractTopLabels::onTrigger(const std::shared_ptr & session->read(flow_file, &tensor_cb); tensorflow::Tensor input; - input.FromProto(*input_tensor_proto); + if (!input.FromProto(*input_tensor_proto)) { + // failure deliberately ignored at this time + // added to avoid warn_unused_result build errors + } auto input_flat = input.flat(); std::vector> scores; @@ -102,7 +105,7 @@ void TFExtractTopLabels::onTrigger(const std::shared_ptr & return a.second > b.second; }); - for (int i = 0; i < 5 && i < scores.size(); i++) { + for (std::size_t i = 0; i < 5 && i < scores.size(); i++) { if (!labels || scores[i].first > labels->size()) { logger_->log_error("Label index is out of range (are the correct labels loaded?); routing to retry..."); session->transfer(flow_file, Retry); diff --git a/extensions/tensorflow/TFExtractTopLabels.h b/extensions/tensorflow/TFExtractTopLabels.h index a1b2211..03ca8ca 100644 --- a/extensions/tensorflow/TFExtractTopLabels.h +++ b/extensions/tensorflow/TFExtractTopLabels.h @@ -44,7 +44,7 @@ class TFExtractTopLabels : public core::Processor { void initialize() override; void onSchedule(core::ProcessContext *context, core::ProcessSessionFactory *sessionFactory) override; - void onTrigger(core::ProcessContext *context, core::ProcessSession *session) override { + void onTrigger(core::ProcessContext* /*context*/, core::ProcessSession* /*session*/) override { logger_->log_error("onTrigger invocation with raw pointers is not implemented"); } void onTrigger(const std::shared_ptr &context, diff --git a/libminifi/test/tensorflow-tests/CMakeLists.txt b/libminifi/test/tensorflow-tests/CMakeLists.txt index c7e68eb..97b45ec 100644 --- a/libminifi/test/tensorflow-tests/CMakeLists.txt +++ b/libminifi/test/tensorflow-tests/CMakeLists.txt @@ -27,7 +27,7 @@ FOREACH(testfile ${TENSORFLOW_INTEGRATION_TESTS}) add_executable("${testfilename}" "${testfile}" "${TEST_DIR}/TestBase.cpp") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/tensorflow") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/standard-processors") - target_include_directories(${testfilename} PRIVATE BEFORE ${TENSORFLOW_INCLUDE_DIRS}) + target_include_directories(${testfilename} SYSTEM PRIVATE BEFORE ${TENSORFLOW_INCLUDE_DIRS}) createTests("${testfilename}") target_wholearchive_library(${testfilename} minifi-tensorflow-extensions) target_wholearchive_library(${testfilename} minifi-standard-processors) diff --git a/libminifi/test/tensorflow-tests/TensorFlowTests.cpp b/libminifi/test/tensorflow-tests/TensorFlowTests.cpp index d4dc27c..6959cfb 100644 --- a/libminifi/test/tensorflow-tests/TensorFlowTests.cpp +++ b/libminifi/test/tensorflow-tests/TensorFlowTests.cpp @@ -120,13 +120,13 @@ TEST_CASE("TensorFlow: Apply Graph", "[tfApplyGraph]") { // NOLINT tensorflow::GraphDef graph; // Write test TensorFlow graph - root.ToGraphDef(&graph); + REQUIRE(root.ToGraphDef(&graph).ok()); std::ofstream in_file_stream(in_graph_file); graph.SerializeToOstream(&in_file_stream); } // Read test TensorFlow graph into TFApplyGraph - plan->runNextProcessor([&get_file, &in_graph_file, &plan](const std::shared_ptr context, + plan->runNextProcessor([&in_graph_file](const std::shared_ptr /*context*/, const std::shared_ptr session) { // Intercept the call so that we can add an attr (won't be required when we have UpdateAttribute processor) auto flow_file = session->create(); @@ -163,7 +163,7 @@ TEST_CASE("TensorFlow: Apply Graph", "[tfApplyGraph]") { // NOLINT tensorflow::TensorProto tensor_proto; tensor_proto.ParseFromIstream(&out_file_stream); tensorflow::Tensor tensor; - tensor.FromProto(tensor_proto); + REQUIRE(tensor.FromProto(tensor_proto)); // Verify output tensor float tensor_val = tensor.flat().data()[0]; @@ -286,7 +286,7 @@ TEST_CASE("TensorFlow: ConvertImageToTensor", "[tfConvertImageToTensor]") { // N tensorflow::TensorProto tensor_proto; tensor_proto.ParseFromIstream(&out_file_stream); tensorflow::Tensor tensor; - tensor.FromProto(tensor_proto); + REQUIRE(tensor.FromProto(tensor_proto)); // Verify output tensor auto shape = tensor.shape(); @@ -358,7 +358,7 @@ TEST_CASE("TensorFlow: Extract Top Labels", "[tfExtractTopLabels]") { // NOLINT } // Read labels - plan->runNextProcessor([&get_file, &in_labels_file, &plan](const std::shared_ptr context, + plan->runNextProcessor([&in_labels_file](const std::shared_ptr /*context*/, const std::shared_ptr session) { // Intercept the call so that we can add an attr (won't be required when we have UpdateAttribute processor) auto flow_file = session->create();