parquet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From u..@apache.org
Subject parquet-cpp git commit: PARQUET-1048: Apache Arrow static transitive dependencies
Date Mon, 10 Jul 2017 17:40:14 GMT
Repository: parquet-cpp
Updated Branches:
  refs/heads/master 5c02c7583 -> 562651e11


PARQUET-1048:  Apache Arrow static transitive dependencies

Author: Deepak Majeti <deepak.majeti@hpe.com>

Closes #367 from majetideepak/PARQUET-1048 and squashes the following commits:

42635f2 [Deepak Majeti] try another script
bb96b28 [Deepak Majeti] disable boost shared
b3a0e75 [Deepak Majeti] enable mem check
db50b65 [Deepak Majeti] build arrow shared only if parquet shared is specified
5fe857e [Deepak Majeti] Remove mem check
68f8c59 [Deepak Majeti] Fix Arrow visibility warnings
0bbd073 [Deepak Majeti] add env vars
e4406ab [Deepak Majeti] Add static linking for travis ci
032c8c7 [Deepak Majeti] Fix linking of benchmarks and examples
bdb5b2f [Deepak Majeti] Add Arrow lib transitive dependencies


Project: http://git-wip-us.apache.org/repos/asf/parquet-cpp/repo
Commit: http://git-wip-us.apache.org/repos/asf/parquet-cpp/commit/562651e1
Tree: http://git-wip-us.apache.org/repos/asf/parquet-cpp/tree/562651e1
Diff: http://git-wip-us.apache.org/repos/asf/parquet-cpp/diff/562651e1

Branch: refs/heads/master
Commit: 562651e11a3a148ead4a27c5ab5ede903f1fd5bb
Parents: 5c02c75
Author: Deepak Majeti <deepak.majeti@hpe.com>
Authored: Mon Jul 10 19:40:10 2017 +0200
Committer: Uwe L. Korn <uwe@apache.org>
Committed: Mon Jul 10 19:40:10 2017 +0200

----------------------------------------------------------------------
 .travis.yml                             |  6 +++
 CMakeLists.txt                          | 39 +++++++++++++-
 benchmarks/CMakeLists.txt               |  5 +-
 ci/travis_script_static.sh              | 76 ++++++++++++++++++++++++++++
 cmake_modules/ThirdpartyToolchain.cmake |  1 +
 examples/CMakeLists.txt                 |  2 +-
 src/parquet/arrow/reader.cc             |  4 +-
 7 files changed, 127 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/562651e1/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index b332cf1..70da9ed 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -45,6 +45,12 @@ matrix:
     env: PARQUET_BUILD_GROUP=toolchain
     before_script:
     script:
+    - $TRAVIS_BUILD_DIR/ci/travis_script_static.sh
+  - compiler: gcc
+    os: linux
+    env: PARQUET_BUILD_GROUP=toolchain
+    before_script:
+    script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_toolchain.sh
   - compiler: clang
     os: linux

http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/562651e1/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 5db2a54..b7a41d8 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -509,8 +509,45 @@ if ("${PARQUET_ARROW_LINKAGE}" STREQUAL "shared")
   set(ARROW_LINK_LIBS
     arrow)
 else()
+  #############################################################
+  # Transitive Library Linkage
+
+  if (NOT DEFINED ENV{BROTLI_STATIC_LIB_ENC} OR
+      NOT DEFINED ENV{BROTLI_STATIC_LIB_DEC} OR
+      NOT DEFINED ENV{BROTLI_STATIC_LIB_COMMON} OR
+      NOT DEFINED ENV{SNAPPY_STATIC_LIB} OR
+      NOT DEFINED ENV{ZLIB_STATIC_LIB})
+    message(FATAL_ERROR "Missing transitive dependencies for Arrow static linking")
+  endif()
+
+  set(BROTLI_STATIC_LIB_ENC "$ENV{BROTLI_STATIC_LIB_ENC}")
+  set(BROTLI_STATIC_LIB_DEC "$ENV{BROTLI_STATIC_LIB_DEC}")
+  set(BROTLI_STATIC_LIB_COMMON "$ENV{BROTLI_STATIC_LIB_COMMON}")
+  set(SNAPPY_STATIC_LIB "$ENV{SNAPPY_STATIC_LIB}")
+  set(ZLIB_STATIC_LIB "$ENV{ZLIB_STATIC_LIB}")
+
+  add_library(brotli_enc STATIC IMPORTED)
+  set_target_properties(brotli_enc PROPERTIES IMPORTED_LOCATION ${BROTLI_STATIC_LIB_ENC})
+  add_library(brotli_dec STATIC IMPORTED)
+  set_target_properties(brotli_dec PROPERTIES IMPORTED_LOCATION ${BROTLI_STATIC_LIB_DEC})
+  add_library(brotli_common STATIC IMPORTED)
+  set_target_properties(brotli_common PROPERTIES IMPORTED_LOCATION ${BROTLI_STATIC_LIB_COMMON})
+  add_library(snappy STATIC IMPORTED)
+  set_target_properties(snappy PROPERTIES IMPORTED_LOCATION ${SNAPPY_STATIC_LIB})
+  add_library(zlib STATIC IMPORTED)
+  set_target_properties(zlib PROPERTIES IMPORTED_LOCATION ${ZLIB_STATIC_LIB})
+
+  set(TRANSITIVE_LINK_LIBS
+    snappy
+    zlib
+    brotli_enc
+    brotli_dec
+    brotli_common
+  )
+
   set(ARROW_LINK_LIBS
-    arrow_static)
+    arrow_static
+    ${TRANSITIVE_LINK_LIBS})
 endif()
 
 #############################################################

http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/562651e1/benchmarks/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt
index 61c1491..43af746 100644
--- a/benchmarks/CMakeLists.txt
+++ b/benchmarks/CMakeLists.txt
@@ -16,13 +16,14 @@
 # under the License.
 
 SET(LINK_LIBS
+  ${ARROW_LINK_LIBS}
   thriftstatic)
 
 if (PARQUET_BUILD_BENCHMARKS)
   add_executable(decode_benchmark decode_benchmark.cc)
 
   # This uses private APIs
-  target_link_libraries(decode_benchmark ${LINK_LIBS}
-    arrow
+  target_link_libraries(decode_benchmark
+    ${LINK_LIBS} 
     parquet_static)
 endif()

http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/562651e1/ci/travis_script_static.sh
----------------------------------------------------------------------
diff --git a/ci/travis_script_static.sh b/ci/travis_script_static.sh
new file mode 100755
index 0000000..de57acc
--- /dev/null
+++ b/ci/travis_script_static.sh
@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License. See accompanying LICENSE file.
+
+set -xe
+
+# Use a C++11 compiler on Linux
+export CC="gcc-4.9"
+export CXX="g++-4.9"
+
+# ----------------------------------------------------------------------
+# Set up external toolchain
+
+MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh"
+export MINICONDA=$HOME/miniconda
+wget -O miniconda.sh $MINICONDA_URL
+bash miniconda.sh -b -p $MINICONDA
+export PATH="$MINICONDA/bin:$PATH"
+export CPP_TOOLCHAIN=$TRAVIS_BUILD_DIR/cpp-toolchain
+
+conda update -y -q conda
+conda config --set auto_update_conda false
+conda info -a
+
+conda config --set show_channel_urls True
+
+# Help with SSL timeouts to S3
+conda config --set remote_connect_timeout_secs 12
+
+conda info -a
+
+conda create -y -q -p $CPP_TOOLCHAIN \
+      boost-cpp thrift-cpp cmake git \
+      -c conda-forge
+
+# ----------------------------------------------------------------------
+
+: ${CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/parquet-build}
+export PARQUET_TEST_DATA=$TRAVIS_BUILD_DIR/data
+export PARQUET_BUILD_TOOLCHAIN=$CPP_TOOLCHAIN
+export LD_LIBRARY_PATH=$CPP_TOOLCHAIN/lib:$LD_LIBRARY_PATH
+export BOOST_ROOT=$CPP_TOOLCHAIN
+export PARQUET_TEST_DATA=$TRAVIS_BUILD_DIR/data
+export SNAPPY_STATIC_LIB=$TRAVIS_BUILD_DIR/parquet-build/arrow_ep-prefix/src/arrow_ep-build/snappy_ep/src/snappy_ep-install/lib/libsnappy.a
+export BROTLI_STATIC_LIB_ENC=$TRAVIS_BUILD_DIR/parquet-build/arrow_ep-prefix/src/arrow_ep-build/brotli_ep/src/brotli_ep-install/lib/x86_64-linux-gnu/libbrotlienc.a
+export BROTLI_STATIC_LIB_DEC=$TRAVIS_BUILD_DIR/parquet-build/arrow_ep-prefix/src/arrow_ep-build/brotli_ep/src/brotli_ep-install/lib/x86_64-linux-gnu/libbrotlidec.a
+export BROTLI_STATIC_LIB_COMMON=$TRAVIS_BUILD_DIR/parquet-build/arrow_ep-prefix/src/arrow_ep-build/brotli_ep/src/brotli_ep-install/lib/x86_64-linux-gnu/libbrotlicommon.a
+export ZLIB_STATIC_LIB=$TRAVIS_BUILD_DIR/parquet-build/arrow_ep-prefix/src/arrow_ep-build/zlib_ep/src/zlib_ep-install/lib/libz.a
+
+
+cmake -DPARQUET_CXXFLAGS=-Werror \
+      -DPARQUET_TEST_MEMCHECK=ON \
+      -DPARQUET_ARROW_LINKAGE="static" \
+      -DPARQUET_BUILD_SHARED=OFF \
+      -DPARQUET_BOOST_USE_SHARED=OFF \
+      -DPARQUET_BUILD_BENCHMARKS=ON \
+      -DPARQUET_BUILD_EXAMPLES=ON \
+      -DPARQUET_GENERATE_COVERAGE=1 \
+      $TRAVIS_BUILD_DIR
+
+pushd $CPP_BUILD_DIR
+
+make -j4 || exit 1
+ctest -VV -L unittest || { cat $TRAVIS_BUILD_DIR/parquet-build/Testing/Temporary/LastTest.log;
exit 1; }
+
+popd

http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/562651e1/cmake_modules/ThirdpartyToolchain.cmake
----------------------------------------------------------------------
diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake
index 716debc..85630a4 100644
--- a/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cmake_modules/ThirdpartyToolchain.cmake
@@ -332,6 +332,7 @@ if (NOT ARROW_FOUND)
     -DCMAKE_INSTALL_LIBDIR=${ARROW_LIB_DIR}
     -DARROW_JEMALLOC=OFF
     -DARROW_IPC=OFF
+    -DARROW_BUILD_SHARED=${PARQUET_BUILD_SHARED}
     -DARROW_BOOST_USE_SHARED=${PARQUET_BOOST_USE_SHARED}
     -DARROW_BUILD_TESTS=OFF)
 

http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/562651e1/examples/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index 594295d..721fa9a 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -17,5 +17,5 @@
 
 if (PARQUET_BUILD_EXECUTABLES)
   add_executable(reader-writer reader-writer.cc)
-  target_link_libraries(reader-writer parquet_shared arrow)
+  target_link_libraries(reader-writer parquet_static)
 endif()

http://git-wip-us.apache.org/repos/asf/parquet-cpp/blob/562651e1/src/parquet/arrow/reader.cc
----------------------------------------------------------------------
diff --git a/src/parquet/arrow/reader.cc b/src/parquet/arrow/reader.cc
index dd1c9d2..addfb85 100644
--- a/src/parquet/arrow/reader.cc
+++ b/src/parquet/arrow/reader.cc
@@ -226,7 +226,7 @@ class ColumnReader::Impl {
 };
 
 // Reader implementation for primitive arrays
-class PrimitiveImpl : public ColumnReader::Impl {
+class PARQUET_NO_EXPORT PrimitiveImpl : public ColumnReader::Impl {
  public:
   PrimitiveImpl(MemoryPool* pool, std::unique_ptr<FileColumnIterator> input)
       : pool_(pool),
@@ -300,7 +300,7 @@ class PrimitiveImpl : public ColumnReader::Impl {
 };
 
 // Reader implementation for struct array
-class StructImpl : public ColumnReader::Impl {
+class PARQUET_NO_EXPORT StructImpl : public ColumnReader::Impl {
  public:
   explicit StructImpl(const std::vector<std::shared_ptr<Impl>>& children,
       int16_t struct_def_level, MemoryPool* pool, const NodePtr& node)


Mime
View raw message