arrow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject arrow git commit: ARROW-383: [C++] Integration testing CLI tool
Date Mon, 21 Nov 2016 22:48:19 GMT
Repository: arrow
Updated Branches:
  refs/heads/master 997f502ce -> f082b1732


ARROW-383: [C++] Integration testing CLI tool

Modeled after Java version in ARROW-367

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #209 from wesm/ARROW-383 and squashes the following commits:

7b29b24 [Wes McKinney] Use git master gflags to avoid valgrind error
f563d3a [Wes McKinney] Build integration test as a normal unit test executable, opt-in to
integration testing
bdf1f7a [Wes McKinney] Call the RunCommand method instead
dbc6aab [Wes McKinney] Add unit tests for the integration validator
ca1eade [Wes McKinney] Clean up includes
1752249 [Wes McKinney] Draft integration testing CLI tool, untested
b773d0d [Wes McKinney] Add gflags external project and json-integration-test executable stub


Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/f082b173
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/f082b173
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/f082b173

Branch: refs/heads/master
Commit: f082b17323354dc2af31f39c15c58b995ba08360
Parents: 997f502
Author: Wes McKinney <wes.mckinney@twosigma.com>
Authored: Mon Nov 21 17:48:09 2016 -0500
Committer: Wes McKinney <wes.mckinney@twosigma.com>
Committed: Mon Nov 21 17:48:09 2016 -0500

----------------------------------------------------------------------
 cpp/CMakeLists.txt                         |  44 +++
 cpp/cmake_modules/FindGFlags.cmake         |  60 ++++
 cpp/src/arrow/io/file.cc                   |   7 +-
 cpp/src/arrow/ipc/CMakeLists.txt           |  27 ++
 cpp/src/arrow/ipc/adapter.h                |   2 +-
 cpp/src/arrow/ipc/json-integration-test.cc | 381 ++++++++++++++++++++++++
 6 files changed, 517 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/arrow/blob/f082b173/cpp/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 0bff752..839ea17 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -26,6 +26,7 @@ include(ExternalProject)
 set(BUILD_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/build-support")
 set(THIRDPARTY_DIR "${CMAKE_SOURCE_DIR}/thirdparty")
 
+set(GFLAGS_VERSION "2.1.2")
 set(GTEST_VERSION "1.7.0")
 set(GBENCHMARK_VERSION "1.0.0")
 set(FLATBUFFERS_VERSION "1.3.0")
@@ -506,6 +507,49 @@ if(ARROW_BUILD_TESTS)
   if(GTEST_VENDORED)
     add_dependencies(gtest googletest_ep)
   endif()
+
+  # gflags (formerly Googleflags) command line parsing
+  if("$ENV{GFLAGS_HOME}" STREQUAL "")
+    if(APPLE)
+      set(GFLAGS_CMAKE_CXX_FLAGS "-fPIC -std=c++11 -stdlib=libc++")
+    else()
+      set(GFLAGS_CMAKE_CXX_FLAGS "-fPIC")
+    endif()
+
+    set(GFLAGS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/gflags_ep-prefix/src/gflags_ep")
+    ExternalProject_Add(gflags_ep
+      GIT_REPOSITORY https://github.com/gflags/gflags.git
+      GIT_TAG cce68f0c9c5d054017425e6e6fd54f696d36e8ee
+      # URL "https://github.com/gflags/gflags/archive/v${GFLAGS_VERSION}.tar.gz"
+      BUILD_IN_SOURCE 1
+      CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
+                 -DCMAKE_INSTALL_PREFIX=${GFLAGS_PREFIX}
+                 -DBUILD_SHARED_LIBS=OFF
+                 -DBUILD_STATIC_LIBS=ON
+                 -DBUILD_PACKAGING=OFF
+                 -DBUILD_TESTING=OFF
+                 -BUILD_CONFIG_TESTS=OFF
+                 -DINSTALL_HEADERS=ON
+                 -DCMAKE_CXX_FLAGS=${GFLAGS_CMAKE_CXX_FLAGS})
+
+    set(GFLAGS_HOME "${GFLAGS_PREFIX}")
+    set(GFLAGS_INCLUDE_DIR "${GFLAGS_PREFIX}/include")
+    set(GFLAGS_STATIC_LIB "${GFLAGS_PREFIX}/lib/libgflags.a")
+    set(GFLAGS_VENDORED 1)
+  else()
+    set(GFLAGS_VENDORED 0)
+    find_package(GFlags REQUIRED)
+  endif()
+
+  message(STATUS "GFlags include dir: ${GFLAGS_INCLUDE_DIR}")
+  message(STATUS "GFlags static library: ${GFLAGS_STATIC_LIB}")
+  include_directories(SYSTEM ${GFLAGS_INCLUDE_DIR})
+  ADD_THIRDPARTY_LIB(gflags
+    STATIC_LIB ${GFLAGS_STATIC_LIB})
+
+  if(GFLAGS_VENDORED)
+    add_dependencies(gflags gflags_ep)
+  endif()
 endif()
 
 if(ARROW_BUILD_BENCHMARKS)

http://git-wip-us.apache.org/repos/asf/arrow/blob/f082b173/cpp/cmake_modules/FindGFlags.cmake
----------------------------------------------------------------------
diff --git a/cpp/cmake_modules/FindGFlags.cmake b/cpp/cmake_modules/FindGFlags.cmake
new file mode 100644
index 0000000..eaea835
--- /dev/null
+++ b/cpp/cmake_modules/FindGFlags.cmake
@@ -0,0 +1,60 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# - Find GFLAGS (gflags.h, libgflags.a, libgflags.so, and libgflags.so.0)
+# This module defines
+#  GFLAGS_INCLUDE_DIR, directory containing headers
+#  GFLAGS_SHARED_LIB, path to libgflags shared library
+#  GFLAGS_STATIC_LIB, path to libgflags static library
+#  GFLAGS_FOUND, whether gflags has been found
+
+if( NOT "$ENV{GFLAGS_HOME}" STREQUAL "")
+    file( TO_CMAKE_PATH "$ENV{GFLAGS_HOME}" _native_path )
+    list( APPEND _gflags_roots ${_native_path} )
+elseif ( GFlags_HOME )
+    list( APPEND _gflags_roots ${GFlags_HOME} )
+endif()
+
+if ( _gflags_roots )
+  find_path(GFLAGS_INCLUDE_DIR NAMES gflags/gflags.h
+    PATHS ${_gflags_roots}
+    NO_DEFAULT_PATH
+    PATH_SUFFIXES "include" )
+  find_library(GFLAGS_SHARED_LIB NAMES gflags
+    PATHS ${_gflags_roots}
+    NO_DEFAULT_PATH
+    PATH_SUFFIXES "lib" )
+  find_library(GFLAGS_SHARED_LIB NAMES libgflags.a
+    PATHS ${_gflags_roots}
+    NO_DEFAULT_PATH
+    PATH_SUFFIXES "lib" )
+else()
+  find_path(GFLAGS_INCLUDE_DIR gflags/gflags.h
+    # make sure we don't accidentally pick up a different version
+    NO_CMAKE_SYSTEM_PATH
+    NO_SYSTEM_ENVIRONMENT_PATH)
+  find_library(GFLAGS_SHARED_LIB gflags
+    NO_CMAKE_SYSTEM_PATH
+    NO_SYSTEM_ENVIRONMENT_PATH)
+  find_library(GFLAGS_STATIC_LIB libgflags.a
+    NO_CMAKE_SYSTEM_PATH
+    NO_SYSTEM_ENVIRONMENT_PATH)
+endif()
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(GFLAGS REQUIRED_VARS
+  GFLAGS_SHARED_LIB GFLAGS_STATIC_LIB GFLAGS_INCLUDE_DIR)

http://git-wip-us.apache.org/repos/asf/arrow/blob/f082b173/cpp/src/arrow/io/file.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/io/file.cc b/cpp/src/arrow/io/file.cc
index 93f0ad9..05fa666 100644
--- a/cpp/src/arrow/io/file.cc
+++ b/cpp/src/arrow/io/file.cc
@@ -186,12 +186,13 @@ static inline Status FileOpenWriteable(const std::string& filename,
int* fd) {
   memcpy(wpath.data(), filename.data(), filename.size());
   memcpy(wpath.data() + nwchars, L"\0", sizeof(wchar_t));
 
-  errno_actual = _wsopen_s(
-      fd, wpath.data(), _O_WRONLY | _O_CREAT | _O_BINARY, _SH_DENYNO, _S_IWRITE);
+  errno_actual = _wsopen_s(fd, wpath.data(), _O_WRONLY | _O_CREAT | _O_BINARY | _O_TRUNC,
+      _SH_DENYNO, _S_IWRITE);
   ret = *fd;
 
 #else
-  ret = *fd = open(filename.c_str(), O_WRONLY | O_CREAT | O_BINARY, ARROW_WRITE_SHMODE);
+  ret = *fd =
+      open(filename.c_str(), O_WRONLY | O_CREAT | O_BINARY | O_TRUNC, ARROW_WRITE_SHMODE);
 #endif
   return CheckOpenResult(ret, errno_actual, filename.c_str(), filename.size());
 }

http://git-wip-us.apache.org/repos/asf/arrow/blob/f082b173/cpp/src/arrow/ipc/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/CMakeLists.txt b/cpp/src/arrow/ipc/CMakeLists.txt
index 6955bcb..f9e7cf7 100644
--- a/cpp/src/arrow/ipc/CMakeLists.txt
+++ b/cpp/src/arrow/ipc/CMakeLists.txt
@@ -85,6 +85,33 @@ ADD_ARROW_TEST(ipc-json-test)
 ARROW_TEST_LINK_LIBRARIES(ipc-json-test
   ${ARROW_IPC_TEST_LINK_LIBS})
 
+ADD_ARROW_TEST(json-integration-test)
+
+if (APPLE)
+  target_link_libraries(json-integration-test
+    arrow_static
+    arrow_io
+    arrow_ipc
+    gflags
+    gtest
+    boost_filesystem_static
+    boost_system_static
+    dl)
+  set_target_properties(json-integration-test
+    PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
+else()
+  target_link_libraries(json-integration-test
+    arrow_static
+    arrow_io
+    arrow_ipc
+    gflags
+    gtest
+    pthread
+    boost_filesystem_static
+    boost_system_static
+    dl)
+endif()
+
 # make clean will delete the generated file
 set_source_files_properties(Metadata_generated.h PROPERTIES GENERATED TRUE)
 

http://git-wip-us.apache.org/repos/asf/arrow/blob/f082b173/cpp/src/arrow/ipc/adapter.h
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/adapter.h b/cpp/src/arrow/ipc/adapter.h
index 3fde18d..b02de28 100644
--- a/cpp/src/arrow/ipc/adapter.h
+++ b/cpp/src/arrow/ipc/adapter.h
@@ -16,7 +16,7 @@
 // under the License.
 
 // Public API for writing and accessing (with zero copy, if possible) Arrow
-// data in shared memory
+// IPC binary formatted data (e.g. in shared memory, or from some other IO source)
 
 #ifndef ARROW_IPC_ADAPTER_H
 #define ARROW_IPC_ADAPTER_H

http://git-wip-us.apache.org/repos/asf/arrow/blob/f082b173/cpp/src/arrow/ipc/json-integration-test.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/json-integration-test.cc b/cpp/src/arrow/ipc/json-integration-test.cc
new file mode 100644
index 0000000..5eff899
--- /dev/null
+++ b/cpp/src/arrow/ipc/json-integration-test.cc
@@ -0,0 +1,381 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <fstream>
+#include <iostream>
+#include <memory>
+#include <string>
+
+#include "gflags/gflags.h"
+#include "gtest/gtest.h"
+
+#include <boost/filesystem.hpp>  // NOLINT
+
+#include "arrow/io/file.h"
+#include "arrow/ipc/file.h"
+#include "arrow/ipc/json.h"
+#include "arrow/schema.h"
+#include "arrow/table.h"
+#include "arrow/test-util.h"
+#include "arrow/util/status.h"
+
+DEFINE_string(arrow, "", "Arrow file name");
+DEFINE_string(json, "", "JSON file name");
+DEFINE_string(mode, "VALIDATE",
+    "Mode of integration testing tool (ARROW_TO_JSON, JSON_TO_ARROW, VALIDATE)");
+DEFINE_bool(integration, false, "Run in integration test mode");
+DEFINE_bool(verbose, true, "Verbose output");
+
+namespace fs = boost::filesystem;
+
+namespace arrow {
+
+bool file_exists(const char* path) {
+  std::ifstream handle(path);
+  return handle.good();
+}
+
+// Convert JSON file to IPC binary format
+static Status ConvertJsonToArrow(
+    const std::string& json_path, const std::string& arrow_path) {
+  std::shared_ptr<io::ReadableFile> in_file;
+  std::shared_ptr<io::FileOutputStream> out_file;
+
+  RETURN_NOT_OK(io::ReadableFile::Open(json_path, &in_file));
+  RETURN_NOT_OK(io::FileOutputStream::Open(arrow_path, &out_file));
+
+  int64_t file_size = 0;
+  RETURN_NOT_OK(in_file->GetSize(&file_size));
+
+  std::shared_ptr<Buffer> json_buffer;
+  RETURN_NOT_OK(in_file->Read(file_size, &json_buffer));
+
+  std::unique_ptr<ipc::JsonReader> reader;
+  RETURN_NOT_OK(ipc::JsonReader::Open(json_buffer, &reader));
+
+  if (FLAGS_verbose) {
+    std::cout << "Found schema: " << reader->schema()->ToString() <<
std::endl;
+  }
+
+  std::shared_ptr<ipc::FileWriter> writer;
+  RETURN_NOT_OK(ipc::FileWriter::Open(out_file.get(), reader->schema(), &writer));
+
+  for (int i = 0; i < reader->num_record_batches(); ++i) {
+    std::shared_ptr<RecordBatch> batch;
+    RETURN_NOT_OK(reader->GetRecordBatch(i, &batch));
+    RETURN_NOT_OK(writer->WriteRecordBatch(batch->columns(), batch->num_rows()));
+  }
+  return writer->Close();
+}
+
+// Convert IPC binary format to JSON
+static Status ConvertArrowToJson(
+    const std::string& arrow_path, const std::string& json_path) {
+  std::shared_ptr<io::ReadableFile> in_file;
+  std::shared_ptr<io::FileOutputStream> out_file;
+
+  RETURN_NOT_OK(io::ReadableFile::Open(arrow_path, &in_file));
+  RETURN_NOT_OK(io::FileOutputStream::Open(json_path, &out_file));
+
+  std::shared_ptr<ipc::FileReader> reader;
+  RETURN_NOT_OK(ipc::FileReader::Open(in_file, &reader));
+
+  if (FLAGS_verbose) {
+    std::cout << "Found schema: " << reader->schema()->ToString() <<
std::endl;
+  }
+
+  std::unique_ptr<ipc::JsonWriter> writer;
+  RETURN_NOT_OK(ipc::JsonWriter::Open(reader->schema(), &writer));
+
+  for (int i = 0; i < reader->num_record_batches(); ++i) {
+    std::shared_ptr<RecordBatch> batch;
+    RETURN_NOT_OK(reader->GetRecordBatch(i, &batch));
+    RETURN_NOT_OK(writer->WriteRecordBatch(batch->columns(), batch->num_rows()));
+  }
+
+  std::string result;
+  RETURN_NOT_OK(writer->Finish(&result));
+  return out_file->Write(reinterpret_cast<const uint8_t*>(result.c_str()),
+      static_cast<int64_t>(result.size()));
+}
+
+static Status ValidateArrowVsJson(
+    const std::string& arrow_path, const std::string& json_path) {
+  // Construct JSON reader
+  std::shared_ptr<io::ReadableFile> json_file;
+  RETURN_NOT_OK(io::ReadableFile::Open(json_path, &json_file));
+
+  int64_t file_size = 0;
+  RETURN_NOT_OK(json_file->GetSize(&file_size));
+
+  std::shared_ptr<Buffer> json_buffer;
+  RETURN_NOT_OK(json_file->Read(file_size, &json_buffer));
+
+  std::unique_ptr<ipc::JsonReader> json_reader;
+  RETURN_NOT_OK(ipc::JsonReader::Open(json_buffer, &json_reader));
+
+  // Construct Arrow reader
+  std::shared_ptr<io::ReadableFile> arrow_file;
+  RETURN_NOT_OK(io::ReadableFile::Open(arrow_path, &arrow_file));
+
+  std::shared_ptr<ipc::FileReader> arrow_reader;
+  RETURN_NOT_OK(ipc::FileReader::Open(arrow_file, &arrow_reader));
+
+  auto json_schema = json_reader->schema();
+  auto arrow_schema = arrow_reader->schema();
+
+  if (!json_schema->Equals(arrow_schema)) {
+    std::stringstream ss;
+    ss << "JSON schema: \n"
+       << json_schema->ToString() << "\n"
+       << "Arrow schema: \n"
+       << arrow_schema->ToString();
+
+    if (FLAGS_verbose) { std::cout << ss.str() << std::endl; }
+    return Status::Invalid("Schemas did not match");
+  }
+
+  const int json_nbatches = json_reader->num_record_batches();
+  const int arrow_nbatches = arrow_reader->num_record_batches();
+
+  if (json_nbatches != arrow_nbatches) {
+    std::stringstream ss;
+    ss << "Different number of record batches: " << json_nbatches << "
(JSON) vs "
+       << arrow_nbatches << " (Arrow)";
+    return Status::Invalid(ss.str());
+  }
+
+  std::shared_ptr<RecordBatch> arrow_batch;
+  std::shared_ptr<RecordBatch> json_batch;
+  for (int i = 0; i < json_nbatches; ++i) {
+    RETURN_NOT_OK(json_reader->GetRecordBatch(i, &json_batch));
+    RETURN_NOT_OK(arrow_reader->GetRecordBatch(i, &arrow_batch));
+
+    if (!json_batch->Equals(*arrow_batch.get())) {
+      std::stringstream ss;
+      ss << "Record batch " << i << " did not match";
+      return Status::Invalid(ss.str());
+    }
+  }
+
+  return Status::OK();
+}
+
+Status RunCommand(const std::string& json_path, const std::string& arrow_path,
+    const std::string& command) {
+  if (json_path == "") { return Status::Invalid("Must specify json file name"); }
+
+  if (arrow_path == "") { return Status::Invalid("Must specify arrow file name"); }
+
+  if (command == "ARROW_TO_JSON") {
+    if (!file_exists(arrow_path.c_str())) {
+      return Status::Invalid("Input file does not exist");
+    }
+
+    return ConvertArrowToJson(arrow_path, json_path);
+  } else if (command == "JSON_TO_ARROW") {
+    if (!file_exists(json_path.c_str())) {
+      return Status::Invalid("Input file does not exist");
+    }
+
+    return ConvertJsonToArrow(json_path, arrow_path);
+  } else if (command == "VALIDATE") {
+    if (!file_exists(json_path.c_str())) {
+      return Status::Invalid("JSON file does not exist");
+    }
+
+    if (!file_exists(arrow_path.c_str())) {
+      return Status::Invalid("Arrow file does not exist");
+    }
+
+    return ValidateArrowVsJson(arrow_path, json_path);
+  } else {
+    std::stringstream ss;
+    ss << "Unknown command: " << command;
+    return Status::Invalid(ss.str());
+  }
+}
+
+static std::string temp_path() {
+  return (fs::temp_directory_path() / fs::unique_path()).native();
+}
+
+class TestJSONIntegration : public ::testing::Test {
+ public:
+  void SetUp() {}
+
+  std::string mkstemp() {
+    auto path = temp_path();
+    tmp_paths_.push_back(path);
+    return path;
+  }
+
+  Status WriteJson(const char* data, const std::string& path) {
+    do {
+      std::shared_ptr<io::FileOutputStream> out;
+      RETURN_NOT_OK(io::FileOutputStream::Open(path, &out));
+      RETURN_NOT_OK(out->Write(
+          reinterpret_cast<const uint8_t*>(data), static_cast<int64_t>(strlen(data))));
+    } while (0);
+    return Status::OK();
+  }
+
+  void TearDown() {
+    for (const std::string path : tmp_paths_) {
+      std::remove(path.c_str());
+    }
+  }
+
+ protected:
+  std::vector<std::string> tmp_paths_;
+};
+
+static const char* JSON_EXAMPLE = R"example(
+{
+  "schema": {
+    "fields": [
+      {
+        "name": "foo",
+        "type": {"name": "int", "isSigned": true, "bitWidth": 32},
+        "nullable": true, "children": [],
+        "typeLayout": [
+          {"type": "VALIDITY", "typeBitWidth": 1},
+          {"type": "DATA", "typeBitWidth": 32}
+        ]
+      },
+      {
+        "name": "bar",
+        "type": {"name": "floatingpoint", "precision": "DOUBLE"},
+        "nullable": true, "children": [],
+        "typeLayout": [
+          {"type": "VALIDITY", "typeBitWidth": 1},
+          {"type": "DATA", "typeBitWidth": 64}
+        ]
+      }
+    ]
+  },
+  "batches": [
+    {
+      "count": 5,
+      "columns": [
+        {
+          "name": "foo",
+          "count": 5,
+          "DATA": [1, 2, 3, 4, 5],
+          "VALIDITY": [1, 0, 1, 1, 1]
+        },
+        {
+          "name": "bar",
+          "count": 5,
+          "DATA": [1.0, 2.0, 3.0, 4.0, 5.0],
+          "VALIDITY": [1, 0, 0, 1, 1]
+        }
+      ]
+    }
+  ]
+}
+)example";
+
+static const char* JSON_EXAMPLE2 = R"example(
+{
+  "schema": {
+    "fields": [
+      {
+        "name": "foo",
+        "type": {"name": "int", "isSigned": true, "bitWidth": 32},
+        "nullable": true, "children": [],
+        "typeLayout": [
+          {"type": "VALIDITY", "typeBitWidth": 1},
+          {"type": "DATA", "typeBitWidth": 32}
+        ]
+      }
+    ]
+  },
+  "batches": [
+    {
+      "count": 5,
+      "columns": [
+        {
+          "name": "foo",
+          "count": 5,
+          "DATA": [1, 2, 3, 4, 5],
+          "VALIDITY": [1, 0, 1, 1, 1]
+        }
+      ]
+    }
+  ]
+}
+)example";
+
+TEST_F(TestJSONIntegration, ConvertAndValidate) {
+  std::string json_path = this->mkstemp();
+  std::string arrow_path = this->mkstemp();
+
+  ASSERT_OK(WriteJson(JSON_EXAMPLE, json_path));
+
+  ASSERT_OK(RunCommand(json_path, arrow_path, "JSON_TO_ARROW"));
+  ASSERT_OK(RunCommand(json_path, arrow_path, "VALIDATE"));
+
+  // Convert and overwrite
+  ASSERT_OK(RunCommand(json_path, arrow_path, "ARROW_TO_JSON"));
+
+  // Convert back to arrow, and validate
+  ASSERT_OK(RunCommand(json_path, arrow_path, "JSON_TO_ARROW"));
+  ASSERT_OK(RunCommand(json_path, arrow_path, "VALIDATE"));
+}
+
+TEST_F(TestJSONIntegration, ErrorStates) {
+  std::string json_path = this->mkstemp();
+  std::string json_path2 = this->mkstemp();
+  std::string arrow_path = this->mkstemp();
+
+  ASSERT_OK(WriteJson(JSON_EXAMPLE, json_path));
+  ASSERT_OK(WriteJson(JSON_EXAMPLE2, json_path2));
+
+  ASSERT_OK(ConvertJsonToArrow(json_path, arrow_path));
+  ASSERT_RAISES(Invalid, ValidateArrowVsJson(arrow_path, json_path2));
+
+  ASSERT_RAISES(IOError, ValidateArrowVsJson("does_not_exist-1234", json_path2));
+  ASSERT_RAISES(IOError, ValidateArrowVsJson(arrow_path, "does_not_exist-1234"));
+
+  ASSERT_RAISES(Invalid, RunCommand("", arrow_path, "VALIDATE"));
+  ASSERT_RAISES(Invalid, RunCommand(json_path, "", "VALIDATE"));
+}
+
+}  // namespace arrow
+
+int main(int argc, char** argv) {
+  gflags::ParseCommandLineFlags(&argc, &argv, true);
+
+  int ret = 0;
+
+  if (FLAGS_integration) {
+    arrow::Status result = arrow::RunCommand(FLAGS_json, FLAGS_arrow, FLAGS_mode);
+    if (!result.ok()) {
+      std::cout << "Error message: " << result.ToString() << std::endl;
+      ret = 1;
+    }
+  } else {
+    ::testing::InitGoogleTest(&argc, argv);
+    ret = RUN_ALL_TESTS();
+  }
+  gflags::ShutDownCommandLineFlags();
+  return ret;
+}


Mime
View raw message