arrow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject arrow git commit: ARROW-1040: [GLib] Support tensor IO
Date Tue, 16 May 2017 14:03:34 GMT
Repository: arrow
Updated Branches:
  refs/heads/master 49c539810 -> e7e8d611c


ARROW-1040: [GLib] Support tensor IO

Author: Kouhei Sutou <kou@clear-code.com>

Closes #697 from kou/glib-support-tensor-io and squashes the following commits:

eca2c3f [Kouhei Sutou] [GLib] Support tensor IO


Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/e7e8d611
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/e7e8d611
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/e7e8d611

Branch: refs/heads/master
Commit: e7e8d611cc415d8667673dc57a5c3c5732b25696
Parents: 49c5398
Author: Kouhei Sutou <kou@clear-code.com>
Authored: Tue May 16 10:03:29 2017 -0400
Committer: Wes McKinney <wes.mckinney@twosigma.com>
Committed: Tue May 16 10:03:29 2017 -0400

----------------------------------------------------------------------
 c_glib/arrow-glib/input-stream.cpp  | 32 +++++++++++++++++++++++++++++++
 c_glib/arrow-glib/input-stream.h    |  4 ++++
 c_glib/arrow-glib/output-stream.cpp | 33 +++++++++++++++++++++++++++++++-
 c_glib/arrow-glib/output-stream.h   |  5 +++++
 c_glib/test/test-tensor.rb          |  9 +++++++++
 5 files changed, 82 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e8d611/c_glib/arrow-glib/input-stream.cpp
----------------------------------------------------------------------
diff --git a/c_glib/arrow-glib/input-stream.cpp b/c_glib/arrow-glib/input-stream.cpp
index b931cf8..d81e4a3 100644
--- a/c_glib/arrow-glib/input-stream.cpp
+++ b/c_glib/arrow-glib/input-stream.cpp
@@ -23,12 +23,14 @@
 
 #include <arrow/io/interfaces.h>
 #include <arrow/io/memory.h>
+#include <arrow/ipc/reader.h>
 
 #include <arrow-glib/buffer.hpp>
 #include <arrow-glib/error.hpp>
 #include <arrow-glib/file.hpp>
 #include <arrow-glib/input-stream.hpp>
 #include <arrow-glib/readable.hpp>
+#include <arrow-glib/tensor.hpp>
 
 G_BEGIN_DECLS
 
@@ -253,6 +255,36 @@ garrow_seekable_input_stream_read_at(GArrowSeekableInputStream *input_stream,
   }
 }
 
+/**
+ * garrow_seekable_input_stream_read_tensor:
+ * @input_stream: A #GArrowSeekableInputStream.
+ * @position: The read start position.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: (transfer full) (nullable):
+ *   #GArrowTensor on success, %NULL on error.
+ *
+ * Since: 0.4.0
+ */
+GArrowTensor *
+garrow_seekable_input_stream_read_tensor(GArrowSeekableInputStream *input_stream,
+                                         gint64 position,
+                                         GError **error)
+{
+  auto arrow_random_access_file =
+    garrow_seekable_input_stream_get_raw(input_stream);
+
+  std::shared_ptr<arrow::Tensor> arrow_tensor;
+  auto status = arrow::ipc::ReadTensor(position,
+                                       arrow_random_access_file.get(),
+                                       &arrow_tensor);
+  if (garrow_error_check(error, status, "[seekable-input-stream][read-tensor]")) {
+    return garrow_tensor_new_raw(&arrow_tensor);
+  } else {
+    return NULL;
+  }
+}
+
 
 G_DEFINE_TYPE(GArrowBufferInputStream,                       \
               garrow_buffer_input_stream,                     \

http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e8d611/c_glib/arrow-glib/input-stream.h
----------------------------------------------------------------------
diff --git a/c_glib/arrow-glib/input-stream.h b/c_glib/arrow-glib/input-stream.h
index 5118828..8a4d362 100644
--- a/c_glib/arrow-glib/input-stream.h
+++ b/c_glib/arrow-glib/input-stream.h
@@ -20,6 +20,7 @@
 #pragma once
 
 #include <arrow-glib/buffer.h>
+#include <arrow-glib/tensor.h>
 
 G_BEGIN_DECLS
 
@@ -123,6 +124,9 @@ GArrowBuffer *garrow_seekable_input_stream_read_at(GArrowSeekableInputStream
*in
                                                    gint64 position,
                                                    gint64 n_bytes,
                                                    GError **error);
+GArrowTensor *garrow_seekable_input_stream_read_tensor(GArrowSeekableInputStream *input_stream,
+                                                       gint64 position,
+                                                       GError **error);
 
 
 #define GARROW_TYPE_BUFFER_INPUT_STREAM         \

http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e8d611/c_glib/arrow-glib/output-stream.cpp
----------------------------------------------------------------------
diff --git a/c_glib/arrow-glib/output-stream.cpp b/c_glib/arrow-glib/output-stream.cpp
index 48c48b8..ffb6fec 100644
--- a/c_glib/arrow-glib/output-stream.cpp
+++ b/c_glib/arrow-glib/output-stream.cpp
@@ -21,13 +21,14 @@
 #  include <config.h>
 #endif
 
-#include <arrow/api.h>
 #include <arrow/io/memory.h>
+#include <arrow/ipc/writer.h>
 
 #include <arrow-glib/buffer.hpp>
 #include <arrow-glib/error.hpp>
 #include <arrow-glib/file.hpp>
 #include <arrow-glib/output-stream.hpp>
+#include <arrow-glib/tensor.hpp>
 #include <arrow-glib/writeable.hpp>
 
 G_BEGIN_DECLS
@@ -168,6 +169,36 @@ garrow_output_stream_class_init(GArrowOutputStreamClass *klass)
   g_object_class_install_property(gobject_class, PROP_OUTPUT_STREAM, spec);
 }
 
+/**
+ * garrow_output_stream_write_tensor:
+ * @stream: A #GArrowWriteable.
+ * @tensor: A #GArrowTensor to be written.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: The number of written bytes on success, -1 on error.
+ *
+ * Since: 0.4.0
+ */
+gint64
+garrow_output_stream_write_tensor(GArrowOutputStream *stream,
+                                  GArrowTensor *tensor,
+                                  GError **error)
+{
+  auto arrow_tensor = garrow_tensor_get_raw(tensor);
+  auto arrow_stream = garrow_output_stream_get_raw(stream);
+  int32_t metadata_length;
+  int64_t body_length;
+  auto status = arrow::ipc::WriteTensor(*arrow_tensor,
+                                        arrow_stream.get(),
+                                        &metadata_length,
+                                        &body_length);
+  if (garrow_error_check(error, status, "[output-stream][write-tensor]")) {
+    return metadata_length + body_length;
+  } else {
+    return -1;
+  }
+}
+
 
 G_DEFINE_TYPE(GArrowFileOutputStream,
               garrow_file_output_stream,

http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e8d611/c_glib/arrow-glib/output-stream.h
----------------------------------------------------------------------
diff --git a/c_glib/arrow-glib/output-stream.h b/c_glib/arrow-glib/output-stream.h
index 48b891c..c86597b 100644
--- a/c_glib/arrow-glib/output-stream.h
+++ b/c_glib/arrow-glib/output-stream.h
@@ -22,6 +22,7 @@
 #include <glib-object.h>
 
 #include <arrow-glib/buffer.h>
+#include <arrow-glib/tensor.h>
 
 G_BEGIN_DECLS
 
@@ -71,6 +72,10 @@ struct _GArrowOutputStreamClass
 
 GType garrow_output_stream_get_type(void) G_GNUC_CONST;
 
+gint64 garrow_output_stream_write_tensor(GArrowOutputStream *stream,
+                                         GArrowTensor *tensor,
+                                         GError **error);
+
 
 #define GARROW_TYPE_FILE_OUTPUT_STREAM          \
   (garrow_file_output_stream_get_type())

http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e8d611/c_glib/test/test-tensor.rb
----------------------------------------------------------------------
diff --git a/c_glib/test/test-tensor.rb b/c_glib/test/test-tensor.rb
index 780c9f1..bdfc657 100644
--- a/c_glib/test/test-tensor.rb
+++ b/c_glib/test/test-tensor.rb
@@ -114,4 +114,13 @@ class TestTensor < Test::Unit::TestCase
       not @tensor.column_major?
     end
   end
+
+  def test_io
+    buffer = Arrow::PoolBuffer.new
+    output = Arrow::BufferOutputStream.new(buffer)
+    output.write_tensor(@tensor)
+    input = Arrow::BufferInputStream.new(buffer)
+    assert_equal(@tensor,
+                 input.read_tensor(0))
+  end
 end


Mime
View raw message