arrow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject arrow git commit: ARROW-945: [GLib] Add a Lua example to show Torch integration
Date Fri, 05 May 2017 14:38:38 GMT
Repository: arrow
Updated Branches:
  refs/heads/master 80b72d43e -> bcf073c3a


ARROW-945: [GLib] Add a Lua example to show Torch integration

Author: Kouhei Sutou <kou@clear-code.com>

Closes #637 from kou/glib-lua-to-torch-tensor and squashes the following commits:

4aba395 [Kouhei Sutou] [GLib] Add a Lua example to show Torch integration


Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/bcf073c3
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/bcf073c3
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/bcf073c3

Branch: refs/heads/master
Commit: bcf073c3aeca872e41f86cee14d2c43598ce3149
Parents: 80b72d4
Author: Kouhei Sutou <kou@clear-code.com>
Authored: Fri May 5 10:38:33 2017 -0400
Committer: Wes McKinney <wes.mckinney@twosigma.com>
Committed: Fri May 5 10:38:33 2017 -0400

----------------------------------------------------------------------
 c_glib/example/lua/Makefile.am                |   1 +
 c_glib/example/lua/README.md                  |   5 +
 c_glib/example/lua/read-stream.lua            |   2 +-
 c_glib/example/lua/stream-to-torch-tensor.lua | 101 +++++++++++++++++++++
 4 files changed, 108 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/arrow/blob/bcf073c3/c_glib/example/lua/Makefile.am
----------------------------------------------------------------------
diff --git a/c_glib/example/lua/Makefile.am b/c_glib/example/lua/Makefile.am
index 9019d24..86bdbed 100644
--- a/c_glib/example/lua/Makefile.am
+++ b/c_glib/example/lua/Makefile.am
@@ -20,5 +20,6 @@ dist_lua_example_DATA =				\
 	README.md				\
 	read-batch.lua				\
 	read-stream.lua				\
+	stream-to-torch-tensor.lua		\
 	write-batch.lua				\
 	write-stream.lua

http://git-wip-us.apache.org/repos/asf/arrow/blob/bcf073c3/c_glib/example/lua/README.md
----------------------------------------------------------------------
diff --git a/c_glib/example/lua/README.md b/c_glib/example/lua/README.md
index d127573..6145bc7 100644
--- a/c_glib/example/lua/README.md
+++ b/c_glib/example/lua/README.md
@@ -43,3 +43,8 @@ Here are example codes in this directory:
 
   * `read-stream.lua`: It shows how to read Arrow array from file in
     stream mode.
+
+  * `stream-to-torch-tensor.lua`: It shows how to read Arrow array
+    from file in stream mode and convert it to
+    [Torch](http://torch.ch/)'s
+    [`Tensor` object](http://torch7.readthedocs.io/en/rtd/tensor/index.html).

http://git-wip-us.apache.org/repos/asf/arrow/blob/bcf073c3/c_glib/example/lua/read-stream.lua
----------------------------------------------------------------------
diff --git a/c_glib/example/lua/read-stream.lua b/c_glib/example/lua/read-stream.lua
index e744bed..987d463 100644
--- a/c_glib/example/lua/read-stream.lua
+++ b/c_glib/example/lua/read-stream.lua
@@ -25,7 +25,7 @@ local reader = Arrow.StreamReader.open(input)
 
 local i = 0
 while true do
-   local record_batch = reader:get_next_record_batch(i)
+   local record_batch = reader:get_next_record_batch()
    if not record_batch then
       break
    end

http://git-wip-us.apache.org/repos/asf/arrow/blob/bcf073c3/c_glib/example/lua/stream-to-torch-tensor.lua
----------------------------------------------------------------------
diff --git a/c_glib/example/lua/stream-to-torch-tensor.lua b/c_glib/example/lua/stream-to-torch-tensor.lua
new file mode 100644
index 0000000..237d759
--- /dev/null
+++ b/c_glib/example/lua/stream-to-torch-tensor.lua
@@ -0,0 +1,101 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+local lgi = require 'lgi'
+local Arrow = lgi.Arrow
+
+local torch = require 'torch'
+
+Arrow.Array.torch_types = function(self)
+   return nil
+end
+
+Arrow.Array.to_torch = function(self)
+   local types = self:torch_types()
+   if not types then
+      return nil
+   end
+
+   local storage_type = types[1]
+   local tensor_type = types[2]
+
+   local size = self:get_length()
+   local storage = storage_type(size)
+   if not storage then
+      return nil
+   end
+
+   for i = 1, size do
+      storage[i] = self:get_value(i - 1)
+   end
+   return tensor_type(storage)
+end
+
+Arrow.UInt8Array.torch_types = function(self)
+   return {torch.ByteStorage, torch.ByteTensor}
+end
+
+Arrow.Int8Array.torch_types = function(self)
+   return {torch.CharStorage, torch.CharTensor}
+end
+
+Arrow.Int16Array.torch_types = function(self)
+   return {torch.ShortStorage, torch.ShortTensor}
+end
+
+Arrow.Int32Array.torch_types = function(self)
+   return {torch.IntStorage, torch.IntTensor}
+end
+
+Arrow.Int64Array.torch_types = function(self)
+   return {torch.LongStorage, torch.LongTensor}
+end
+
+Arrow.FloatArray.torch_types = function(self)
+   return {torch.FloatStorage, torch.FloatTensor}
+end
+
+Arrow.DoubleArray.torch_types = function(self)
+   return {torch.DoubleStorage, torch.DoubleTensor}
+end
+
+
+local input_path = arg[1] or "/tmp/stream.arrow";
+
+local input = Arrow.MemoryMappedInputStream.new(input_path)
+local reader = Arrow.StreamReader.open(input)
+
+local i = 0
+while true do
+   local record_batch = reader:get_next_record_batch()
+   if not record_batch then
+      break
+   end
+
+   print(string.rep("=", 40))
+   print("record-batch["..i.."]:")
+   for j = 0, record_batch:get_n_columns() - 1 do
+      local column = record_batch:get_column(j)
+      local column_name = record_batch:get_column_name(j)
+      print("  "..column_name..":")
+      print(column:to_torch())
+   end
+
+   i = i + 1
+end
+
+input:close()


Mime
View raw message