Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 77874200D4A for ; Tue, 14 Nov 2017 04:54:10 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 75E54160C07; Tue, 14 Nov 2017 03:54:10 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 4F68F160BF3 for ; Tue, 14 Nov 2017 04:54:08 +0100 (CET) Received: (qmail 9373 invoked by uid 500); 14 Nov 2017 03:54:07 -0000 Mailing-List: contact commits-help@arrow.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@arrow.apache.org Delivered-To: mailing list commits@arrow.apache.org Received: (qmail 9359 invoked by uid 99); 14 Nov 2017 03:54:07 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 14 Nov 2017 03:54:07 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id 92F3581907; Tue, 14 Nov 2017 03:54:06 +0000 (UTC) Date: Tue, 14 Nov 2017 03:54:06 +0000 To: "commits@arrow.apache.org" Subject: [arrow] branch master updated: ARROW-1802: [GLib] Support arrow-gpu MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Message-ID: <151063164641.815.8725384317296571682@gitbox.apache.org> From: wesm@apache.org X-Git-Host: gitbox.apache.org X-Git-Repo: arrow X-Git-Refname: refs/heads/master X-Git-Reftype: branch X-Git-Oldrev: 6f8e2873c3931ff279b05a5e517a87637eb13d05 X-Git-Newrev: 8f2d15256d6b3a9da569797f363518abf50d23f5 X-Git-Rev: 8f2d15256d6b3a9da569797f363518abf50d23f5 X-Git-NotificationType: ref_changed_plus_diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated archived-at: Tue, 14 Nov 2017 03:54:10 -0000 This is an automated email from the ASF dual-hosted git repository. wesm pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/arrow.git The following commit(s) were added to refs/heads/master by this push: new 8f2d152 ARROW-1802: [GLib] Support arrow-gpu 8f2d152 is described below commit 8f2d15256d6b3a9da569797f363518abf50d23f5 Author: Kouhei Sutou AuthorDate: Mon Nov 13 22:53:58 2017 -0500 ARROW-1802: [GLib] Support arrow-gpu arrow-gpu isn't required. If `arrow-gpu.pc` isn't installed, GPU support is just ignored. Author: Kouhei Sutou Closes #1313 from kou/glib-gpu and squashes the following commits: b36d491a [Kouhei Sutou] [GLib] Support arrow-gpu --- c_glib/.gitignore | 1 + c_glib/Makefile.am | 1 + c_glib/arrow-glib/Makefile.am | 8 +- c_glib/arrow-glib/buffer.h | 82 +- c_glib/arrow-glib/input-stream.h | 136 +-- c_glib/arrow-glib/meson.build | 33 +- c_glib/arrow-glib/output-stream.h | 47 +- c_glib/arrow-glib/readable.cpp | 4 +- c_glib/arrow-glib/readable.hpp | 1 + c_glib/arrow-gpu-glib/Makefile.am | 109 +++ .../arrow-gpu-glib.h} | 18 +- .../arrow-gpu-glib.hpp} | 18 +- .../arrow-gpu-glib.pc.in} | 13 +- c_glib/arrow-gpu-glib/cuda.cpp | 941 +++++++++++++++++++++ c_glib/arrow-gpu-glib/cuda.h | 181 ++++ c_glib/arrow-gpu-glib/cuda.hpp | 54 ++ c_glib/arrow-gpu-glib/meson.build | 80 ++ c_glib/configure.ac | 26 +- c_glib/doc/Makefile.am | 2 +- c_glib/doc/reference/Makefile.am | 11 + c_glib/doc/reference/arrow-glib-docs.sgml | 10 + c_glib/doc/reference/meson.build | 23 +- c_glib/meson.build | 9 +- c_glib/test/run-test.rb | 6 + c_glib/test/run-test.sh | 37 +- c_glib/test/test-gpu-cuda.rb | 144 ++++ 26 files changed, 1683 insertions(+), 312 deletions(-) diff --git a/c_glib/.gitignore b/c_glib/.gitignore index 03bb0fe..2719147 100644 --- a/c_glib/.gitignore +++ b/c_glib/.gitignore @@ -41,6 +41,7 @@ Makefile.in /arrow-glib/enums.h /arrow-glib/stamp-* /arrow-glib/*.pc +/arrow-gpu-glib/*.pc /example/build /example/read-batch /example/read-stream diff --git a/c_glib/Makefile.am b/c_glib/Makefile.am index 577b749..4cc70e5 100644 --- a/c_glib/Makefile.am +++ b/c_glib/Makefile.am @@ -19,6 +19,7 @@ ACLOCAL_AMFLAGS = -I m4 ${ACLOCAL_FLAGS} SUBDIRS = \ arrow-glib \ + arrow-gpu-glib \ doc \ example \ tool diff --git a/c_glib/arrow-glib/Makefile.am b/c_glib/arrow-glib/Makefile.am index bf68ec4..5ecb1a6 100644 --- a/c_glib/arrow-glib/Makefile.am +++ b/c_glib/arrow-glib/Makefile.am @@ -203,20 +203,18 @@ pkgconfigdir = $(libdir)/pkgconfig pkgconfig_DATA = \ arrow-glib.pc -# GObject Introspection +if HAVE_INTROSPECTION -include $(INTROSPECTION_MAKEFILE) INTROSPECTION_GIRS = INTROSPECTION_SCANNER_ARGS = INTROSPECTION_COMPILER_ARGS = -if HAVE_INTROSPECTION Arrow-1.0.gir: libarrow-glib.la Arrow_1_0_gir_PACKAGES = \ - gobject-2.0 \ gio-2.0 -Arrow_1_0_gir_EXPORT_PACKAGES = arrow +Arrow_1_0_gir_EXPORT_PACKAGES = \ + arrow-glib Arrow_1_0_gir_INCLUDES = \ - GObject-2.0 \ Gio-2.0 Arrow_1_0_gir_CFLAGS = \ $(AM_CPPFLAGS) diff --git a/c_glib/arrow-glib/buffer.h b/c_glib/arrow-glib/buffer.h index b3f3a2c..300bb4f 100644 --- a/c_glib/arrow-glib/buffer.h +++ b/c_glib/arrow-glib/buffer.h @@ -19,44 +19,21 @@ #pragma once -#include +#include G_BEGIN_DECLS -#define GARROW_TYPE_BUFFER \ - (garrow_buffer_get_type()) -#define GARROW_BUFFER(obj) \ - (G_TYPE_CHECK_INSTANCE_CAST((obj), GARROW_TYPE_BUFFER, GArrowBuffer)) -#define GARROW_BUFFER_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_CAST((klass), GARROW_TYPE_BUFFER, GArrowBufferClass)) -#define GARROW_IS_BUFFER(obj) \ - (G_TYPE_CHECK_INSTANCE_TYPE((obj), GARROW_TYPE_BUFFER)) -#define GARROW_IS_BUFFER_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_TYPE((klass), GARROW_TYPE_BUFFER)) -#define GARROW_BUFFER_GET_CLASS(obj) \ - (G_TYPE_INSTANCE_GET_CLASS((obj), GARROW_TYPE_BUFFER, GArrowBufferClass)) - -typedef struct _GArrowBuffer GArrowBuffer; -typedef struct _GArrowBufferClass GArrowBufferClass; - -/** - * GArrowBuffer: - * - * It wraps `arrow::Buffer`. - */ -struct _GArrowBuffer -{ - /*< private >*/ - GObject parent_instance; -}; - +#define GARROW_TYPE_BUFFER (garrow_buffer_get_type()) +G_DECLARE_DERIVABLE_TYPE(GArrowBuffer, + garrow_buffer, + GARROW, + BUFFER, + GObject) struct _GArrowBufferClass { GObjectClass parent_class; }; -GType garrow_buffer_get_type (void) G_GNUC_CONST; - GArrowBuffer *garrow_buffer_new (const guint8 *data, gint64 size); gboolean garrow_buffer_equal (GArrowBuffer *buffer, @@ -80,49 +57,16 @@ GArrowBuffer *garrow_buffer_slice (GArrowBuffer *buffer, gint64 size); -#define GARROW_TYPE_MUTABLE_BUFFER \ - (garrow_mutable_buffer_get_type()) -#define GARROW_MUTABLE_BUFFER(obj) \ - (G_TYPE_CHECK_INSTANCE_CAST((obj), \ - GARROW_TYPE_MUTABLE_BUFFER, \ - GArrowMutableBuffer)) -#define GARROW_MUTABLE_BUFFER_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_CAST((klass), \ - GARROW_TYPE_MUTABLE_BUFFER, \ - GArrowMutableBufferClass)) -#define GARROW_IS_MUTABLE_BUFFER(obj) \ - (G_TYPE_CHECK_INSTANCE_TYPE((obj), GARROW_TYPE_MUTABLE_BUFFER)) -#define GARROW_IS_MUTABLE_BUFFER_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_TYPE((klass), GARROW_TYPE_MUTABLE_BUFFER)) -#define GARROW_MUTABLE_BUFFER_GET_CLASS(obj) \ - (G_TYPE_INSTANCE_GET_CLASS((obj), \ - GARROW_TYPE_MUTABLE_BUFFER, \ - GArrowMutableBufferClass)) - -typedef struct _GArrowMutableBuffer GArrowMutableBuffer; -#ifndef __GTK_DOC_IGNORE__ -typedef struct _GArrowMutableBufferClass GArrowMutableBufferClass; -#endif - -/** - * GArrowMutableBuffer: - * - * It wraps `arrow::MutableBuffer`. - */ -struct _GArrowMutableBuffer -{ - /*< private >*/ - GArrowBuffer parent_instance; -}; - -#ifndef __GTK_DOC_IGNORE__ +#define GARROW_TYPE_MUTABLE_BUFFER (garrow_mutable_buffer_get_type()) +G_DECLARE_DERIVABLE_TYPE(GArrowMutableBuffer, + garrow_mutable_buffer, + GARROW, + MUTABLE_BUFFER, + GArrowBuffer) struct _GArrowMutableBufferClass { GArrowBufferClass parent_class; }; -#endif - -GType garrow_mutable_buffer_get_type(void) G_GNUC_CONST; GArrowMutableBuffer *garrow_mutable_buffer_new (guint8 *data, gint64 size); diff --git a/c_glib/arrow-glib/input-stream.h b/c_glib/arrow-glib/input-stream.h index 12c7ae7..c2068d6 100644 --- a/c_glib/arrow-glib/input-stream.h +++ b/c_glib/arrow-glib/input-stream.h @@ -26,98 +26,28 @@ G_BEGIN_DECLS -#define GARROW_TYPE_INPUT_STREAM \ - (garrow_input_stream_get_type()) -#define GARROW_INPUT_STREAM(obj) \ - (G_TYPE_CHECK_INSTANCE_CAST((obj), \ - GARROW_TYPE_INPUT_STREAM, \ - GArrowInputStream)) -#define GARROW_INPUT_STREAM_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_CAST((klass), \ - GARROW_TYPE_INPUT_STREAM, \ - GArrowInputStreamClass)) -#define GARROW_IS_INPUT_STREAM(obj) \ - (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ - GARROW_TYPE_INPUT_STREAM)) -#define GARROW_IS_INPUT_STREAM_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_TYPE((klass), \ - GARROW_TYPE_INPUT_STREAM)) -#define GARROW_INPUT_STREAM_GET_CLASS(obj) \ - (G_TYPE_INSTANCE_GET_CLASS((obj), \ - GARROW_TYPE_INPUT_STREAM, \ - GArrowInputStreamClass)) - -typedef struct _GArrowInputStream GArrowInputStream; -#ifndef __GTK_DOC_IGNORE__ -typedef struct _GArrowInputStreamClass GArrowInputStreamClass; -#endif - -/** - * GArrowInputStream: - * - * It wraps `arrow::io::InputStream`. - */ -struct _GArrowInputStream -{ - /*< private >*/ - GObject parent_instance; -}; - -#ifndef __GTK_DOC_IGNORE__ +#define GARROW_TYPE_INPUT_STREAM (garrow_input_stream_get_type()) +G_DECLARE_DERIVABLE_TYPE(GArrowInputStream, + garrow_input_stream, + GARROW, + INPUT_STREAM, + GObject) struct _GArrowInputStreamClass { GObjectClass parent_class; }; -#endif - -GType garrow_input_stream_get_type(void) G_GNUC_CONST; - #define GARROW_TYPE_SEEKABLE_INPUT_STREAM \ (garrow_seekable_input_stream_get_type()) -#define GARROW_SEEKABLE_INPUT_STREAM(obj) \ - (G_TYPE_CHECK_INSTANCE_CAST((obj), \ - GARROW_TYPE_SEEKABLE_INPUT_STREAM, \ - GArrowSeekableInputStream)) -#define GARROW_SEEKABLE_INPUT_STREAM_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_CAST((klass), \ - GARROW_TYPE_SEEKABLE_INPUT_STREAM, \ - GArrowSeekableInputStreamClass)) -#define GARROW_IS_SEEKABLE_INPUT_STREAM(obj) \ - (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ - GARROW_TYPE_SEEKABLE_INPUT_STREAM)) -#define GARROW_IS_SEEKABLE_INPUT_STREAM_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_TYPE((klass), \ - GARROW_TYPE_SEEKABLE_INPUT_STREAM)) -#define GARROW_SEEKABLE_INPUT_STREAM_GET_CLASS(obj) \ - (G_TYPE_INSTANCE_GET_CLASS((obj), \ - GARROW_TYPE_SEEKABLE_INPUT_STREAM, \ - GArrowSeekableInputStreamClass)) - -typedef struct _GArrowSeekableInputStream GArrowSeekableInputStream; -#ifndef __GTK_DOC_IGNORE__ -typedef struct _GArrowSeekableInputStreamClass GArrowSeekableInputStreamClass; -#endif - -/** - * GArrowSeekableInputStream: - * - * It wraps `arrow::io::RandomAccessFile`. - */ -struct _GArrowSeekableInputStream -{ - /*< private >*/ - GArrowInputStream parent_instance; -}; - -#ifndef __GTK_DOC_IGNORE__ +G_DECLARE_DERIVABLE_TYPE(GArrowSeekableInputStream, + garrow_seekable_input_stream, + GARROW, + SEEKABLE_INPUT_STREAM, + GArrowInputStream) struct _GArrowSeekableInputStreamClass { GArrowInputStreamClass parent_class; }; -#endif - -GType garrow_seekable_input_stream_get_type(void) G_GNUC_CONST; guint64 garrow_seekable_input_stream_get_size(GArrowSeekableInputStream *input_stream, GError **error); @@ -133,49 +63,15 @@ GArrowTensor *garrow_seekable_input_stream_read_tensor(GArrowSeekableInputStream #define GARROW_TYPE_BUFFER_INPUT_STREAM \ (garrow_buffer_input_stream_get_type()) -#define GARROW_BUFFER_INPUT_STREAM(obj) \ - (G_TYPE_CHECK_INSTANCE_CAST((obj), \ - GARROW_TYPE_BUFFER_INPUT_STREAM, \ - GArrowBufferInputStream)) -#define GARROW_BUFFER_INPUT_STREAM_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_CAST((klass), \ - GARROW_TYPE_BUFFER_INPUT_STREAM, \ - GArrowBufferInputStreamClass)) -#define GARROW_IS_BUFFER_INPUT_STREAM(obj) \ - (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ - GARROW_TYPE_BUFFER_INPUT_STREAM)) -#define GARROW_IS_BUFFER_INPUT_STREAM_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_TYPE((klass), \ - GARROW_TYPE_BUFFER_INPUT_STREAM)) -#define GARROW_BUFFER_INPUT_STREAM_GET_CLASS(obj) \ - (G_TYPE_INSTANCE_GET_CLASS((obj), \ - GARROW_TYPE_BUFFER_INPUT_STREAM, \ - GArrowBufferInputStreamClass)) - -typedef struct _GArrowBufferInputStream GArrowBufferInputStream; -#ifndef __GTK_DOC_IGNORE__ -typedef struct _GArrowBufferInputStreamClass GArrowBufferInputStreamClass; -#endif - -/** - * GArrowBufferInputStream: - * - * It wraps `arrow::io::BufferReader`. - */ -struct _GArrowBufferInputStream -{ - /*< private >*/ - GArrowSeekableInputStream parent_instance; -}; - -#ifndef __GTK_DOC_IGNORE__ +G_DECLARE_DERIVABLE_TYPE(GArrowBufferInputStream, + garrow_buffer_input_stream, + GARROW, + BUFFER_INPUT_STREAM, + GArrowSeekableInputStream) struct _GArrowBufferInputStreamClass { GArrowSeekableInputStreamClass parent_class; }; -#endif - -GType garrow_buffer_input_stream_get_type(void) G_GNUC_CONST; GArrowBufferInputStream *garrow_buffer_input_stream_new(GArrowBuffer *buffer); diff --git a/c_glib/arrow-glib/meson.build b/c_glib/arrow-glib/meson.build index 464a002..aeec417 100644 --- a/c_glib/arrow-glib/meson.build +++ b/c_glib/arrow-glib/meson.build @@ -179,22 +179,23 @@ pkgconfig.generate(filebase: meson.project_name(), name: 'Apache Arrow GLib', description: 'C API for Apache Arrow based on GLib', version: version, - requires: ['gobject-2.0', 'arrow'], + requires: ['gio-2.0', 'arrow'], libraries: [libarrow_glib], subdirs: ['arrow-glib']) -gnome.generate_gir(libarrow_glib, - sources: sources + c_headers + enums, - namespace: 'Arrow', - nsversion: api_version, - identifier_prefix: 'GArrow', - symbol_prefix: 'garrow', - export_packages: 'arrow-glib', - includes: [ - 'GObject-2.0', - 'Gio-2.0', - ], - install: true, - extra_args: [ - '--warn-all', - ]) +arrow_glib_gir = gnome.generate_gir(libarrow_glib, + sources: sources + c_headers + enums, + namespace: 'Arrow', + nsversion: api_version, + identifier_prefix: 'GArrow', + symbol_prefix: 'garrow', + export_packages: 'arrow-glib', + includes: [ + 'GObject-2.0', + 'Gio-2.0', + ], + install: true, + extra_args: [ + '--warn-all', + ]) +arrow_glib_gir_dependency = declare_dependency(sources: arrow_glib_gir) diff --git a/c_glib/arrow-glib/output-stream.h b/c_glib/arrow-glib/output-stream.h index e42ebcd..195a97a 100644 --- a/c_glib/arrow-glib/output-stream.h +++ b/c_glib/arrow-glib/output-stream.h @@ -26,51 +26,16 @@ G_BEGIN_DECLS -#define GARROW_TYPE_OUTPUT_STREAM \ - (garrow_output_stream_get_type()) -#define GARROW_OUTPUT_STREAM(obj) \ - (G_TYPE_CHECK_INSTANCE_CAST((obj), \ - GARROW_TYPE_OUTPUT_STREAM, \ - GArrowOutputStream)) -#define GARROW_OUTPUT_STREAM_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_CAST((klass), \ - GARROW_TYPE_OUTPUT_STREAM, \ - GArrowOutputStreamClass)) -#define GARROW_IS_OUTPUT_STREAM(obj) \ - (G_TYPE_CHECK_INSTANCE_TYPE((obj), \ - GARROW_TYPE_OUTPUT_STREAM)) -#define GARROW_IS_OUTPUT_STREAM_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_TYPE((klass), \ - GARROW_TYPE_OUTPUT_STREAM)) -#define GARROW_OUTPUT_STREAM_GET_CLASS(obj) \ - (G_TYPE_INSTANCE_GET_CLASS((obj), \ - GARROW_TYPE_OUTPUT_STREAM, \ - GArrowOutputStreamClass)) - -typedef struct _GArrowOutputStream GArrowOutputStream; -#ifndef __GTK_DOC_IGNORE__ -typedef struct _GArrowOutputStreamClass GArrowOutputStreamClass; -#endif - -/** - * GArrowOutputStream: - * - * It wraps `arrow::io::OutputStream`. - */ -struct _GArrowOutputStream -{ - /*< private >*/ - GObject parent_instance; -}; - -#ifndef __GTK_DOC_IGNORE__ +#define GARROW_TYPE_OUTPUT_STREAM (garrow_output_stream_get_type()) +G_DECLARE_DERIVABLE_TYPE(GArrowOutputStream, + garrow_output_stream, + GARROW, + OUTPUT_STREAM, + GObject) struct _GArrowOutputStreamClass { GObjectClass parent_class; }; -#endif - -GType garrow_output_stream_get_type(void) G_GNUC_CONST; gint64 garrow_output_stream_write_tensor(GArrowOutputStream *stream, GArrowTensor *tensor, diff --git a/c_glib/arrow-glib/readable.cpp b/c_glib/arrow-glib/readable.cpp index 6a9023e..33f98d9 100644 --- a/c_glib/arrow-glib/readable.cpp +++ b/c_glib/arrow-glib/readable.cpp @@ -45,6 +45,7 @@ G_DEFINE_INTERFACE(GArrowReadable, static void garrow_readable_default_init (GArrowReadableInterface *iface) { + iface->new_raw = garrow_buffer_new_raw; } /** @@ -66,7 +67,8 @@ garrow_readable_read(GArrowReadable *readable, std::shared_ptr arrow_buffer; auto status = arrow_readable->Read(n_bytes, &arrow_buffer); if (garrow_error_check(error, status, "[io][readable][read]")) { - return garrow_buffer_new_raw(&arrow_buffer); + auto *iface = GARROW_READABLE_GET_IFACE(readable); + return iface->new_raw(&arrow_buffer); } else { return NULL; } diff --git a/c_glib/arrow-glib/readable.hpp b/c_glib/arrow-glib/readable.hpp index c241c77..ce77701 100644 --- a/c_glib/arrow-glib/readable.hpp +++ b/c_glib/arrow-glib/readable.hpp @@ -32,6 +32,7 @@ struct _GArrowReadableInterface { GTypeInterface parent_iface; + GArrowBuffer *(*new_raw)(std::shared_ptr *arrow_buffer); std::shared_ptr (*get_raw)(GArrowReadable *file); }; diff --git a/c_glib/arrow-gpu-glib/Makefile.am b/c_glib/arrow-gpu-glib/Makefile.am new file mode 100644 index 0000000..ec96159 --- /dev/null +++ b/c_glib/arrow-gpu-glib/Makefile.am @@ -0,0 +1,109 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +CLEANFILES = + +EXTRA_DIST = \ + meson.build + +AM_CPPFLAGS = \ + -I$(top_builddir) \ + -I$(top_srcdir) + +if HAVE_ARROW_GPU +lib_LTLIBRARIES = \ + libarrow-gpu-glib.la + +libarrow_gpu_glib_la_CXXFLAGS = \ + $(GLIB_CFLAGS) \ + $(ARROW_CFLAGS) \ + $(ARROW_GPU_CFLAGS) \ + $(GARROW_CXXFLAGS) + +libarrow_gpu_glib_la_LIBADD = \ + $(GLIB_LIBS) \ + $(ARROW_LIBS) \ + $(ARROW_GPU_LIBS) \ + ../arrow-glib/libarrow-glib.la + +libarrow_gpu_glib_la_headers = \ + arrow-gpu-glib.h \ + cuda.h + +libarrow_gpu_glib_la_sources = \ + cuda.cpp \ + $(libarrow_gpu_glib_la_headers) + +libarrow_gpu_glib_la_cpp_headers = \ + arrow-gpu-glib.hpp \ + cuda.hpp + +libarrow_gpu_glib_la_SOURCES = \ + $(libarrow_gpu_glib_la_sources) \ + $(libarrow_gpu_glib_la_cpp_headers) + +arrow_gpu_glib_includedir = \ + $(includedir)/arrow-gpu-glib +arrow_gpu_glib_include_HEADERS = \ + $(libarrow_gpu_glib_la_headers) \ + $(libarrow_gpu_glib_la_cpp_headers) + +pkgconfigdir = $(libdir)/pkgconfig +pkgconfig_DATA = \ + arrow-gpu-glib.pc + +if HAVE_INTROSPECTION +-include $(INTROSPECTION_MAKEFILE) +INTROSPECTION_GIRS = +INTROSPECTION_SCANNER_ARGS = +INTROSPECTION_SCANNER_ENV = \ + PKG_CONFIG_PATH=${abs_builddir}/../arrow-glib:$${PKG_CONFIG_PATH} +INTROSPECTION_COMPILER_ARGS = \ + --includedir=$(abs_builddir)/../arrow-glib + +ArrowGPU-1.0.gir: libarrow-gpu-glib.la +ArrowGPU_1_0_gir_PACKAGES = \ + arrow-glib +ArrowGPU_1_0_gir_EXPORT_PACKAGES = \ + arrow-gpu-glib +ArrowGPU_1_0_gir_INCLUDES = \ + Arrow-1.0 +ArrowGPU_1_0_gir_CFLAGS = \ + $(AM_CPPFLAGS) +ArrowGPU_1_0_gir_LIBS = \ + $(abs_builddir)/../arrow-glib/libarrow-glib.la \ + libarrow-gpu-glib.la +ArrowGPU_1_0_gir_FILES = \ + $(libarrow_gpu_glib_la_sources) +ArrowGPU_1_0_gir_SCANNERFLAGS = \ + --warn-all \ + --add-include-path=$(abs_builddir)/../arrow-glib \ + --identifier-prefix=GArrowGPU \ + --symbol-prefix=garrow_gpu +INTROSPECTION_GIRS += ArrowGPU-1.0.gir + +girdir = $(datadir)/gir-1.0 +gir_DATA = $(INTROSPECTION_GIRS) + +typelibdir = $(libdir)/girepository-1.0 +typelib_DATA = $(INTROSPECTION_GIRS:.gir=.typelib) + +CLEANFILES += \ + $(gir_DATA) \ + $(typelib_DATA) +endif +endif diff --git a/c_glib/arrow-glib/readable.hpp b/c_glib/arrow-gpu-glib/arrow-gpu-glib.h similarity index 68% copy from c_glib/arrow-glib/readable.hpp copy to c_glib/arrow-gpu-glib/arrow-gpu-glib.h index c241c77..1538c9a 100644 --- a/c_glib/arrow-glib/readable.hpp +++ b/c_glib/arrow-gpu-glib/arrow-gpu-glib.h @@ -19,20 +19,6 @@ #pragma once -#include +#include -#include - -/** - * GArrowReadableInterface: - * - * It wraps `arrow::io::Readable`. - */ -struct _GArrowReadableInterface -{ - GTypeInterface parent_iface; - - std::shared_ptr (*get_raw)(GArrowReadable *file); -}; - -std::shared_ptr garrow_readable_get_raw(GArrowReadable *readable); +#include diff --git a/c_glib/arrow-glib/readable.hpp b/c_glib/arrow-gpu-glib/arrow-gpu-glib.hpp similarity index 68% copy from c_glib/arrow-glib/readable.hpp copy to c_glib/arrow-gpu-glib/arrow-gpu-glib.hpp index c241c77..92017d8 100644 --- a/c_glib/arrow-glib/readable.hpp +++ b/c_glib/arrow-gpu-glib/arrow-gpu-glib.hpp @@ -19,20 +19,6 @@ #pragma once -#include +#include -#include - -/** - * GArrowReadableInterface: - * - * It wraps `arrow::io::Readable`. - */ -struct _GArrowReadableInterface -{ - GTypeInterface parent_iface; - - std::shared_ptr (*get_raw)(GArrowReadable *file); -}; - -std::shared_ptr garrow_readable_get_raw(GArrowReadable *readable); +#include diff --git a/c_glib/doc/Makefile.am b/c_glib/arrow-gpu-glib/arrow-gpu-glib.pc.in similarity index 74% copy from c_glib/doc/Makefile.am copy to c_glib/arrow-gpu-glib/arrow-gpu-glib.pc.in index 85c1d51..38a6bae 100644 --- a/c_glib/doc/Makefile.am +++ b/c_glib/arrow-gpu-glib/arrow-gpu-glib.pc.in @@ -15,5 +15,14 @@ # specific language governing permissions and limitations # under the License. -SUBDIRS = \ - reference +prefix=@prefix@ +exec_prefix=@exec_prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: Apache Arrow GPU GLib +Description: C API for Apache Arrow GPU based on GLib +Version: @VERSION@ +Libs: -L${libdir} -larrow-gpu-glib +Cflags: -I${includedir} +Requires: arrow-glib diff --git a/c_glib/arrow-gpu-glib/cuda.cpp b/c_glib/arrow-gpu-glib/cuda.cpp new file mode 100644 index 0000000..c2a9af5 --- /dev/null +++ b/c_glib/arrow-gpu-glib/cuda.cpp @@ -0,0 +1,941 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifdef HAVE_CONFIG_H +# include +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include + +G_BEGIN_DECLS + +/** + * SECTION: cuda + * @section_id: cuda-classes + * @title: CUDA related classes + * @include: arrow-gpu-glib/arrow-gpu-glib.h + * + * The following classes provide CUDA support for Apache Arrow data. + * + * #GArrowGPUCUDADeviceManager is the starting point. You need at + * least one #GArrowGPUCUDAContext to process Apache Arrow data on + * NVIDIA GPU. + * + * #GArrowGPUCUDAContext is a class to keep context for one GPU. You + * need to create #GArrowGPUCUDAContext for each GPU that you want to + * use. You can create #GArrowGPUCUDAContext by + * garrow_gpu_cuda_device_manager_get_context(). + * + * #GArrowGPUCUDABuffer is a class for data on GPU. You can copy data + * on GPU to/from CPU by garrow_gpu_cuda_buffer_copy_to_host() and + * garrow_gpu_cuda_buffer_copy_from_host(). You can share data on GPU + * with other processes by garrow_gpu_cuda_buffer_export() and + * garrow_gpu_cuda_buffer_new_ipc(). + * + * #GArrowGPUCUDAHostBuffer is a class for data on CPU that is + * directly accessible from GPU. + * + * #GArrowGPUCUDAIPCMemoryHandle is a class to share data on GPU with + * other processes. You can export your data on GPU to other processes + * by garrow_gpu_cuda_buffer_export() and + * garrow_gpu_cuda_ipc_memory_handle_new(). You can import other + * process data on GPU by garrow_gpu_cuda_ipc_memory_handle_new() and + * garrow_gpu_cuda_buffer_new_ipc(). + * + * #GArrowGPUCUDABufferInputStream is a class to read data in + * #GArrowGPUCUDABuffer. + * + * #GArrowGPUCUDABufferOutputStream is a class to write data into + * #GArrowGPUCUDABuffer. + */ + +G_DEFINE_TYPE(GArrowGPUCUDADeviceManager, + garrow_gpu_cuda_device_manager, + G_TYPE_OBJECT) + +static void +garrow_gpu_cuda_device_manager_init(GArrowGPUCUDADeviceManager *object) +{ +} + +static void +garrow_gpu_cuda_device_manager_class_init(GArrowGPUCUDADeviceManagerClass *klass) +{ +} + +/** + * garrow_gpu_cuda_device_manager_new: + * @error: (nullable): Return location for a #GError or %NULL. + * + * Returns: A newly created #GArrowGPUCUDADeviceManager on success, + * %NULL on error. + * + * Since: 0.8.0 + */ +GArrowGPUCUDADeviceManager * +garrow_gpu_cuda_device_manager_new(GError **error) +{ + arrow::gpu::CudaDeviceManager *manager; + auto status = arrow::gpu::CudaDeviceManager::GetInstance(&manager); + if (garrow_error_check(error, status, "[gpu][cuda][device-manager][new]")) { + auto manager = g_object_new(GARROW_GPU_TYPE_CUDA_DEVICE_MANAGER, + NULL); + return GARROW_GPU_CUDA_DEVICE_MANAGER(manager); + } else { + return NULL; + } +} + +/** + * garrow_gpu_cuda_device_manager_get_context: + * @manager: A #GArrowGPUCUDADeviceManager. + * @gpu_number: A GPU device number for the target context. + * @error: (nullable): Return location for a #GError or %NULL. + * + * Returns: (transfer full): A newly created #GArrowGPUCUDAContext on + * success, %NULL on error. Contexts for the same GPU device number + * share the same data internally. + * + * Since: 0.8.0 + */ +GArrowGPUCUDAContext * +garrow_gpu_cuda_device_manager_get_context(GArrowGPUCUDADeviceManager *manager, + gint gpu_number, + GError **error) +{ + arrow::gpu::CudaDeviceManager *arrow_manager; + arrow::gpu::CudaDeviceManager::GetInstance(&arrow_manager); + std::shared_ptr context; + auto status = arrow_manager->GetContext(gpu_number, &context); + if (garrow_error_check(error, status, + "[gpu][cuda][device-manager][get-context]]")) { + return garrow_gpu_cuda_context_new_raw(&context); + } else { + return NULL; + } +} + +/** + * garrow_gpu_cuda_device_manager_get_n_devices: + * @manager: A #GArrowGPUCUDADeviceManager. + * + * Returns: The number of GPU devices. + * + * Since: 0.8.0 + */ +gsize +garrow_gpu_cuda_device_manager_get_n_devices(GArrowGPUCUDADeviceManager *manager) +{ + arrow::gpu::CudaDeviceManager *arrow_manager; + arrow::gpu::CudaDeviceManager::GetInstance(&arrow_manager); + return arrow_manager->num_devices(); +} + + +typedef struct GArrowGPUCUDAContextPrivate_ { + std::shared_ptr context; +} GArrowGPUCUDAContextPrivate; + +enum { + PROP_CONTEXT = 1 +}; + +G_DEFINE_TYPE_WITH_PRIVATE(GArrowGPUCUDAContext, + garrow_gpu_cuda_context, + G_TYPE_OBJECT) + +#define GARROW_GPU_CUDA_CONTEXT_GET_PRIVATE(object) \ + static_cast( \ + garrow_gpu_cuda_context_get_instance_private( \ + GARROW_GPU_CUDA_CONTEXT(object))) + +static void +garrow_gpu_cuda_context_finalize(GObject *object) +{ + auto priv = GARROW_GPU_CUDA_CONTEXT_GET_PRIVATE(object); + + priv->context = nullptr; + + G_OBJECT_CLASS(garrow_gpu_cuda_context_parent_class)->finalize(object); +} + +static void +garrow_gpu_cuda_context_set_property(GObject *object, + guint prop_id, + const GValue *value, + GParamSpec *pspec) +{ + auto priv = GARROW_GPU_CUDA_CONTEXT_GET_PRIVATE(object); + + switch (prop_id) { + case PROP_CONTEXT: + priv->context = + *static_cast *>(g_value_get_pointer(value)); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec); + break; + } +} + +static void +garrow_gpu_cuda_context_get_property(GObject *object, + guint prop_id, + GValue *value, + GParamSpec *pspec) +{ + switch (prop_id) { + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec); + break; + } +} + +static void +garrow_gpu_cuda_context_init(GArrowGPUCUDAContext *object) +{ +} + +static void +garrow_gpu_cuda_context_class_init(GArrowGPUCUDAContextClass *klass) +{ + GParamSpec *spec; + + auto gobject_class = G_OBJECT_CLASS(klass); + + gobject_class->finalize = garrow_gpu_cuda_context_finalize; + gobject_class->set_property = garrow_gpu_cuda_context_set_property; + gobject_class->get_property = garrow_gpu_cuda_context_get_property; + + /** + * GArrowGPUCUDAContext:context: + * + * Since: 0.8.0 + */ + spec = g_param_spec_pointer("context", + "Context", + "The raw std::shared_ptr *", + static_cast(G_PARAM_WRITABLE | + G_PARAM_CONSTRUCT_ONLY)); + g_object_class_install_property(gobject_class, PROP_CONTEXT, spec); +} + +/** + * garrow_gpu_cuda_context_get_allocated_size: + * @context: A #GArrowGPUCUDAContext. + * + * Returns: The allocated memory by this context in bytes. + * + * Since: 0.8.0 + */ +gint64 +garrow_gpu_cuda_context_get_allocated_size(GArrowGPUCUDAContext *context) +{ + auto arrow_context = garrow_gpu_cuda_context_get_raw(context); + return arrow_context->bytes_allocated(); +} + + +G_DEFINE_TYPE(GArrowGPUCUDABuffer, + garrow_gpu_cuda_buffer, + GARROW_TYPE_BUFFER) + +static void +garrow_gpu_cuda_buffer_init(GArrowGPUCUDABuffer *object) +{ +} + +static void +garrow_gpu_cuda_buffer_class_init(GArrowGPUCUDABufferClass *klass) +{ +} + +/** + * garrow_gpu_cuda_buffer_new: + * @context: A #GArrowGPUCUDAContext. + * @size: The number of bytes to be allocated on GPU device for this context. + * @error: (nullable): Return location for a #GError or %NULL. + * + * Returns: (transfer full): A newly created #GArrowGPUCUDABuffer on + * success, %NULL on error. + * + * Since: 0.8.0 + */ +GArrowGPUCUDABuffer * +garrow_gpu_cuda_buffer_new(GArrowGPUCUDAContext *context, + gint64 size, + GError **error) +{ + auto arrow_context = garrow_gpu_cuda_context_get_raw(context); + std::shared_ptr arrow_buffer; + auto status = arrow_context->Allocate(size, &arrow_buffer); + if (garrow_error_check(error, status, "[gpu][cuda][buffer][new]")) { + return garrow_gpu_cuda_buffer_new_raw(&arrow_buffer); + } else { + return NULL; + } +} + +/** + * garrow_gpu_cuda_buffer_new_ipc: + * @context: A #GArrowGPUCUDAContext. + * @handle: A #GArrowGPUCUDAIPCMemoryHandle to be communicated. + * @error: (nullable): Return location for a #GError or %NULL. + * + * Returns: (transfer full): A newly created #GArrowGPUCUDABuffer on + * success, %NULL on error. The buffer has data from the IPC target. + * + * Since: 0.8.0 + */ +GArrowGPUCUDABuffer * +garrow_gpu_cuda_buffer_new_ipc(GArrowGPUCUDAContext *context, + GArrowGPUCUDAIPCMemoryHandle *handle, + GError **error) +{ + auto arrow_context = garrow_gpu_cuda_context_get_raw(context); + auto arrow_handle = garrow_gpu_cuda_ipc_memory_handle_get_raw(handle); + std::shared_ptr arrow_buffer; + auto status = arrow_context->OpenIpcBuffer(*arrow_handle, &arrow_buffer); + if (garrow_error_check(error, status, + "[gpu][cuda][buffer][new-ipc]")) { + return garrow_gpu_cuda_buffer_new_raw(&arrow_buffer); + } else { + return NULL; + } +} + +/** + * garrow_gpu_cuda_buffer_new_record_batch: + * @context: A #GArrowGPUCUDAContext. + * @record_batch: A #GArrowRecordBatch to be serialized. + * @error: (nullable): Return location for a #GError or %NULL. + * + * Returns: (transfer full): A newly created #GArrowGPUCUDABuffer on + * success, %NULL on error. The buffer has serialized record batch + * data. + * + * Since: 0.8.0 + */ +GArrowGPUCUDABuffer * +garrow_gpu_cuda_buffer_new_record_batch(GArrowGPUCUDAContext *context, + GArrowRecordBatch *record_batch, + GError **error) +{ + auto arrow_context = garrow_gpu_cuda_context_get_raw(context); + auto arrow_record_batch = garrow_record_batch_get_raw(record_batch); + std::shared_ptr arrow_buffer; + auto status = arrow::gpu::SerializeRecordBatch(*arrow_record_batch, + arrow_context.get(), + &arrow_buffer); + if (garrow_error_check(error, status, + "[gpu][cuda][buffer][new-record-batch]")) { + return garrow_gpu_cuda_buffer_new_raw(&arrow_buffer); + } else { + return NULL; + } +} + +/** + * garrow_gpu_cuda_buffer_copy_to_host: + * @buffer: A #GArrowGPUCUDABuffer. + * @position: The offset of memory on GPU device to be copied. + * @size: The size of memory on GPU device to be copied in bytes. + * @error: (nullable): Return location for a #GError or %NULL. + * + * Returns: (transfer full): A #GBytes that have copied memory on CPU + * host on success, %NULL on error. + * + * Since: 0.8.0 + */ +GBytes * +garrow_gpu_cuda_buffer_copy_to_host(GArrowGPUCUDABuffer *buffer, + gint64 position, + gint64 size, + GError **error) +{ + auto arrow_buffer = garrow_gpu_cuda_buffer_get_raw(buffer); + auto data = static_cast(g_malloc(size)); + auto status = arrow_buffer->CopyToHost(position, size, data); + if (garrow_error_check(error, status, "[gpu][cuda][buffer][copy-to-host]")) { + return g_bytes_new_take(data, size); + } else { + g_free(data); + return NULL; + } +} + +/** + * garrow_gpu_cuda_buffer_copy_from_host: + * @buffer: A #GArrowGPUCUDABuffer. + * @data: (array length=size): Data on CPU host to be copied. + * @size: The size of data on CPU host to be copied in bytes. + * @error: (nullable): Return location for a #GError or %NULL. + * + * Returns: %TRUE on success, %FALSE if there was an error. + * + * Since: 0.8.0 + */ +gboolean +garrow_gpu_cuda_buffer_copy_from_host(GArrowGPUCUDABuffer *buffer, + const guint8 *data, + gint64 size, + GError **error) +{ + auto arrow_buffer = garrow_gpu_cuda_buffer_get_raw(buffer); + auto status = arrow_buffer->CopyFromHost(0, data, size); + return garrow_error_check(error, + status, + "[gpu][cuda][buffer][copy-from-host]"); +} + +/** + * garrow_gpu_cuda_buffer_export: + * @buffer: A #GArrowGPUCUDABuffer. + * @error: (nullable): Return location for a #GError or %NULL. + * + * Returns: (transfer full): A newly created + * #GArrowGPUCUDAIPCMemoryHandle to handle the exported buffer on + * success, %NULL on error + * + * Since: 0.8.0 + */ +GArrowGPUCUDAIPCMemoryHandle * +garrow_gpu_cuda_buffer_export(GArrowGPUCUDABuffer *buffer, GError **error) +{ + auto arrow_buffer = garrow_gpu_cuda_buffer_get_raw(buffer); + std::unique_ptr arrow_handle; + auto status = arrow_buffer->ExportForIpc(&arrow_handle); + if (garrow_error_check(error, status, "[gpu][cuda][buffer][export-for-ipc]")) { + return garrow_gpu_cuda_ipc_memory_handle_new_raw(arrow_handle.release()); + } else { + return NULL; + } +} + +/** + * garrow_gpu_cuda_buffer_get_context: + * @buffer: A #GArrowGPUCUDABuffer. + * + * Returns: (transfer full): A newly created #GArrowGPUCUDAContext for the + * buffer. Contexts for the same buffer share the same data internally. + * + * Since: 0.8.0 + */ +GArrowGPUCUDAContext * +garrow_gpu_cuda_buffer_get_context(GArrowGPUCUDABuffer *buffer) +{ + auto arrow_buffer = garrow_gpu_cuda_buffer_get_raw(buffer); + auto arrow_context = arrow_buffer->context(); + return garrow_gpu_cuda_context_new_raw(&arrow_context); +} + +/** + * garrow_gpu_cuda_buffer_read_record_batch: + * @buffer: A #GArrowGPUCUDABuffer. + * @schema: A #GArrowSchema for record batch. + * @error: (nullable): Return location for a #GError or %NULL. + * + * Returns: (transfer full): A newly created #GArrowRecordBatch on + * success, %NULL on error. The record batch data is located on GPU. + * + * Since: 0.8.0 + */ +GArrowRecordBatch * +garrow_gpu_cuda_buffer_read_record_batch(GArrowGPUCUDABuffer *buffer, + GArrowSchema *schema, + GError **error) +{ + auto arrow_buffer = garrow_gpu_cuda_buffer_get_raw(buffer); + auto arrow_schema = garrow_schema_get_raw(schema); + auto pool = arrow::default_memory_pool(); + std::shared_ptr arrow_record_batch; + auto status = arrow::gpu::ReadRecordBatch(arrow_schema, + arrow_buffer, + pool, + &arrow_record_batch); + if (garrow_error_check(error, status, + "[gpu][cuda][buffer][read-record-batch]")) { + return garrow_record_batch_new_raw(&arrow_record_batch); + } else { + return NULL; + } +} + + +G_DEFINE_TYPE(GArrowGPUCUDAHostBuffer, + garrow_gpu_cuda_host_buffer, + GARROW_TYPE_MUTABLE_BUFFER) + +static void +garrow_gpu_cuda_host_buffer_init(GArrowGPUCUDAHostBuffer *object) +{ +} + +static void +garrow_gpu_cuda_host_buffer_class_init(GArrowGPUCUDAHostBufferClass *klass) +{ +} + +/** + * garrow_gpu_cuda_host_buffer_new: + * @size: The number of bytes to be allocated on CPU host. + * @error: (nullable): Return location for a #GError or %NULL. + * + * Returns: A newly created #GArrowGPUCUDAHostBuffer on success, + * %NULL on error. The allocated memory is accessible from GPU + * device for the @context. + * + * Since: 0.8.0 + */ +GArrowGPUCUDAHostBuffer * +garrow_gpu_cuda_host_buffer_new(gint64 size, GError **error) +{ + arrow::gpu::CudaDeviceManager *manager; + auto status = arrow::gpu::CudaDeviceManager::GetInstance(&manager); + std::shared_ptr arrow_buffer; + status = manager->AllocateHost(size, &arrow_buffer); + if (garrow_error_check(error, status, "[gpu][cuda][host-buffer][new]")) { + return garrow_gpu_cuda_host_buffer_new_raw(&arrow_buffer); + } else { + return NULL; + } +} + + +typedef struct GArrowGPUCUDAIPCMemoryHandlePrivate_ { + arrow::gpu::CudaIpcMemHandle *ipc_memory_handle; +} GArrowGPUCUDAIPCMemoryHandlePrivate; + +enum { + PROP_IPC_MEMORY_HANDLE = 1 +}; + +G_DEFINE_TYPE_WITH_PRIVATE(GArrowGPUCUDAIPCMemoryHandle, + garrow_gpu_cuda_ipc_memory_handle, + G_TYPE_OBJECT) + +#define GARROW_GPU_CUDA_IPC_MEMORY_HANDLE_GET_PRIVATE(object) \ + static_cast( \ + garrow_gpu_cuda_ipc_memory_handle_get_instance_private( \ + GARROW_GPU_CUDA_IPC_MEMORY_HANDLE(object))) + +static void +garrow_gpu_cuda_ipc_memory_handle_finalize(GObject *object) +{ + auto priv = GARROW_GPU_CUDA_IPC_MEMORY_HANDLE_GET_PRIVATE(object); + + delete priv->ipc_memory_handle; + + G_OBJECT_CLASS(garrow_gpu_cuda_ipc_memory_handle_parent_class)->finalize(object); +} + +static void +garrow_gpu_cuda_ipc_memory_handle_set_property(GObject *object, + guint prop_id, + const GValue *value, + GParamSpec *pspec) +{ + auto priv = GARROW_GPU_CUDA_IPC_MEMORY_HANDLE_GET_PRIVATE(object); + + switch (prop_id) { + case PROP_IPC_MEMORY_HANDLE: + priv->ipc_memory_handle = + static_cast(g_value_get_pointer(value)); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec); + break; + } +} + +static void +garrow_gpu_cuda_ipc_memory_handle_get_property(GObject *object, + guint prop_id, + GValue *value, + GParamSpec *pspec) +{ + switch (prop_id) { + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec); + break; + } +} + +static void +garrow_gpu_cuda_ipc_memory_handle_init(GArrowGPUCUDAIPCMemoryHandle *object) +{ +} + +static void +garrow_gpu_cuda_ipc_memory_handle_class_init(GArrowGPUCUDAIPCMemoryHandleClass *klass) +{ + GParamSpec *spec; + + auto gobject_class = G_OBJECT_CLASS(klass); + + gobject_class->finalize = garrow_gpu_cuda_ipc_memory_handle_finalize; + gobject_class->set_property = garrow_gpu_cuda_ipc_memory_handle_set_property; + gobject_class->get_property = garrow_gpu_cuda_ipc_memory_handle_get_property; + + /** + * GArrowGPUCUDAIPCMemoryHandle:ipc-memory-handle: + * + * Since: 0.8.0 + */ + spec = g_param_spec_pointer("ipc-memory-handle", + "IPC Memory Handle", + "The raw arrow::gpu::CudaIpcMemHandle *", + static_cast(G_PARAM_WRITABLE | + G_PARAM_CONSTRUCT_ONLY)); + g_object_class_install_property(gobject_class, PROP_IPC_MEMORY_HANDLE, spec); +} + +/** + * garrow_gpu_cuda_ipc_memory_handle_new: + * @data: (array length=size): A serialized #GArrowGPUCUDAIPCMemoryHandle. + * @size: The size of data. + * @error: (nullable): Return location for a #GError or %NULL. + * + * Returns: (transfer full): A newly created #GArrowGPUCUDAIPCMemoryHandle + * on success, %NULL on error. + * + * Since: 0.8.0 + */ +GArrowGPUCUDAIPCMemoryHandle * +garrow_gpu_cuda_ipc_memory_handle_new(const guint8 *data, + gsize size, + GError **error) +{ + std::unique_ptr arrow_handle; + auto status = arrow::gpu::CudaIpcMemHandle::FromBuffer(data, &arrow_handle); + if (garrow_error_check(error, status, + "[gpu][cuda][ipc-memory-handle][new]")) { + return garrow_gpu_cuda_ipc_memory_handle_new_raw(arrow_handle.release()); + } else { + return NULL; + } +} + +/** + * garrow_gpu_cuda_ipc_memory_handle_serialize: + * @handle: A #GArrowGPUCUDAIPCMemoryHandle. + * @error: (nullable): Return location for a #GError or %NULL. + * + * Returns: (transfer full): A newly created #GArrowBuffer on success, + * %NULL on error. The buffer has serialized @handle. The serialized + * @handle can be deserialized by garrow_gpu_cuda_ipc_memory_handle_new() + * in other process. + * + * Since: 0.8.0 + */ +GArrowBuffer * +garrow_gpu_cuda_ipc_memory_handle_serialize(GArrowGPUCUDAIPCMemoryHandle *handle, + GError **error) +{ + auto arrow_handle = garrow_gpu_cuda_ipc_memory_handle_get_raw(handle); + std::shared_ptr arrow_buffer; + auto status = arrow_handle->Serialize(arrow::default_memory_pool(), + &arrow_buffer); + if (garrow_error_check(error, status, + "[gpu][cuda][ipc-memory-handle][serialize]")) { + return garrow_buffer_new_raw(&arrow_buffer); + } else { + return NULL; + } +} + +GArrowBuffer * +garrow_gpu_cuda_buffer_input_stream_new_raw_readable_interface(std::shared_ptr *arrow_buffer) +{ + auto buffer = GARROW_BUFFER(g_object_new(GARROW_GPU_TYPE_CUDA_BUFFER, + "buffer", arrow_buffer, + NULL)); + return buffer; +} + +static std::shared_ptr +garrow_gpu_cuda_buffer_input_stream_get_raw_readable_interface(GArrowReadable *readable) +{ + auto input_stream = GARROW_INPUT_STREAM(readable); + auto arrow_input_stream = garrow_input_stream_get_raw(input_stream); + return arrow_input_stream; +} + +static void +garrow_gpu_cuda_buffer_input_stream_readable_interface_init(GArrowReadableInterface *iface) +{ + iface->new_raw = + garrow_gpu_cuda_buffer_input_stream_new_raw_readable_interface; + iface->get_raw = + garrow_gpu_cuda_buffer_input_stream_get_raw_readable_interface; +} + +G_DEFINE_TYPE_WITH_CODE( + GArrowGPUCUDABufferInputStream, + garrow_gpu_cuda_buffer_input_stream, + GARROW_TYPE_BUFFER_INPUT_STREAM, + G_IMPLEMENT_INTERFACE( + GARROW_TYPE_READABLE, + garrow_gpu_cuda_buffer_input_stream_readable_interface_init)) + +static void +garrow_gpu_cuda_buffer_input_stream_init(GArrowGPUCUDABufferInputStream *object) +{ +} + +static void +garrow_gpu_cuda_buffer_input_stream_class_init(GArrowGPUCUDABufferInputStreamClass *klass) +{ +} + +/** + * garrow_gpu_cuda_buffer_input_stream_new: + * @buffer: A #GArrowGPUCUDABuffer. + * + * Returns: (transfer full): A newly created + * #GArrowGPUCUDABufferInputStream. + * + * Since: 0.8.0 + */ +GArrowGPUCUDABufferInputStream * +garrow_gpu_cuda_buffer_input_stream_new(GArrowGPUCUDABuffer *buffer) +{ + auto arrow_buffer = garrow_gpu_cuda_buffer_get_raw(buffer); + auto arrow_reader = + std::make_shared(arrow_buffer); + return garrow_gpu_cuda_buffer_input_stream_new_raw(&arrow_reader); +} + + +G_DEFINE_TYPE(GArrowGPUCUDABufferOutputStream, + garrow_gpu_cuda_buffer_output_stream, + GARROW_TYPE_OUTPUT_STREAM) + +static void +garrow_gpu_cuda_buffer_output_stream_init(GArrowGPUCUDABufferOutputStream *object) +{ +} + +static void +garrow_gpu_cuda_buffer_output_stream_class_init(GArrowGPUCUDABufferOutputStreamClass *klass) +{ +} + +/** + * garrow_gpu_cuda_buffer_output_stream_new: + * @buffer: A #GArrowGPUCUDABuffer. + * + * Returns: (transfer full): A newly created + * #GArrowGPUCUDABufferOutputStream. + * + * Since: 0.8.0 + */ +GArrowGPUCUDABufferOutputStream * +garrow_gpu_cuda_buffer_output_stream_new(GArrowGPUCUDABuffer *buffer) +{ + auto arrow_buffer = garrow_gpu_cuda_buffer_get_raw(buffer); + auto arrow_writer = + std::make_shared(arrow_buffer); + return garrow_gpu_cuda_buffer_output_stream_new_raw(&arrow_writer); +} + +/** + * garrow_gpu_cuda_buffer_output_stream_set_buffer_size: + * @stream: A #GArrowGPUCUDABufferOutputStream. + * @size: A size of CPU buffer in bytes. + * @error: (nullable): Return location for a #GError or %NULL. + * + * Returns: %TRUE on success, %FALSE if there was an error. + * + * Sets CPU buffer size. to limit `cudaMemcpy()` calls. If CPU buffer + * size is `0`, buffering is disabled. + * + * The default is `0`. + * + * Since: 0.8.0 + */ +gboolean +garrow_gpu_cuda_buffer_output_stream_set_buffer_size(GArrowGPUCUDABufferOutputStream *stream, + gint64 size, + GError **error) +{ + auto arrow_stream = garrow_gpu_cuda_buffer_output_stream_get_raw(stream); + auto status = arrow_stream->SetBufferSize(size); + return garrow_error_check(error, + status, + "[gpu][cuda][buffer-output-stream][set-buffer-size]"); +} + +/** + * garrow_gpu_cuda_buffer_output_stream_get_buffer_size: + * @stream: A #GArrowGPUCUDABufferOutputStream. + * + * Returns: The CPU buffer size in bytes. + * + * See garrow_gpu_cuda_buffer_output_stream_set_buffer_size() for CPU + * buffer size details. + * + * Since: 0.8.0 + */ +gint64 +garrow_gpu_cuda_buffer_output_stream_get_buffer_size(GArrowGPUCUDABufferOutputStream *stream) +{ + auto arrow_stream = garrow_gpu_cuda_buffer_output_stream_get_raw(stream); + return arrow_stream->buffer_size(); +} + +/** + * garrow_gpu_cuda_buffer_output_stream_get_buffered_size: + * @stream: A #GArrowGPUCUDABufferOutputStream. + * + * Returns: The size of buffered data in bytes. + * + * Since: 0.8.0 + */ +gint64 +garrow_gpu_cuda_buffer_output_stream_get_buffered_size(GArrowGPUCUDABufferOutputStream *stream) +{ + auto arrow_stream = garrow_gpu_cuda_buffer_output_stream_get_raw(stream); + return arrow_stream->num_bytes_buffered(); +} + + +G_END_DECLS + +GArrowGPUCUDAContext * +garrow_gpu_cuda_context_new_raw(std::shared_ptr *arrow_context) +{ + return GARROW_GPU_CUDA_CONTEXT(g_object_new(GARROW_GPU_TYPE_CUDA_CONTEXT, + "context", arrow_context, + NULL)); +} + +std::shared_ptr +garrow_gpu_cuda_context_get_raw(GArrowGPUCUDAContext *context) +{ + if (!context) + return nullptr; + + auto priv = GARROW_GPU_CUDA_CONTEXT_GET_PRIVATE(context); + return priv->context; +} + +GArrowGPUCUDAIPCMemoryHandle * +garrow_gpu_cuda_ipc_memory_handle_new_raw(arrow::gpu::CudaIpcMemHandle *arrow_handle) +{ + auto handle = g_object_new(GARROW_GPU_TYPE_CUDA_IPC_MEMORY_HANDLE, + "ipc-memory-handle", arrow_handle, + NULL); + return GARROW_GPU_CUDA_IPC_MEMORY_HANDLE(handle); +} + +arrow::gpu::CudaIpcMemHandle * +garrow_gpu_cuda_ipc_memory_handle_get_raw(GArrowGPUCUDAIPCMemoryHandle *handle) +{ + if (!handle) + return nullptr; + + auto priv = GARROW_GPU_CUDA_IPC_MEMORY_HANDLE_GET_PRIVATE(handle); + return priv->ipc_memory_handle; +} + +GArrowGPUCUDABuffer * +garrow_gpu_cuda_buffer_new_raw(std::shared_ptr *arrow_buffer) +{ + return GARROW_GPU_CUDA_BUFFER(g_object_new(GARROW_GPU_TYPE_CUDA_BUFFER, + "buffer", arrow_buffer, + NULL)); +} + +std::shared_ptr +garrow_gpu_cuda_buffer_get_raw(GArrowGPUCUDABuffer *buffer) +{ + if (!buffer) + return nullptr; + + auto arrow_buffer = garrow_buffer_get_raw(GARROW_BUFFER(buffer)); + return std::static_pointer_cast(arrow_buffer); +} + +GArrowGPUCUDAHostBuffer * +garrow_gpu_cuda_host_buffer_new_raw(std::shared_ptr *arrow_buffer) +{ + auto buffer = g_object_new(GARROW_GPU_TYPE_CUDA_HOST_BUFFER, + "buffer", arrow_buffer, + NULL); + return GARROW_GPU_CUDA_HOST_BUFFER(buffer); +} + +std::shared_ptr +garrow_gpu_cuda_host_buffer_get_raw(GArrowGPUCUDAHostBuffer *buffer) +{ + if (!buffer) + return nullptr; + + auto arrow_buffer = garrow_buffer_get_raw(GARROW_BUFFER(buffer)); + return std::static_pointer_cast(arrow_buffer); +} + +GArrowGPUCUDABufferInputStream * +garrow_gpu_cuda_buffer_input_stream_new_raw(std::shared_ptr *arrow_reader) +{ + auto input_stream = g_object_new(GARROW_GPU_TYPE_CUDA_BUFFER_INPUT_STREAM, + "input-stream", arrow_reader, + NULL); + return GARROW_GPU_CUDA_BUFFER_INPUT_STREAM(input_stream); +} + +std::shared_ptr +garrow_gpu_cuda_buffer_input_stream_get_raw(GArrowGPUCUDABufferInputStream *input_stream) +{ + if (!input_stream) + return nullptr; + + auto arrow_reader = + garrow_input_stream_get_raw(GARROW_INPUT_STREAM(input_stream)); + return std::static_pointer_cast(arrow_reader); +} + +GArrowGPUCUDABufferOutputStream * +garrow_gpu_cuda_buffer_output_stream_new_raw(std::shared_ptr *arrow_writer) +{ + auto output_stream = g_object_new(GARROW_GPU_TYPE_CUDA_BUFFER_OUTPUT_STREAM, + "output-stream", arrow_writer, + NULL); + return GARROW_GPU_CUDA_BUFFER_OUTPUT_STREAM(output_stream); +} + +std::shared_ptr +garrow_gpu_cuda_buffer_output_stream_get_raw(GArrowGPUCUDABufferOutputStream *output_stream) +{ + if (!output_stream) + return nullptr; + + auto arrow_writer = + garrow_output_stream_get_raw(GARROW_OUTPUT_STREAM(output_stream)); + return std::static_pointer_cast(arrow_writer); +} diff --git a/c_glib/arrow-gpu-glib/cuda.h b/c_glib/arrow-gpu-glib/cuda.h new file mode 100644 index 0000000..7c615a1 --- /dev/null +++ b/c_glib/arrow-gpu-glib/cuda.h @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include + +G_BEGIN_DECLS + +#define GARROW_GPU_TYPE_CUDA_DEVICE_MANAGER \ + (garrow_gpu_cuda_device_manager_get_type()) +G_DECLARE_DERIVABLE_TYPE(GArrowGPUCUDADeviceManager, + garrow_gpu_cuda_device_manager, + GARROW_GPU, + CUDA_DEVICE_MANAGER, + GObject) +struct _GArrowGPUCUDADeviceManagerClass +{ + GObjectClass parent_class; +}; + +#define GARROW_GPU_TYPE_CUDA_CONTEXT (garrow_gpu_cuda_context_get_type()) +G_DECLARE_DERIVABLE_TYPE(GArrowGPUCUDAContext, + garrow_gpu_cuda_context, + GARROW_GPU, + CUDA_CONTEXT, + GObject) +struct _GArrowGPUCUDAContextClass +{ + GObjectClass parent_class; +}; + +#define GARROW_GPU_TYPE_CUDA_BUFFER (garrow_gpu_cuda_buffer_get_type()) +G_DECLARE_DERIVABLE_TYPE(GArrowGPUCUDABuffer, + garrow_gpu_cuda_buffer, + GARROW_GPU, + CUDA_BUFFER, + GArrowBuffer) +struct _GArrowGPUCUDABufferClass +{ + GArrowBufferClass parent_class; +}; + +#define GARROW_GPU_TYPE_CUDA_HOST_BUFFER (garrow_gpu_cuda_host_buffer_get_type()) +G_DECLARE_DERIVABLE_TYPE(GArrowGPUCUDAHostBuffer, + garrow_gpu_cuda_host_buffer, + GARROW_GPU, + CUDA_HOST_BUFFER, + GArrowMutableBuffer) +struct _GArrowGPUCUDAHostBufferClass +{ + GArrowMutableBufferClass parent_class; +}; + +#define GARROW_GPU_TYPE_CUDA_IPC_MEMORY_HANDLE \ + (garrow_gpu_cuda_ipc_memory_handle_get_type()) +G_DECLARE_DERIVABLE_TYPE(GArrowGPUCUDAIPCMemoryHandle, + garrow_gpu_cuda_ipc_memory_handle, + GARROW_GPU, + CUDA_IPC_MEMORY_HANDLE, + GObject) +struct _GArrowGPUCUDAIPCMemoryHandleClass +{ + GObjectClass parent_class; +}; + +#define GARROW_GPU_TYPE_CUDA_BUFFER_INPUT_STREAM \ + (garrow_gpu_cuda_buffer_input_stream_get_type()) +G_DECLARE_DERIVABLE_TYPE(GArrowGPUCUDABufferInputStream, + garrow_gpu_cuda_buffer_input_stream, + GARROW_GPU, + CUDA_BUFFER_INPUT_STREAM, + GArrowBufferInputStream) +struct _GArrowGPUCUDABufferInputStreamClass +{ + GArrowBufferInputStreamClass parent_class; +}; + +#define GARROW_GPU_TYPE_CUDA_BUFFER_OUTPUT_STREAM \ + (garrow_gpu_cuda_buffer_output_stream_get_type()) +G_DECLARE_DERIVABLE_TYPE(GArrowGPUCUDABufferOutputStream, + garrow_gpu_cuda_buffer_output_stream, + GARROW_GPU, + CUDA_BUFFER_OUTPUT_STREAM, + GArrowOutputStream) +struct _GArrowGPUCUDABufferOutputStreamClass +{ + GArrowOutputStreamClass parent_class; +}; + +GArrowGPUCUDADeviceManager * +garrow_gpu_cuda_device_manager_new(GError **error); + +GArrowGPUCUDAContext * +garrow_gpu_cuda_device_manager_get_context(GArrowGPUCUDADeviceManager *manager, + gint gpu_number, + GError **error); +gsize +garrow_gpu_cuda_device_manager_get_n_devices(GArrowGPUCUDADeviceManager *manager); + +gint64 +garrow_gpu_cuda_context_get_allocated_size(GArrowGPUCUDAContext *context); + + +GArrowGPUCUDABuffer * +garrow_gpu_cuda_buffer_new(GArrowGPUCUDAContext *context, + gint64 size, + GError **error); +GArrowGPUCUDABuffer * +garrow_gpu_cuda_buffer_new_ipc(GArrowGPUCUDAContext *context, + GArrowGPUCUDAIPCMemoryHandle *handle, + GError **error); +GArrowGPUCUDABuffer * +garrow_gpu_cuda_buffer_new_record_batch(GArrowGPUCUDAContext *context, + GArrowRecordBatch *record_batch, + GError **error); +GBytes * +garrow_gpu_cuda_buffer_copy_to_host(GArrowGPUCUDABuffer *buffer, + gint64 position, + gint64 size, + GError **error); +gboolean +garrow_gpu_cuda_buffer_copy_from_host(GArrowGPUCUDABuffer *buffer, + const guint8 *data, + gint64 size, + GError **error); +GArrowGPUCUDAIPCMemoryHandle * +garrow_gpu_cuda_buffer_export(GArrowGPUCUDABuffer *buffer, + GError **error); +GArrowGPUCUDAContext * +garrow_gpu_cuda_buffer_get_context(GArrowGPUCUDABuffer *buffer); +GArrowRecordBatch * +garrow_gpu_cuda_buffer_read_record_batch(GArrowGPUCUDABuffer *buffer, + GArrowSchema *schema, + GError **error); + + +GArrowGPUCUDAHostBuffer * +garrow_gpu_cuda_host_buffer_new(gint64 size, GError **error); + +GArrowGPUCUDAIPCMemoryHandle * +garrow_gpu_cuda_ipc_memory_handle_new(const guint8 *data, + gsize size, + GError **error); + +GArrowBuffer * +garrow_gpu_cuda_ipc_memory_handle_serialize(GArrowGPUCUDAIPCMemoryHandle *handle, + GError **error); + +GArrowGPUCUDABufferInputStream * +garrow_gpu_cuda_buffer_input_stream_new(GArrowGPUCUDABuffer *buffer); + +GArrowGPUCUDABufferOutputStream * +garrow_gpu_cuda_buffer_output_stream_new(GArrowGPUCUDABuffer *buffer); + +gboolean +garrow_gpu_cuda_buffer_output_stream_set_buffer_size(GArrowGPUCUDABufferOutputStream *stream, + gint64 size, + GError **error); +gint64 +garrow_gpu_cuda_buffer_output_stream_get_buffer_size(GArrowGPUCUDABufferOutputStream *stream); +gint64 +garrow_gpu_cuda_buffer_output_stream_get_buffered_size(GArrowGPUCUDABufferOutputStream *stream); + +G_END_DECLS diff --git a/c_glib/arrow-gpu-glib/cuda.hpp b/c_glib/arrow-gpu-glib/cuda.hpp new file mode 100644 index 0000000..3eeff8b --- /dev/null +++ b/c_glib/arrow-gpu-glib/cuda.hpp @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include + +#include + +GArrowGPUCUDAContext * +garrow_gpu_cuda_context_new_raw(std::shared_ptr *arrow_context); +std::shared_ptr +garrow_gpu_cuda_context_get_raw(GArrowGPUCUDAContext *context); + +GArrowGPUCUDAIPCMemoryHandle * +garrow_gpu_cuda_ipc_memory_handle_new_raw(arrow::gpu::CudaIpcMemHandle *arrow_handle); +arrow::gpu::CudaIpcMemHandle * +garrow_gpu_cuda_ipc_memory_handle_get_raw(GArrowGPUCUDAIPCMemoryHandle *handle); + +GArrowGPUCUDABuffer * +garrow_gpu_cuda_buffer_new_raw(std::shared_ptr *arrow_buffer); +std::shared_ptr +garrow_gpu_cuda_buffer_get_raw(GArrowGPUCUDABuffer *buffer); + +GArrowGPUCUDAHostBuffer * +garrow_gpu_cuda_host_buffer_new_raw(std::shared_ptr *arrow_buffer); +std::shared_ptr +garrow_gpu_cuda_host_buffer_get_raw(GArrowGPUCUDAHostBuffer *buffer); + +GArrowGPUCUDABufferInputStream * +garrow_gpu_cuda_buffer_input_stream_new_raw(std::shared_ptr *arrow_reader); +std::shared_ptr +garrow_gpu_cuda_buffer_input_stream_get_raw(GArrowGPUCUDABufferInputStream *input_stream); + +GArrowGPUCUDABufferOutputStream * +garrow_gpu_cuda_buffer_output_stream_new_raw(std::shared_ptr *arrow_writer); +std::shared_ptr +garrow_gpu_cuda_buffer_output_stream_get_raw(GArrowGPUCUDABufferOutputStream *output_stream); diff --git a/c_glib/arrow-gpu-glib/meson.build b/c_glib/arrow-gpu-glib/meson.build new file mode 100644 index 0000000..00c7f07 --- /dev/null +++ b/c_glib/arrow-gpu-glib/meson.build @@ -0,0 +1,80 @@ +# -*- indent-tabs-mode: nil -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +sources = files( + 'cuda.cpp', +) + +c_headers = files( + 'arrow-gpu-glib.h', + 'cuda.h', +) + +cpp_headers = files( + 'arrow-gpu-glib.hpp', + 'cuda.hpp', +) + +headers = c_headers + cpp_headers +install_headers(headers, subdir: 'arrow-gpu-glib') + + +dependencies = [ + arrow_gpu_dependency, + libarrow_glib_dependency, +] +libarrow_gpu_glib = library('arrow-gpu-glib', + sources: sources, + install: true, + dependencies: dependencies, + include_directories: [ + root_inc, + ], + soversion: so_version, + version: library_version) +libarrow_gpu_glib_dependency = declare_dependency(link_with: libarrow_gpu_glib, + include_directories: [ + root_inc, + ], + dependencies: dependencies) + +pkgconfig.generate(filebase: 'arrow-gpu-glib', + name: 'Apache Arrow GPU GLib', + description: 'C API for Apache Arrow GPU based on GLib', + version: version, + requires: ['arrow-glib', 'arrow-gpu'], + libraries: [libarrow_gpu_glib], + subdirs: ['arrow-gpu-glib']) + +gnome.generate_gir(libarrow_gpu_glib, + dependencies: arrow_glib_gir_dependency, + sources: sources + c_headers, + namespace: 'ArrowGPU', + nsversion: api_version, + identifier_prefix: 'GArrowGPU', + symbol_prefix: 'garrow_gpu', + export_packages: 'arrow-gpu-glib', + includes: [ + 'Arrow-1.0', + ], + install: true, + extra_args: [ + '--warn-all', + '--include-uninstalled=./arrow-glib/Arrow-1.0.gir', + ]) diff --git a/c_glib/configure.ac b/c_glib/configure.ac index 5db4352..c6fa019 100644 --- a/c_glib/configure.ac +++ b/c_glib/configure.ac @@ -77,18 +77,34 @@ AC_ARG_WITH(arrow-cpp-build-dir, [GARROW_ARROW_CPP_BUILD_DIR=""]) if test "x$GARROW_ARROW_CPP_BUILD_DIR" = "x"; then PKG_CHECK_MODULES([ARROW], [arrow arrow-compute]) + PKG_CHECK_MODULES([ARROW_GPU], + [arrow-gpu], + [HAVE_ARROW_GPU=yes], + [HAVE_ARROW_GPU=no]) else ARROW_INCLUDE_DIR="\$(abs_top_srcdir)/../cpp/src" ARROW_LIB_DIR="${GARROW_ARROW_CPP_BUILD_DIR}/${GARROW_ARROW_CPP_BUILD_TYPE}" ARROW_CFLAGS="-I${ARROW_INCLUDE_DIR}" - ARROW_LIBS="-L${ARROW_LIB_DIR} -larrow" - - AC_SUBST(ARROW_LIB_DIR) - AC_SUBST(ARROW_CFLAGS) AC_SUBST(ARROW_LIBS) + + ARROW_GPU_CFLAGS="" + if test -f "${GARROW_ARROW_CPP_BUILD_DIR}/src/arrow/gpu/arrow-gpu.pc"; then + HAVE_ARROW_GPU=yes + ARROW_GPU_LIBS="-larrow_gpu" + else + HAVE_ARROW_GPU=no + ARROW_GPU_LIBS="" + fi + AC_SUBST(ARROW_GPU_CFLAGS) + AC_SUBST(ARROW_GPU_LIBS) +fi + +AM_CONDITIONAL([HAVE_ARROW_GPU], [test "$HAVE_ARROW_GPU" = "yes"]) +if test "$HAVE_ARROW_GPU" = "yes"; then + AC_DEFINE(HAVE_ARROW_GPU, [1], [Define to 1 if Apache Arrow supports GPU.]) fi exampledir="\$(datadir)/arrow-glib/example" @@ -98,6 +114,8 @@ AC_CONFIG_FILES([ Makefile arrow-glib/Makefile arrow-glib/arrow-glib.pc + arrow-gpu-glib/Makefile + arrow-gpu-glib/arrow-gpu-glib.pc doc/Makefile doc/reference/Makefile doc/reference/xml/Makefile diff --git a/c_glib/doc/Makefile.am b/c_glib/doc/Makefile.am index 85c1d51..1d491ab 100644 --- a/c_glib/doc/Makefile.am +++ b/c_glib/doc/Makefile.am @@ -16,4 +16,4 @@ # under the License. SUBDIRS = \ - reference + reference diff --git a/c_glib/doc/reference/Makefile.am b/c_glib/doc/reference/Makefile.am index 45b11f0..896aff5 100644 --- a/c_glib/doc/reference/Makefile.am +++ b/c_glib/doc/reference/Makefile.am @@ -51,6 +51,17 @@ AM_CFLAGS = \ GTKDOC_LIBS = \ $(top_builddir)/arrow-glib/libarrow-glib.la +if HAVE_ARROW_GPU +DOC_SOURCE_DIR += \ + $(top_srcdir)/arrow-gpu-glib +HFILE_GLOB += \ + $(top_srcdir)/arrow-gpu-glib/*.h +CFILE_GLOB += \ + $(top_srcdir)/arrow-gpu-glib/*.cpp +GTKDOC_LIBS += \ + $(top_builddir)/arrow-gpu-glib/libarrow-gpu-glib.la +endif + include $(srcdir)/gtk-doc.make CLEANFILES += \ diff --git a/c_glib/doc/reference/arrow-glib-docs.sgml b/c_glib/doc/reference/arrow-glib-docs.sgml index a504ef1..e267ea2 100644 --- a/c_glib/doc/reference/arrow-glib-docs.sgml +++ b/c_glib/doc/reference/arrow-glib-docs.sgml @@ -125,6 +125,16 @@ + + Object Hierarchy diff --git a/c_glib/doc/reference/meson.build b/c_glib/doc/reference/meson.build index 08936da..4c9552e 100644 --- a/c_glib/doc/reference/meson.build +++ b/c_glib/doc/reference/meson.build @@ -32,13 +32,26 @@ glib_prefix = dependency('glib-2.0').get_pkgconfig_variable('prefix') glib_doc_path = join_paths(glib_prefix, 'share', 'gtk-doc', 'html') doc_path = join_paths(data_dir, meson.project_name(), 'gtk-doc', 'html') +source_directories = [ + join_paths(meson.source_root(), 'arrow-glib'), + join_paths(meson.build_root(), 'arrow-glib'), +] +dependencies = [ + libarrow_glib_dependency, +] +if arrow_gpu_dependency.found() + source_directories += [ + join_paths(meson.source_root(), 'arrow-gpu-glib'), + join_paths(meson.build_root(), 'arrow-gpu-glib'), + ] + dependencies += [ + libarrow_gpu_glib_dependency, + ] +endif gnome.gtkdoc(meson.project_name(), main_xml: meson.project_name() + '-docs.sgml', - src_dir: [ - join_paths(meson.source_root(), 'arrow-glib'), - join_paths(meson.build_root(), 'arrow-glib'), - ], - dependencies: libarrow_glib_dependency, + src_dir: source_directories, + dependencies: dependencies, gobject_typesfile: meson.project_name() + '.types', scan_args: [ '--rebuild-types', diff --git a/c_glib/meson.build b/c_glib/meson.build index 1fa64ba..9fe1b8c 100644 --- a/c_glib/meson.build +++ b/c_glib/meson.build @@ -49,6 +49,10 @@ pkgconfig = import('pkgconfig') root_inc = include_directories('.') subdir('arrow-glib') +arrow_gpu_dependency = dependency('arrow-gpu', required: false) +if arrow_gpu_dependency.found() + subdir('arrow-gpu-glib') +endif subdir('example') if get_option('enable_gtk_doc') @@ -58,4 +62,7 @@ endif run_test = find_program('test/run-test.sh') test('unit test', run_test, - env: ['ARROW_GLIB_TYPELIB_DIR=@0@/arrow-glib'.format(meson.build_root())]) + env: [ + 'ARROW_GLIB_TYPELIB_DIR=@0@/arrow-glib'.format(meson.build_root()), + 'ARROW_GPU_GLIB_TYPELIB_DIR=@0@/arrow-gpu-glib'.format(meson.build_root()), + ]) diff --git a/c_glib/test/run-test.rb b/c_glib/test/run-test.rb index 3451bd2..392c56f 100755 --- a/c_glib/test/run-test.rb +++ b/c_glib/test/run-test.rb @@ -37,6 +37,12 @@ module Arrow end end +begin + ArrowGPU = GI.load("ArrowGPU") +rescue GObjectIntrospection::RepositoryError::TypelibNotFound +end + +require "rbconfig" require "tempfile" require_relative "helper/buildable" require_relative "helper/omittable" diff --git a/c_glib/test/run-test.sh b/c_glib/test/run-test.sh index 19ccf07..d563e85 100755 --- a/c_glib/test/run-test.sh +++ b/c_glib/test/run-test.sh @@ -20,27 +20,34 @@ test_dir="$(cd $(dirname $0); pwd)" build_dir="$(cd .; pwd)" -arrow_glib_build_dir="${build_dir}/arrow-glib/" -libtool_dir="${arrow_glib_build_dir}/.libs" -if [ -d "${libtool_dir}" ]; then - LD_LIBRARY_PATH="${libtool_dir}:${LD_LIBRARY_PATH}" -else - if [ -d "${arrow_glib_build_dir}" ]; then - LD_LIBRARY_PATH="${arrow_glib_build_dir}:${LD_LIBRARY_PATH}" +modules="arrow-glib arrow-gpu-glib" + +for module in ${modules}; do + module_build_dir="${build_dir}/${module}" + libtool_dir="${module_build_dir}/.libs" + if [ -d "${libtool_dir}" ]; then + LD_LIBRARY_PATH="${libtool_dir}:${LD_LIBRARY_PATH}" + else + if [ -d "${module_build_dir}" ]; then + LD_LIBRARY_PATH="${module_build_dir}:${LD_LIBRARY_PATH}" + fi fi -fi +done if [ -f "Makefile" -a "${NO_MAKE}" != "yes" ]; then make -j8 > /dev/null || exit $? fi -arrow_glib_typelib_dir="${ARROW_GLIB_TYPELIB_DIR}" -if [ -z "${arrow_glib_typelib_dir}" ]; then - arrow_glib_typelib_dir="${build_dir}/arrow-glib" -fi +for module in ${modules}; do + MODULE_TYPELIB_DIR_VAR_NAME="$(echo ${module} | tr a-z- A-Z_)_TYPELIB_DIR" + module_typelib_dir=$(eval "echo \${${MODULE_TYPELIB_DIR_VAR_NAME}}") + if [ -z "${module_typelib_dir}" ]; then + module_typelib_dir="${build_dir}/${module}" + fi -if [ -d "${arrow_glib_typelib_dir}" ]; then - GI_TYPELIB_PATH="${arrow_glib_typelib_dir}:${GI_TYPELIB_PATH}" -fi + if [ -d "${module_typelib_dir}" ]; then + GI_TYPELIB_PATH="${module_typelib_dir}:${GI_TYPELIB_PATH}" + fi +done ${GDB} ruby ${test_dir}/run-test.rb "$@" diff --git a/c_glib/test/test-gpu-cuda.rb b/c_glib/test/test-gpu-cuda.rb new file mode 100644 index 0000000..c710ef2 --- /dev/null +++ b/c_glib/test/test-gpu-cuda.rb @@ -0,0 +1,144 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class TestGPUCUDA < Test::Unit::TestCase + include Helper::Buildable + + def setup + omit("Arrow GPU is required") unless defined?(::ArrowGPU) + @manager = ArrowGPU::CUDADeviceManager.new + omit("At least one GPU is required") if @manager.n_devices.zero? + @context = @manager.get_context(0) + end + + sub_test_case("Context") do + def test_allocated_size + allocated_size_before = @context.allocated_size + size = 128 + buffer = ArrowGPU::CUDABuffer.new(@context, size) + assert_equal(size, + @context.allocated_size - allocated_size_before) + end + end + + sub_test_case("Buffer") do + def setup + super + @buffer = ArrowGPU::CUDABuffer.new(@context, 128) + end + + def test_copy + @buffer.copy_from_host("Hello World") + assert_equal("llo W", @buffer.copy_to_host(2, 5).to_s) + end + + def test_export + @buffer.copy_from_host("Hello World") + handle = @buffer.export + serialized_handle = handle.serialize.data + Tempfile.open("arrow-gpu-cuda-export") do |output| + pid = spawn(RbConfig.ruby, "-e", <<-SCRIPT) +require "gi" + +Gio = GI.load("Gio") +Arrow = GI.load("Arrow") +ArrowGPU = GI.load("ArrowGPU") + +manager = ArrowGPU::CUDADeviceManager.new +context = manager.get_context(0) +serialized_handle = #{serialized_handle.to_s.dump} +handle = ArrowGPU::CUDAIPCMemoryHandle.new(serialized_handle) +buffer = ArrowGPU::CUDABuffer.new(context, handle) +File.open(#{output.path.dump}, "w") do |output| + output.print(buffer.copy_to_host(0, 6).to_s) +end + SCRIPT + Process.waitpid(pid) + assert_equal("Hello ", output.read) + end + end + + def test_context + assert_equal(@context.allocated_size, + @buffer.context.allocated_size) + end + + def test_record_batch + field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new) + schema = Arrow::Schema.new([field]) + columns = [ + build_boolean_array([true]), + ] + cpu_record_batch = Arrow::RecordBatch.new(schema, 1, columns) + + buffer = ArrowGPU::CUDABuffer.new(@context, cpu_record_batch) + gpu_record_batch = buffer.read_record_batch(schema) + assert_equal(cpu_record_batch.n_rows, + gpu_record_batch.n_rows) + end + end + + sub_test_case("HostBuffer") do + def test_new + buffer = ArrowGPU::CUDAHostBuffer.new(128) + assert_equal(128, buffer.size) + end + end + + sub_test_case("BufferInputStream") do + def test_new + buffer = ArrowGPU::CUDABuffer.new(@context, 128) + buffer.copy_from_host("Hello World") + stream = ArrowGPU::CUDABufferInputStream.new(buffer) + begin + assert_equal("Hello Worl", stream.read(5).copy_to_host(0, 10).to_s) + ensure + stream.close + end + end + end + + sub_test_case("BufferOutputStream") do + def setup + super + @buffer = ArrowGPU::CUDABuffer.new(@context, 128) + @buffer.copy_from_host("\x00" * @buffer.size) + @stream = ArrowGPU::CUDABufferOutputStream.new(@buffer) + end + + def cleanup + super + @stream.close + end + + def test_new + @stream.write("Hello World") + assert_equal("Hello World", @buffer.copy_to_host(0, 11).to_s) + end + + def test_buffer + assert_equal(0, @stream.buffer_size) + @stream.buffer_size = 5 + assert_equal(5, @stream.buffer_size) + @stream.write("Hell") + assert_equal(4, @stream.buffered_size) + assert_equal("\x00" * 5, @buffer.copy_to_host(0, 5).to_s) + @stream.write("o") + assert_equal("Hello", @buffer.copy_to_host(0, 5).to_s) + end + end +end -- To stop receiving notification emails like this one, please contact ['"commits@arrow.apache.org" '].