arrow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject [23/30] arrow-site git commit: Update Python and GLib API docs for 0.6.0
Date Wed, 16 Aug 2017 21:33:17 GMT
http://git-wip-us.apache.org/repos/asf/arrow-site/blob/4d4a3202/docs/c_glib/writer-classes.html
----------------------------------------------------------------------
diff --git a/docs/c_glib/writer-classes.html b/docs/c_glib/writer-classes.html
new file mode 100644
index 0000000..6b048f7
--- /dev/null
+++ b/docs/c_glib/writer-classes.html
@@ -0,0 +1,585 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
+<html>
+<head>
+<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title> Reference Manual: Writer classes</title>
+<meta name="generator" content="DocBook XSL Stylesheets V1.78.1">
+<link rel="home" href="index.html" title="Reference Manual">
+<link rel="up" href="writer.html" title="Writer">
+<link rel="prev" href="writer.html" title="Writer">
+<link rel="next" href="object-tree.html" title="Object Hierarchy">
+<meta name="generator" content="GTK-Doc V1.20 (XML mode)">
+<link rel="stylesheet" href="style.css" type="text/css">
+</head>
+<body bgcolor="white" text="black" link="#0000FF" vlink="#840084" alink="#0000FF">
+<table class="navigation" id="top" width="100%" summary="Navigation header" cellpadding="2" cellspacing="10"><tr valign="middle">
+<td width="100%" align="left" class="shortcuts">
+<a href="#" class="shortcut">Top</a><span id="nav_description"> <span class="dim">|</span> 
+                  <a href="#writer-classes.description" class="shortcut">Description</a></span><span id="nav_hierarchy"> <span class="dim">|</span> 
+                  <a href="#writer-classes.object-hierarchy" class="shortcut">Object Hierarchy</a></span><span id="nav_properties"> <span class="dim">|</span> 
+                  <a href="#writer-classes.properties" class="shortcut">Properties</a></span>
+</td>
+<td><a accesskey="h" href="index.html"><img src="home.png" width="16" height="16" border="0" alt="Home"></a></td>
+<td><a accesskey="u" href="writer.html"><img src="up.png" width="16" height="16" border="0" alt="Up"></a></td>
+<td><a accesskey="p" href="writer.html"><img src="left.png" width="16" height="16" border="0" alt="Prev"></a></td>
+<td><a accesskey="n" href="object-tree.html"><img src="right.png" width="16" height="16" border="0" alt="Next"></a></td>
+</tr></table>
+<div class="refentry">
+<a name="writer-classes"></a><div class="titlepage"></div>
+<div class="refnamediv"><table width="100%"><tr>
+<td valign="top">
+<h2><span class="refentrytitle"><a name="writer-classes.top_of_page"></a>Writer classes</span></h2>
+<p>Writer classes</p>
+</td>
+<td class="gallery_image" valign="top" align="right"></td>
+</tr></table></div>
+<div class="refsect1">
+<a name="writer-classes.functions"></a><h2>Functions</h2>
+<div class="informaltable"><table width="100%" border="0">
+<colgroup>
+<col width="150px" class="functions_return">
+<col class="functions_name">
+</colgroup>
+<tbody>
+<tr>
+<td class="function_type">
+<GTKDOCLINK HREF="gboolean"><span class="returnvalue">gboolean</span></GTKDOCLINK>
+</td>
+<td class="function_name">
+<a class="link" href="writer-classes.html#garrow-record-batch-writer-write-record-batch" title="garrow_record_batch_writer_write_record_batch ()">garrow_record_batch_writer_write_record_batch</a> <span class="c_punctuation">()</span>
+</td>
+</tr>
+<tr>
+<td class="function_type">
+<GTKDOCLINK HREF="gboolean"><span class="returnvalue">gboolean</span></GTKDOCLINK>
+</td>
+<td class="function_name">
+<a class="link" href="writer-classes.html#garrow-record-batch-writer-close" title="garrow_record_batch_writer_close ()">garrow_record_batch_writer_close</a> <span class="c_punctuation">()</span>
+</td>
+</tr>
+<tr>
+<td class="function_type">
+<a class="link" href="writer-classes.html#GArrowRecordBatchStreamWriter"><span class="returnvalue">GArrowRecordBatchStreamWriter</span></a> *
+</td>
+<td class="function_name">
+<a class="link" href="writer-classes.html#garrow-record-batch-stream-writer-new" title="garrow_record_batch_stream_writer_new ()">garrow_record_batch_stream_writer_new</a> <span class="c_punctuation">()</span>
+</td>
+</tr>
+<tr>
+<td class="function_type">
+<a class="link" href="writer-classes.html#GArrowRecordBatchFileWriter"><span class="returnvalue">GArrowRecordBatchFileWriter</span></a> *
+</td>
+<td class="function_name">
+<a class="link" href="writer-classes.html#garrow-record-batch-file-writer-new" title="garrow_record_batch_file_writer_new ()">garrow_record_batch_file_writer_new</a> <span class="c_punctuation">()</span>
+</td>
+</tr>
+<tr>
+<td class="function_type">
+<a class="link" href="writer-classes.html#GArrowFeatherFileWriter"><span class="returnvalue">GArrowFeatherFileWriter</span></a> *
+</td>
+<td class="function_name">
+<a class="link" href="writer-classes.html#garrow-feather-file-writer-new" title="garrow_feather_file_writer_new ()">garrow_feather_file_writer_new</a> <span class="c_punctuation">()</span>
+</td>
+</tr>
+<tr>
+<td class="function_type">
+<GTKDOCLINK HREF="void"><span class="returnvalue">void</span></GTKDOCLINK>
+</td>
+<td class="function_name">
+<a class="link" href="writer-classes.html#garrow-feather-file-writer-set-description" title="garrow_feather_file_writer_set_description ()">garrow_feather_file_writer_set_description</a> <span class="c_punctuation">()</span>
+</td>
+</tr>
+<tr>
+<td class="function_type">
+<GTKDOCLINK HREF="void"><span class="returnvalue">void</span></GTKDOCLINK>
+</td>
+<td class="function_name">
+<a class="link" href="writer-classes.html#garrow-feather-file-writer-set-n-rows" title="garrow_feather_file_writer_set_n_rows ()">garrow_feather_file_writer_set_n_rows</a> <span class="c_punctuation">()</span>
+</td>
+</tr>
+<tr>
+<td class="function_type">
+<GTKDOCLINK HREF="gboolean"><span class="returnvalue">gboolean</span></GTKDOCLINK>
+</td>
+<td class="function_name">
+<a class="link" href="writer-classes.html#garrow-feather-file-writer-append" title="garrow_feather_file_writer_append ()">garrow_feather_file_writer_append</a> <span class="c_punctuation">()</span>
+</td>
+</tr>
+<tr>
+<td class="function_type">
+<GTKDOCLINK HREF="gboolean"><span class="returnvalue">gboolean</span></GTKDOCLINK>
+</td>
+<td class="function_name">
+<a class="link" href="writer-classes.html#garrow-feather-file-writer-close" title="garrow_feather_file_writer_close ()">garrow_feather_file_writer_close</a> <span class="c_punctuation">()</span>
+</td>
+</tr>
+</tbody>
+</table></div>
+</div>
+<div class="refsect1">
+<a name="writer-classes.properties"></a><h2>Properties</h2>
+<div class="informaltable"><table border="0">
+<colgroup>
+<col width="150px" class="properties_type">
+<col width="300px" class="properties_name">
+<col width="200px" class="properties_flags">
+</colgroup>
+<tbody>
+<tr>
+<td class="property_type"><GTKDOCLINK HREF="gpointer"><span class="type">gpointer</span></GTKDOCLINK></td>
+<td class="property_name"><a class="link" href="writer-classes.html#GArrowRecordBatchWriter--record-batch-writer" title="The “record-batch-writer” property">record-batch-writer</a></td>
+<td class="property_flags">Write / Construct Only</td>
+</tr>
+<tr>
+<td class="property_type"><GTKDOCLINK HREF="gpointer"><span class="type">gpointer</span></GTKDOCLINK></td>
+<td class="property_name"><a class="link" href="writer-classes.html#GArrowFeatherFileWriter--feather-table-writer" title="The “feather-table-writer” property">feather-table-writer</a></td>
+<td class="property_flags">Write / Construct Only</td>
+</tr>
+</tbody>
+</table></div>
+</div>
+<a name="GArrowRecordBatchWriter"></a><a name="GArrowRecordBatchStreamWriter"></a><a name="GArrowRecordBatchFileWriter"></a><a name="GArrowFeatherFileWriter"></a><div class="refsect1">
+<a name="writer-classes.other"></a><h2>Types and Values</h2>
+<div class="informaltable"><table width="100%" border="0">
+<colgroup>
+<col width="150px" class="name">
+<col class="description">
+</colgroup>
+<tbody>
+<tr>
+<td class="datatype_keyword">struct</td>
+<td class="function_name"><a class="link" href="writer-classes.html#GArrowRecordBatchWriter-struct" title="struct GArrowRecordBatchWriter">GArrowRecordBatchWriter</a></td>
+</tr>
+<tr>
+<td class="datatype_keyword">struct</td>
+<td class="function_name"><a class="link" href="writer-classes.html#GArrowRecordBatchStreamWriter-struct" title="struct GArrowRecordBatchStreamWriter">GArrowRecordBatchStreamWriter</a></td>
+</tr>
+<tr>
+<td class="datatype_keyword">struct</td>
+<td class="function_name"><a class="link" href="writer-classes.html#GArrowRecordBatchFileWriter-struct" title="struct GArrowRecordBatchFileWriter">GArrowRecordBatchFileWriter</a></td>
+</tr>
+<tr>
+<td class="datatype_keyword">struct</td>
+<td class="function_name"><a class="link" href="writer-classes.html#GArrowFeatherFileWriter-struct" title="struct GArrowFeatherFileWriter">GArrowFeatherFileWriter</a></td>
+</tr>
+</tbody>
+</table></div>
+</div>
+<div class="refsect1">
+<a name="writer-classes.object-hierarchy"></a><h2>Object Hierarchy</h2>
+<pre class="screen">    <GTKDOCLINK HREF="GObject">GObject</GTKDOCLINK>
+    <span class="lineart">├──</span> GArrowFeatherFileWriter
+    <span class="lineart">├──</span> <a class="link" href="writer-classes.html#GArrowRecordBatchWriter">GArrowRecordBatchWriter</a>
+    <span class="lineart">│</span>   <span class="lineart">╰──</span> GArrowRecordBatchStreamWriter
+    <span class="lineart">│</span>       <span class="lineart">╰──</span> <a class="link" href="writer-classes.html#GArrowRecordBatchFileWriter">GArrowRecordBatchFileWriter</a>
+    <span class="lineart">╰──</span> GArrowRecordBatchWriter
+        <span class="lineart">╰──</span> <a class="link" href="writer-classes.html#GArrowRecordBatchStreamWriter">GArrowRecordBatchStreamWriter</a>
+            <span class="lineart">╰──</span> GArrowRecordBatchFileWriter
+</pre>
+</div>
+<div class="refsect1">
+<a name="writer-classes.includes"></a><h2>Includes</h2>
+<pre class="synopsis">#include &lt;arrow-glib/arrow-glib.h&gt;
+</pre>
+</div>
+<div class="refsect1">
+<a name="writer-classes.description"></a><h2>Description</h2>
+<p><a class="link" href="writer-classes.html#GArrowRecordBatchWriter"><span class="type">GArrowRecordBatchWriter</span></a> is a base class for writing record batches
+in stream format into output.</p>
+<p><a class="link" href="writer-classes.html#GArrowRecordBatchStreamWriter"><span class="type">GArrowRecordBatchStreamWriter</span></a> is a base class for writing record
+batches in stream format into output synchronously.</p>
+<p><a class="link" href="writer-classes.html#GArrowRecordBatchFileWriter"><span class="type">GArrowRecordBatchFileWriter</span></a> is a class for writing record
+batches in file format into output.</p>
+<p><a class="link" href="writer-classes.html#GArrowFeatherFileWriter"><span class="type">GArrowFeatherFileWriter</span></a> is a class for writing arrays
+in Feather file format into output.</p>
+</div>
+<div class="refsect1">
+<a name="writer-classes.functions_details"></a><h2>Functions</h2>
+<div class="refsect2">
+<a name="garrow-record-batch-writer-write-record-batch"></a><h3>garrow_record_batch_writer_write_record_batch ()</h3>
+<pre class="programlisting"><GTKDOCLINK HREF="gboolean"><span class="returnvalue">gboolean</span></GTKDOCLINK>
+garrow_record_batch_writer_write_record_batch
+                               (<em class="parameter"><code><a class="link" href="writer-classes.html#GArrowRecordBatchWriter"><span class="type">GArrowRecordBatchWriter</span></a> *writer</code></em>,
+                                <em class="parameter"><code><a class="link" href="GArrowRecordBatch.html" title="GArrowRecordBatch"><span class="type">GArrowRecordBatch</span></a> *record_batch</code></em>,
+                                <em class="parameter"><code><GTKDOCLINK HREF="GError"><span class="type">GError</span></GTKDOCLINK> **error</code></em>);</pre>
+<div class="refsect3">
+<a name="id-1.4.4.2.13.2.4"></a><h4>Parameters</h4>
+<div class="informaltable"><table width="100%" border="0">
+<colgroup>
+<col width="150px" class="parameters_name">
+<col class="parameters_description">
+<col width="200px" class="parameters_annotations">
+</colgroup>
+<tbody>
+<tr>
+<td class="parameter_name"><p>writer</p></td>
+<td class="parameter_description"><p>A <a class="link" href="writer-classes.html#GArrowRecordBatchWriter"><span class="type">GArrowRecordBatchWriter</span></a>.</p></td>
+<td class="parameter_annotations"> </td>
+</tr>
+<tr>
+<td class="parameter_name"><p>record_batch</p></td>
+<td class="parameter_description"><p>The record batch to be written.</p></td>
+<td class="parameter_annotations"> </td>
+</tr>
+<tr>
+<td class="parameter_name"><p>error</p></td>
+<td class="parameter_description"><p> Return locatipcn for a <GTKDOCLINK HREF="GError"><span class="type">GError</span></GTKDOCLINK> or <GTKDOCLINK HREF="NULL:CAPS"><code class="literal">NULL</code></GTKDOCLINK>. </p></td>
+<td class="parameter_annotations"><span class="annotation">[nullable]</span></td>
+</tr>
+</tbody>
+</table></div>
+</div>
+<div class="refsect3">
+<a name="id-1.4.4.2.13.2.5"></a><h4>Returns</h4>
+<p> <GTKDOCLINK HREF="TRUE:CAPS"><code class="literal">TRUE</code></GTKDOCLINK> on success, <GTKDOCLINK HREF="FALSE:CAPS"><code class="literal">FALSE</code></GTKDOCLINK> if there was an error.</p>
+<p></p>
+</div>
+<p class="since">Since 0.4.0</p>
+</div>
+<hr>
+<div class="refsect2">
+<a name="garrow-record-batch-writer-close"></a><h3>garrow_record_batch_writer_close ()</h3>
+<pre class="programlisting"><GTKDOCLINK HREF="gboolean"><span class="returnvalue">gboolean</span></GTKDOCLINK>
+garrow_record_batch_writer_close (<em class="parameter"><code><a class="link" href="writer-classes.html#GArrowRecordBatchWriter"><span class="type">GArrowRecordBatchWriter</span></a> *writer</code></em>,
+                                  <em class="parameter"><code><GTKDOCLINK HREF="GError"><span class="type">GError</span></GTKDOCLINK> **error</code></em>);</pre>
+<div class="refsect3">
+<a name="id-1.4.4.2.13.3.4"></a><h4>Parameters</h4>
+<div class="informaltable"><table width="100%" border="0">
+<colgroup>
+<col width="150px" class="parameters_name">
+<col class="parameters_description">
+<col width="200px" class="parameters_annotations">
+</colgroup>
+<tbody>
+<tr>
+<td class="parameter_name"><p>writer</p></td>
+<td class="parameter_description"><p>A <a class="link" href="writer-classes.html#GArrowRecordBatchWriter"><span class="type">GArrowRecordBatchWriter</span></a>.</p></td>
+<td class="parameter_annotations"> </td>
+</tr>
+<tr>
+<td class="parameter_name"><p>error</p></td>
+<td class="parameter_description"><p> Return locatipcn for a <GTKDOCLINK HREF="GError"><span class="type">GError</span></GTKDOCLINK> or <GTKDOCLINK HREF="NULL:CAPS"><code class="literal">NULL</code></GTKDOCLINK>. </p></td>
+<td class="parameter_annotations"><span class="annotation">[nullable]</span></td>
+</tr>
+</tbody>
+</table></div>
+</div>
+<div class="refsect3">
+<a name="id-1.4.4.2.13.3.5"></a><h4>Returns</h4>
+<p> <GTKDOCLINK HREF="TRUE:CAPS"><code class="literal">TRUE</code></GTKDOCLINK> on success, <GTKDOCLINK HREF="FALSE:CAPS"><code class="literal">FALSE</code></GTKDOCLINK> if there was an error.</p>
+<p></p>
+</div>
+<p class="since">Since 0.4.0</p>
+</div>
+<hr>
+<div class="refsect2">
+<a name="garrow-record-batch-stream-writer-new"></a><h3>garrow_record_batch_stream_writer_new ()</h3>
+<pre class="programlisting"><a class="link" href="writer-classes.html#GArrowRecordBatchStreamWriter"><span class="returnvalue">GArrowRecordBatchStreamWriter</span></a> *
+garrow_record_batch_stream_writer_new (<em class="parameter"><code><a class="link" href="output-stream-classes.html#GArrowOutputStream"><span class="type">GArrowOutputStream</span></a> *sink</code></em>,
+                                       <em class="parameter"><code><a class="link" href="GArrowSchema.html" title="GArrowSchema"><span class="type">GArrowSchema</span></a> *schema</code></em>,
+                                       <em class="parameter"><code><GTKDOCLINK HREF="GError"><span class="type">GError</span></GTKDOCLINK> **error</code></em>);</pre>
+<div class="refsect3">
+<a name="id-1.4.4.2.13.4.4"></a><h4>Parameters</h4>
+<div class="informaltable"><table width="100%" border="0">
+<colgroup>
+<col width="150px" class="parameters_name">
+<col class="parameters_description">
+<col width="200px" class="parameters_annotations">
+</colgroup>
+<tbody>
+<tr>
+<td class="parameter_name"><p>sink</p></td>
+<td class="parameter_description"><p>The output of the writer.</p></td>
+<td class="parameter_annotations"> </td>
+</tr>
+<tr>
+<td class="parameter_name"><p>schema</p></td>
+<td class="parameter_description"><p>The schema of the writer.</p></td>
+<td class="parameter_annotations"> </td>
+</tr>
+<tr>
+<td class="parameter_name"><p>error</p></td>
+<td class="parameter_description"><p> Return locatipcn for a <GTKDOCLINK HREF="GError"><span class="type">GError</span></GTKDOCLINK> or <GTKDOCLINK HREF="NULL:CAPS"><code class="literal">NULL</code></GTKDOCLINK>. </p></td>
+<td class="parameter_annotations"><span class="annotation">[nullable]</span></td>
+</tr>
+</tbody>
+</table></div>
+</div>
+<div class="refsect3">
+<a name="id-1.4.4.2.13.4.5"></a><h4>Returns</h4>
+<p> A newly created <a class="link" href="writer-classes.html#GArrowRecordBatchStreamWriter"><span class="type">GArrowRecordBatchStreamWriter</span></a>
+or <GTKDOCLINK HREF="NULL:CAPS"><code class="literal">NULL</code></GTKDOCLINK> on error. </p>
+<p><span class="annotation">[nullable]</span></p>
+</div>
+<p class="since">Since 0.4.0</p>
+</div>
+<hr>
+<div class="refsect2">
+<a name="garrow-record-batch-file-writer-new"></a><h3>garrow_record_batch_file_writer_new ()</h3>
+<pre class="programlisting"><a class="link" href="writer-classes.html#GArrowRecordBatchFileWriter"><span class="returnvalue">GArrowRecordBatchFileWriter</span></a> *
+garrow_record_batch_file_writer_new (<em class="parameter"><code><a class="link" href="output-stream-classes.html#GArrowOutputStream"><span class="type">GArrowOutputStream</span></a> *sink</code></em>,
+                                     <em class="parameter"><code><a class="link" href="GArrowSchema.html" title="GArrowSchema"><span class="type">GArrowSchema</span></a> *schema</code></em>,
+                                     <em class="parameter"><code><GTKDOCLINK HREF="GError"><span class="type">GError</span></GTKDOCLINK> **error</code></em>);</pre>
+<div class="refsect3">
+<a name="id-1.4.4.2.13.5.4"></a><h4>Parameters</h4>
+<div class="informaltable"><table width="100%" border="0">
+<colgroup>
+<col width="150px" class="parameters_name">
+<col class="parameters_description">
+<col width="200px" class="parameters_annotations">
+</colgroup>
+<tbody>
+<tr>
+<td class="parameter_name"><p>sink</p></td>
+<td class="parameter_description"><p>The output of the writer.</p></td>
+<td class="parameter_annotations"> </td>
+</tr>
+<tr>
+<td class="parameter_name"><p>schema</p></td>
+<td class="parameter_description"><p>The schema of the writer.</p></td>
+<td class="parameter_annotations"> </td>
+</tr>
+<tr>
+<td class="parameter_name"><p>error</p></td>
+<td class="parameter_description"><p> Return locatipcn for a <GTKDOCLINK HREF="GError"><span class="type">GError</span></GTKDOCLINK> or <GTKDOCLINK HREF="NULL:CAPS"><code class="literal">NULL</code></GTKDOCLINK>. </p></td>
+<td class="parameter_annotations"><span class="annotation">[nullable]</span></td>
+</tr>
+</tbody>
+</table></div>
+</div>
+<div class="refsect3">
+<a name="id-1.4.4.2.13.5.5"></a><h4>Returns</h4>
+<p> A newly created <a class="link" href="writer-classes.html#GArrowRecordBatchFileWriter"><span class="type">GArrowRecordBatchFileWriter</span></a>
+or <GTKDOCLINK HREF="NULL:CAPS"><code class="literal">NULL</code></GTKDOCLINK> on error. </p>
+<p><span class="annotation">[nullable]</span></p>
+</div>
+<p class="since">Since 0.4.0</p>
+</div>
+<hr>
+<div class="refsect2">
+<a name="garrow-feather-file-writer-new"></a><h3>garrow_feather_file_writer_new ()</h3>
+<pre class="programlisting"><a class="link" href="writer-classes.html#GArrowFeatherFileWriter"><span class="returnvalue">GArrowFeatherFileWriter</span></a> *
+garrow_feather_file_writer_new (<em class="parameter"><code><a class="link" href="output-stream-classes.html#GArrowOutputStream"><span class="type">GArrowOutputStream</span></a> *sink</code></em>,
+                                <em class="parameter"><code><GTKDOCLINK HREF="GError"><span class="type">GError</span></GTKDOCLINK> **error</code></em>);</pre>
+<div class="refsect3">
+<a name="id-1.4.4.2.13.6.4"></a><h4>Parameters</h4>
+<div class="informaltable"><table width="100%" border="0">
+<colgroup>
+<col width="150px" class="parameters_name">
+<col class="parameters_description">
+<col width="200px" class="parameters_annotations">
+</colgroup>
+<tbody>
+<tr>
+<td class="parameter_name"><p>sink</p></td>
+<td class="parameter_description"><p>The output of the writer.</p></td>
+<td class="parameter_annotations"> </td>
+</tr>
+<tr>
+<td class="parameter_name"><p>error</p></td>
+<td class="parameter_description"><p> Return locatipcn for a <GTKDOCLINK HREF="GError"><span class="type">GError</span></GTKDOCLINK> or <GTKDOCLINK HREF="NULL:CAPS"><code class="literal">NULL</code></GTKDOCLINK>. </p></td>
+<td class="parameter_annotations"><span class="annotation">[nullable]</span></td>
+</tr>
+</tbody>
+</table></div>
+</div>
+<div class="refsect3">
+<a name="id-1.4.4.2.13.6.5"></a><h4>Returns</h4>
+<p> A newly created <a class="link" href="writer-classes.html#GArrowFeatherFileWriter"><span class="type">GArrowFeatherFileWriter</span></a>
+or <GTKDOCLINK HREF="NULL:CAPS"><code class="literal">NULL</code></GTKDOCLINK> on error. </p>
+<p><span class="annotation">[nullable]</span></p>
+</div>
+<p class="since">Since 0.4.0</p>
+</div>
+<hr>
+<div class="refsect2">
+<a name="garrow-feather-file-writer-set-description"></a><h3>garrow_feather_file_writer_set_description ()</h3>
+<pre class="programlisting"><GTKDOCLINK HREF="void"><span class="returnvalue">void</span></GTKDOCLINK>
+garrow_feather_file_writer_set_description
+                               (<em class="parameter"><code><a class="link" href="writer-classes.html#GArrowFeatherFileWriter"><span class="type">GArrowFeatherFileWriter</span></a> *writer</code></em>,
+                                <em class="parameter"><code>const <GTKDOCLINK HREF="gchar"><span class="type">gchar</span></GTKDOCLINK> *description</code></em>);</pre>
+<div class="refsect3">
+<a name="id-1.4.4.2.13.7.4"></a><h4>Parameters</h4>
+<div class="informaltable"><table width="100%" border="0">
+<colgroup>
+<col width="150px" class="parameters_name">
+<col class="parameters_description">
+<col width="200px" class="parameters_annotations">
+</colgroup>
+<tbody>
+<tr>
+<td class="parameter_name"><p>writer</p></td>
+<td class="parameter_description"><p>A <a class="link" href="writer-classes.html#GArrowFeatherFileWriter"><span class="type">GArrowFeatherFileWriter</span></a>.</p></td>
+<td class="parameter_annotations"> </td>
+</tr>
+<tr>
+<td class="parameter_name"><p>description</p></td>
+<td class="parameter_description"><p>The description of the file.</p></td>
+<td class="parameter_annotations"> </td>
+</tr>
+</tbody>
+</table></div>
+</div>
+<p class="since">Since 0.4.0</p>
+</div>
+<hr>
+<div class="refsect2">
+<a name="garrow-feather-file-writer-set-n-rows"></a><h3>garrow_feather_file_writer_set_n_rows ()</h3>
+<pre class="programlisting"><GTKDOCLINK HREF="void"><span class="returnvalue">void</span></GTKDOCLINK>
+garrow_feather_file_writer_set_n_rows (<em class="parameter"><code><a class="link" href="writer-classes.html#GArrowFeatherFileWriter"><span class="type">GArrowFeatherFileWriter</span></a> *writer</code></em>,
+                                       <em class="parameter"><code><GTKDOCLINK HREF="gint64"><span class="type">gint64</span></GTKDOCLINK> n_rows</code></em>);</pre>
+<div class="refsect3">
+<a name="id-1.4.4.2.13.8.4"></a><h4>Parameters</h4>
+<div class="informaltable"><table width="100%" border="0">
+<colgroup>
+<col width="150px" class="parameters_name">
+<col class="parameters_description">
+<col width="200px" class="parameters_annotations">
+</colgroup>
+<tbody>
+<tr>
+<td class="parameter_name"><p>writer</p></td>
+<td class="parameter_description"><p>A <a class="link" href="writer-classes.html#GArrowFeatherFileWriter"><span class="type">GArrowFeatherFileWriter</span></a>.</p></td>
+<td class="parameter_annotations"> </td>
+</tr>
+<tr>
+<td class="parameter_name"><p>n_rows</p></td>
+<td class="parameter_description"><p>The number of rows in the file.</p></td>
+<td class="parameter_annotations"> </td>
+</tr>
+</tbody>
+</table></div>
+</div>
+<p class="since">Since 0.4.0</p>
+</div>
+<hr>
+<div class="refsect2">
+<a name="garrow-feather-file-writer-append"></a><h3>garrow_feather_file_writer_append ()</h3>
+<pre class="programlisting"><GTKDOCLINK HREF="gboolean"><span class="returnvalue">gboolean</span></GTKDOCLINK>
+garrow_feather_file_writer_append (<em class="parameter"><code><a class="link" href="writer-classes.html#GArrowFeatherFileWriter"><span class="type">GArrowFeatherFileWriter</span></a> *writer</code></em>,
+                                   <em class="parameter"><code>const <GTKDOCLINK HREF="gchar"><span class="type">gchar</span></GTKDOCLINK> *name</code></em>,
+                                   <em class="parameter"><code><a class="link" href="array-classes.html#GArrowArray"><span class="type">GArrowArray</span></a> *array</code></em>,
+                                   <em class="parameter"><code><GTKDOCLINK HREF="GError"><span class="type">GError</span></GTKDOCLINK> **error</code></em>);</pre>
+<div class="refsect3">
+<a name="id-1.4.4.2.13.9.4"></a><h4>Parameters</h4>
+<div class="informaltable"><table width="100%" border="0">
+<colgroup>
+<col width="150px" class="parameters_name">
+<col class="parameters_description">
+<col width="200px" class="parameters_annotations">
+</colgroup>
+<tbody>
+<tr>
+<td class="parameter_name"><p>writer</p></td>
+<td class="parameter_description"><p>A <a class="link" href="writer-classes.html#GArrowFeatherFileWriter"><span class="type">GArrowFeatherFileWriter</span></a>.</p></td>
+<td class="parameter_annotations"> </td>
+</tr>
+<tr>
+<td class="parameter_name"><p>name</p></td>
+<td class="parameter_description"><p>The name of the array to be appended.</p></td>
+<td class="parameter_annotations"> </td>
+</tr>
+<tr>
+<td class="parameter_name"><p>array</p></td>
+<td class="parameter_description"><p>The array to be appended.</p></td>
+<td class="parameter_annotations"> </td>
+</tr>
+<tr>
+<td class="parameter_name"><p>error</p></td>
+<td class="parameter_description"><p> Return locatipcn for a <GTKDOCLINK HREF="GError"><span class="type">GError</span></GTKDOCLINK> or <GTKDOCLINK HREF="NULL:CAPS"><code class="literal">NULL</code></GTKDOCLINK>. </p></td>
+<td class="parameter_annotations"><span class="annotation">[nullable]</span></td>
+</tr>
+</tbody>
+</table></div>
+</div>
+<div class="refsect3">
+<a name="id-1.4.4.2.13.9.5"></a><h4>Returns</h4>
+<p> <GTKDOCLINK HREF="TRUE:CAPS"><code class="literal">TRUE</code></GTKDOCLINK> on success, <GTKDOCLINK HREF="FALSE:CAPS"><code class="literal">FALSE</code></GTKDOCLINK> if there was an error.</p>
+<p></p>
+</div>
+<p class="since">Since 0.4.0</p>
+</div>
+<hr>
+<div class="refsect2">
+<a name="garrow-feather-file-writer-close"></a><h3>garrow_feather_file_writer_close ()</h3>
+<pre class="programlisting"><GTKDOCLINK HREF="gboolean"><span class="returnvalue">gboolean</span></GTKDOCLINK>
+garrow_feather_file_writer_close (<em class="parameter"><code><a class="link" href="writer-classes.html#GArrowFeatherFileWriter"><span class="type">GArrowFeatherFileWriter</span></a> *writer</code></em>,
+                                  <em class="parameter"><code><GTKDOCLINK HREF="GError"><span class="type">GError</span></GTKDOCLINK> **error</code></em>);</pre>
+<div class="refsect3">
+<a name="id-1.4.4.2.13.10.4"></a><h4>Parameters</h4>
+<div class="informaltable"><table width="100%" border="0">
+<colgroup>
+<col width="150px" class="parameters_name">
+<col class="parameters_description">
+<col width="200px" class="parameters_annotations">
+</colgroup>
+<tbody>
+<tr>
+<td class="parameter_name"><p>writer</p></td>
+<td class="parameter_description"><p>A <a class="link" href="writer-classes.html#GArrowFeatherFileWriter"><span class="type">GArrowFeatherFileWriter</span></a>.</p></td>
+<td class="parameter_annotations"> </td>
+</tr>
+<tr>
+<td class="parameter_name"><p>error</p></td>
+<td class="parameter_description"><p> Return locatipcn for a <GTKDOCLINK HREF="GError"><span class="type">GError</span></GTKDOCLINK> or <GTKDOCLINK HREF="NULL:CAPS"><code class="literal">NULL</code></GTKDOCLINK>. </p></td>
+<td class="parameter_annotations"><span class="annotation">[nullable]</span></td>
+</tr>
+</tbody>
+</table></div>
+</div>
+<div class="refsect3">
+<a name="id-1.4.4.2.13.10.5"></a><h4>Returns</h4>
+<p> <GTKDOCLINK HREF="TRUE:CAPS"><code class="literal">TRUE</code></GTKDOCLINK> on success, <GTKDOCLINK HREF="FALSE:CAPS"><code class="literal">FALSE</code></GTKDOCLINK> if there was an error.</p>
+<p></p>
+</div>
+<p class="since">Since 0.4.0</p>
+</div>
+</div>
+<div class="refsect1">
+<a name="writer-classes.other_details"></a><h2>Types and Values</h2>
+<div class="refsect2">
+<a name="GArrowRecordBatchWriter-struct"></a><h3>struct GArrowRecordBatchWriter</h3>
+<pre class="programlisting">struct GArrowRecordBatchWriter;</pre>
+<p>It wraps <code class="literal">arrow::ipc::RecordBatchWriter</code>.</p>
+</div>
+<hr>
+<div class="refsect2">
+<a name="GArrowRecordBatchStreamWriter-struct"></a><h3>struct GArrowRecordBatchStreamWriter</h3>
+<pre class="programlisting">struct GArrowRecordBatchStreamWriter;</pre>
+<p>It wraps <code class="literal">arrow::ipc::RecordBatchStreamWriter</code>.</p>
+</div>
+<hr>
+<div class="refsect2">
+<a name="GArrowRecordBatchFileWriter-struct"></a><h3>struct GArrowRecordBatchFileWriter</h3>
+<pre class="programlisting">struct GArrowRecordBatchFileWriter;</pre>
+<p>It wraps <code class="literal">arrow::ipc::RecordBatchFileWriter</code>.</p>
+</div>
+<hr>
+<div class="refsect2">
+<a name="GArrowFeatherFileWriter-struct"></a><h3>struct GArrowFeatherFileWriter</h3>
+<pre class="programlisting">struct GArrowFeatherFileWriter;</pre>
+<p>It wraps <code class="literal">arrow::ipc::feather::TableWriter</code>.</p>
+</div>
+</div>
+<div class="refsect1">
+<a name="writer-classes.property-details"></a><h2>Property Details</h2>
+<div class="refsect2">
+<a name="GArrowRecordBatchWriter--record-batch-writer"></a><h3>The <code class="literal">“record-batch-writer”</code> property</h3>
+<pre class="programlisting">  “record-batch-writer”      <GTKDOCLINK HREF="gpointer"><span class="type">gpointer</span></GTKDOCLINK></pre>
+<p>The raw std::shared&lt;arrow::ipc::RecordBatchWriter&gt; *.</p>
+<p>Flags: Write / Construct Only</p>
+</div>
+<hr>
+<div class="refsect2">
+<a name="GArrowFeatherFileWriter--feather-table-writer"></a><h3>The <code class="literal">“feather-table-writer”</code> property</h3>
+<pre class="programlisting">  “feather-table-writer”     <GTKDOCLINK HREF="gpointer"><span class="type">gpointer</span></GTKDOCLINK></pre>
+<p>The raw std::shared&lt;arrow::ipc::feather::TableWriter&gt; *.</p>
+<p>Flags: Write / Construct Only</p>
+</div>
+</div>
+</div>
+<div class="footer">
+<hr>
+          Generated by GTK-Doc V1.20</div>
+</body>
+</html>

http://git-wip-us.apache.org/repos/asf/arrow-site/blob/4d4a3202/docs/c_glib/writer.html
----------------------------------------------------------------------
diff --git a/docs/c_glib/writer.html b/docs/c_glib/writer.html
index ccb09c1..ea5d730 100644
--- a/docs/c_glib/writer.html
+++ b/docs/c_glib/writer.html
@@ -6,8 +6,8 @@
 <meta name="generator" content="DocBook XSL Stylesheets V1.78.1">
 <link rel="home" href="index.html" title="Reference Manual">
 <link rel="up" href="ipc.html" title="Part III. IPC">
-<link rel="prev" href="GArrowStreamReader.html" title="GArrowStreamReader">
-<link rel="next" href="GArrowFileWriter.html" title="GArrowFileWriter">
+<link rel="prev" href="reader-classes.html" title="Reader classes">
+<link rel="next" href="writer-classes.html" title="Writer classes">
 <meta name="generator" content="GTK-Doc V1.20 (XML mode)">
 <link rel="stylesheet" href="style.css" type="text/css">
 </head>
@@ -16,20 +16,15 @@
 <td width="100%" align="left" class="shortcuts"></td>
 <td><a accesskey="h" href="index.html"><img src="home.png" width="16" height="16" border="0" alt="Home"></a></td>
 <td><a accesskey="u" href="ipc.html"><img src="up.png" width="16" height="16" border="0" alt="Up"></a></td>
-<td><a accesskey="p" href="GArrowStreamReader.html"><img src="left.png" width="16" height="16" border="0" alt="Prev"></a></td>
-<td><a accesskey="n" href="GArrowFileWriter.html"><img src="right.png" width="16" height="16" border="0" alt="Next"></a></td>
+<td><a accesskey="p" href="reader-classes.html"><img src="left.png" width="16" height="16" border="0" alt="Prev"></a></td>
+<td><a accesskey="n" href="writer-classes.html"><img src="right.png" width="16" height="16" border="0" alt="Next"></a></td>
 </tr></table>
 <div class="chapter">
 <div class="titlepage"><div><div><h2 class="title">
 <a name="writer"></a>Writer</h2></div></div></div>
-<div class="toc"><dl class="toc">
-<dt>
-<span class="refentrytitle"><a href="GArrowFileWriter.html">GArrowFileWriter</a></span><span class="refpurpose"> — File writer class</span>
-</dt>
-<dt>
-<span class="refentrytitle"><a href="GArrowStreamWriter.html">GArrowStreamWriter</a></span><span class="refpurpose"> — Stream writer class</span>
-</dt>
-</dl></div>
+<div class="toc"><dl class="toc"><dt>
+<span class="refentrytitle"><a href="writer-classes.html">Writer classes</a></span><span class="refpurpose"></span>
+</dt></dl></div>
 </div>
 <div class="footer">
 <hr>

http://git-wip-us.apache.org/repos/asf/arrow-site/blob/4d4a3202/docs/python/_modules/index.html
----------------------------------------------------------------------
diff --git a/docs/python/_modules/index.html b/docs/python/_modules/index.html
index 48c9094..6070788 100644
--- a/docs/python/_modules/index.html
+++ b/docs/python/_modules/index.html
@@ -71,7 +71,8 @@
 <li class="toctree-l1"><a class="reference internal" href="../memory.html">Memory and IO Interfaces</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../data.html">In-Memory Data Model</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../ipc.html">IPC: Fast Streaming and Serialization</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../filesystems.html">Filesystem Interfaces</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../filesystems.html">File System Interfaces</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../plasma.html">The Plasma In-Memory Object Store</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../pandas.html">Using PyArrow with pandas</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../parquet.html">Reading and Writing the Apache Parquet Format</a></li>
 <li class="toctree-l1"><a class="reference internal" href="../api.html">API Reference</a></li>
@@ -123,9 +124,12 @@
     <div class="col-md-12 content">
       
   <h1>All modules for which code is available</h1>
-<ul><li><a href="pyarrow/ipc.html">pyarrow.ipc</a></li>
+<ul><li><a href="pyarrow/filesystem.html">pyarrow.filesystem</a></li>
+<li><a href="pyarrow/hdfs.html">pyarrow.hdfs</a></li>
+<li><a href="pyarrow/ipc.html">pyarrow.ipc</a></li>
 <li><a href="pyarrow/lib.html">pyarrow.lib</a></li>
 <li><a href="pyarrow/parquet.html">pyarrow.parquet</a></li>
+<li><a href="pyarrow/plasma.html">pyarrow.plasma</a></li>
 </ul>
 
     </div>

http://git-wip-us.apache.org/repos/asf/arrow-site/blob/4d4a3202/docs/python/_modules/pyarrow/filesystem.html
----------------------------------------------------------------------
diff --git a/docs/python/_modules/pyarrow/filesystem.html b/docs/python/_modules/pyarrow/filesystem.html
new file mode 100644
index 0000000..b194357
--- /dev/null
+++ b/docs/python/_modules/pyarrow/filesystem.html
@@ -0,0 +1,472 @@
+<!DOCTYPE html>
+
+
+<html xmlns="http://www.w3.org/1999/xhtml">
+  <head>
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+    
+    <title>pyarrow.filesystem &#8212; pyarrow  documentation</title>
+    
+    <link rel="stylesheet" href="../../_static/bootstrap-sphinx.css" type="text/css" />
+    <link rel="stylesheet" href="../../_static/pygments.css" type="text/css" />
+    
+    <script type="text/javascript">
+      var DOCUMENTATION_OPTIONS = {
+        URL_ROOT:    '../../',
+        VERSION:     '',
+        COLLAPSE_INDEX: false,
+        FILE_SUFFIX: '.html',
+        HAS_SOURCE:  true,
+        SOURCELINK_SUFFIX: '.txt'
+      };
+    </script>
+    <script type="text/javascript" src="../../_static/jquery.js"></script>
+    <script type="text/javascript" src="../../_static/underscore.js"></script>
+    <script type="text/javascript" src="../../_static/doctools.js"></script>
+    <script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script>
+    <script type="text/javascript" src="../../_static/js/jquery-1.11.0.min.js"></script>
+    <script type="text/javascript" src="../../_static/js/jquery-fix.js"></script>
+    <script type="text/javascript" src="../../_static/bootstrap-3.3.6/js/bootstrap.min.js"></script>
+    <script type="text/javascript" src="../../_static/bootstrap-sphinx.js"></script>
+    <link rel="index" title="Index" href="../../genindex.html" />
+    <link rel="search" title="Search" href="../../search.html" />
+<meta charset='utf-8'>
+<meta http-equiv='X-UA-Compatible' content='IE=edge,chrome=1'>
+<meta name='viewport' content='width=device-width, initial-scale=1.0, maximum-scale=1'>
+<meta name="apple-mobile-web-app-capable" content="yes">
+
+  </head>
+  <body role="document">
+
+  <div id="navbar" class="navbar navbar-default navbar-fixed-top">
+    <div class="container">
+      <div class="navbar-header">
+        <!-- .btn-navbar is used as the toggle for collapsed navbar content -->
+        <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".nav-collapse">
+          <span class="icon-bar"></span>
+          <span class="icon-bar"></span>
+          <span class="icon-bar"></span>
+        </button>
+        <a class="navbar-brand" href="../../index.html">
+          pyarrow</a>
+        <span class="navbar-text navbar-version pull-left"><b></b></span>
+      </div>
+
+        <div class="collapse navbar-collapse nav-collapse">
+          <ul class="nav navbar-nav">
+            
+            
+              <li class="dropdown globaltoc-container">
+  <a role="button"
+     id="dLabelGlobalToc"
+     data-toggle="dropdown"
+     data-target="#"
+     href="../../index.html">Site <b class="caret"></b></a>
+  <ul class="dropdown-menu globaltoc"
+      role="menu"
+      aria-labelledby="dLabelGlobalToc"><p class="caption"><span class="caption-text">Getting Started</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../../install.html">Install PyArrow</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../development.html">Development</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../memory.html">Memory and IO Interfaces</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../data.html">In-Memory Data Model</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../ipc.html">IPC: Fast Streaming and Serialization</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../filesystems.html">File System Interfaces</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../plasma.html">The Plasma In-Memory Object Store</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../pandas.html">Using PyArrow with pandas</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../parquet.html">Reading and Writing the Apache Parquet Format</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../api.html">API Reference</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../getting_involved.html">Getting Involved</a></li>
+</ul>
+</ul>
+</li>
+              
+                <li class="dropdown">
+  <a role="button"
+     id="dLabelLocalToc"
+     data-toggle="dropdown"
+     data-target="#"
+     href="#">Page <b class="caret"></b></a>
+  <ul class="dropdown-menu localtoc"
+      role="menu"
+      aria-labelledby="dLabelLocalToc"></ul>
+</li>
+              
+            
+            
+              
+                
+              
+            
+            
+            
+            
+              <li class="hidden-sm"></li>
+            
+          </ul>
+
+          
+            
+<form class="navbar-form navbar-right" action="../../search.html" method="get">
+ <div class="form-group">
+  <input type="text" name="q" class="form-control" placeholder="Search" />
+ </div>
+  <input type="hidden" name="check_keywords" value="yes" />
+  <input type="hidden" name="area" value="default" />
+</form>
+          
+        </div>
+    </div>
+  </div>
+
+<div class="container">
+  <div class="row">
+    <div class="col-md-12 content">
+      
+  <h1>Source code for pyarrow.filesystem</h1><div class="highlight"><pre>
+<span></span><span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements.  See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership.  The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># &quot;License&quot;); you may not use this file except in compliance</span>
+<span class="c1"># with the License.  You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1">#   http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># &quot;AS IS&quot; BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied.  See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">os.path</span> <span class="k">import</span> <span class="n">join</span> <span class="k">as</span> <span class="n">pjoin</span>
+<span class="kn">import</span> <span class="nn">os</span>
+<span class="kn">import</span> <span class="nn">posixpath</span>
+
+<span class="kn">from</span> <span class="nn">pyarrow.util</span> <span class="k">import</span> <span class="n">implements</span>
+
+
+<span class="k">class</span> <span class="nc">FileSystem</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Abstract filesystem interface</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">def</span> <span class="nf">cat</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Return contents of file as a bytes object</span>
+
+<span class="sd">        Returns</span>
+<span class="sd">        -------</span>
+<span class="sd">        contents : bytes</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="s1">&#39;rb&#39;</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
+            <span class="k">return</span> <span class="n">f</span><span class="o">.</span><span class="n">read</span><span class="p">()</span>
+
+    <span class="k">def</span> <span class="nf">ls</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Return list of file paths</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">raise</span> <span class="ne">NotImplementedError</span>
+
+    <span class="k">def</span> <span class="nf">delete</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">recursive</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Delete the indicated file or directory</span>
+
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        path : string</span>
+<span class="sd">        recursive : boolean, default False</span>
+<span class="sd">            If True, also delete child paths for directories</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">raise</span> <span class="ne">NotImplementedError</span>
+
+    <span class="k">def</span> <span class="nf">disk_usage</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Compute bytes used by all contents under indicated path in file tree</span>
+
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        path : string</span>
+<span class="sd">            Can be a file path or directory</span>
+
+<span class="sd">        Returns</span>
+<span class="sd">        -------</span>
+<span class="sd">        usage : int</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">path_info</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">stat</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
+        <span class="k">if</span> <span class="n">path_info</span><span class="p">[</span><span class="s1">&#39;kind&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="s1">&#39;file&#39;</span><span class="p">:</span>
+            <span class="k">return</span> <span class="n">path_info</span><span class="p">[</span><span class="s1">&#39;size&#39;</span><span class="p">]</span>
+
+        <span class="n">total</span> <span class="o">=</span> <span class="mi">0</span>
+        <span class="k">for</span> <span class="n">root</span><span class="p">,</span> <span class="n">directories</span><span class="p">,</span> <span class="n">files</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">walk</span><span class="p">(</span><span class="n">path</span><span class="p">):</span>
+            <span class="k">for</span> <span class="n">child_path</span> <span class="ow">in</span> <span class="n">files</span><span class="p">:</span>
+                <span class="n">abspath</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_path_join</span><span class="p">(</span><span class="n">root</span><span class="p">,</span> <span class="n">child_path</span><span class="p">)</span>
+                <span class="n">total</span> <span class="o">+=</span> <span class="bp">self</span><span class="o">.</span><span class="n">stat</span><span class="p">(</span><span class="n">abspath</span><span class="p">)[</span><span class="s1">&#39;size&#39;</span><span class="p">]</span>
+
+        <span class="k">return</span> <span class="n">total</span>
+
+    <span class="k">def</span> <span class="nf">_path_join</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">):</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">pathsep</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">args</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">stat</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+
+<span class="sd">        Returns</span>
+<span class="sd">        -------</span>
+<span class="sd">        stat : dict</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span><span class="s1">&#39;FileSystem.stat&#39;</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">rm</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">recursive</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Alias for FileSystem.delete</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">delete</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">recursive</span><span class="o">=</span><span class="n">recursive</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">mv</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">new_path</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Alias for FileSystem.rename</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">rename</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">new_path</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">rename</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">new_path</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Rename file, like UNIX mv command</span>
+
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        path : string</span>
+<span class="sd">            Path to alter</span>
+<span class="sd">        new_path : string</span>
+<span class="sd">            Path to move to</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span><span class="s1">&#39;FileSystem.rename&#39;</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">mkdir</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">create_parents</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
+        <span class="k">raise</span> <span class="ne">NotImplementedError</span>
+
+    <span class="k">def</span> <span class="nf">exists</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+        <span class="k">raise</span> <span class="ne">NotImplementedError</span>
+
+    <span class="k">def</span> <span class="nf">isdir</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Return True if path is a directory</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">raise</span> <span class="ne">NotImplementedError</span>
+
+    <span class="k">def</span> <span class="nf">isfile</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Return True if path is a file</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">raise</span> <span class="ne">NotImplementedError</span>
+
+    <span class="k">def</span> <span class="nf">read_parquet</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">columns</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">metadata</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">schema</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                     <span class="n">nthreads</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">use_pandas_metadata</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Read Parquet data from path in file system. Can read from a single file</span>
+<span class="sd">        or a directory of files</span>
+
+<span class="sd">        Parameters</span>
+<span class="sd">        ----------</span>
+<span class="sd">        path : str</span>
+<span class="sd">            Single file path or directory</span>
+<span class="sd">        columns : List[str], optional</span>
+<span class="sd">            Subset of columns to read</span>
+<span class="sd">        metadata : pyarrow.parquet.FileMetaData</span>
+<span class="sd">            Known metadata to validate files against</span>
+<span class="sd">        schema : pyarrow.parquet.Schema</span>
+<span class="sd">            Known schema to validate files against. Alternative to metadata</span>
+<span class="sd">            argument</span>
+<span class="sd">        nthreads : int, default 1</span>
+<span class="sd">            Number of columns to read in parallel. If &gt; 1, requires that the</span>
+<span class="sd">            underlying file source is threadsafe</span>
+<span class="sd">        use_pandas_metadata : boolean, default False</span>
+<span class="sd">            If True and file has custom pandas schema metadata, ensure that</span>
+<span class="sd">            index columns are also loaded</span>
+
+<span class="sd">        Returns</span>
+<span class="sd">        -------</span>
+<span class="sd">        table : pyarrow.Table</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="kn">from</span> <span class="nn">pyarrow.parquet</span> <span class="k">import</span> <span class="n">ParquetDataset</span>
+        <span class="n">dataset</span> <span class="o">=</span> <span class="n">ParquetDataset</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">schema</span><span class="o">=</span><span class="n">schema</span><span class="p">,</span> <span class="n">metadata</span><span class="o">=</span><span class="n">metadata</span><span class="p">,</span>
+                                 <span class="n">filesystem</span><span class="o">=</span><span class="bp">self</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">dataset</span><span class="o">.</span><span class="n">read</span><span class="p">(</span><span class="n">columns</span><span class="o">=</span><span class="n">columns</span><span class="p">,</span> <span class="n">nthreads</span><span class="o">=</span><span class="n">nthreads</span><span class="p">,</span>
+                            <span class="n">use_pandas_metadata</span><span class="o">=</span><span class="n">use_pandas_metadata</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">open</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">mode</span><span class="o">=</span><span class="s1">&#39;rb&#39;</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Open file for reading or writing</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">raise</span> <span class="ne">NotImplementedError</span>
+
+    <span class="nd">@property</span>
+    <span class="k">def</span> <span class="nf">pathsep</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="k">return</span> <span class="s1">&#39;/&#39;</span>
+
+
+<div class="viewcode-block" id="LocalFileSystem"><a class="viewcode-back" href="../../generated/pyarrow.LocalFileSystem.html#pyarrow.LocalFileSystem">[docs]</a><span class="k">class</span> <span class="nc">LocalFileSystem</span><span class="p">(</span><span class="n">FileSystem</span><span class="p">):</span>
+
+    <span class="n">_instance</span> <span class="o">=</span> <span class="kc">None</span>
+
+    <span class="nd">@classmethod</span>
+<div class="viewcode-block" id="LocalFileSystem.get_instance"><a class="viewcode-back" href="../../generated/pyarrow.LocalFileSystem.html#pyarrow.LocalFileSystem.get_instance">[docs]</a>    <span class="k">def</span> <span class="nf">get_instance</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span>
+        <span class="k">if</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_instance</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="bp">cls</span><span class="o">.</span><span class="n">_instance</span> <span class="o">=</span> <span class="n">LocalFileSystem</span><span class="p">()</span>
+        <span class="k">return</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_instance</span></div>
+
+    <span class="nd">@implements</span><span class="p">(</span><span class="n">FileSystem</span><span class="o">.</span><span class="n">ls</span><span class="p">)</span>
+<div class="viewcode-block" id="LocalFileSystem.ls"><a class="viewcode-back" href="../../generated/pyarrow.LocalFileSystem.html#pyarrow.LocalFileSystem.ls">[docs]</a>    <span class="k">def</span> <span class="nf">ls</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+        <span class="k">return</span> <span class="nb">sorted</span><span class="p">(</span><span class="n">pjoin</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">x</span><span class="p">)</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">os</span><span class="o">.</span><span class="n">listdir</span><span class="p">(</span><span class="n">path</span><span class="p">))</span></div>
+
+    <span class="nd">@implements</span><span class="p">(</span><span class="n">FileSystem</span><span class="o">.</span><span class="n">mkdir</span><span class="p">)</span>
+<div class="viewcode-block" id="LocalFileSystem.mkdir"><a class="viewcode-back" href="../../generated/pyarrow.LocalFileSystem.html#pyarrow.LocalFileSystem.mkdir">[docs]</a>    <span class="k">def</span> <span class="nf">mkdir</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">create_parents</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
+        <span class="k">if</span> <span class="n">create_parents</span><span class="p">:</span>
+            <span class="n">os</span><span class="o">.</span><span class="n">makedirs</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">os</span><span class="o">.</span><span class="n">mkdir</span><span class="p">(</span><span class="n">path</span><span class="p">)</span></div>
+
+    <span class="nd">@implements</span><span class="p">(</span><span class="n">FileSystem</span><span class="o">.</span><span class="n">isdir</span><span class="p">)</span>
+<div class="viewcode-block" id="LocalFileSystem.isdir"><a class="viewcode-back" href="../../generated/pyarrow.LocalFileSystem.html#pyarrow.LocalFileSystem.isdir">[docs]</a>    <span class="k">def</span> <span class="nf">isdir</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+        <span class="k">return</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isdir</span><span class="p">(</span><span class="n">path</span><span class="p">)</span></div>
+
+    <span class="nd">@implements</span><span class="p">(</span><span class="n">FileSystem</span><span class="o">.</span><span class="n">isfile</span><span class="p">)</span>
+<div class="viewcode-block" id="LocalFileSystem.isfile"><a class="viewcode-back" href="../../generated/pyarrow.LocalFileSystem.html#pyarrow.LocalFileSystem.isfile">[docs]</a>    <span class="k">def</span> <span class="nf">isfile</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+        <span class="k">return</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isfile</span><span class="p">(</span><span class="n">path</span><span class="p">)</span></div>
+
+    <span class="nd">@implements</span><span class="p">(</span><span class="n">FileSystem</span><span class="o">.</span><span class="n">exists</span><span class="p">)</span>
+<div class="viewcode-block" id="LocalFileSystem.exists"><a class="viewcode-back" href="../../generated/pyarrow.LocalFileSystem.html#pyarrow.LocalFileSystem.exists">[docs]</a>    <span class="k">def</span> <span class="nf">exists</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+        <span class="k">return</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">path</span><span class="p">)</span></div>
+
+    <span class="nd">@implements</span><span class="p">(</span><span class="n">FileSystem</span><span class="o">.</span><span class="n">open</span><span class="p">)</span>
+<div class="viewcode-block" id="LocalFileSystem.open"><a class="viewcode-back" href="../../generated/pyarrow.LocalFileSystem.html#pyarrow.LocalFileSystem.open">[docs]</a>    <span class="k">def</span> <span class="nf">open</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">mode</span><span class="o">=</span><span class="s1">&#39;rb&#39;</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Open file for reading or writing</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="nb">open</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">mode</span><span class="o">=</span><span class="n">mode</span><span class="p">)</span></div>
+
+    <span class="nd">@property</span>
+    <span class="k">def</span> <span class="nf">pathsep</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="k">return</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">sep</span>
+
+<div class="viewcode-block" id="LocalFileSystem.walk"><a class="viewcode-back" href="../../generated/pyarrow.LocalFileSystem.html#pyarrow.LocalFileSystem.walk">[docs]</a>    <span class="k">def</span> <span class="nf">walk</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">top_dir</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Directory tree generator, see os.walk</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="n">os</span><span class="o">.</span><span class="n">walk</span><span class="p">(</span><span class="n">top_dir</span><span class="p">)</span></div></div>
+
+
+<span class="k">class</span> <span class="nc">DaskFileSystem</span><span class="p">(</span><span class="n">FileSystem</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Wraps s3fs Dask filesystem implementation like s3fs, gcsfs, etc.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">fs</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">fs</span> <span class="o">=</span> <span class="n">fs</span>
+
+    <span class="nd">@implements</span><span class="p">(</span><span class="n">FileSystem</span><span class="o">.</span><span class="n">isdir</span><span class="p">)</span>
+    <span class="k">def</span> <span class="nf">isdir</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+        <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span><span class="s2">&quot;Unsupported file system API&quot;</span><span class="p">)</span>
+
+    <span class="nd">@implements</span><span class="p">(</span><span class="n">FileSystem</span><span class="o">.</span><span class="n">isfile</span><span class="p">)</span>
+    <span class="k">def</span> <span class="nf">isfile</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+        <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span><span class="s2">&quot;Unsupported file system API&quot;</span><span class="p">)</span>
+
+    <span class="nd">@implements</span><span class="p">(</span><span class="n">FileSystem</span><span class="o">.</span><span class="n">delete</span><span class="p">)</span>
+    <span class="k">def</span> <span class="nf">delete</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">recursive</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">fs</span><span class="o">.</span><span class="n">rm</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">recursive</span><span class="o">=</span><span class="n">recursive</span><span class="p">)</span>
+
+    <span class="nd">@implements</span><span class="p">(</span><span class="n">FileSystem</span><span class="o">.</span><span class="n">mkdir</span><span class="p">)</span>
+    <span class="k">def</span> <span class="nf">mkdir</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">fs</span><span class="o">.</span><span class="n">mkdir</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
+
+    <span class="nd">@implements</span><span class="p">(</span><span class="n">FileSystem</span><span class="o">.</span><span class="n">open</span><span class="p">)</span>
+    <span class="k">def</span> <span class="nf">open</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">mode</span><span class="o">=</span><span class="s1">&#39;rb&#39;</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Open file for reading or writing</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">fs</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">mode</span><span class="o">=</span><span class="n">mode</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">ls</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">detail</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">fs</span><span class="o">.</span><span class="n">ls</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">detail</span><span class="o">=</span><span class="n">detail</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">walk</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">top_path</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Directory tree generator, like os.walk</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">fs</span><span class="o">.</span><span class="n">walk</span><span class="p">(</span><span class="n">top_path</span><span class="p">)</span>
+
+
+<span class="k">class</span> <span class="nc">S3FSWrapper</span><span class="p">(</span><span class="n">DaskFileSystem</span><span class="p">):</span>
+
+    <span class="nd">@implements</span><span class="p">(</span><span class="n">FileSystem</span><span class="o">.</span><span class="n">isdir</span><span class="p">)</span>
+    <span class="k">def</span> <span class="nf">isdir</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+        <span class="k">try</span><span class="p">:</span>
+            <span class="n">contents</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">fs</span><span class="o">.</span><span class="n">ls</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
+            <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">contents</span><span class="p">)</span> <span class="o">==</span> <span class="mi">1</span> <span class="ow">and</span> <span class="n">contents</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="n">path</span><span class="p">:</span>
+                <span class="k">return</span> <span class="kc">False</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="k">return</span> <span class="kc">True</span>
+        <span class="k">except</span> <span class="ne">OSError</span><span class="p">:</span>
+            <span class="k">return</span> <span class="kc">False</span>
+
+    <span class="nd">@implements</span><span class="p">(</span><span class="n">FileSystem</span><span class="o">.</span><span class="n">isfile</span><span class="p">)</span>
+    <span class="k">def</span> <span class="nf">isfile</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+        <span class="k">try</span><span class="p">:</span>
+            <span class="n">contents</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">fs</span><span class="o">.</span><span class="n">ls</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
+            <span class="k">return</span> <span class="nb">len</span><span class="p">(</span><span class="n">contents</span><span class="p">)</span> <span class="o">==</span> <span class="mi">1</span> <span class="ow">and</span> <span class="n">contents</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="n">path</span>
+        <span class="k">except</span> <span class="ne">OSError</span><span class="p">:</span>
+            <span class="k">return</span> <span class="kc">False</span>
+
+    <span class="k">def</span> <span class="nf">walk</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">refresh</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Directory tree generator, like os.walk</span>
+
+<span class="sd">        Generator version of what is in s3fs, which yields a flattened list of</span>
+<span class="sd">        files</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">path</span> <span class="o">=</span> <span class="n">path</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;s3://&#39;</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span>
+        <span class="n">directories</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
+        <span class="n">files</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
+
+        <span class="k">for</span> <span class="n">key</span> <span class="ow">in</span> <span class="nb">list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">fs</span><span class="o">.</span><span class="n">_ls</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">refresh</span><span class="o">=</span><span class="n">refresh</span><span class="p">)):</span>
+            <span class="n">path</span> <span class="o">=</span> <span class="n">key</span><span class="p">[</span><span class="s1">&#39;Key&#39;</span><span class="p">]</span>
+            <span class="k">if</span> <span class="n">key</span><span class="p">[</span><span class="s1">&#39;StorageClass&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="s1">&#39;DIRECTORY&#39;</span><span class="p">:</span>
+                <span class="n">directories</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
+            <span class="k">elif</span> <span class="n">key</span><span class="p">[</span><span class="s1">&#39;StorageClass&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="s1">&#39;BUCKET&#39;</span><span class="p">:</span>
+                <span class="k">pass</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="n">files</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
+
+        <span class="c1"># s3fs creates duplicate &#39;DIRECTORY&#39; entries</span>
+        <span class="n">files</span> <span class="o">=</span> <span class="nb">sorted</span><span class="p">([</span><span class="n">posixpath</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="n">f</span><span class="p">)[</span><span class="mi">1</span><span class="p">]</span> <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="n">files</span>
+                        <span class="k">if</span> <span class="n">f</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">directories</span><span class="p">])</span>
+        <span class="n">directories</span> <span class="o">=</span> <span class="nb">sorted</span><span class="p">([</span><span class="n">posixpath</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="n">x</span><span class="p">)[</span><span class="mi">1</span><span class="p">]</span>
+                              <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">directories</span><span class="p">])</span>
+
+        <span class="k">yield</span> <span class="n">path</span><span class="p">,</span> <span class="n">directories</span><span class="p">,</span> <span class="n">files</span>
+
+        <span class="k">for</span> <span class="n">directory</span> <span class="ow">in</span> <span class="n">directories</span><span class="p">:</span>
+            <span class="k">for</span> <span class="n">tup</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">walk</span><span class="p">(</span><span class="n">directory</span><span class="p">,</span> <span class="n">refresh</span><span class="o">=</span><span class="n">refresh</span><span class="p">):</span>
+                <span class="k">yield</span> <span class="n">tup</span>
+</pre></div>
+
+    </div>
+      
+  </div>
+</div>
+<footer class="footer">
+  <div class="container">
+    <p class="pull-right">
+      <a href="#">Back to top</a>
+      
+    </p>
+    <p>
+        &copy; Copyright 2016-2017 Apache Software Foundation.<br/>
+      Created using <a href="http://sphinx-doc.org/">Sphinx</a> 1.5.5.<br/>
+    </p>
+  </div>
+</footer>
+  </body>
+</html>
\ No newline at end of file


Mime
View raw message