arrow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject [1/7] arrow-site git commit: Update to new Jekyll-rendered website
Date Sun, 23 Apr 2017 22:11:23 GMT
Repository: arrow-site
Updated Branches:
  refs/heads/asf-site 355ede334 -> 97fc1851b


http://git-wip-us.apache.org/repos/asf/arrow-site/blob/97fc1851/docs/ipc.html
----------------------------------------------------------------------
diff --git a/docs/ipc.html b/docs/ipc.html
new file mode 100644
index 0000000..c84d7c8
--- /dev/null
+++ b/docs/ipc.html
@@ -0,0 +1,279 @@
+<!DOCTYPE html>
+<html lang="en-US">
+  <head>
+    <meta charset="UTF-8">
+    <title>Arrow Messaging and IPC</title>
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <meta name="generator" content="Jekyll v3.4.3">
+    <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags -->
+    <link rel="icon" type="image/x-icon" href="/favicon.ico">
+
+    <title>Apache Arrow Homepage</title>
+    <link rel="stylesheet" href="//fonts.googleapis.com/css?family=Lato:300,300italic,400,400italic,700,700italic,900">
+
+    <link href="/css/main.css" rel="stylesheet">
+    <script src="https://code.jquery.com/jquery-3.2.1.min.js"
+            integrity="sha256-hwg4gsxgFZhOsEEamdOYGBf13FyQuiTwlAQgxVSNgt4="
+            crossorigin="anonymous"></script>
+    <script src="/assets/javascripts/bootstrap.min.js"></script>
+  </head>
+
+
+<body class="wrap">
+  <div class="container">
+    <nav class="navbar navbar-default">
+  <div class="container-fluid">
+    <div class="navbar-header">
+      <a class="navbar-brand" href="/">Apache Arrow&#8482;&nbsp;&nbsp;&nbsp;</a>
+    </div>
+
+    <!-- Collect the nav links, forms, and other content for toggling -->
+    <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
+      <ul class="nav navbar-nav">
+        <li class="dropdown">
+          <a href="#" class="dropdown-toggle" data-toggle="dropdown"
+             role="button" aria-haspopup="true"
+             aria-expanded="false">Specification<span class="caret"></span>
+          </a>
+          <ul class="dropdown-menu">
+            <li><a href="/docs/memory_layout.html">Memory Layout</a></li>
+            <li><a href="/docs/metadata.html">Metadata</a></li>
+            <li><a href="/docs/ipc.html">Messaging / IPC</a></li>
+          </ul>
+        </li>
+
+        <li class="dropdown">
+          <a href="#" class="dropdown-toggle" data-toggle="dropdown"
+             role="button" aria-haspopup="true"
+             aria-expanded="false">Code Documentation<span class="caret"></span>
+          </a>
+          <ul class="dropdown-menu">
+            <li><a href="/docs/python">Python</a></li>
+            <li><a href="/docs/java">Java</a></li>
+          </ul>
+        </li>
+        <!-- <li><a href="/blog">Blog</a></li> -->
+        <li class="dropdown">
+          <a href="#" class="dropdown-toggle" data-toggle="dropdown"
+             role="button" aria-haspopup="true"
+             aria-expanded="false">ASF Links<span class="caret"></span>
+          </a>
+          <ul class="dropdown-menu">
+            <li><a href="http://www.apache.org/">ASF Website</a></li>
+            <li><a href="http://www.apache.org/licenses/">License</a></li>
+            <li><a href="http://www.apache.org/foundation/sponsorship.html">Donate</a></li>
+            <li><a href="http://www.apache.org/foundation/thanks.html">Thanks</a></li>
+            <li><a href="http://www.apache.org/security/">Security</a></li>
+          </ul>
+        </li>
+      </ul>
+      <a href="http://www.apache.org/">
+        <img style="float:right;" src="/img/asf_logo.svg" width="120px"/>
+      </a>
+      </div><!-- /.navbar-collapse -->
+    </div>
+  </nav>
+
+
+    <!--
+
+-->
+
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+<h1 id="interprocess-messaging--communication-ipc">Interprocess messaging / communication (IPC)</h1>
+
+<h2 id="encapsulated-message-format">Encapsulated message format</h2>
+
+<p>Data components in the stream and file formats are represented as encapsulated
+<em>messages</em> consisting of:</p>
+
+<ul>
+  <li>A length prefix indicating the metadata size</li>
+  <li>The message metadata as a <a href="https://github.com/google]/flatbuffers">Flatbuffer</a></li>
+  <li>Padding bytes to an 8-byte boundary</li>
+  <li>The message body</li>
+</ul>
+
+<p>Schematically, we have:</p>
+
+<div class="highlighter-rouge"><pre class="highlight"><code>&lt;metadata_size: int32&gt;
+&lt;metadata_flatbuffer: bytes&gt;
+&lt;padding&gt;
+&lt;message body&gt;
+</code></pre>
+</div>
+
+<p>The <code class="highlighter-rouge">metadata_size</code> includes the size of the flatbuffer plus padding. The
+<code class="highlighter-rouge">Message</code> flatbuffer includes a version number, the particular message (as a
+flatbuffer union), and the size of the message body:</p>
+
+<div class="highlighter-rouge"><pre class="highlight"><code>table Message {
+  version: org.apache.arrow.flatbuf.MetadataVersion;
+  header: MessageHeader;
+  bodyLength: long;
+}
+</code></pre>
+</div>
+
+<p>Currently, we support 4 types of messages:</p>
+
+<ul>
+  <li>Schema</li>
+  <li>RecordBatch</li>
+  <li>DictionaryBatch</li>
+  <li>Tensor</li>
+</ul>
+
+<h2 id="streaming-format">Streaming format</h2>
+
+<p>We provide a streaming format for record batches. It is presented as a sequence
+of encapsulated messages, each of which follows the format above. The schema
+comes first in the stream, and it is the same for all of the record batches
+that follow. If any fields in the schema are dictionary-encoded, one or more
+<code class="highlighter-rouge">DictionaryBatch</code> messages will follow the schema.</p>
+
+<div class="highlighter-rouge"><pre class="highlight"><code>&lt;SCHEMA&gt;
+&lt;DICTIONARY 0&gt;
+...
+&lt;DICTIONARY k - 1&gt;
+&lt;RECORD BATCH 0&gt;
+...
+&lt;RECORD BATCH n - 1&gt;
+&lt;EOS [optional]: int32&gt;
+</code></pre>
+</div>
+
+<p>When a stream reader implementation is reading a stream, after each message, it
+may read the next 4 bytes to know how large the message metadata that follows
+is. Once the message flatbuffer is read, you can then read the message body.</p>
+
+<p>The stream writer can signal end-of-stream (EOS) either by writing a 0 length
+as an <code class="highlighter-rouge">int32</code> or simply closing the stream interface.</p>
+
+<h2 id="file-format">File format</h2>
+
+<p>We define a “file format” supporting random access in a very similar format to
+the streaming format. The file starts and ends with a magic string <code class="highlighter-rouge">ARROW1</code>
+(plus padding). What follows in the file is identical to the stream format. At
+the end of the file, we write a <em>footer</em> including offsets and sizes for each
+of the data blocks in the file, so that random access is possible. See
+<a href="https://github.com/apache/arrow/blob/master/format/File.fbs">format/File.fbs</a> for the precise details of the file footer.</p>
+
+<p>Schematically we have:</p>
+
+<div class="highlighter-rouge"><pre class="highlight"><code>&lt;magic number "ARROW1"&gt;
+&lt;empty padding bytes [to 8 byte boundary]&gt;
+&lt;STREAMING FORMAT&gt;
+&lt;FOOTER&gt;
+&lt;FOOTER SIZE: int32&gt;
+&lt;magic number "ARROW1"&gt;
+</code></pre>
+</div>
+
+<h3 id="recordbatch-body-structure">RecordBatch body structure</h3>
+
+<p>The <code class="highlighter-rouge">RecordBatch</code> metadata contains a depth-first (pre-order) flattened set of
+field metadata and physical memory buffers (some comments from <a href="https://github.com/apache/arrow/blob/master/format/Message.fbs">Message.fbs</a>
+have been shortened / removed):</p>
+
+<div class="highlighter-rouge"><pre class="highlight"><code>table RecordBatch {
+  length: long;
+  nodes: [FieldNode];
+  buffers: [Buffer];
+}
+
+struct FieldNode {
+  length: long;
+  null_count: long;
+}
+
+struct Buffer {
+  /// The shared memory page id where this buffer is located. Currently this is
+  /// not used
+  page: int;
+
+  /// The relative offset into the shared memory page where the bytes for this
+  /// buffer starts
+  offset: long;
+
+  /// The absolute length (in bytes) of the memory buffer. The memory is found
+  /// from offset (inclusive) to offset + length (non-inclusive).
+  length: long;
+}
+</code></pre>
+</div>
+
+<p>In the context of a file, the <code class="highlighter-rouge">page</code> is not used, and the <code class="highlighter-rouge">Buffer</code> offsets use
+as a frame of reference the start of the message body. So, while in a general
+IPC setting these offsets may be anyplace in one or more shared memory regions,
+in the file format the offsets start from 0.</p>
+
+<p>The location of a record batch and the size of the metadata block as well as
+the body of buffers is stored in the file footer:</p>
+
+<div class="highlighter-rouge"><pre class="highlight"><code>struct Block {
+  offset: long;
+  metaDataLength: int;
+  bodyLength: long;
+}
+</code></pre>
+</div>
+
+<p>Some notes about this</p>
+
+<ul>
+  <li>The <code class="highlighter-rouge">Block</code> offset indicates the starting byte of the record batch.</li>
+  <li>The metadata length includes the flatbuffer size, the record batch metadata
+flatbuffer, and any padding bytes</li>
+</ul>
+
+<h3 id="dictionary-batches">Dictionary Batches</h3>
+
+<p>Dictionary batches have not yet been implemented, while they are provided for
+in the metadata. For the time being, the <code class="highlighter-rouge">DICTIONARY</code> segments shown above in
+the file do not appear in any of the file implementations.</p>
+
+<h3 id="tensor-multi-dimensional-array-message-format">Tensor (Multi-dimensional Array) Message Format</h3>
+
+<p>The <code class="highlighter-rouge">Tensor</code> message types provides a way to write a multidimensional array of
+fixed-size values (such as a NumPy ndarray) using Arrow’s shared memory
+tools. Arrow implementations in general are not required to implement this data
+format, though we provide a reference implementation in C++.</p>
+
+<p>When writing a standalone encapsulated tensor message, we use the format as
+indicated above, but additionally align the starting offset (if writing to a
+shared memory region) to be a multiple of 8:</p>
+
+<div class="highlighter-rouge"><pre class="highlight"><code>&lt;PADDING&gt;
+&lt;metadata size: int32&gt;
+&lt;metadata&gt;
+&lt;tensor body&gt;
+</code></pre>
+</div>
+
+
+
+    <hr></hr>
+
+    <footer class="footer">
+  <p>Apache Arrow, Arrow, Apache, the Apache feather logo, and the Apache Arrow project logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</p>
+  <p>&copy; 2017 Apache Software Foundation</p>
+</footer>
+
+  </div>
+</body>
+</html>

http://git-wip-us.apache.org/repos/asf/arrow-site/blob/97fc1851/docs/memory_layout.html
----------------------------------------------------------------------
diff --git a/docs/memory_layout.html b/docs/memory_layout.html
new file mode 100644
index 0000000..05fe86e
--- /dev/null
+++ b/docs/memory_layout.html
@@ -0,0 +1,751 @@
+<!DOCTYPE html>
+<html lang="en-US">
+  <head>
+    <meta charset="UTF-8">
+    <title>Physical Memory Layout</title>
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <meta name="generator" content="Jekyll v3.4.3">
+    <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags -->
+    <link rel="icon" type="image/x-icon" href="/favicon.ico">
+
+    <title>Apache Arrow Homepage</title>
+    <link rel="stylesheet" href="//fonts.googleapis.com/css?family=Lato:300,300italic,400,400italic,700,700italic,900">
+
+    <link href="/css/main.css" rel="stylesheet">
+    <script src="https://code.jquery.com/jquery-3.2.1.min.js"
+            integrity="sha256-hwg4gsxgFZhOsEEamdOYGBf13FyQuiTwlAQgxVSNgt4="
+            crossorigin="anonymous"></script>
+    <script src="/assets/javascripts/bootstrap.min.js"></script>
+  </head>
+
+
+<body class="wrap">
+  <div class="container">
+    <nav class="navbar navbar-default">
+  <div class="container-fluid">
+    <div class="navbar-header">
+      <a class="navbar-brand" href="/">Apache Arrow&#8482;&nbsp;&nbsp;&nbsp;</a>
+    </div>
+
+    <!-- Collect the nav links, forms, and other content for toggling -->
+    <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
+      <ul class="nav navbar-nav">
+        <li class="dropdown">
+          <a href="#" class="dropdown-toggle" data-toggle="dropdown"
+             role="button" aria-haspopup="true"
+             aria-expanded="false">Specification<span class="caret"></span>
+          </a>
+          <ul class="dropdown-menu">
+            <li><a href="/docs/memory_layout.html">Memory Layout</a></li>
+            <li><a href="/docs/metadata.html">Metadata</a></li>
+            <li><a href="/docs/ipc.html">Messaging / IPC</a></li>
+          </ul>
+        </li>
+
+        <li class="dropdown">
+          <a href="#" class="dropdown-toggle" data-toggle="dropdown"
+             role="button" aria-haspopup="true"
+             aria-expanded="false">Code Documentation<span class="caret"></span>
+          </a>
+          <ul class="dropdown-menu">
+            <li><a href="/docs/python">Python</a></li>
+            <li><a href="/docs/java">Java</a></li>
+          </ul>
+        </li>
+        <!-- <li><a href="/blog">Blog</a></li> -->
+        <li class="dropdown">
+          <a href="#" class="dropdown-toggle" data-toggle="dropdown"
+             role="button" aria-haspopup="true"
+             aria-expanded="false">ASF Links<span class="caret"></span>
+          </a>
+          <ul class="dropdown-menu">
+            <li><a href="http://www.apache.org/">ASF Website</a></li>
+            <li><a href="http://www.apache.org/licenses/">License</a></li>
+            <li><a href="http://www.apache.org/foundation/sponsorship.html">Donate</a></li>
+            <li><a href="http://www.apache.org/foundation/thanks.html">Thanks</a></li>
+            <li><a href="http://www.apache.org/security/">Security</a></li>
+          </ul>
+        </li>
+      </ul>
+      <a href="http://www.apache.org/">
+        <img style="float:right;" src="/img/asf_logo.svg" width="120px"/>
+      </a>
+      </div><!-- /.navbar-collapse -->
+    </div>
+  </nav>
+
+
+    <!--
+
+-->
+
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+<h1 id="arrow-physical-memory-layout">Arrow: Physical memory layout</h1>
+
+<h2 id="definitions--terminology">Definitions / Terminology</h2>
+
+<p>Since different projects have used differents words to describe various
+concepts, here is a small glossary to help disambiguate.</p>
+
+<ul>
+  <li>Array: a sequence of values with known length all having the same type.</li>
+  <li>Slot or array slot: a single logical value in an array of some particular data type</li>
+  <li>Contiguous memory region: a sequential virtual address space with a given
+length. Any byte can be reached via a single pointer offset less than the
+region’s length.</li>
+  <li>Contiguous memory buffer: A contiguous memory region that stores
+a multi-value component of an Array.  Sometimes referred to as just “buffer”.</li>
+  <li>Primitive type: a data type that occupies a fixed-size memory slot specified
+in bit width or byte width</li>
+  <li>Nested or parametric type: a data type whose full structure depends on one or
+more other child relative types. Two fully-specified nested types are equal
+if and only if their child types are equal. For example, <code class="highlighter-rouge">List&lt;U&gt;</code> is distinct
+from <code class="highlighter-rouge">List&lt;V&gt;</code> iff U and V are different relative types.</li>
+  <li>Relative type or simply type (unqualified): either a specific primitive type
+or a fully-specified nested type. When we say slot we mean a relative type
+value, not necessarily any physical storage region.</li>
+  <li>Logical type: A data type that is implemented using some relative (physical)
+type. For example, a Decimal value stored in 16 bytes could be stored in a
+primitive array with slot size 16 bytes. Similarly, strings can be stored as
+<code class="highlighter-rouge">List&lt;1-byte&gt;</code>.</li>
+  <li>Parent and child arrays: names to express relationships between physical
+value arrays in a nested type structure. For example, a <code class="highlighter-rouge">List&lt;T&gt;</code>-type parent
+array has a T-type array as its child (see more on lists below).</li>
+  <li>Leaf node or leaf: A primitive value array that may or may not be a child
+array of some array with a nested type.</li>
+</ul>
+
+<h2 id="requirements-goals-and-non-goals">Requirements, goals, and non-goals</h2>
+
+<p>Base requirements</p>
+
+<ul>
+  <li>A physical memory layout enabling zero-deserialization data interchange
+amongst a variety of systems handling flat and nested columnar data, including
+such systems as Spark, Drill, Impala, Kudu, Ibis, Spark, ODBC protocols, and
+proprietary systems that utilize the open source components.</li>
+  <li>All array slots are accessible in constant time, with complexity growing
+linearly in the nesting level</li>
+  <li>Capable of representing fully-materialized and decoded / decompressed <a href="https://parquet.apache.org/documentation/latest/">Parquet</a>
+data</li>
+  <li>All contiguous memory buffers are aligned at 64-byte boundaries and padded to a multiple of 64 bytes.</li>
+  <li>Any relative type can have null slots</li>
+  <li>Arrays are immutable once created. Implementations can provide APIs to mutate
+an array, but applying mutations will require a new array data structure to
+be built.</li>
+  <li>Arrays are relocatable (e.g. for RPC/transient storage) without pointer
+swizzling. Another way of putting this is that contiguous memory regions can
+be migrated to a different address space (e.g. via a memcpy-type of
+operation) without altering their contents.</li>
+</ul>
+
+<h2 id="goals-for-this-document">Goals (for this document)</h2>
+
+<ul>
+  <li>To describe relative types (physical value types and a preliminary set of
+nested types) sufficient for an unambiguous implementation</li>
+  <li>Memory layout and random access patterns for each relative type</li>
+  <li>Null value representation</li>
+</ul>
+
+<h2 id="non-goals-for-this-document">Non-goals (for this document)</h2>
+
+<ul>
+  <li>To enumerate or specify logical types that can be implemented as primitive
+(fixed-width) value types. For example: signed and unsigned integers,
+floating point numbers, boolean, exact decimals, date and time types,
+CHAR(K), VARCHAR(K), etc.</li>
+  <li>To specify standardized metadata or a data layout for RPC or transient file
+storage.</li>
+  <li>To define a selection or masking vector construct</li>
+  <li>Implementation-specific details</li>
+  <li>Details of a user or developer C/C++/Java API.</li>
+  <li>Any “table” structure composed of named arrays each having their own type or
+any other structure that composes arrays.</li>
+  <li>Any memory management or reference counting subsystem</li>
+  <li>To enumerate or specify types of encodings or compression support</li>
+</ul>
+
+<h2 id="byte-order-endianness">Byte Order (<a href="https://en.wikipedia.org/wiki/Endianness">Endianness</a>)</h2>
+
+<p>The Arrow format is little endian by default.
+The Schema metadata has an endianness field indicating endianness of RecordBatches.
+Typically this is the endianness of the system where the RecordBatch was generated.
+The main use case is exchanging RecordBatches between systems with the same Endianness.
+At first we will return an error when trying to read a Schema with an endianness
+that does not match the underlying system. The reference implementation is focused on
+Little Endian and provides tests for it. Eventually we may provide automatic conversion
+via byte swapping.</p>
+
+<h2 id="alignment-and-padding">Alignment and Padding</h2>
+
+<p>As noted above, all buffers are intended to be aligned in memory at 64 byte
+boundaries and padded to a length that is a multiple of 64 bytes.  The alignment
+requirement follows best practices for optimized memory access:</p>
+
+<ul>
+  <li>Elements in numeric arrays will be guaranteed to be retrieved via aligned access.</li>
+  <li>On some architectures alignment can help limit partially used cache lines.</li>
+  <li>64 byte alignment is recommended by the <a href="https://software.intel.com/en-us/articles/practical-intel-avx-optimization-on-2nd-generation-intel-core-processors">Intel performance guide</a> for
+data-structures over 64 bytes (which will be a common case for Arrow Arrays).</li>
+</ul>
+
+<p>Requiring padding to a multiple of 64 bytes allows for using <a href="https://software.intel.com/en-us/node/600110">SIMD</a> instructions
+consistently in loops without additional conditional checks.
+This should allow for simpler and more efficient code.<br />
+The specific padding length was chosen because it matches the largest known
+SIMD instruction registers available as of April 2016 (Intel AVX-512).
+Guaranteed padding can also allow certain compilers
+to generate more optimized code directly (e.g. One can safely use Intel’s
+<code class="highlighter-rouge">-qopt-assume-safe-padding</code>).</p>
+
+<p>Unless otherwise noted, padded bytes do not need to have a specific value.</p>
+
+<h2 id="array-lengths">Array lengths</h2>
+
+<p>Any array has a known and fixed length, stored as a 32-bit signed integer, so a
+maximum of 2<sup>31</sup> - 1 elements. We choose a signed int32 for a couple reasons:</p>
+
+<ul>
+  <li>Enhance compatibility with Java and client languages which may have varying
+quality of support for unsigned integers.</li>
+  <li>To encourage developers to compose smaller arrays (each of which contains
+contiguous memory in its leaf nodes) to create larger array structures
+possibly exceeding 2<sup>31</sup> - 1 elements, as opposed to allocating very large
+contiguous memory blocks.</li>
+</ul>
+
+<h2 id="null-count">Null count</h2>
+
+<p>The number of null value slots is a property of the physical array and
+considered part of the data structure. The null count is stored as a 32-bit
+signed integer, as it may be as large as the array length.</p>
+
+<h2 id="null-bitmaps">Null bitmaps</h2>
+
+<p>Any relative type can have null value slots, whether primitive or nested type.</p>
+
+<p>An array with nulls must have a contiguous memory buffer, known as the null (or
+validity) bitmap, whose length is a multiple of 64 bytes (as discussed above)<br />
+and large enough to have at least 1 bit for each array
+slot.</p>
+
+<p>Whether any array slot is valid (non-null) is encoded in the respective bits of
+this bitmap. A 1 (set bit) for index <code class="highlighter-rouge">j</code> indicates that the value is not null,
+while a 0 (bit not set) indicates that it is null. Bitmaps are to be
+initialized to be all unset at allocation time (this includes padding).</p>
+
+<div class="highlighter-rouge"><pre class="highlight"><code>is_valid[j] -&gt; bitmap[j / 8] &amp; (1 &lt;&lt; (j % 8))
+</code></pre>
+</div>
+
+<p>We use <a href="https://en.wikipedia.org/wiki/Bit_numbering">least-significant bit (LSB) numbering</a> (also known as
+bit-endianness). This means that within a group of 8 bits, we read
+right-to-left:</p>
+
+<div class="highlighter-rouge"><pre class="highlight"><code>values = [0, 1, null, 2, null, 3]
+
+bitmap
+j mod 8   7  6  5  4  3  2  1  0
+          0  0  1  0  1  0  1  1
+</code></pre>
+</div>
+
+<p>Arrays having a 0 null count may choose to not allocate the null
+bitmap. Implementations may choose to always allocate one anyway as a matter of
+convenience, but this should be noted when memory is being shared.</p>
+
+<p>Nested type arrays have their own null bitmap and null count regardless of
+the null count and null bits of their child arrays.</p>
+
+<h2 id="primitive-value-arrays">Primitive value arrays</h2>
+
+<p>A primitive value array represents a fixed-length array of values each having
+the same physical slot width typically measured in bytes, though the spec also
+provides for bit-packed types (e.g. boolean values encoded in bits).</p>
+
+<p>Internally, the array contains a contiguous memory buffer whose total size is
+equal to the slot width multiplied by the array length. For bit-packed types,
+the size is rounded up to the nearest byte.</p>
+
+<p>The associated null bitmap is contiguously allocated (as described above) but
+does not need to be adjacent in memory to the values buffer.</p>
+
+<h3 id="example-layout-int32-array">Example Layout: Int32 Array</h3>
+<p>For example a primitive array of int32s:</p>
+
+<p>[1, 2, null, 4, 8]</p>
+
+<p>Would look like:</p>
+
+<div class="highlighter-rouge"><pre class="highlight"><code>* Length: 5, Null count: 1
+* Null bitmap buffer:
+
+  |Byte 0 (validity bitmap) | Bytes 1-63            |
+  |-------------------------|-----------------------|
+  |00011011                 | 0 (padding)           |
+
+* Value Buffer:
+
+  |Bytes 0-3   | Bytes 4-7   | Bytes 8-11  | Bytes 12-15 | Bytes 16-19 | Bytes 20-63 |
+  |------------|-------------|-------------|-------------|-------------|-------------|
+  | 1          | 2           | unspecified | 4           | 8           | unspecified |
+</code></pre>
+</div>
+
+<h3 id="example-layout-non-null-int32-array">Example Layout: Non-null int32 Array</h3>
+
+<p>[1, 2, 3, 4, 8] has two possible layouts:</p>
+
+<div class="highlighter-rouge"><pre class="highlight"><code>* Length: 5, Null count: 0
+* Null bitmap buffer:
+
+  | Byte 0 (validity bitmap) | Bytes 1-63            |
+  |--------------------------|-----------------------|
+  | 00011111                 | 0 (padding)           |
+
+* Value Buffer:
+
+  |Bytes 0-3   | Bytes 4-7   | Bytes 8-11  | bytes 12-15 | bytes 16-19 | Bytes 20-63 |
+  |------------|-------------|-------------|-------------|-------------|-------------|
+  | 1          | 2           | 3           | 4           | 8           | unspecified |
+</code></pre>
+</div>
+
+<p>or with the bitmap elided:</p>
+
+<div class="highlighter-rouge"><pre class="highlight"><code>* Length 5, Null count: 0
+* Null bitmap buffer: Not required
+* Value Buffer:
+
+  |Bytes 0-3   | Bytes 4-7   | Bytes 8-11  | bytes 12-15 | bytes 16-19 | Bytes 20-63 |
+  |------------|-------------|-------------|-------------|-------------|-------------|
+  | 1          | 2           | 3           | 4           | 8           | unspecified |
+</code></pre>
+</div>
+
+<h2 id="list-type">List type</h2>
+
+<p>List is a nested type in which each array slot contains a variable-size
+sequence of values all having the same relative type (heterogeneity can be
+achieved through unions, described later).</p>
+
+<p>A list type is specified like <code class="highlighter-rouge">List&lt;T&gt;</code>, where <code class="highlighter-rouge">T</code> is any relative type
+(primitive or nested).</p>
+
+<p>A list-array is represented by the combination of the following:</p>
+
+<ul>
+  <li>A values array, a child array of type T. T may also be a nested type.</li>
+  <li>An offsets buffer containing 32-bit signed integers with length equal to the
+length of the top-level array plus one. Note that this limits the size of the
+values array to 2<sup>31</sup>-1.</li>
+</ul>
+
+<p>The offsets array encodes a start position in the values array, and the length
+of the value in each slot is computed using the first difference with the next
+element in the offsets array. For example. the position and length of slot j is
+computed as:</p>
+
+<div class="highlighter-rouge"><pre class="highlight"><code>slot_position = offsets[j]
+slot_length = offsets[j + 1] - offsets[j]  // (for 0 &lt;= j &lt; length)
+</code></pre>
+</div>
+
+<p>The first value in the offsets array is 0, and the last element is the length
+of the values array.</p>
+
+<h3 id="example-layout-listchar-array">Example Layout: <code class="highlighter-rouge">List&lt;Char&gt;</code> Array</h3>
+<p>Let’s consider an example, the type <code class="highlighter-rouge">List&lt;Char&gt;</code>, where Char is a 1-byte
+logical type.</p>
+
+<p>For an array of length 4 with respective values:</p>
+
+<p>[[‘j’, ‘o’, ‘e’], null, [‘m’, ‘a’, ‘r’, ‘k’], []]</p>
+
+<p>will have the following representation:</p>
+
+<div class="highlighter-rouge"><pre class="highlight"><code>* Length: 4, Null count: 1
+* Null bitmap buffer:
+
+  | Byte 0 (validity bitmap) | Bytes 1-63            |
+  |--------------------------|-----------------------|
+  | 00001101                 | 0 (padding)           |
+
+* Offsets buffer (int32)
+
+  | Bytes 0-3  | Bytes 4-7   | Bytes 8-11  | Bytes 12-15 | Bytes 16-19 | Bytes 20-63 |
+  |------------|-------------|-------------|-------------|-------------|-------------|
+  | 0          | 3           | 3           | 7           | 7           | unspecified |
+
+* Values array (char array):
+  * Length: 7,  Null count: 0
+  * Null bitmap buffer: Not required
+
+    | Bytes 0-7  | Bytes 8-63  |
+    |------------|-------------|
+    | joemark    | unspecified |
+</code></pre>
+</div>
+
+<h3 id="example-layout-listlistbyte">Example Layout: <code class="highlighter-rouge">List&lt;List&lt;byte&gt;&gt;</code></h3>
+<p>[[[1, 2], [3, 4]], [[5, 6, 7], null, [8]], [[9, 10]]]</p>
+
+<p>will be be represented as follows:</p>
+
+<div class="highlighter-rouge"><pre class="highlight"><code>* Length 3
+* Nulls count: 0
+* Null bitmap buffer: Not required
+* Offsets buffer (int32)
+
+  | Bytes 0-3  | Bytes 4-7  | Bytes 8-11 | Bytes 12-15 | Bytes 16-63 |
+  |------------|------------|------------|-------------|-------------|
+  | 0          |  2         |  5         |  6          | unspecified |
+
+* Values array (`List&lt;byte&gt;`)
+  * Length: 6, Null count: 1
+  * Null bitmap buffer:
+
+    | Byte 0 (validity bitmap) | Bytes 1-63  |
+    |--------------------------|-------------|
+    | 00110111                 | 0 (padding) |
+
+  * Offsets buffer (int32)
+
+    | Bytes 0-28           | Bytes 29-63 |
+    |----------------------|-------------|
+    | 0, 2, 4, 7, 7, 8, 10 | unspecified |
+
+  * Values array (bytes):
+    * Length: 10, Null count: 0
+    * Null bitmap buffer: Not required
+
+      | Bytes 0-9                     | Bytes 10-63 |
+      |-------------------------------|-------------|
+      | 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 | unspecified |
+</code></pre>
+</div>
+
+<h2 id="struct-type">Struct type</h2>
+
+<p>A struct is a nested type parameterized by an ordered sequence of relative
+types (which can all be distinct), called its fields.</p>
+
+<p>Typically the fields have names, but the names and their types are part of the
+type metadata, not the physical memory layout.</p>
+
+<p>A struct array does not have any additional allocated physical storage for its values.
+A struct array must still have an allocated null bitmap, if it has one or more null values.</p>
+
+<p>Physically, a struct type has one child array for each field.</p>
+
+<p>For example, the struct (field names shown here as strings for illustration
+purposes)</p>
+
+<div class="highlighter-rouge"><pre class="highlight"><code>Struct &lt;
+  name: String (= List&lt;char&gt;),
+  age: Int32
+&gt;
+</code></pre>
+</div>
+
+<p>has two child arrays, one List<char> array (layout as above) and one 4-byte
+primitive value array having Int32 logical type.</char></p>
+
+<h3 id="example-layout-structlistchar-int32">Example Layout: <code class="highlighter-rouge">Struct&lt;List&lt;char&gt;, Int32&gt;</code>:</h3>
+<p>The layout for [{‘joe’, 1}, {null, 2}, null, {‘mark’, 4}] would be:</p>
+
+<div class="highlighter-rouge"><pre class="highlight"><code>* Length: 4, Null count: 1
+* Null bitmap buffer:
+
+  | Byte 0 (validity bitmap) | Bytes 1-7   | Bytes 8-63  |
+  |--------------------------|-------------|-------------|
+  | 00001011                 | 0 (padding) | unspecified |
+
+* Children arrays:
+  * field-0 array (`List&lt;char&gt;`):
+    * Length: 4, Null count: 1
+    * Null bitmap buffer:
+
+      | Byte 0 (validity bitmap) | Bytes 1-7             |
+      |--------------------------|-----------------------|
+      | 00001101                 | 0 (padding)           |
+
+    * Offsets buffer:
+
+      | Bytes 0-19     |
+      |----------------|
+      | 0, 3, 3, 6, 10 |
+
+     * Values array:
+        * Length: 10, Null count: 0
+        * Null bitmap buffer: Not required
+
+        * Value buffer:
+
+          | Bytes 0-9      |
+          |----------------|
+          | joebobmark     |
+
+  * field-1 array (int32 array):
+    * Length: 4, Null count: 0
+    * Null bitmap buffer: Not required
+    * Value Buffer:
+
+      | Bytes 0-15     |
+      |----------------|
+      | 1, 2, 3, 4     |
+
+</code></pre>
+</div>
+
+<p>While a struct does not have physical storage for each of its semantic slots
+(i.e. each scalar C-like struct), an entire struct slot can be set to null via
+the null bitmap. Any of the child field arrays can have null values according
+to their respective independent null bitmaps.
+This implies that for a particular struct slot the null bitmap for the struct
+array might indicate a null slot when one or more of its child arrays has a
+non-null value in their corresponding slot.  When reading the struct array the
+parent null bitmap is authoritative.
+This is illustrated in the example above, the child arrays have valid entries
+for the null struct but are ‘hidden’ from the consumer by the parent array’s
+null bitmap.  However, when treated independently corresponding
+values of the children array will be non-null.</p>
+
+<h2 id="dense-union-type">Dense union type</h2>
+
+<p>A dense union is semantically similar to a struct, and contains an ordered
+sequence of relative types. While a struct contains multiple arrays, a union is
+semantically a single array in which each slot can have a different type.</p>
+
+<p>The union types may be named, but like structs this will be a matter of the
+metadata and will not affect the physical memory layout.</p>
+
+<p>We define two distinct union types that are optimized for different use
+cases. This first, the dense union, represents a mixed-type array with 5 bytes
+of overhead for each value. Its physical layout is as follows:</p>
+
+<ul>
+  <li>One child array for each relative type</li>
+  <li>Types buffer: A buffer of 8-bit signed integers, enumerated from 0 corresponding
+to each type.  A union with more then 127 possible types can be modeled as a
+union of unions.</li>
+  <li>Offsets buffer: A buffer of signed int32 values indicating the relative offset
+into the respective child array for the type in a given slot. The respective
+offsets for each child value array must be in order / increasing.</li>
+</ul>
+
+<p>Critically, the dense union allows for minimal overhead in the ubiquitous
+union-of-structs with non-overlapping-fields use case (<code class="highlighter-rouge">Union&lt;s1: Struct1, s2:
+Struct2, s3: Struct3, ...&gt;</code>)</p>
+
+<h3 id="example-layout-dense-union">Example Layout: Dense union</h3>
+
+<p>An example layout for logical union of:
+<code class="highlighter-rouge">Union&lt;f: float, i: int32&gt;</code> having the values:
+[{f=1.2}, null, {f=3.4}, {i=5}]</p>
+
+<div class="highlighter-rouge"><pre class="highlight"><code>* Length: 4, Null count: 1
+* Null bitmap buffer:
+  |Byte 0 (validity bitmap) | Bytes 1-63            |
+  |-------------------------|-----------------------|
+  |00001101                 | 0 (padding)           |
+
+* Types buffer:
+
+  |Byte 0   | Byte 1      | Byte 2   | Byte 3   | Bytes 4-63  |
+  |---------|-------------|----------|----------|-------------|
+  | 0       | unspecified | 0        | 1        | unspecified |
+
+* Offset buffer:
+
+  |Byte 0-3 | Byte 4-7    | Byte 8-11 | Byte 12-15 | Bytes 16-63 |
+  |---------|-------------|-----------|------------|-------------|
+  | 0       | unspecified | 1         | 0          | unspecified |
+
+* Children arrays:
+  * Field-0 array (f: float):
+    * Length: 2, nulls: 0
+    * Null bitmap buffer: Not required
+
+    * Value Buffer:
+
+      | Bytes 0-7 | Bytes 8-63  |
+      |-----------|-------------|
+      | 1.2, 3.4  | unspecified |
+
+
+  * Field-1 array (i: int32):
+    * Length: 1, nulls: 0
+    * Null bitmap buffer: Not required
+
+    * Value Buffer:
+
+      | Bytes 0-3 | Bytes 4-63  |
+      |-----------|-------------|
+      | 5         | unspecified |
+</code></pre>
+</div>
+
+<h2 id="sparse-union-type">Sparse union type</h2>
+
+<p>A sparse union has the same structure as a dense union, with the omission of
+the offsets array. In this case, the child arrays are each equal in length to
+the length of the union.</p>
+
+<p>While a sparse union may use significantly more space compared with a dense
+union, it has some advantages that may be desirable in certain use cases:</p>
+
+<ul>
+  <li>A sparse union is more amenable to vectorized expression evaluation in some use cases.</li>
+  <li>Equal-length arrays can be interpreted as a union by only defining the types array.</li>
+</ul>
+
+<h3 id="example-layout-sparseunionu0-int32-u1-float-u2-listchar">Example layout: <code class="highlighter-rouge">SparseUnion&lt;u0: Int32, u1: Float, u2: List&lt;Char&gt;&gt;</code></h3>
+
+<p>For the union array:</p>
+
+<p>[{u0=5}, {u1=1.2}, {u2=’joe’}, {u1=3.4}, {u0=4}, {u2=’mark’}]</p>
+
+<p>will have the following layout:</p>
+<div class="highlighter-rouge"><pre class="highlight"><code>* Length: 6, Null count: 0
+* Null bitmap buffer: Not required
+
+* Types buffer:
+
+ | Byte 0     | Byte 1      | Byte 2      | Byte 3      | Byte 4      | Byte 5       | Bytes  6-63           |
+ |------------|-------------|-------------|-------------|-------------|--------------|-----------------------|
+ | 0          | 1           | 2           | 1           | 0           | 2            | unspecified (padding) |
+
+* Children arrays:
+
+  * u0 (Int32):
+    * Length: 6, Null count: 4
+    * Null bitmap buffer:
+
+      |Byte 0 (validity bitmap) | Bytes 1-63            |
+      |-------------------------|-----------------------|
+      |00010001                 | 0 (padding)           |
+
+    * Value buffer:
+
+      |Bytes 0-3   | Bytes 4-7   | Bytes 8-11  | Bytes 12-15 | Bytes 16-19 | Bytes 20-23  | Bytes 24-63           |
+      |------------|-------------|-------------|-------------|-------------|--------------|-----------------------|
+      | 1          | unspecified | unspecified | unspecified | 4           |  unspecified | unspecified (padding) |
+
+  * u1 (float):
+    * Length: 6, Null count: 4
+    * Null bitmap buffer:
+
+      |Byte 0 (validity bitmap) | Bytes 1-63            |
+      |-------------------------|-----------------------|
+      |00001010                 | 0 (padding)           |
+
+    * Value buffer:
+
+      |Bytes 0-3    | Bytes 4-7   | Bytes 8-11  | Bytes 12-15 | Bytes 16-19 | Bytes 20-23  | Bytes 24-63           |
+      |-------------|-------------|-------------|-------------|-------------|--------------|-----------------------|
+      | unspecified |  1.2        | unspecified | 3.4         | unspecified |  unspecified | unspecified (padding) |
+
+  * u2 (`List&lt;char&gt;`)
+    * Length: 6, Null count: 4
+    * Null bitmap buffer:
+
+      | Byte 0 (validity bitmap) | Bytes 1-63            |
+      |--------------------------|-----------------------|
+      | 00100100                 | 0 (padding)           |
+
+    * Offsets buffer (int32)
+
+      | Bytes 0-3  | Bytes 4-7   | Bytes 8-11  | Bytes 12-15 | Bytes 16-19 | Bytes 20-23 | Bytes 24-27 | Bytes 28-63 |
+      |------------|-------------|-------------|-------------|-------------|-------------|-------------|-------------|
+      | 0          | 0           | 0           | 3           | 3           | 3           | 7           | unspecified |
+
+    * Values array (char array):
+      * Length: 7,  Null count: 0
+      * Null bitmap buffer: Not required
+
+        | Bytes 0-7  | Bytes 8-63            |
+        |------------|-----------------------|
+        | joemark    | unspecified (padding) |
+</code></pre>
+</div>
+
+<p>Note that nested types in a sparse union must be internally consistent
+(e.g. see the List in the diagram), i.e. random access at any index j
+on any child array will not cause an error.
+In other words, the array for the nested type must be valid if it is
+reinterpreted as a non-nested array.</p>
+
+<p>Similar to structs, a particular child array may have a non-null slot
+even if the null bitmap of the parent union array indicates the slot is
+null.  Additionally, a child array may have a non-null slot even if
+the the types array indicates that a slot contains a different type at the index.</p>
+
+<h2 id="dictionary-encoding">Dictionary encoding</h2>
+
+<p>When a field is dictionary encoded, the values are represented by an array of Int32 representing the index of the value in the dictionary.
+The Dictionary is received as a DictionaryBacth whose id is referenced by a dictionary attribute defined in the metadata (Message.fbs) in the Field table.
+The dictionary has the same layout as the type of the field would dictate. Each entry in the dictionary can be accessed by its index in the DictionaryBatch.
+When a Schema references a Dictionary id, it must send a DictionaryBatch for this id before any RecordBatch.</p>
+
+<p>As an example, you could have the following data:</p>
+<div class="highlighter-rouge"><pre class="highlight"><code>type: List&lt;String&gt;
+
+[
+ ['a', 'b'],
+ ['a', 'b'],
+ ['a', 'b'],
+ ['c', 'd', 'e'],
+ ['c', 'd', 'e'],
+ ['c', 'd', 'e'],
+ ['c', 'd', 'e'],
+ ['a', 'b']
+]
+</code></pre>
+</div>
+<p>In dictionary-encoded form, this could appear as:</p>
+<div class="highlighter-rouge"><pre class="highlight"><code>data List&lt;String&gt; (dictionary-encoded, dictionary id i)
+indices: [0, 0, 0, 1, 1, 1, 0]
+
+dictionary i
+
+type: List&lt;String&gt;
+
+[
+ ['a', 'b'],
+ ['c', 'd', 'e'],
+]
+</code></pre>
+</div>
+
+<h2 id="references">References</h2>
+
+<p>Apache Drill Documentation - <a href="https://drill.apache.org/docs/value-vectors/">Value Vectors</a></p>
+
+
+
+    <hr></hr>
+
+    <footer class="footer">
+  <p>Apache Arrow, Arrow, Apache, the Apache feather logo, and the Apache Arrow project logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</p>
+  <p>&copy; 2017 Apache Software Foundation</p>
+</footer>
+
+  </div>
+</body>
+</html>

http://git-wip-us.apache.org/repos/asf/arrow-site/blob/97fc1851/docs/metadata.html
----------------------------------------------------------------------
diff --git a/docs/metadata.html b/docs/metadata.html
new file mode 100644
index 0000000..e8da1b8
--- /dev/null
+++ b/docs/metadata.html
@@ -0,0 +1,526 @@
+<!DOCTYPE html>
+<html lang="en-US">
+  <head>
+    <meta charset="UTF-8">
+    <title>Arrow Metadata</title>
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <meta name="generator" content="Jekyll v3.4.3">
+    <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags -->
+    <link rel="icon" type="image/x-icon" href="/favicon.ico">
+
+    <title>Apache Arrow Homepage</title>
+    <link rel="stylesheet" href="//fonts.googleapis.com/css?family=Lato:300,300italic,400,400italic,700,700italic,900">
+
+    <link href="/css/main.css" rel="stylesheet">
+    <script src="https://code.jquery.com/jquery-3.2.1.min.js"
+            integrity="sha256-hwg4gsxgFZhOsEEamdOYGBf13FyQuiTwlAQgxVSNgt4="
+            crossorigin="anonymous"></script>
+    <script src="/assets/javascripts/bootstrap.min.js"></script>
+  </head>
+
+
+<body class="wrap">
+  <div class="container">
+    <nav class="navbar navbar-default">
+  <div class="container-fluid">
+    <div class="navbar-header">
+      <a class="navbar-brand" href="/">Apache Arrow&#8482;&nbsp;&nbsp;&nbsp;</a>
+    </div>
+
+    <!-- Collect the nav links, forms, and other content for toggling -->
+    <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
+      <ul class="nav navbar-nav">
+        <li class="dropdown">
+          <a href="#" class="dropdown-toggle" data-toggle="dropdown"
+             role="button" aria-haspopup="true"
+             aria-expanded="false">Specification<span class="caret"></span>
+          </a>
+          <ul class="dropdown-menu">
+            <li><a href="/docs/memory_layout.html">Memory Layout</a></li>
+            <li><a href="/docs/metadata.html">Metadata</a></li>
+            <li><a href="/docs/ipc.html">Messaging / IPC</a></li>
+          </ul>
+        </li>
+
+        <li class="dropdown">
+          <a href="#" class="dropdown-toggle" data-toggle="dropdown"
+             role="button" aria-haspopup="true"
+             aria-expanded="false">Code Documentation<span class="caret"></span>
+          </a>
+          <ul class="dropdown-menu">
+            <li><a href="/docs/python">Python</a></li>
+            <li><a href="/docs/java">Java</a></li>
+          </ul>
+        </li>
+        <!-- <li><a href="/blog">Blog</a></li> -->
+        <li class="dropdown">
+          <a href="#" class="dropdown-toggle" data-toggle="dropdown"
+             role="button" aria-haspopup="true"
+             aria-expanded="false">ASF Links<span class="caret"></span>
+          </a>
+          <ul class="dropdown-menu">
+            <li><a href="http://www.apache.org/">ASF Website</a></li>
+            <li><a href="http://www.apache.org/licenses/">License</a></li>
+            <li><a href="http://www.apache.org/foundation/sponsorship.html">Donate</a></li>
+            <li><a href="http://www.apache.org/foundation/thanks.html">Thanks</a></li>
+            <li><a href="http://www.apache.org/security/">Security</a></li>
+          </ul>
+        </li>
+      </ul>
+      <a href="http://www.apache.org/">
+        <img style="float:right;" src="/img/asf_logo.svg" width="120px"/>
+      </a>
+      </div><!-- /.navbar-collapse -->
+    </div>
+  </nav>
+
+
+    <!--
+
+-->
+
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+<h1 id="metadata-logical-types-schemas-data-headers">Metadata: Logical types, schemas, data headers</h1>
+
+<p>This is documentation for the Arrow metadata specification, which enables
+systems to communicate the</p>
+
+<ul>
+  <li>
+    <p>Logical array types (which are implemented using the physical memory layouts
+specified in <a href="https://github.com/apache/arrow/blob/master/format/Layout.md">Layout.md</a>)</p>
+  </li>
+  <li>
+    <p>Schemas for table-like collections of Arrow data structures</p>
+  </li>
+  <li>
+    <p>“Data headers” indicating the physical locations of memory buffers sufficient
+to reconstruct a Arrow data structures without copying memory.</p>
+  </li>
+</ul>
+
+<h2 id="canonical-implementation">Canonical implementation</h2>
+
+<p>We are using <a href="http://github.com/google/flatbuffers">Flatbuffers</a> for low-overhead reading and writing of the Arrow
+metadata. See <a href="https://github.com/apache/arrow/blob/master/format/Message.fbs">Message.fbs</a>.</p>
+
+<h2 id="schemas">Schemas</h2>
+
+<p>The <code class="highlighter-rouge">Schema</code> type describes a table-like structure consisting of any number of
+Arrow arrays, each of which can be interpreted as a column in the table. A
+schema by itself does not describe the physical structure of any particular set
+of data.</p>
+
+<p>A schema consists of a sequence of <strong>fields</strong>, which are metadata describing
+the columns. The Flatbuffers IDL for a field is:</p>
+
+<div class="highlighter-rouge"><pre class="highlight"><code>table Field {
+  // Name is not required, in i.e. a List
+  name: string;
+  nullable: bool;
+  type: Type;
+  // present only if the field is dictionary encoded
+  // will point to a dictionary provided by a DictionaryBatch message
+  dictionary: long;
+  // children apply only to Nested data types like Struct, List and Union
+  children: [Field];
+  /// layout of buffers produced for this type (as derived from the Type)
+  /// does not include children
+  /// each recordbatch will return instances of those Buffers.
+  layout: [ VectorLayout ];
+  // User-defined metadata
+  custom_metadata: [ KeyValue ];
+}
+</code></pre>
+</div>
+
+<p>The <code class="highlighter-rouge">type</code> is the logical type of the field. Nested types, such as List,
+Struct, and Union, have a sequence of child fields.</p>
+
+<p>a JSON representation of the schema is also provided:
+Field:</p>
+<div class="highlighter-rouge"><pre class="highlight"><code><span class="p">{</span><span class="w">
+  </span><span class="nt">"name"</span><span class="w"> </span><span class="p">:</span><span class="w"> </span><span class="s2">"name_of_the_field"</span><span class="p">,</span><span class="w">
+  </span><span class="nt">"nullable"</span><span class="w"> </span><span class="p">:</span><span class="w"> </span><span class="kc">false</span><span class="p">,</span><span class="w">
+  </span><span class="nt">"type"</span><span class="w"> </span><span class="p">:</span><span class="w"> </span><span class="err">/*</span><span class="w"> </span><span class="err">Type</span><span class="w"> </span><span class="err">*/</span><span class="p">,</span><span class="w">
+  </span><span class="nt">"children"</span><span class="w"> </span><span class="p">:</span><span class="w"> </span><span class="p">[</span><span class="w"> </span><span class="err">/*</span><span class="w"> </span><span class="err">Field</span><span class="w"> </span><span class="err">*/</span><span class="w"> </span><span class="p">],</span><span class="w">
+  </span><span class="nt">"typeLayout"</span><span class="w"> </span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="w">
+    </span><span class="nt">"vectors"</span><span class="w"> </span><span class="p">:</span><span class="w"> </span><span class="p">[</span><span class="w"> </span><span class="err">/*</span><span class="w"> </span><span class="err">VectorLayout</span><span class="w"> </span><span class="err">*/</span><span class="w"> </span><span class="p">]</span><span class="w">
+  </span><span class="p">}</span><span class="w">
+</span><span class="p">}</span><span class="w">
+</span></code></pre>
+</div>
+<p>VectorLayout:</p>
+<div class="highlighter-rouge"><pre class="highlight"><code><span class="p">{</span><span class="w">
+  </span><span class="nt">"type"</span><span class="w"> </span><span class="p">:</span><span class="w"> </span><span class="s2">"DATA|OFFSET|VALIDITY|TYPE"</span><span class="p">,</span><span class="w">
+  </span><span class="nt">"typeBitWidth"</span><span class="w"> </span><span class="p">:</span><span class="w"> </span><span class="err">/*</span><span class="w"> </span><span class="err">int</span><span class="w"> </span><span class="err">*/</span><span class="w">
+</span><span class="p">}</span><span class="w">
+</span></code></pre>
+</div>
+<p>Type:</p>
+<div class="highlighter-rouge"><pre class="highlight"><code><span class="p">{</span><span class="w">
+  </span><span class="nt">"name"</span><span class="w"> </span><span class="p">:</span><span class="w"> </span><span class="s2">"null|struct|list|union|int|floatingpoint|utf8|binary|fixedsizebinary|bool|decimal|date|time|timestamp|interval"</span><span class="w">
+  </span><span class="err">//</span><span class="w"> </span><span class="err">fields</span><span class="w"> </span><span class="err">as</span><span class="w"> </span><span class="err">defined</span><span class="w"> </span><span class="err">in</span><span class="w"> </span><span class="err">the</span><span class="w"> </span><span class="err">Flatbuffer</span><span class="w"> </span><span class="err">depending</span><span class="w"> </span><span class="err">on</span><span class="w"> </span><span class="err">the</span><span class="w"> </span><span class="err">type</span><span class="w"> </span><span class="err">name</span><span class="w">
+</span><span class="p">}</span><span class="w">
+</span></code></pre>
+</div>
+<p>Union:</p>
+<div class="highlighter-rouge"><pre class="highlight"><code><span class="p">{</span><span class="w">
+  </span><span class="nt">"name"</span><span class="w"> </span><span class="p">:</span><span class="w"> </span><span class="s2">"union"</span><span class="p">,</span><span class="w">
+  </span><span class="nt">"mode"</span><span class="w"> </span><span class="p">:</span><span class="w"> </span><span class="s2">"Sparse|Dense"</span><span class="p">,</span><span class="w">
+  </span><span class="nt">"typeIds"</span><span class="w"> </span><span class="p">:</span><span class="w"> </span><span class="p">[</span><span class="w"> </span><span class="err">/*</span><span class="w"> </span><span class="err">integer</span><span class="w"> </span><span class="err">*/</span><span class="w"> </span><span class="p">]</span><span class="w">
+</span><span class="p">}</span><span class="w">
+</span></code></pre>
+</div>
+
+<p>The <code class="highlighter-rouge">typeIds</code> field in the Union are the codes used to denote each type, which
+may be different from the index of the child array. This is so that the union
+type ids do not have to be enumerated from 0.</p>
+
+<p>Int:</p>
+<div class="highlighter-rouge"><pre class="highlight"><code><span class="p">{</span><span class="w">
+  </span><span class="nt">"name"</span><span class="w"> </span><span class="p">:</span><span class="w"> </span><span class="s2">"int"</span><span class="p">,</span><span class="w">
+  </span><span class="nt">"bitWidth"</span><span class="w"> </span><span class="p">:</span><span class="w"> </span><span class="err">/*</span><span class="w"> </span><span class="err">integer</span><span class="w"> </span><span class="err">*/</span><span class="p">,</span><span class="w">
+  </span><span class="nt">"isSigned"</span><span class="w"> </span><span class="p">:</span><span class="w"> </span><span class="err">/*</span><span class="w"> </span><span class="err">boolean</span><span class="w"> </span><span class="err">*/</span><span class="w">
+</span><span class="p">}</span><span class="w">
+</span></code></pre>
+</div>
+<p>FloatingPoint:</p>
+<div class="highlighter-rouge"><pre class="highlight"><code><span class="p">{</span><span class="w">
+  </span><span class="nt">"name"</span><span class="w"> </span><span class="p">:</span><span class="w"> </span><span class="s2">"floatingpoint"</span><span class="p">,</span><span class="w">
+  </span><span class="nt">"precision"</span><span class="w"> </span><span class="p">:</span><span class="w"> </span><span class="s2">"HALF|SINGLE|DOUBLE"</span><span class="w">
+</span><span class="p">}</span><span class="w">
+</span></code></pre>
+</div>
+<p>Decimal:</p>
+<div class="highlighter-rouge"><pre class="highlight"><code><span class="p">{</span><span class="w">
+  </span><span class="nt">"name"</span><span class="w"> </span><span class="p">:</span><span class="w"> </span><span class="s2">"decimal"</span><span class="p">,</span><span class="w">
+  </span><span class="nt">"precision"</span><span class="w"> </span><span class="p">:</span><span class="w"> </span><span class="err">/*</span><span class="w"> </span><span class="err">integer</span><span class="w"> </span><span class="err">*/</span><span class="p">,</span><span class="w">
+  </span><span class="nt">"scale"</span><span class="w"> </span><span class="p">:</span><span class="w"> </span><span class="err">/*</span><span class="w"> </span><span class="err">integer</span><span class="w"> </span><span class="err">*/</span><span class="w">
+</span><span class="p">}</span><span class="w">
+</span></code></pre>
+</div>
+
+<p>Timestamp:</p>
+
+<div class="highlighter-rouge"><pre class="highlight"><code><span class="p">{</span><span class="w">
+  </span><span class="nt">"name"</span><span class="w"> </span><span class="p">:</span><span class="w"> </span><span class="s2">"timestamp"</span><span class="p">,</span><span class="w">
+  </span><span class="nt">"unit"</span><span class="w"> </span><span class="p">:</span><span class="w"> </span><span class="s2">"SECOND|MILLISECOND|MICROSECOND|NANOSECOND"</span><span class="w">
+</span><span class="p">}</span><span class="w">
+</span></code></pre>
+</div>
+
+<p>Date:</p>
+
+<div class="highlighter-rouge"><pre class="highlight"><code><span class="p">{</span><span class="w">
+  </span><span class="nt">"name"</span><span class="w"> </span><span class="p">:</span><span class="w"> </span><span class="s2">"date"</span><span class="p">,</span><span class="w">
+  </span><span class="nt">"unit"</span><span class="w"> </span><span class="p">:</span><span class="w"> </span><span class="s2">"DAY|MILLISECOND"</span><span class="w">
+</span><span class="p">}</span><span class="w">
+</span></code></pre>
+</div>
+
+<p>Time:</p>
+
+<div class="highlighter-rouge"><pre class="highlight"><code><span class="p">{</span><span class="w">
+  </span><span class="nt">"name"</span><span class="w"> </span><span class="p">:</span><span class="w"> </span><span class="s2">"time"</span><span class="p">,</span><span class="w">
+  </span><span class="nt">"unit"</span><span class="w"> </span><span class="p">:</span><span class="w"> </span><span class="s2">"SECOND|MILLISECOND|MICROSECOND|NANOSECOND"</span><span class="p">,</span><span class="w">
+  </span><span class="nt">"bitWidth"</span><span class="p">:</span><span class="w"> </span><span class="err">/*</span><span class="w"> </span><span class="err">integer:</span><span class="w"> </span><span class="mi">32</span><span class="w"> </span><span class="err">or</span><span class="w"> </span><span class="mi">64</span><span class="w"> </span><span class="err">*/</span><span class="w">
+</span><span class="p">}</span><span class="w">
+</span></code></pre>
+</div>
+
+<p>Interval:</p>
+
+<div class="highlighter-rouge"><pre class="highlight"><code><span class="p">{</span><span class="w">
+  </span><span class="nt">"name"</span><span class="w"> </span><span class="p">:</span><span class="w"> </span><span class="s2">"interval"</span><span class="p">,</span><span class="w">
+  </span><span class="nt">"unit"</span><span class="w"> </span><span class="p">:</span><span class="w"> </span><span class="s2">"YEAR_MONTH|DAY_TIME"</span><span class="w">
+</span><span class="p">}</span><span class="w">
+</span></code></pre>
+</div>
+<p>Schema:</p>
+<div class="highlighter-rouge"><pre class="highlight"><code><span class="p">{</span><span class="w">
+  </span><span class="nt">"fields"</span><span class="w"> </span><span class="p">:</span><span class="w"> </span><span class="p">[</span><span class="w">
+    </span><span class="err">/*</span><span class="w"> </span><span class="err">Field</span><span class="w"> </span><span class="err">*/</span><span class="w">
+  </span><span class="p">]</span><span class="w">
+</span><span class="p">}</span><span class="w">
+</span></code></pre>
+</div>
+
+<h2 id="record-data-headers">Record data headers</h2>
+
+<p>A record batch is a collection of top-level named, equal length Arrow arrays
+(or vectors). If one of the arrays contains nested data, its child arrays are
+not required to be the same length as the top-level arrays.</p>
+
+<p>One can be thought of as a realization of a particular schema. The metadata
+describing a particular record batch is called a “data header”. Here is the
+Flatbuffers IDL for a record batch data header</p>
+
+<div class="highlighter-rouge"><pre class="highlight"><code>table RecordBatch {
+  length: long;
+  nodes: [FieldNode];
+  buffers: [Buffer];
+}
+</code></pre>
+</div>
+
+<p>The <code class="highlighter-rouge">RecordBatch</code> metadata provides for record batches with length exceeding
+2^31 - 1, but Arrow implementations are not required to implement support
+beyond this size.</p>
+
+<p>The <code class="highlighter-rouge">nodes</code> and <code class="highlighter-rouge">buffers</code> fields are produced by a depth-first traversal /
+flattening of a schema (possibly containing nested types) for a given in-memory
+data set.</p>
+
+<h3 id="buffers">Buffers</h3>
+
+<p>A buffer is metadata describing a contiguous memory region relative to some
+virtual address space. This may include:</p>
+
+<ul>
+  <li>Shared memory, e.g. a memory-mapped file</li>
+  <li>An RPC message received in-memory</li>
+  <li>Data in a file</li>
+</ul>
+
+<p>The key form of the Buffer type is:</p>
+
+<div class="highlighter-rouge"><pre class="highlight"><code>struct Buffer {
+  offset: long;
+  length: long;
+}
+</code></pre>
+</div>
+
+<p>In the context of a record batch, each field has some number of buffers
+associated with it, which are derived from their physical memory layout.</p>
+
+<p>Each logical type (separate from its children, if it is a nested type) has a
+deterministic number of buffers associated with it. These will be specified in
+the logical types section.</p>
+
+<h3 id="field-metadata">Field metadata</h3>
+
+<p>The <code class="highlighter-rouge">FieldNode</code> values contain metadata about each level in a nested type
+hierarchy.</p>
+
+<div class="highlighter-rouge"><pre class="highlight"><code>struct FieldNode {
+  /// The number of value slots in the Arrow array at this level of a nested
+  /// tree
+  length: long;
+
+  /// The number of observed nulls.
+  null_count: lohng;
+}
+</code></pre>
+</div>
+
+<p>The <code class="highlighter-rouge">FieldNode</code> metadata provides for fields with length exceeding 2^31 - 1,
+but Arrow implementations are not required to implement support for large
+arrays.</p>
+
+<h2 id="flattening-of-nested-data">Flattening of nested data</h2>
+
+<p>Nested types are flattened in the record batch in depth-first order. When
+visiting each field in the nested type tree, the metadata is appended to the
+top-level <code class="highlighter-rouge">fields</code> array and the buffers associated with that field (but not
+its children) are appended to the <code class="highlighter-rouge">buffers</code> array.</p>
+
+<p>For example, let’s consider the schema</p>
+
+<div class="highlighter-rouge"><pre class="highlight"><code>col1: Struct&lt;a: Int32, b: List&lt;Int64&gt;, c: Float64&gt;
+col2: Utf8
+</code></pre>
+</div>
+
+<p>The flattened version of this is:</p>
+
+<div class="highlighter-rouge"><pre class="highlight"><code>FieldNode 0: Struct name='col1'
+FieldNode 1: Int32 name=a'
+FieldNode 2: List name='b'
+FieldNode 3: Int64 name='item'  # arbitrary
+FieldNode 4: Float64 name='c'
+FieldNode 5: Utf8 name='col2'
+</code></pre>
+</div>
+
+<p>For the buffers produced, we would have the following (as described in more
+detail for each type below):</p>
+
+<div class="highlighter-rouge"><pre class="highlight"><code>buffer 0: field 0 validity bitmap
+
+buffer 1: field 1 validity bitmap
+buffer 2: field 1 values &lt;int32_t*&gt;
+
+buffer 3: field 2 validity bitmap
+buffer 4: field 2 list offsets &lt;int32_t*&gt;
+
+buffer 5: field 3 validity bitmap
+buffer 6: field 3 values &lt;int64_t*&gt;
+
+buffer 7: field 4 validity bitmap
+buffer 8: field 4 values &lt;double*&gt;
+
+buffer 9: field 5 validity bitmap
+buffer 10: field 5 offsets &lt;int32_t*&gt;
+buffer 11: field 5 data &lt;uint8_t*&gt;
+</code></pre>
+</div>
+
+<h2 id="logical-types">Logical types</h2>
+
+<p>A logical type consists of a type name and metadata along with an explicit
+mapping to a physical memory representation. These may fall into some different
+categories:</p>
+
+<ul>
+  <li>Types represented as fixed-width primitive arrays (for example: C-style
+integers and floating point numbers)</li>
+  <li>Types having equivalent memory layout to a physical nested type (e.g. strings
+use the list representation, but logically are not nested types)</li>
+</ul>
+
+<h3 id="integers">Integers</h3>
+
+<p>In the first version of Arrow we provide the standard 8-bit through 64-bit size
+standard C integer types, both signed and unsigned:</p>
+
+<ul>
+  <li>Signed types: Int8, Int16, Int32, Int64</li>
+  <li>Unsigned types: UInt8, UInt16, UInt32, UInt64</li>
+</ul>
+
+<p>The IDL looks like:</p>
+
+<div class="highlighter-rouge"><pre class="highlight"><code>table Int {
+  bitWidth: int;
+  is_signed: bool;
+}
+</code></pre>
+</div>
+
+<p>The integer endianness is currently set globally at the schema level. If a
+schema is set to be little-endian, then all integer types occurring within must
+be little-endian. Integers that are part of other data representations, such as
+list offsets and union types, must have the same endianness as the entire
+record batch.</p>
+
+<h3 id="floating-point-numbers">Floating point numbers</h3>
+
+<p>We provide 3 types of floating point numbers as fixed bit-width primitive array</p>
+
+<ul>
+  <li>Half precision, 16-bit width</li>
+  <li>Single precision, 32-bit width</li>
+  <li>Double precision, 64-bit width</li>
+</ul>
+
+<p>The IDL looks like:</p>
+
+<div class="highlighter-rouge"><pre class="highlight"><code>enum Precision:int {HALF, SINGLE, DOUBLE}
+
+table FloatingPoint {
+  precision: Precision;
+}
+</code></pre>
+</div>
+
+<h3 id="boolean">Boolean</h3>
+
+<p>The Boolean logical type is represented as a 1-bit wide primitive physical
+type. The bits are numbered using least-significant bit (LSB) ordering.</p>
+
+<p>Like other fixed bit-width primitive types, boolean data appears as 2 buffers
+in the data header (one bitmap for the validity vector and one for the values).</p>
+
+<h3 id="list">List</h3>
+
+<p>The <code class="highlighter-rouge">List</code> logical type is the logical (and identically-named) counterpart to
+the List physical type.</p>
+
+<p>In data header form, the list field node contains 2 buffers:</p>
+
+<ul>
+  <li>Validity bitmap</li>
+  <li>List offsets</li>
+</ul>
+
+<p>The buffers associated with a list’s child field are handled recursively
+according to the child logical type (e.g. <code class="highlighter-rouge">List&lt;Utf8&gt;</code> vs. <code class="highlighter-rouge">List&lt;Boolean&gt;</code>).</p>
+
+<h3 id="utf8-and-binary">Utf8 and Binary</h3>
+
+<p>We specify two logical types for variable length bytes:</p>
+
+<ul>
+  <li><code class="highlighter-rouge">Utf8</code> data is unicode values with UTF-8 encoding</li>
+  <li><code class="highlighter-rouge">Binary</code> is any other variable length bytes</li>
+</ul>
+
+<p>These types both have the same memory layout as the nested type <code class="highlighter-rouge">List&lt;UInt8&gt;</code>,
+with the constraint that the inner bytes can contain no null values. From a
+logical type perspective they are primitive, not nested types.</p>
+
+<p>In data header form, while <code class="highlighter-rouge">List&lt;UInt8&gt;</code> would appear as 2 field nodes (<code class="highlighter-rouge">List</code>
+and <code class="highlighter-rouge">UInt8</code>) and 4 buffers (2 for each of the nodes, as per above), these types
+have a simplified representation single field node (of <code class="highlighter-rouge">Utf8</code> or <code class="highlighter-rouge">Binary</code>
+logical type, which have no children) and 3 buffers:</p>
+
+<ul>
+  <li>Validity bitmap</li>
+  <li>List offsets</li>
+  <li>Byte data</li>
+</ul>
+
+<h3 id="decimal">Decimal</h3>
+
+<p>TBD</p>
+
+<h3 id="timestamp">Timestamp</h3>
+
+<p>All timestamps are stored as a 64-bit integer, with one of four unit
+resolutions: second, millisecond, microsecond, and nanosecond.</p>
+
+<h3 id="date">Date</h3>
+
+<p>We support two different date types:</p>
+
+<ul>
+  <li>Days since the UNIX epoch as a 32-bit integer</li>
+  <li>Milliseconds since the UNIX epoch as a 64-bit integer</li>
+</ul>
+
+<h3 id="time">Time</h3>
+
+<p>Time supports the same unit resolutions: second, millisecond, microsecond, and
+nanosecond. We represent time as the smallest integer accommodating the
+indicated unit. For second and millisecond: 32-bit, for the others 64-bit.</p>
+
+<h2 id="dictionary-encoding">Dictionary encoding</h2>
+
+
+
+    <hr></hr>
+
+    <footer class="footer">
+  <p>Apache Arrow, Arrow, Apache, the Apache feather logo, and the Apache Arrow project logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</p>
+  <p>&copy; 2017 Apache Software Foundation</p>
+</footer>
+
+  </div>
+</body>
+</html>

http://git-wip-us.apache.org/repos/asf/arrow-site/blob/97fc1851/feed.xml
----------------------------------------------------------------------
diff --git a/feed.xml b/feed.xml
new file mode 100644
index 0000000..8794a38
--- /dev/null
+++ b/feed.xml
@@ -0,0 +1 @@
+<?xml version="1.0" encoding="utf-8"?><feed xmlns="http://www.w3.org/2005/Atom" ><generator uri="https://jekyllrb.com/" version="3.4.3">Jekyll</generator><link href="/feed.xml" rel="self" type="application/atom+xml" /><link href="/" rel="alternate" type="text/html" /><updated>2017-04-23T18:10:37-04:00</updated><id>/</id></feed>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/arrow-site/blob/97fc1851/index.html
----------------------------------------------------------------------
diff --git a/index.html b/index.html
index 533faa9..b8552a8 100644
--- a/index.html
+++ b/index.html
@@ -1,43 +1,82 @@
 <!DOCTYPE html>
-<html lang="en">
+<html lang="en-US">
   <head>
-    <meta charset="utf-8">
+    <meta charset="UTF-8">
+    <title></title>
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
     <meta name="viewport" content="width=device-width, initial-scale=1">
+    <meta name="generator" content="Jekyll v3.4.3">
     <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags -->
-    <meta name="description" content="">
-    <meta name="author" content="">
-    <link rel="icon" href="../../favicon.ico">
+    <link rel="icon" type="image/x-icon" href="/favicon.ico">
 
     <title>Apache Arrow Homepage</title>
+    <link rel="stylesheet" href="//fonts.googleapis.com/css?family=Lato:300,300italic,400,400italic,700,700italic,900">
 
-    <!-- Bootstrap core CSS -->
-    <link href="css/bootstrap.min.css" rel="stylesheet">
+    <link href="/css/main.css" rel="stylesheet">
+    <script src="https://code.jquery.com/jquery-3.2.1.min.js"
+            integrity="sha256-hwg4gsxgFZhOsEEamdOYGBf13FyQuiTwlAQgxVSNgt4="
+            crossorigin="anonymous"></script>
+    <script src="/assets/javascripts/bootstrap.min.js"></script>
+  </head>
 
-    <!-- IE10 viewport hack for Surface/desktop Windows 8 bug -->
-    <link href="css/ie10-viewport-bug-workaround.css" rel="stylesheet">
 
-    <!-- Custom styles for this template -->
-    <link href="css/jumbotron-narrow.css" rel="stylesheet">
+<body class="wrap">
+  <div class="container">
+    <nav class="navbar navbar-default">
+  <div class="container-fluid">
+    <div class="navbar-header">
+      <a class="navbar-brand" href="/">Apache Arrow&#8482;&nbsp;&nbsp;&nbsp;</a>
+    </div>
 
-    <!-- Just for debugging purposes. Don't actually copy these 2 lines! -->
-    <!--[if lt IE 9]><script src="/js/ie8-responsive-file-warning.js"></script><![endif]-->
-    <script src="js/ie-emulation-modes-warning.js"></script>
+    <!-- Collect the nav links, forms, and other content for toggling -->
+    <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
+      <ul class="nav navbar-nav">
+        <li class="dropdown">
+          <a href="#" class="dropdown-toggle" data-toggle="dropdown"
+             role="button" aria-haspopup="true"
+             aria-expanded="false">Specification<span class="caret"></span>
+          </a>
+          <ul class="dropdown-menu">
+            <li><a href="/docs/memory_layout.html">Memory Layout</a></li>
+            <li><a href="/docs/metadata.html">Metadata</a></li>
+            <li><a href="/docs/ipc.html">Messaging / IPC</a></li>
+          </ul>
+        </li>
 
-    <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries -->
-    <!--[if lt IE 9]>
-      <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script>
-      <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
-    <![endif]-->
-  </head>
+        <li class="dropdown">
+          <a href="#" class="dropdown-toggle" data-toggle="dropdown"
+             role="button" aria-haspopup="true"
+             aria-expanded="false">Code Documentation<span class="caret"></span>
+          </a>
+          <ul class="dropdown-menu">
+            <li><a href="/docs/python">Python</a></li>
+            <li><a href="/docs/java">Java</a></li>
+          </ul>
+        </li>
+        <!-- <li><a href="/blog">Blog</a></li> -->
+        <li class="dropdown">
+          <a href="#" class="dropdown-toggle" data-toggle="dropdown"
+             role="button" aria-haspopup="true"
+             aria-expanded="false">ASF Links<span class="caret"></span>
+          </a>
+          <ul class="dropdown-menu">
+            <li><a href="http://www.apache.org/">ASF Website</a></li>
+            <li><a href="http://www.apache.org/licenses/">License</a></li>
+            <li><a href="http://www.apache.org/foundation/sponsorship.html">Donate</a></li>
+            <li><a href="http://www.apache.org/foundation/thanks.html">Thanks</a></li>
+            <li><a href="http://www.apache.org/security/">Security</a></li>
+          </ul>
+        </li>
+      </ul>
+      <a href="http://www.apache.org/">
+        <img style="float:right;" src="/img/asf_logo.svg" width="120px"/>
+      </a>
+      </div><!-- /.navbar-collapse -->
+    </div>
+  </nav>
 
-  <body>
 
-    <div class="container">
-      <div class="header clearfix">
-        <h3 class="text-muted"><a href="index.html">Apache Arrow</a>&#8482;&nbsp;&nbsp;&nbsp;<a type="button" class="btn btn-default" href="events/">Upcoming Events</a>
-          <a href="http://www.apache.org/"><img style="float:right;" src="img/asf_logo.svg" width="120px"/><a/></h3>
-      </div>
+        <div class="container">
       <div class="jumbotron">
         <h1>Apache Arrow</h1>
         <p class="lead">Powering Columnar In-Memory Analytics</p>
@@ -45,13 +84,6 @@
           <a class="btn btn-lg btn-success" href="mailto:dev-subscribe@arrow.apache.org" role="button">Join Mailing List</a>
         </p>
       </div>
-      <div>
-  <a type="button" class="btn btn-default" href="http://www.apache.org/">apache foundation</a>
-  <a type="button" class="btn btn-default" href="http://www.apache.org/licenses/">license</a>
-  <a type="button" class="btn btn-default" href="http://www.apache.org/foundation/sponsorship.html">donate</a>
-  <a type="button" class="btn btn-default" href="http://www.apache.org/foundation/thanks.html">thanks</a>
-  <a type="button" class="btn btn-default" href="http://www.apache.org/security/">security</a>
-      </div>
       <div class="row">
         <div class="col-lg-4">
           <h2>Fast</h2>
@@ -93,8 +125,9 @@
         </div>
     </div>
 <h2>Performance Advantage of Columnar In-Memory</h2>
-    <img src="img/simd.png" alt="SIMD" style="width:100%" />
-
+<div align="center">
+  <img src="img/simd.png" alt="SIMD" style="width:60%" />
+</div>
 <h2>Advantages of a Common Data Layer</h2>
 
 <div class="row">
@@ -208,18 +241,16 @@
 </tr>
 </tbody></table>
 
+    </div> <!-- /container -->
+  </body>
+</html>
 
 
-<footer class="footer">
+    <footer class="footer">
   <p>Apache Arrow, Arrow, Apache, the Apache feather logo, and the Apache Arrow project logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</p>
-        <p>&copy; 2017 Apache Software Foundation</p>
-      </footer>
-
-    </div> <!-- /container -->
-
+  <p>&copy; 2017 Apache Software Foundation</p>
+</footer>
 
-    <!-- IE10 viewport hack for Surface/desktop Windows 8 bug -->
-    <script src="js/ie10-viewport-bug-workaround.js"></script>
-  </body>
+  </div>
+</body>
 </html>
-


Mime
View raw message