incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [38/50] [abbrv] git commit: Updating documenation and how to generate it.
Date Sun, 03 Nov 2013 15:20:32 GMT
Updating documenation and how to generate it.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/2d743c68
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/2d743c68
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/2d743c68

Branch: refs/heads/0.3.0-lucene-upgrade
Commit: 2d743c68ae058fc7bafa56ebfa789359a892f5c1
Parents: 6e35701
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Thu Oct 31 15:00:42 2013 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Thu Oct 31 15:04:10 2013 -0400

----------------------------------------------------------------------
 blur-util/generate_docs.sh                      |  19 ++
 .../blur/doc/CreateBlurServerSetupHtmlPage.java | 114 +++++++
 .../src/main/resources/blur-default.properties  |   4 +-
 docs/cluster-setup.base.html                    | 297 +++++++++++++++++++
 docs/cluster-setup.html                         | 178 +----------
 5 files changed, 442 insertions(+), 170 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2d743c68/blur-util/generate_docs.sh
----------------------------------------------------------------------
diff --git a/blur-util/generate_docs.sh b/blur-util/generate_docs.sh
new file mode 100755
index 0000000..2239917
--- /dev/null
+++ b/blur-util/generate_docs.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+mvn exec:java -Dexec.mainClass="org.apache.blur.doc.CreateBlurServerSetupHtmlPage" -Dexec.args="../docs/cluster-setup.base.html
../docs/cluster-setup.html"

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2d743c68/blur-util/src/main/java/org/apache/blur/doc/CreateBlurServerSetupHtmlPage.java
----------------------------------------------------------------------
diff --git a/blur-util/src/main/java/org/apache/blur/doc/CreateBlurServerSetupHtmlPage.java
b/blur-util/src/main/java/org/apache/blur/doc/CreateBlurServerSetupHtmlPage.java
new file mode 100644
index 0000000..bcba22b
--- /dev/null
+++ b/blur-util/src/main/java/org/apache/blur/doc/CreateBlurServerSetupHtmlPage.java
@@ -0,0 +1,114 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.blur.doc;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.PrintWriter;
+import java.util.HashMap;
+import java.util.Map;
+
+public class CreateBlurServerSetupHtmlPage {
+
+  public static void main(String[] args) throws IOException {
+    InputStream inputStream = CreateBlurServerSetupHtmlPage.class.getResourceAsStream("/blur-default.properties");
+    BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream));
+    String line;
+    String prevLine = null;
+
+    String key = "|||General-Server-Properties|||";
+    Map<String, StringBuffer> map = new HashMap<String, StringBuffer>();
+    while ((line = reader.readLine()) != null) {
+      line = line.trim();
+      if (line.equals("### Shard Server Configuration")) {
+        key = "|||Shard-Server-Properties|||";
+      } else if (line.equals("### Controller Server Configuration")) {
+        key = "|||Controller-Server-Properties|||";
+      }
+      if (!line.startsWith("#") && !line.isEmpty()) {
+        System.out.println(prevLine);
+        System.out.println(line);
+        String desc = getDesc(prevLine);
+        String name = getName(line);
+        String value = getValue(line);
+        StringBuffer buffer = map.get(key);
+        if (buffer == null) {
+          buffer = new StringBuffer();
+          map.put(key, buffer);
+        }
+        buffer.append("<tr><td>").append(name);
+        if (!value.trim().isEmpty()) {
+          buffer.append(" (").append(value).append(")");
+        }
+        buffer.append("</td><td>").append(desc).append("</td></tr>");
+      }
+      prevLine = line;
+    }
+    reader.close();
+    String source = args[0];
+    String dest = args[1];
+    replaceValuesInFile(source, dest, map);
+  }
+
+  private static void replaceValuesInFile(String s, String o, Map<String, StringBuffer>
replacements)
+      throws IOException {
+
+    File source = new File(s);
+    File output = new File(o);
+
+    PrintWriter writer = new PrintWriter(output);
+
+    BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(source)));
+    String line;
+    while ((line = reader.readLine()) != null) {
+      StringBuffer newData = replacements.get(line);
+      if (newData != null) {
+        writer.println(newData.toString());
+      } else {
+        writer.println(line);
+      }
+    }
+    writer.close();
+    reader.close();
+
+  }
+
+  private static String getValue(String line) {
+    int index = line.indexOf('=');
+    if (index < 0) {
+      throw new RuntimeException();
+    }
+    return line.substring(index + 1);
+  }
+
+  private static String getName(String line) {
+    int index = line.indexOf('=');
+    if (index < 0) {
+      throw new RuntimeException();
+    }
+    return line.substring(0, index);
+  }
+
+  private static String getDesc(String prevLine) {
+    return prevLine.substring(1).trim();
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2d743c68/blur-util/src/main/resources/blur-default.properties
----------------------------------------------------------------------
diff --git a/blur-util/src/main/resources/blur-default.properties b/blur-util/src/main/resources/blur-default.properties
index 280eded..0d0c7bd 100644
--- a/blur-util/src/main/resources/blur-default.properties
+++ b/blur-util/src/main/resources/blur-default.properties
@@ -28,7 +28,7 @@ blur.query.max.record.fetch=1000
 # Setup metric reporter
 blur.metrics.reporters=
 
-# Shard Properties
+### Shard Server Configuration
 
 # The hostname for the shard, if blank the hostname is automatically detected
 blur.shard.hostname=
@@ -163,7 +163,7 @@ blur.gui.shard.port=40090
 
 
 
-# Controller Properties
+### Controller Server Configuration
 
 # Sets the hostname for the controller, if blank the hostname is automatically detected
 blur.controller.hostname=

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2d743c68/docs/cluster-setup.base.html
----------------------------------------------------------------------
diff --git a/docs/cluster-setup.base.html b/docs/cluster-setup.base.html
new file mode 100644
index 0000000..4672326
--- /dev/null
+++ b/docs/cluster-setup.base.html
@@ -0,0 +1,297 @@
+<!DOCTYPE html>
+<!-- 
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+  
+       http://www.apache.org/licenses/LICENSE-2.0
+  
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<html>
+  <head>
+    <title>Cluster Setup - Apache Blur (Incubator) Documentation</title>
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <!-- Bootstrap -->
+    <link href="resources/css/bootstrap.min.css" rel="stylesheet" media="screen">
+    <link href="resources/css/bs-docs.css" rel="stylesheet" media="screen">
+  </head>
+  <body>
+    <div class="navbar navbar-inverse navbar-fixed-top">
+      <div class="container">
+        <div class="navbar-header">
+          <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">
+            <span class="icon-bar"></span>
+            <span class="icon-bar"></span>
+            <span class="icon-bar"></span>
+          </button>
+          <a class="navbar-brand" href="http://incubator.apache.org/blur">Apache Blur
(Incubator)</a>
+        </div>
+        <div class="collapse navbar-collapse">
+          <ul class="nav navbar-nav">
+	        <li><a href="index.html">Main</a></li>
+            <li><a href="getting-started.html">Getting Started</a></li>
+            <li><a href="data-model.html">Data Model</a></li>
+            <li class="active"><a href="cluster-setup.html">Cluster Setup</a></li>
+            <li><a href="using-blur.html">Using Blur</a></li>
+            <li><a href="Blur.html">Blur API</a></li>
+            <li><a href="site/index.html" target="_blank">Maven Site</a></li>
+            <li><a href="site/apidocs/index.html" target="_blank">Javadocs</a></li>
+          </ul>
+        </div>
+      </div>
+    </div>
+    <div class="container bs-docs-container">
+      <div class="row">
+        <div class="col-md-3">
+          <div class="bs-sidebar hidden-print affix" role="complementary">
+            <ul class="nav bs-sidenav">
+	          <li>
+                <a href="#general">General Configuration</a>
+                <ul class="nav">
+                  <li><a href="#general-blur-site">blur-site.properties</a></li>
+                  <li><a href="#general-hadoop">Hadoop</a></li>
+                </ul>
+              </li>
+              <li>
+                <a href="#controller">Controller Server Configuration</a>
+                <ul class="nav">
+                  <li><a href="#controller-blur-site">blur-site.properties</a></li>
+                  <li><a href="#controller-blur-env">blur-env.sh</a></li>
+                </ul>
+              </li>
+              <li>
+                <a href="#shard">Shard Server Configuration</a>
+                <ul class="nav">
+                  <li><a href="#shard-blur-site">blur-site.properties</a></li>
+                  <li><a href="#shard-blur-env">blur-env.sh</a></li>
+                  <li><a href="#block-cache">Block Cache Configuration</a></li>
+                </ul>
+              </li>
+              <li>
+                <a href="#metrics">Metrics</a>
+                <ul class="nav">
+                  <li><a href="#shard-mbean">Shard Server - MBean</a></li>
+                  <li><a href="#reporters">Other Reporters</a></li>
+                </ul>
+              </li>
+            </ul>
+          </div>
+        </div>
+        <div class="col-md-9" role="main">
+          <section>
+            <div class="page-header">
+              <h1 id="general">General Configuration</h1>
+            </div>
+<p>
+The basic cluster setup involves editing the blur-site.properties and the blur-env.sh 
+files in the $BLUR_HOME/conf directory. It is recommended that a standalone ZooKeeper 
+be setup. Also a modern version of Hadoop with append support is required for proper data
+management (the write ahead log requires the sync operation).
+
+<div class="bs-callout bs-callout-warning"><h4>Caution</h4>If you setup
a standalone ZooKeeper
+you will need to configure Blur to NOT manage the ZooKeeper.  You will need to edit blur-env.sh
+file:
+<pre><code class="bash">export BLUR_MANAGE_ZK=false</code></pre>
+</div>
+</p>
+            <h3 id="general-blur-site">blur-site.properties</h3>
+            <p>
+
+<pre>
+<code class="bash"># The ZooKeeper connection string, consider adding a root path to
the string, it
+# can help when upgrading Blur.
+# Example: zknode1:2181,zknode2:2181,zknode3:2181/blur-0.2.1
+#
+# NOTE: If you provide the root path "/blur-0.2.1", that will have to be manually
+# created before Blur will start.
+
+blur.zookeeper.connection=127.0.0.1
+
+# If you are only going to run a single shard cluster then leave this as default.
+
+blur.cluster.name=default
+
+# Sets the default table location in hdfs.  If left null or omitted the table uri property
in 
+# the table descriptor will be required for all tables.
+
+blur.cluster.default.table.uri=hdfs://namenode/blur/tables</code>
+</pre>
+            </p>
+            <h3 id="general-hadoop">Hadoop</h3>
+<p>
+The current version of Blur has Hadoop 1.2.1 embedded in the &quot;apache-blur-*/lib/hadoop-1.2.1&quot;
path.  However if 
+you are using a different version of Hadoop or want Blur to use the Hadoop configuration
in your installed 
+version you will need to set the &quot;HADOOP_HOME&quot; environment variable in
the 
+&quot;blur-env.sh&quot; script found in &quot;apache-blur-*/conf/&quot;.
+<pre>
+<code class="bash"># Edit the blur-env.sh
+export HADOOP_HOME=&lt;path to your Hadoop install directory&gt;</code>
+</pre>
+</p>
+	      </section>
+          <section>
+            <div class="page-header">
+              <h1 id="controller">Controller Server Configuration</h1>
+            </div>
+            <h3 id="controller-blur-site">blur-site.properties</h3>
+            <p>
+              These are the default settings for the shard server that can be overridden
in the blur-site.properties file. Consider increasing the various thread pool counts (*.thread.count).
The blur.controller.server.remote.thread.count is very important to increase for larger clusters,
basically one thread is used per shard server per query. Some production clusters have set
this thread pool to 2000 or more threads.
+            </p>
+<h4>Properties</h4>
+<table class="table-bordered table-striped table-condensed">
+<tr><td>Property</td><td>Description</td></tr>
+|||Controller-Server-Properties|||
+</table>
+            <h3 id="controller-blur-env">blur-env.sh</h3>
+            <pre><code class="bash"># JAVA JVM OPTIONS for the controller servers,
jvm tuning parameters are placed here.
+# Consider adding the -XX:OnOutOfMemoryError="kill -9 %p" option to kill jvms that are failing
due to memory issues.
+export BLUR_CONTROLLER_JVM_OPTIONS="-Xmx1024m -Djava.net.preferIPv4Stack=true "
+
+# Time to sleep between controller server commands.
+export BLUR_CONTROLLER_SLEEP=0.1
+
+# The of controller servers to spawn per machine.
+export BLUR_NUMBER_OF_CONTROLLER_SERVER_INSTANCES_PER_MACHINE=1</code></pre>
+          </section>
+          <section>
+            <div class="page-header">
+              <h1 id="shard">Shard Server Configuration</h1>
+            </div>
+<h3>Minimum Settings to Configure</h3>
+<p>
+It is highly recommended that the ulimits are increase on the server specifically:
+<ul>
+<li>open files</li>
+<li>max user processes</li>
+</ul>
+<br/>
+In Hadoop the dfs.datanode.max.xcievers should be increased to at least 4096 if not more.
+<pre>
+<code class="bash">&lt;property&gt;
+    &lt;name&gt;dfs.datanode.max.xcievers&lt;/name&gt;
+    &lt;value&gt;4096&lt;/value&gt;
+&lt;/property&gt;</code></pre>
+<br/>
+In blur-env.sh set the cache memory for the shard processes.  DO NOT over allocate this will

+likely crash your server.
+<pre><code class="bash">-XX:MaxDirectMemorySize=13g</code></pre>
+<div class="bs-callout bs-callout-warning"><h4>Caution</h4>
+Swap can kill java perform, you may want to consider disabling swap.</div>
+
+</p>
+            <h3 id="shard-blur-site">blur-site.properties</h3>
+            <p>
+              These are the default settings for the shard server that can be overridden
in the blur-site.properties file. Consider increasing the various thread pool counts (*.thread.count).
Also the blur.max.clause.count sets the BooleanQuery max clause count for Lucene queries.
+            </p>
+
+			<h4>Properties</h4>
+			<table class="table-bordered table-striped table-condensed">
+			<tr><td>Property</td><td>Description</td></tr>
+|||Shard-Server-Properties|||
+			</table>
+
+            <h3 id="shard-blur-env">blur-env.sh</h3>
+            <pre><code class="bash"># JAVA JVM OPTIONS for the shard servers,
jvm tuning parameters are placed here.
+export BLUR_SHARD_JVM_OPTIONS="-Xmx1024m -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=256m
"
+
+# Time to sleep between shard server commands.
+export BLUR_SHARD_SLEEP=0.1
+
+# The of shard servers to spawn per machine.
+export BLUR_NUMBER_OF_SHARD_SERVER_INSTANCES_PER_MACHINE=1</code></pre>
+
+            <h3 id="block-cache">Block Cache Configuration</h3>
+            <h4>Why</h4>
+            <p>HDFS is a great filesystem for streaming large amounts data across large
scale clusters. However the random access latency is typically the same performance you would
get in reading from a local drive if the data you are trying to access is not in the operating
systems file cache. In other words every access to HDFS is similar to a local read with a
cache miss. There have been great performance boosts in HDFS over the past few years but it
still can't perform at the level that a search engine needs.</p>
+            <p>Now you might be thinking that Lucene reads from the local hard drive
and performs great, so why wouldn't HDFS perform fairly well on it's own? However most of
time the Lucene index files are cached by the operating system's file system cache. So Blur
has it's own file system cache allows it to perform low latency data look-ups against HDFS.</p>
+            <h4>How</h4>
+            <p>On shard server start-up Blur creates 1 or more block cache slabs blur.shard.blockcache.slab.count
that are each 128 MB in size. These slabs can be allocated on or off the heap blur.shard.blockcache.direct.memory.allocation.
Each slab is broken up into 16,384 blocks with each block size being 8K. Then on the heap
there is a concurrent LRU cache that tracks what blocks of what files are in which slab(s)
at what offset. So the more slabs of cache you create the more entries there will be in the
LRU thus more heap.</p>
+            <h4>Configuration</h4>
+            <p>Scenario:
+
+            Say the shard server(s) that you are planning to run Blur on have 32G of ram.
These machines are probably also running HDFS data nodes as well with very high xcievers (dfs.datanode.max.xcievers
in hdfs-site.xml) say 8K. If the data nodes are configured with 1G of heap then they may consume
up to 4G of memory due to the high thread count because of the xcievers. Next let's say you
configure Blur to 4G of heap as well, and you want to use 12G of off heap cache.</p>
+            <h5>Auto Configuration</h5>
+            <p>In the blur-env.sh file you would need to change BLUR_SHARD_JVM_OPTIONS
to include "-XX:MaxDirectMemorySize=12g" and possibly "-XX:+UseLargePages" depending on your
Linux setup. If you leave the blur.shard.blockcache.slab.count to the default -1 the shard
startup will automatically detect the -XX:MaxDirectMemorySize size and automatically use almost
all of the memory. By default the JVM has 64m in reserve for direct memory so by default Blur
leaves at least that amount available to the JVM.</p>
+            <h5>Custom Configuration</h5>
+            <p>Again in the blur-env.sh file you would need to change BLUR_SHARD_JVM_OPTIONS
to include "-XX:MaxDirectMemorySize=13g" and possibly "-XX:+UseLargePages" depending on your
Linux setup. I set the MaxDirectMemorySize to more than 12G to make sure we don't hit the
maximum limit and cause a OOM exception, this does not reserve 13G it's a control to not allow
more than that. Below is a working example, it also contains GC logging and GC configuration:</p>
+            <pre><code class="bash">export BLUR_SHARD_JVM_OPTIONS="-XX:MaxDirectMemorySize=13g
\
+            -XX:+UseLargePages \
+            -Xms4g \
+            -Xmx4g \
+            -Xmn512m \
+            -XX:+UseCompressedOops \
+            -XX:+UseConcMarkSweepGC \
+            -XX:+CMSIncrementalMode \
+            -XX:CMSIncrementalDutyCycleMin=10 \
+            -XX:CMSIncrementalDutyCycle=50 \
+            -XX:ParallelGCThreads=8 \
+            -XX:+UseParNewGC \
+            -XX:MaxGCPauseMillis=200 \
+            -XX:GCTimeRatio=10 \
+            -XX:+DisableExplicitGC \
+            -verbose:gc \
+            -XX:+PrintGCDetails \
+            -XX:+PrintGCDateStamps \
+            -Xloggc:$BLUR_HOME/logs/gc-blur-shard-server_`date +%Y%m%d_%H%M%S`.log"</code></pre>
+            <p>Next you will need to setup blur-site.properties by changing blur.shard.blockcache.slab.count
to 96. This is telling blur to allocate 96 128MB slabs of memory at shard server start-up.
Note, that the first time you do this that the shard servers may take long time to allocate
the memory. This is because the OS could be using most of that memory for it's own filesystem
caching and it will need to unload it which may cause some IO due the cache synching to disk.</p>
+            <p>Also the blur.shard.blockcache.direct.memory.allocation is set to true
by default, this will tell the JVM to try and allocate the memory off heap. If you want to
run the slabs in the heap (which is not recommended) set this value to false.</p>
+          </section>
+          <section>
+            <div class="page-header">
+              <h1 id="metrics">Metrics</h1>
+            </div>
+            <p class="lead">Internally Blur uses the Metrics library from Coda Hale
(<a href="http://metrics.codahale.com/">http://metrics.codahale.com/</a>). So
by default all metrics are available through JMX here is a screenshot of what is available
in the Shard server.</p>
+            <h3 id="shard-mbean">Shard Server - MBean Screenshot</h3>
+            <img src="resources/img/BlurShardServer.png" style="max-width:1000px"/>
+            <h3 id="reporters">Configuring Other Reporters</h3>
+            <p class="lead">New reporters can be added configured in the blur-site.properties.
Multiple reporters can be configured.</p>
+            <h4>Example</h4>
+            <pre><code class="bash">blur.metrics.reporters=GangliaReporter
+blur.metrics.reporter.ganglia.period=3
+blur.metrics.reporter.ganglia.unit=SECONDS
+blur.metrics.reporter.ganglia.host=ganglia1
+blur.metrics.reporter.ganglia.port=8649</code></pre>
+            <h4>Reporters to Enable</h4>
+            <pre><code class="bash">blur.metrics.reporters=[ConsoleReporter,CsvReporter,GangliaReporter,GraphiteReporter]</code></pre>
+            <h4>ConsoleReporter</h4>
+            <pre><code class="bash">blur.metrics.reporter.console.period=[5]
+blur.metrics.reporter.console.unit=[NANOSECONDS,MICROSECONDS,MILLISECONDS,SECONDS,MINUTES,HOURS,DAYS]</code></pre>
+            <h4>CsvReporter</h4>
+            <pre><code class="bash">blur.metrics.reporter.csv.period=[5]
+blur.metrics.reporter.csv.unit=[NANOSECONDS,MICROSECONDS,MILLISECONDS,SECONDS,MINUTES,HOURS,DAYS]
+blur.metrics.reporter.csv.outputDir=[.]</code></pre>
+            <h4>GangliaReporter</h4>
+            <pre><code class="bash">blur.metrics.reporter.ganglia.period=[5]
+blur.metrics.reporter.ganglia.unit=[NANOSECONDS,MICROSECONDS,MILLISECONDS,SECONDS,MINUTES,HOURS,DAYS]
+blur.metrics.reporter.ganglia.host=[localhost]
+blur.metrics.reporter.ganglia.port=[-1]
+blur.metrics.reporter.ganglia.prefix=[""]
+blur.metrics.reporter.ganglia.compressPackageNames=[false]</code></pre>
+            <h4>GraphiteReporter</h4>
+            <pre><code class="bash">blur.metrics.reporter.graphite.period=[5]
+blur.metrics.reporter.graphite.unit=[NANOSECONDS,MICROSECONDS,MILLISECONDS,SECONDS,MINUTES,HOURS,DAYS]
+blur.metrics.reporter.graphite.host=[localhost]
+blur.metrics.reporter.graphite.port=[-1]
+blur.metrics.reporter.graphite.prefix=[""]</code></pre>
+          </section>
+        </div>
+      </div>
+    </div>
+    
+    <!-- jQuery (necessary for Bootstrap's JavaScript plugins) -->
+    <script src="resources/js/jquery-2.0.3.min.js"></script>
+    <!-- Include all compiled plugins (below), or include individual files as needed -->
+    <script src="resources/js/bootstrap.min.js"></script>
+    <!-- Enable responsive features in IE8 with Respond.js (https://github.com/scottjehl/Respond)
-->
+    <script src="resources/js/respond.min.js"></script>
+    <script src="resources/js/docs.js"></script>
+  </body>
+</html>

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2d743c68/docs/cluster-setup.html
----------------------------------------------------------------------
diff --git a/docs/cluster-setup.html b/docs/cluster-setup.html
index ad5eb37..c29a4ea 100644
--- a/docs/cluster-setup.html
+++ b/docs/cluster-setup.html
@@ -145,77 +145,11 @@ export HADOOP_HOME=&lt;path to your Hadoop install directory&gt;</code>
             <p>
               These are the default settings for the shard server that can be overridden
in the blur-site.properties file. Consider increasing the various thread pool counts (*.thread.count).
The blur.controller.server.remote.thread.count is very important to increase for larger clusters,
basically one thread is used per shard server per query. Some production clusters have set
this thread pool to 2000 or more threads.
             </p>
-<pre>
-<code class="bash"># Sets the hostname for the controller, if blank the hostname is
automatically detected
-blur.controller.hostname=
-
-# The binding address of the controller
-blur.controller.bind.address=0.0.0.0
-
-# The default binding port of the controller server
-blur.controller.bind.port=40010
-
-# The connection timeout, NOTE: this will be the maximum amount of time you can wait for
a query.
-blur.controller.shard.connection.timeout=60000
-
-# The number of threads used for thrift requests
-blur.controller.server.thrift.thread.count=32
-
-# The number of threads used for remote thrift requests to
-# the shards server.  This should be a large number.
-blur.controller.server.remote.thread.count=64
-
-# The number of threads used for selector processing inside the thrift server.
-blur.controller.thrift.selector.threads=2
-
-# The maximum number of bytes used for reading requests in the thrift server.
-blur.controller.thrift.max.read.buffer.bytes=9223372036854775807
-
-# The size of the blocking queue per selector thread for passing accepted connections to
the selector thread.
-blur.controller.thrift.accept.queue.size.per.thread=4
-
-# The number of hits to fetch per request to the shard servers
-blur.controller.remote.fetch.count=100
-
-# The max number of retries to the shard server when there
-# is an error during fetch
-blur.controller.retry.max.fetch.retries=3
-
-# The max number of retries to the shard server when there
-# is an error during mutate
-blur.controller.retry.max.mutate.retries=3
-
-# The max number of retries to the shard server when there
-# is an error during all other request
-blur.controller.retry.max.default.retries=3
-
-# The starting backoff delay for the first retry for a
-# fetch errors
-blur.controller.retry.fetch.delay=500
-
-# The starting backoff delay for the first retry for a
-# mutate errors
-blur.controller.retry.mutate.delay=500
-
-# The starting backoff delay for the first retry for a
-# all other request errors
-blur.controller.retry.default.delay=500
-
-# The ending backoff delay for the last retry for a
-# fetch errors
-blur.controller.retry.max.fetch.delay=2000
-
-# The ending backoff delay for the last retry for a
-# mutate errors
-blur.controller.retry.max.mutate.delay=2000
-
-# The ending backoff delay for the last retry for a
-# all other request errors
-blur.controller.retry.max.default.delay=2000
-
-# The http status page port for the controller server
-blur.gui.controller.port=40080</code>
-</pre>
+<h4>Properties</h4>
+<table class="table-bordered table-striped table-condensed">
+<tr><td>Property</td><td>Description</td></tr>
+<tr><td>blur.controller.hostname</td><td>Sets the hostname for the
controller, if blank the hostname is automatically detected</td></tr><tr><td>blur.controller.bind.address
(0.0.0.0)</td><td>The binding address of the controller</td></tr><tr><td>blur.controller.bind.port
(40010)</td><td>The default binding port of the controller server</td></tr><tr><td>blur.controller.shard.connection.timeout
(60000)</td><td>The connection timeout, NOTE: this will be the maximum amount
of time you can wait for a query.</td></tr><tr><td>blur.controller.server.thrift.thread.count
(32)</td><td>The number of threads used for thrift requests</td></tr><tr><td>blur.controller.server.remote.thread.count
(64)</td><td>The number of threads used for remote thrift requests to the shards
server.  This should be a large number.</td></tr><tr><td>blur.controller.thrift.selector.threads
(2)</td><td>The number of threads used for selector processing inside the thrift
server.</td></tr><tr><td>blur.controller.thrift.ma
 x.read.buffer.bytes (9223372036854775807)</td><td>The maximum number of bytes
used for reading requests in the thrift server.</td></tr><tr><td>blur.controller.thrift.accept.queue.size.per.thread
(4)</td><td>The size of the blocking queue per selector thread for passing accepted
connections to the selector thread.</td></tr><tr><td>blur.controller.remote.fetch.count
(100)</td><td>The number of hits to fetch per request to the shard servers</td></tr><tr><td>blur.controller.retry.max.fetch.retries
(3)</td><td>The max number of retries to the shard server when there is an error
during fetch</td></tr><tr><td>blur.controller.retry.max.mutate.retries
(3)</td><td>The max number of retries to the shard server when there is an error
during mutate</td></tr><tr><td>blur.controller.retry.max.default.retries
(3)</td><td>The max number of retries to the shard server when there is an error
during all other request</td></tr><tr><td>blur.controller.retry.fetch.delay
(500)</td><td>The starting backoff 
 delay for the first retry for a fetch errors</td></tr><tr><td>blur.controller.retry.mutate.delay
(500)</td><td>The starting backoff delay for the first retry for a mutate errors</td></tr><tr><td>blur.controller.retry.default.delay
(500)</td><td>The starting backoff delay for the first retry for a all other request
errors</td></tr><tr><td>blur.controller.retry.max.fetch.delay (2000)</td><td>The
ending backoff delay for the last retry for a fetch errors</td></tr><tr><td>blur.controller.retry.max.mutate.delay
(2000)</td><td>The ending backoff delay for the last retry for a mutate errors</td></tr><tr><td>blur.controller.retry.max.default.delay
(2000)</td><td>The ending backoff delay for the last retry for a all other request
errors</td></tr><tr><td>blur.gui.controller.port (40080)</td><td>The
http status page port for the controller server</td></tr>
+</table>
             <h3 id="controller-blur-env">blur-env.sh</h3>
             <pre><code class="bash"># JAVA JVM OPTIONS for the controller servers,
jvm tuning parameters are placed here.
 # Consider adding the -XX:OnOutOfMemoryError="kill -9 %p" option to kill jvms that are failing
due to memory issues.
@@ -257,105 +191,13 @@ Swap can kill java perform, you may want to consider disabling swap.</div>
             <p>
               These are the default settings for the shard server that can be overridden
in the blur-site.properties file. Consider increasing the various thread pool counts (*.thread.count).
Also the blur.max.clause.count sets the BooleanQuery max clause count for Lucene queries.
             </p>
-<pre>
-<code class="bash"># The hostname for the shard, if blank the hostname is automatically
detected
-blur.shard.hostname=
-
-# The binding address of the shard
-blur.shard.bind.address=0.0.0.0
-
-# The default binding port of the shard server
-blur.shard.bind.port=40020
-
-# The number of fetcher threads
-blur.shard.data.fetch.thread.count=8
 
-# The number of the thrift threads
-blur.shard.server.thrift.thread.count=8
+			<h4>Properties</h4>
+			<table class="table-bordered table-striped table-condensed">
+			<tr><td>Property</td><td>Description</td></tr>
+<tr><td>blur.shard.hostname</td><td>The hostname for the shard, if
blank the hostname is automatically detected</td></tr><tr><td>blur.shard.bind.address
(0.0.0.0)</td><td>The binding address of the shard</td></tr><tr><td>blur.shard.bind.port
(40020)</td><td>The default binding port of the shard server</td></tr><tr><td>blur.shard.data.fetch.thread.count
(8)</td><td>The number of fetcher threads</td></tr><tr><td>blur.shard.server.thrift.thread.count
(8)</td><td>The number of the thrift threads</td></tr><tr><td>blur.shard.thrift.selector.threads
(2)</td><td>The number of threads used for selector processing inside the thrift
server.</td></tr><tr><td>blur.shard.thrift.max.read.buffer.bytes (9223372036854775807)</td><td>The
maximum number of bytes used for reading requests in the thrift server.</td></tr><tr><td>blur.shard.thrift.accept.queue.size.per.thread
(4)</td><td>The size of the blocking queue per selector thread for passing accepted
connections to the selector thread.</td></tr><tr
 ><td>blur.shard.opener.thread.count (8)</td><td>The number of threads
that are used for opening indexes</td></tr><tr><td>blur.shard.cache.max.querycache.elements
(128)</td><td>The number of cached queries</td></tr><tr><td>blur.shard.cache.max.timetolive
(60000)</td><td>The time to live for the cache query</td></tr><tr><td>blur.shard.filter.cache.class
(org.apache.blur.manager.DefaultBlurFilterCache)</td><td>Default implementation
of the blur cache filter, which is a pass through filter that does nothing</td></tr><tr><td>blur.shard.index.warmup.class
(org.apache.blur.manager.indexserver.DefaultBlurIndexWarmup)</td><td>Default Blur
index warmup class that warms the fields provided in the table descriptor</td></tr><tr><td>blur.shard.index.warmup.throttle
(30000000)</td><td>Throttles the warmup to 30MB/s across all the warmup threads</td></tr><tr><td>blur.shard.block.cache.version
(v2)</td><td>By default the v2 version of the block cache is enabled</td></tr><tr><td>blur.shard.block.cach
 e.total.size</td><td>By default the total amount of memory block cache will use
is -XX:MaxDirectMemorySize - 64 MiB</td></tr><tr><td>blur.shard.blockcache.direct.memory.allocation
(true)</td><td>v1 version of block cache only. By default the block cache using
off heap memory</td></tr><tr><td>blur.shard.blockcache.slab.count
(-1)</td><td>v1 version of block cache only. The slabs in the blockcache are automatically
configured by default (-1) otherwise 1 slab equals 128MB.  The auto config is detected through
the MaxDirectoryMemorySize provided to the JVM</td></tr><tr><td>blur.shard.block.cache.v2.fileBufferSize
(8192)</td><td>v2 version of block cache only. File buffer size, this is the buffer
size used to read and write to data to HDFS.  For production this will likely be increased.</td></tr><tr><td>blur.shard.block.cache.v2.cacheBlockSize
(8192)</td><td>v2 version of block cache only. The is the size of the blocks in
the off heap cache, it is good practice to have this match 'blur.s
 hard.block.cache.v2.fileBufferSize'.  For production this will likely be increased.</td></tr><tr><td>blur.shard.block.cache.v2.cacheBlockSize.filter
(33554432)</td><td>blur.shard.block.cache.v2.cacheBlockSize.<ext>=</td></tr><tr><td>blur.shard.block.cache.v2.store
(OFF_HEAP)</td><td>v2 version of block cache only. This is used to control if
the block are created on or off heap.  Values are OFF_HEAP | ON_HEAP</td></tr><tr><td>blur.shard.block.cache.v2.read.cache.ext</td><td>v2
version of block cache only. This specifies what file types should be cached during reads.
 Comma delimited list.</td></tr><tr><td>blur.shard.block.cache.v2.read.nocache.ext
(fdt)</td><td>v2 version of block cache only. This specifies what file types should
NOT be cached during reads.  Comma delimited list.</td></tr><tr><td>blur.shard.block.cache.v2.read.default
(true)</td><td>v2 version of block cache only. This specifies the default behavior
if a file type is not specified in the cache or nocache lists during
  reads.  Values true | false</td></tr><tr><td>blur.shard.block.cache.v2.write.cache.ext</td><td>v2
version of block cache only. This specifies what file types should be cached during writes.
 Comma delimited list.</td></tr><tr><td>blur.shard.block.cache.v2.write.nocache.ext
(fdt)</td><td>v2 version of block cache only. This specifies what file types should
NOT be cached during writes.  Comma delimited list.</td></tr><tr><td>blur.shard.block.cache.v2.write.default
(true)</td><td>v2 version of block cache only. This specifies the default behavior
if a file type is not specified in the cache or nocache lists during writes.  Values true
| false</td></tr><tr><td>blur.shard.buffercache.1024 (8192)</td><td>The
number of 1K byte buffers</td></tr><tr><td>blur.shard.buffercache.8192
(8192)</td><td>The number of 8K byte buffers</td></tr><tr><td>blur.shard.safemodedelay
(5000)</td><td>The number of milliseconds to wait for the cluster to settle once
changes have ceased</td></tr><tr><td>blur.sha
 rd.time.between.commits (30000)</td><td>The default time between index commits</td></tr><tr><td>blur.shard.time.between.refreshs
(3000)</td><td>The default time between index refreshs</td></tr><tr><td>blur.shard.merge.thread.count
(3)</td><td>The max number of threads used during index merges</td></tr><tr><td>blur.max.clause.count
(1024)</td><td>The maximum number of clauses in a BooleanQuery</td></tr><tr><td>blur.indexmanager.search.thread.count
(8)</td><td>The number of thread used for parallel searching in the index manager</td></tr><tr><td>blur.indexmanager.mutate.thread.count
(8)</td><td>The number of thread used for parallel mutating in the index manager</td></tr><tr><td>blur.shard.internal.search.thread.count
(8)</td><td>The number of threads used for parallel searching in the index searchers</td></tr><tr><td>blur.shard.warmup.thread.count
(8)</td><td>Number of threads used for warming up the index</td></tr><tr><td>blur.shard.fetchcount
(100)</td><td>The fetch count per Lucen
 e search, this fetches pointers to hits</td></tr><tr><td>blur.max.heap.per.row.fetch
(10000000)</td><td>Heap limit on row fetch, once this limit has been reached the
request will return</td></tr><tr><td>blur.max.records.per.row.fetch.request
(1000)</td><td>The maximum number of records in a single row fetch</td></tr><tr><td>blur.gui.shard.port
(40090)</td><td>The http status page port for the shard server</td></tr>
+			</table>
 
-# The number of threads used for selector processing inside the thrift server.
-blur.shard.thrift.selector.threads=2
-
-# The maximum number of bytes used for reading requests in the thrift server.
-blur.shard.thrift.max.read.buffer.bytes=9223372036854775807
-
-# The size of the blocking queue per selector thread for passing accepted connections to
the selector thread.
-blur.shard.thrift.accept.queue.size.per.thread=4
-
-# The number of threads that are used for opening indexes
-blur.shard.opener.thread.count=8
-
-# The number of cached queries
-blur.shard.cache.max.querycache.elements=128
-
-# The time to live for the cache query
-blur.shard.cache.max.timetolive=60000
-
-# Default implementation of the blur cache filter, which is 
-# a pass through filter that does nothing
-blur.shard.filter.cache.class=org.apache.blur.manager.DefaultBlurFilterCache
-
-# Default Blur index warmup class that warms the fields provided
-# in the table descriptor
-blur.shard.index.warmup.class=org.apache.blur.manager.indexserver.DefaultBlurIndexWarmup
-
-# Throttles the warmup to 30MB/s across all the warmup threads
-blur.shard.index.warmup.throttle=30000000
-
-# By default the v2 version of the block cache is enabled
-blur.shard.block.cache.version=v2
-
-# v1 version of block cache only. By default the block cache using
-# off heap memory
-blur.shard.blockcache.direct.memory.allocation=true
-
-# v1 version of block cache only. The slabs in the blockcache are 
-# automatically configured by default (-1) otherwise 1 slab equals 
-# 128MB.  The auto config is detected through the MaxDirectoryMemorySize 
-# provided to the JVM
-blur.shard.blockcache.slab.count=-1
-
-# The number of 1K byte buffers
-blur.shard.buffercache.1024=8192
-
-# The number of 8K byte buffers
-blur.shard.buffercache.8192=8192
-
-# The number of milliseconds to wait for the cluster to settle
-# once changes have ceased
-blur.shard.safemodedelay=5000
-
-# The default time between index commits
-blur.shard.time.between.commits=30000
-
-# The default time between index refreshs
-blur.shard.time.between.refreshs=3000
-
-# The maximum number of clauses in a BooleanQuery
-blur.max.clause.count=1024
-
-# The number of thread used for parallel searching in the index manager
-blur.indexmanager.search.thread.count=8
-
-# The number of threads used for parallel searching in the index searchers
-blur.shard.internal.search.thread.count=8
-
-# Number of threads used for warming up the index
-blur.shard.warmup.thread.count=8
-
-# The fetch count per Lucene search, this fetches pointers to hits
-blur.shard.fetchcount=100
-
-# Heap limit on row fetch, once this limit has been reached the
-# request will return
-blur.max.heap.per.row.fetch=10000000
-
-# The maximum number of records in a single row fetch
-blur.max.records.per.row.fetch.request=1000
-
-# The http status page port for the shard server
-blur.gui.shard.port=40090</code>
-</pre>
             <h3 id="shard-blur-env">blur-env.sh</h3>
             <pre><code class="bash"># JAVA JVM OPTIONS for the shard servers,
jvm tuning parameters are placed here.
 export BLUR_SHARD_JVM_OPTIONS="-Xmx1024m -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=256m
"


Mime
View raw message