incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [1/2] git commit: Cleanup removing old files, documenting the blur-default file.
Date Sun, 18 Aug 2013 20:36:34 GMT
Updated Branches:
  refs/heads/master a10c12597 -> 9db2f8411


Cleanup removing old files, documenting the blur-default file.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/2716101a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/2716101a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/2716101a

Branch: refs/heads/master
Commit: 2716101a42ca50bc1105a71907a6d45134bd338b
Parents: a10c125
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Sun Aug 18 16:21:11 2013 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Sun Aug 18 16:21:11 2013 -0400

----------------------------------------------------------------------
 .../src/main/resources/blur-default.properties  | 112 +++++++++++++++--
 .../main/resources/hadoop-metrics.properties    |  43 -------
 .../src/main/scripts/conf/blur-site.properties  |  14 ++-
 .../main/scripts/conf/hadoop-metrics.properties |  28 -----
 docs/cluster-setup.html                         | 124 +++++++++++++++++--
 docs/data-model.html                            |  27 +++-
 6 files changed, 250 insertions(+), 98 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2716101a/blur-util/src/main/resources/blur-default.properties
----------------------------------------------------------------------
diff --git a/blur-util/src/main/resources/blur-default.properties b/blur-util/src/main/resources/blur-default.properties
index ba5c838..098cc1a 100644
--- a/blur-util/src/main/resources/blur-default.properties
+++ b/blur-util/src/main/resources/blur-default.properties
@@ -13,58 +13,152 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# The zookeeper session timeout
+blur.zookeeper.timeout=90000
+
+# The maximum number of results that can be fetched in a single request
+blur.query.max.results.fetch=1000
+
+# The maximum number of rows that can be fetched in a single request
+blur.query.max.row.fetch=100
+
+# The maximum number of records that can be fetched in a single request
+blur.query.max.record.fetch=1000
+
+# Setup metric reporter
+blur.metrics.reporters=
+
+# Shard Properties
+
+# The hostname for the shard, if blank the hostname is automatically detected
 blur.shard.hostname=
+
+# The binding address of the shard
 blur.shard.bind.address=0.0.0.0
+
+# The default binding port of the shard server
 blur.shard.bind.port=40020
+
+# The number of fetcher threads
 blur.shard.data.fetch.thread.count=8
+
+# The number of the thrift threads
 blur.shard.server.thrift.thread.count=8
+
+# The number of threads that are used for opening indexes
 blur.shard.opener.thread.count=8
+
+# The number of cached queries
 blur.shard.cache.max.querycache.elements=128
+
+# The time to live for the cache query
 blur.shard.cache.max.timetolive=60000
+
+# Default implementation of the blur cache filter, which is a pass through filter that does
nothing
 blur.shard.filter.cache.class=org.apache.blur.manager.DefaultBlurFilterCache
+
+# Default Blur index warmup class that warms the fields provided in the table descriptor
 blur.shard.index.warmup.class=org.apache.blur.manager.indexserver.DefaultBlurIndexWarmup
+
+# Throttles the warmup to 30MB/s across all the warmup threads
 blur.shard.index.warmup.throttle=30000000
+
+# By default the block cache using off heap memory
 blur.shard.blockcache.direct.memory.allocation=true
+
+# The slabs in the blockcache are automatically configured by default (-1) otherwise 1 slab
equals 128MB
+# The auto config is detected through the MaxDirectoryMemorySize provided to the JVM
 blur.shard.blockcache.slab.count=-1
+
+# The number of 1K byte buffers
 blur.shard.buffercache.1024=8192
+
+# The number of 8K byte buffers
 blur.shard.buffercache.8192=8192
+
+# The number of milliseconds to wait for the cluster to settle once changes have ceased
 blur.shard.safemodedelay=5000
+
+# The default time between index commits
 blur.shard.time.between.commits=30000
+
+# The default time between index refreshs
 blur.shard.time.between.refreshs=3000
+
+# The maximum number of clauses in a BooleanQuery
 blur.max.clause.count=1024
+
+# The number of thread used for parallel searching in the index manager
 blur.indexmanager.search.thread.count=8
+
+# The number of threads used for parallel searching in the index searchers
 blur.shard.internal.search.thread.count=8
+
+# Number of threads used for warming up the index
 blur.shard.warmup.thread.count=8
+
+# The fetch count per Lucene search, this fetches pointers to hits
 blur.shard.fetchcount=100
+
+# Heap limit on row fetch, once this limit has been reached the request will return
 blur.max.heap.per.row.fetch=10000000
+
+# The maximum number of records in a single row fetch
 blur.max.records.per.row.fetch.request=1000
 
+# The http status page port for the shard server
+blur.gui.shard.port=40090
+
+
+# Controller Properties
+
+# Sets the hostname for the controller, if blank the hostname is automatically detected
 blur.controller.hostname=
+
+# The binding address of the controller
 blur.controller.bind.address=0.0.0.0
+
+# The default binding port of the controller server
 blur.controller.bind.port=40010
+
+# The number of threads used for thrift requests
 blur.controller.server.thrift.thread.count=32
+
+# The number of threads used for remote thrift requests to the shards server.  This should
be a large number.
 blur.controller.server.remote.thread.count=64
+
+# The number of hits to fetch per request to the shard servers
 blur.controller.remote.fetch.count=100
 
+# The max number of retries to the shard server when there is an error during fetch
 blur.controller.retry.max.fetch.retries=3
+
+# The max number of retries to the shard server when there is an error during mutate
 blur.controller.retry.max.mutate.retries=3
+
+# The max number of retries to the shard server when there is an error during all other request
 blur.controller.retry.max.default.retries=3
+
+# The starting backoff delay for the first retry for a fetch errors
 blur.controller.retry.fetch.delay=500
+
+# The starting backoff delay for the first retry for a mutate errors
 blur.controller.retry.mutate.delay=500
+
+# The starting backoff delay for the first retry for a all other request errors
 blur.controller.retry.default.delay=500
+
+# The ending backoff delay for the last retry for a fetch errors
 blur.controller.retry.max.fetch.delay=2000
-blur.controller.retry.max.mutate.delay=2000
-blur.controller.retry.max.default.delay=2000
 
-blur.query.max.results.fetch=1000
-blur.query.max.row.fetch=100
-blur.query.max.record.fetch=1000
+# The ending backoff delay for the last retry for a mutate errors
+blur.controller.retry.max.mutate.delay=2000
 
-blur.zookeeper.system.time.tolerance=3000
-blur.zookeeper.timeout=90000
+# The ending backoff delay for the last retry for a all other request errors
+blur.controller.retry.max.default.delay=2000
 
+# The http status page port for the controller server
 blur.gui.controller.port=40080
-blur.gui.shard.port=40090
 
-blur.metrics.reporters=
+
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2716101a/blur-util/src/main/resources/hadoop-metrics.properties
----------------------------------------------------------------------
diff --git a/blur-util/src/main/resources/hadoop-metrics.properties b/blur-util/src/main/resources/hadoop-metrics.properties
deleted file mode 100644
index 5f4bb56..0000000
--- a/blur-util/src/main/resources/hadoop-metrics.properties
+++ /dev/null
@@ -1,43 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Configuration of the "blur" context for null
-blur.class=org.apache.hadoop.metrics.spi.NullContext
-
-# Configuration of the "blur" context for file
-# blur.class=org.apache.hadoop.metrics.file.FileContext
-# blur.period=10
-# blur.fileName=/tmp/blurmetrics.log
-
-# Configuration of the "blur" context for ganglia
-# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
-# blur.class=org.apache.hadoop.metrics.ganglia.GangliaContext
-# blur.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
-# blur.period=10
-# blur.servers=localhost:8649
-
-
-# Configuration of the "jvm" context for null
-jvm.class=org.apache.hadoop.metrics.spi.NullContext
-
-# Configuration of the "jvm" context for file
-# jvm.class=org.apache.hadoop.metrics.file.FileContext
-# jvm.period=10
-# jvm.fileName=/tmp/jvmmetrics.log
-
-# Configuration of the "jvm" context for ganglia
-# jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext
-# jvm.period=10
-# jvm.servers=localhost:8649

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2716101a/distribution/src/main/scripts/conf/blur-site.properties
----------------------------------------------------------------------
diff --git a/distribution/src/main/scripts/conf/blur-site.properties b/distribution/src/main/scripts/conf/blur-site.properties
index a5b33ad..3064f67 100644
--- a/distribution/src/main/scripts/conf/blur-site.properties
+++ b/distribution/src/main/scripts/conf/blur-site.properties
@@ -13,9 +13,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-#Zookeeper connection string
-blur.zookeeper.connection=localhost
-
-#The cluster name
+blur.zookeeper.connection=127.0.0.1
 blur.cluster.name=default
 
+#
+# Sets the default table location in hdfs.  If left null the table uri property in 
+# the table descriptor will be required for all tables.
+#
+# blur.cluster.<cluster name here>.table.uri=<hdfs uri>
+#
+# Example: 
+# blur.cluster.default.table.uri=hdfs://namenode/blur/tables
+#

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2716101a/distribution/src/main/scripts/conf/hadoop-metrics.properties
----------------------------------------------------------------------
diff --git a/distribution/src/main/scripts/conf/hadoop-metrics.properties b/distribution/src/main/scripts/conf/hadoop-metrics.properties
deleted file mode 100644
index dba5756..0000000
--- a/distribution/src/main/scripts/conf/hadoop-metrics.properties
+++ /dev/null
@@ -1,28 +0,0 @@
-# Configuration of the "blur" context for null
-blur.class=org.apache.hadoop.metrics.spi.NullContext
-
-# Configuration of the "blur" context for file
-# blur.class=org.apache.hadoop.metrics.file.FileContext
-# blur.period=10
-# blur.fileName=/tmp/blurmetrics.log
-
-# Configuration of the "blur" context for ganglia
-# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
-# blur.class=org.apache.hadoop.metrics.ganglia.GangliaContext
-# blur.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
-# blur.period=10
-# blur.servers=localhost:8649
-
-
-# Configuration of the "jvm" context for null
-jvm.class=org.apache.hadoop.metrics.spi.NullContext
-
-# Configuration of the "jvm" context for file
-# jvm.class=org.apache.hadoop.metrics.file.FileContext
-# jvm.period=10
-# jvm.fileName=/tmp/jvmmetrics.log
-
-# Configuration of the "jvm" context for ganglia
-# jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext
-# jvm.period=10
-# jvm.servers=localhost:8649

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2716101a/docs/cluster-setup.html
----------------------------------------------------------------------
diff --git a/docs/cluster-setup.html b/docs/cluster-setup.html
index 9168581..39854a9 100644
--- a/docs/cluster-setup.html
+++ b/docs/cluster-setup.html
@@ -84,30 +84,67 @@
             <p>
               These are the default settings for the shard server that can be overridden
in the blur-site.properties file. Consider increasing the various thread pool counts (*.thread.count).
The blur.controller.server.remote.thread.count is very important to increase for larger clusters,
basically one thread is used per shard server per query. Some production cluster have used
set this thread pool to 2000 or more threads.
             </p>
-            <pre><code class="bash">blur.controller.hostname=
+<pre>
+<code class="bash">
+# Sets the hostname for the controller, if blank the hostname is automatically detected
+blur.controller.hostname=
+
+# The binding address of the controller
 blur.controller.bind.address=0.0.0.0
+
+# The default binding port of the controller server
 blur.controller.bind.port=40010
+
+# The number of threads used for thrift requests
 blur.controller.server.thrift.thread.count=32
+
+# The number of threads used for remote thrift requests to
+# the shards server.  This should be a large number.
 blur.controller.server.remote.thread.count=64
+
+# The number of hits to fetch per request to the shard servers
 blur.controller.remote.fetch.count=100
 
+# The max number of retries to the shard server when there
+# is an error during fetch
 blur.controller.retry.max.fetch.retries=3
+
+# The max number of retries to the shard server when there
+# is an error during mutate
 blur.controller.retry.max.mutate.retries=3
+
+# The max number of retries to the shard server when there
+# is an error during all other request
 blur.controller.retry.max.default.retries=3
+
+# The starting backoff delay for the first retry for a
+# fetch errors
 blur.controller.retry.fetch.delay=500
+
+# The starting backoff delay for the first retry for a
+# mutate errors
 blur.controller.retry.mutate.delay=500
+
+# The starting backoff delay for the first retry for a
+# all other request errors
 blur.controller.retry.default.delay=500
+
+# The ending backoff delay for the last retry for a
+# fetch errors
 blur.controller.retry.max.fetch.delay=2000
+
+# The ending backoff delay for the last retry for a
+# mutate errors
 blur.controller.retry.max.mutate.delay=2000
-blur.controller.retry.max.default.delay=2000
 
-blur.query.max.results.fetch=1000
-blur.query.max.row.fetch=100
-blur.query.max.record.fetch=1000
+# The ending backoff delay for the last retry for a
+# all other request errors
+blur.controller.retry.max.default.delay=2000
 
+# The http status page port for the controller server
 blur.gui.controller.port=40080
-
-blur.metrics.reporters=</code></pre>
+</code>
+</pre>
             <h3 id="controller-blur-env">blur-env.sh</h3>
             <pre><code class="bash"># JAVA JVM OPTIONS for the controller servers,
jvm tuning parameters are placed here.
 # Consider adding the -XX:OnOutOfMemoryError="kill -9 %p" option to kill jvms that are failing
due to memory issues.
@@ -127,33 +164,94 @@ export BLUR_NUMBER_OF_CONTROLLER_SERVER_INSTANCES_PER_MACHINE=1</code></pre>
             <p>
               These are the default settings for the shard server that can be overridden
in the blur-site.properties file. Consider increasing the various thread pool counts (*.thread.count).
Also the blur.max.clause.count sets the BooleanQuery max clause count for Lucene queries.
             </p>
-            <pre><code class="bash">blur.shard.hostname=
+<pre>
+<code class="bash">
+# The hostname for the shard, if blank the hostname is automatically detected
+blur.shard.hostname=
+
+# The binding address of the shard
 blur.shard.bind.address=0.0.0.0
+
+# The default binding port of the shard server
 blur.shard.bind.port=40020
+
+# The number of fetcher threads
 blur.shard.data.fetch.thread.count=8
+
+# The number of the thrift threads
 blur.shard.server.thrift.thread.count=8
+
+# The number of threads that are used for opening indexes
 blur.shard.opener.thread.count=8
+
+# The number of cached queries
 blur.shard.cache.max.querycache.elements=128
+
+# The time to live for the cache query
 blur.shard.cache.max.timetolive=60000
+
+# Default implementation of the blur cache filter, which is 
+# a pass through filter that does nothing
 blur.shard.filter.cache.class=org.apache.blur.manager.DefaultBlurFilterCache
+
+# Default Blur index warmup class that warms the fields provided
+# in the table descriptor
 blur.shard.index.warmup.class=org.apache.blur.manager.indexserver.DefaultBlurIndexWarmup
+
+# Throttles the warmup to 30MB/s across all the warmup threads
+blur.shard.index.warmup.throttle=30000000
+
+# By default the block cache using off heap memory
 blur.shard.blockcache.direct.memory.allocation=true
+
+# The slabs in the blockcache are automatically configured by 
+# default (-1) otherwise 1 slab equals 128MB. The auto config
+# is detected through the MaxDirectoryMemorySize provided to 
+# the JVM
 blur.shard.blockcache.slab.count=-1
+
+# The number of 1K byte buffers
 blur.shard.buffercache.1024=8192
+
+# The number of 8K byte buffers
 blur.shard.buffercache.8192=8192
+
+# The number of milliseconds to wait for the cluster to settle
+# once changes have ceased
 blur.shard.safemodedelay=5000
+
+# The default time between index commits
 blur.shard.time.between.commits=30000
+
+# The default time between index refreshs
 blur.shard.time.between.refreshs=3000
+
+# The maximum number of clauses in a BooleanQuery
 blur.max.clause.count=1024
+
+# The number of thread used for parallel searching in the index manager
 blur.indexmanager.search.thread.count=8
 
-blur.query.max.results.fetch=1000
-blur.query.max.row.fetch=100
-blur.query.max.record.fetch=1000
+# The number of threads used for parallel searching in the index searchers
+blur.shard.internal.search.thread.count=8
 
-blur.gui.shard.port=40090
+# Number of threads used for warming up the index
+blur.shard.warmup.thread.count=8
 
-blur.metrics.reporters=</code></pre>
+# The fetch count per Lucene search, this fetches pointers to hits
+blur.shard.fetchcount=100
+
+# Heap limit on row fetch, once this limit has been reached the
+# request will return
+blur.max.heap.per.row.fetch=10000000
+
+# The maximum number of records in a single row fetch
+blur.max.records.per.row.fetch.request=1000
+
+# The http status page port for the shard server
+blur.gui.shard.port=40090
+</code>
+</pre>
             <h3 id="shard-blur-env">blur-env.sh</h3>
             <pre><code class="bash"># JAVA JVM OPTIONS for the shard servers,
jvm tuning parameters are placed here.
 export BLUR_SHARD_JVM_OPTIONS="-Xmx1024m -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=256m
"

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/2716101a/docs/data-model.html
----------------------------------------------------------------------
diff --git a/docs/data-model.html b/docs/data-model.html
index 1528ed5..6eeca72 100644
--- a/docs/data-model.html
+++ b/docs/data-model.html
@@ -58,7 +58,32 @@
                   <li><a href="#rows">Rows</a></li>
                 </ul>
               </li>
-              <li><a href="#querying">Querying</a></li>
+              <li>
+				<a href="#querying">Querying</a>
+				<ul class="nav">
+                  <li><a href="#string_query">String / Text</a></li>
+                  <li><a href="#numeric_query">Numeric</a></li>
+                  <li><a href="#spatial_query">Spatial</a></li>
+                </ul>
+			  </li>
+              <li>
+				<a href="#types">Types</a>
+				<ul class="nav">
+                  <li><a href="#text_type">Text</a></li>
+                  <li><a href="#string_type">String</a></li>
+                  <li><a href="#long_type">Long</a></li>
+                  <li><a href="#int_type">Integer</a></li>
+                  <li><a href="#float_type">Float</a></li>
+                  <li><a href="#double_type">Double</a></li>
+                  <li><a href="#stored_type">Stored</a></li>
+	  			  <a href="#spatial_type">Spatial</a>
+				  <ul class="nav">
+                    <li><a href="#spatial_pointvector">Point Vector</a></li>
+                    <li><a href="#spatial_termprefix">Term Prefix</a></li>
+                    <li><a href="#spatial_recursiveprefix">Recursive Prefix</a></li>
+                  </ul>
+                </ul>
+			  </li>
             </ul>
           </div>
         </div>


Mime
View raw message