ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jlun...@apache.org
Subject [17/50] [abbrv] ambari git commit: AMBARI-21459. Add lucene index migration script to infra solr client package (oleewere)
Date Sun, 23 Jul 2017 23:33:50 GMT
AMBARI-21459. Add lucene index migration script to infra solr client package (oleewere)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f072dd21
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f072dd21
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f072dd21

Branch: refs/heads/branch-feature-AMBARI-14714
Commit: f072dd2156e83d8a487ce0c3229c1ae22788c6be
Parents: e799f52
Author: oleewere <oleewere@gmail.com>
Authored: Wed Jul 12 21:04:54 2017 +0200
Committer: oleewere <oleewere@gmail.com>
Committed: Sat Jul 15 19:04:53 2017 +0200

----------------------------------------------------------------------
 ambari-infra/ambari-infra-solr-client/build.xml |   1 +
 ambari-infra/ambari-infra-solr-client/pom.xml   |  10 ++
 .../src/main/resources/solrIndexHelper.sh       | 156 +++++++++++++++++++
 3 files changed, 167 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/f072dd21/ambari-infra/ambari-infra-solr-client/build.xml
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-solr-client/build.xml b/ambari-infra/ambari-infra-solr-client/build.xml
index a54e336..9b8b6cc 100644
--- a/ambari-infra/ambari-infra-solr-client/build.xml
+++ b/ambari-infra/ambari-infra-solr-client/build.xml
@@ -35,6 +35,7 @@
     </copy>
     <copy todir="target/package" includeEmptyDirs="no">
       <fileset file="src/main/resources/solrCloudCli.sh"/>
+      <fileset file="src/main/resources/solrIndexHelper.sh"/>
     </copy>
     <copy todir="target/package" includeEmptyDirs="no">
       <fileset file="src/main/resources/log4j.properties"/>

http://git-wip-us.apache.org/repos/asf/ambari/blob/f072dd21/ambari-infra/ambari-infra-solr-client/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-solr-client/pom.xml b/ambari-infra/ambari-infra-solr-client/pom.xml
index d103003..3818aba 100644
--- a/ambari-infra/ambari-infra-solr-client/pom.xml
+++ b/ambari-infra/ambari-infra-solr-client/pom.xml
@@ -36,6 +36,16 @@
       <version>${solr.version}</version>
     </dependency>
     <dependency>
+      <groupId>org.apache.lucene</groupId>
+      <artifactId>lucene-core</artifactId>
+      <version>${solr.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.lucene</groupId>
+      <artifactId>lucene-backward-codecs</artifactId>
+      <version>${solr.version}</version>
+    </dependency>
+    <dependency>
       <groupId>org.apache.zookeeper</groupId>
       <artifactId>zookeeper</artifactId>
       <version>3.4.9</version>

http://git-wip-us.apache.org/repos/asf/ambari/blob/f072dd21/ambari-infra/ambari-infra-solr-client/src/main/resources/solrIndexHelper.sh
----------------------------------------------------------------------
diff --git a/ambari-infra/ambari-infra-solr-client/src/main/resources/solrIndexHelper.sh b/ambari-infra/ambari-infra-solr-client/src/main/resources/solrIndexHelper.sh
new file mode 100755
index 0000000..12e6a77
--- /dev/null
+++ b/ambari-infra/ambari-infra-solr-client/src/main/resources/solrIndexHelper.sh
@@ -0,0 +1,156 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+JVM="java"
+sdir="`dirname \"$0\"`"
+: ${JAVA_HOME:?"Please set the JAVA_HOME for lucene index migration!"}
+
+function print_help() {
+  cat << EOF
+
+   Usage: solrIndexHelper.sh [<command>] [<arguments with flags>]
+
+   commands:
+     upgrade-index            Check and upgrade solr index data in core directories.
+     run-check-index-tool     call 'java -cp ... org.apache.lucene.index.IndexUpgrader' directly
+     run-upgrade-index-tool   call 'java -cp ... org.apache.lucene.index.CheckIndex' directly
+     help                     print usage
+
+
+   upgrade-index command arguments:
+     -d, --index-data-dir <DIRECTORY>        Location of the solr cores (e.g.: /opt/ambari_infra_solr/data)
+     -c, --core-filter <FILTER1,FILTER2>     Comma separated name filters of core directoies
(default: hadoop_logs,audit_logs,history)
+     -f, --force                             Force to start index upgrade, even is the version
is at least 6.
+
+EOF
+}
+
+function upgrade_core() {
+  local INDEX_DIR=${1:?"usage: <index_base_dir> e.g.: /opt/ambari_infra_solr/data"}
+  local FORCE_UPDATE=${2:?"usage <force_update_flag> e.g.: true"}
+  local SOLR_CORE_FILTERS=${3:?"usage: <comma separated core filters> e.g.: hadoop_logs,audit_logs,history"}
+
+  SOLR_CORE_FILTER_ARR=$(echo $SOLR_CORE_FILTERS | sed "s/,/ /g")
+
+  for coll in $SOLR_CORE_FILTER_ARR; do
+    if [[ "$1" == *"$coll"* ]]; then
+      echo "Core '$1' dir name contains $coll (core filter)'";
+      version=$(PATH=$JAVA_HOME/bin:$PATH $JVM -classpath "$sdir/libs/lucene-core-6.6.0.jar:$sdir/libs/lucene-backward-codecs-6.6.0.jar"
org.apache.lucene.index.CheckIndex -fast $1|grep "   version="|sed -e 's/.*=//g'|head -1)
+      if [ -z $version ] ; then
+        echo "Core '$1' - Empty index?"
+        return
+      fi
+      majorVersion=$(echo $version|cut -c 1)
+      if [ $majorVersion -ge 6 ] && [ $FORCE_UPDATE == "false" ] ; then
+        echo "Core '$1' - Already on version $version, not upgrading. Use -f or --force option
to run upgrade anyway."
+      else
+        echo "Core '$1' - Index version is $version, upgrading ..."
+        PATH=$JAVA_HOME/bin:$PATH $JVM -classpath "$sdir/libs/lucene-core-6.6.0.jar:$sdir/libs/lucene-backward-codecs-6.6.0.jar"
org.apache.lucene.index.IndexUpgrader -delete-prior-commits $1
+        echo "Upgrading core '$1' has finished"
+      fi
+    fi
+  done
+}
+
+function upgrade_index() {
+  while [[ $# -gt 0 ]]
+    do
+      key="$1"
+      case $key in
+        -c|--core-filters)
+          local SOLR_CORE_FILTERS="$2"
+          shift 2
+        ;;
+        -f|--force)
+          local FORCE_UPDATE="true"
+          shift
+        ;;
+        -d|--index-data-dir)
+          local INDEX_DIR="$2"
+          shift 2
+        ;;
+        *)
+          echo "Unknown option: $1"
+          exit 1
+        ;;
+      esac
+  done
+  if [[ -z "$INDEX_DIR" ]] ; then
+    echo "Index data dirctory option is required (-d or --index-data-dir). Exiting..."
+    exit 1
+  fi
+
+  if [[ -z "$SOLR_CORE_FILTERS" ]] ; then
+    SOLR_CORE_FILTERS="hadoop_logs,audit_logs,history"
+  fi
+
+  if [[ -z "$FORCE_UPDATE" ]] ; then
+    FORCE_UPDATE="false"
+  else
+    echo "NOTE: Forcing index upgrade is set."
+  fi
+
+  CORES=$(for replica_dir in `find $INDEX_DIR -name data`; do dirname $replica_dir; done);
+  if [[ -z "$CORES" ]] ; then
+    echo "No indices found on path $INDEX_DIR"
+  else
+      for c in $CORES ; do
+        if find $c/data -maxdepth 1 -type d -name 'index*' 1> /dev/null 2>&1; then
+          name=$(echo $c | sed -e 's/.*\///g')
+          abspath=$(cd "$(dirname "$c")"; pwd)/$(basename "$c")
+          find $c/data -maxdepth 1 -type d -name 'index*' | while read indexDir; do
+          echo "Checking core $name - $abspath"
+          upgrade_core "$indexDir" "$FORCE_UPDATE" "$SOLR_CORE_FILTERS"
+          done
+        else
+          echo "No index folder found for $name"
+        fi
+      done
+      echo "DONE"
+  fi
+}
+
+function upgrade_index_tool() {
+  # see: https://cwiki.apache.org/confluence/display/solr/IndexUpgrader+Tool
+  PATH=$JAVA_HOME/bin:$PATH $JVM -classpath "$sdir/libs/lucene-core-6.6.0.jar:$sdir/libs/lucene-backward-codecs-6.6.0.jar"
org.apache.lucene.index.IndexUpgrader ${@}
+}
+
+function check_index_tool() {
+  PATH=$JAVA_HOME/bin:$PATH $JVM -classpath "$sdir/libs/lucene-core-6.6.0.jar:$sdir/libs/lucene-backward-codecs-6.6.0.jar"
org.apache.lucene.index.CheckIndex ${@}
+}
+
+function main() {
+  command="$1"
+  case $command in
+   "upgrade-index")
+     upgrade_index "${@:2}"
+     ;;
+   "run-check-index-tool")
+     check_index_tool "${@:2}"
+     ;;
+   "run-upgrade-index-tool")
+     upgrade_index_tool "${@:2}"
+     ;;
+   "help")
+     print_help
+     ;;
+   *)
+   echo "Available commands: (upgrade-index | run-check-index-tool | run-upgrade-index-tool
| help)"
+   ;;
+   esac
+}
+
+main ${1+"$@"}


Mime
View raw message