Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 359C6200CE5 for ; Fri, 7 Jul 2017 16:57:54 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 34546169459; Fri, 7 Jul 2017 14:57:54 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id A442016943F for ; Fri, 7 Jul 2017 16:57:51 +0200 (CEST) Received: (qmail 81653 invoked by uid 500); 7 Jul 2017 14:57:50 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 81442 invoked by uid 99); 7 Jul 2017 14:57:50 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 07 Jul 2017 14:57:50 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 8E887F551F; Fri, 7 Jul 2017 14:57:48 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit From: ababiichuk@apache.org To: commits@ambari.apache.org Date: Fri, 07 Jul 2017 14:58:21 -0000 Message-Id: <32160acf278f457693cef598ac706c3c@git.apache.org> In-Reply-To: <82c0e00da5374e5d9abf4a536d47fa7d@git.apache.org> References: <82c0e00da5374e5d9abf4a536d47fa7d@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [34/36] ambari git commit: AMBARI-21400. Upgrade Infra Solr version from 5.5.2 to 6.6.x (oleewere) archived-at: Fri, 07 Jul 2017 14:57:54 -0000 AMBARI-21400. Upgrade Infra Solr version from 5.5.2 to 6.6.x (oleewere) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a795f38c Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a795f38c Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a795f38c Branch: refs/heads/branch-feature-logsearch-ui Commit: a795f38cd49c2c55c837b0daecaa63a67015d0eb Parents: 1939dab Author: oleewere Authored: Thu Jul 6 20:28:11 2017 +0200 Committer: oleewere Committed: Fri Jul 7 14:25:24 2017 +0200 ---------------------------------------------------------------------- .../libraries/functions/solr_cloud_util.py | 10 + .../src/main/resources/solr | 826 +++++++++++++++---- .../ambari/infra/solr/AmbariSolrCloudCLI.java | 14 + .../infra/solr/AmbariSolrCloudClient.java | 8 + .../commands/RemoveAdminHandlersCommand.java | 46 ++ .../commands/SetClusterPropertyZkCommand.java | 6 +- .../InfraRuleBasedAuthorizationPluginTest.java | 5 + ambari-infra/pom.xml | 2 +- .../configsets/audit_logs/conf/solrconfig.xml | 3 +- .../configsets/hadoop_logs/conf/solrconfig.xml | 3 +- .../main/configsets/history/conf/solrconfig.xml | 3 +- .../logsearch/dao/SolrSchemaFieldDao.java | 2 +- ambari-logsearch/docker/Dockerfile | 2 +- ambari-logsearch/docker/bin/start.sh | 4 +- ambari-logsearch/pom.xml | 2 +- .../server/upgrade/UpgradeCatalog300.java | 18 + .../0.1.0/package/scripts/params.py | 3 + .../0.1.0/package/scripts/setup_infra_solr.py | 17 +- .../properties/audit_logs-solrconfig.xml.j2 | 3 +- .../properties/service_logs-solrconfig.xml.j2 | 3 +- .../server/upgrade/UpgradeCatalog300Test.java | 33 + .../stacks/2.4/AMBARI_INFRA/test_infra_solr.py | 3 + 22 files changed, 836 insertions(+), 180 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/a795f38c/ambari-common/src/main/python/resource_management/libraries/functions/solr_cloud_util.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/solr_cloud_util.py b/ambari-common/src/main/python/resource_management/libraries/functions/solr_cloud_util.py index 1c5432b..12356ed 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/solr_cloud_util.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/solr_cloud_util.py @@ -190,6 +190,16 @@ def secure_solr_znode(zookeeper_quorum, solr_znode, jaas_file, java64_home, sasl secure_solr_znode_cmd = format('{solr_cli_prefix} --secure-solr-znode --jaas-file {jaas_file} --sasl-users {sasl_users_str}') Execute(secure_solr_znode_cmd) +def remove_admin_handlers(zookeeper_quorum, solr_znode, java64_home, collection, jaas_file, retry = 5, interval = 10): + """ + Remove "solr.admin.AdminHandlers" request handler from collection config. Required for migrating to Solr 6 from Solr 5. + """ + solr_cli_prefix = __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home) + remove_admin_handlers_cmd = format('{solr_cli_prefix} --remove-admin-handlers --collection {collection} --retry {retry} --interval {interval}') + if jaas_file is not None: + remove_admin_handlers_cmd+=format(' --jaas-file {jaas_file}') + Execute(remove_admin_handlers_cmd) + def default_config(config, name, default_value): subdicts = filter(None, name.split('/')) if not config: http://git-wip-us.apache.org/repos/asf/ambari/blob/a795f38c/ambari-infra/ambari-infra-assembly/src/main/resources/solr ---------------------------------------------------------------------- diff --git a/ambari-infra/ambari-infra-assembly/src/main/resources/solr b/ambari-infra/ambari-infra-assembly/src/main/resources/solr old mode 100644 new mode 100755 index bf504d9..6f2de8f --- a/ambari-infra/ambari-infra-assembly/src/main/resources/solr +++ b/ambari-infra/ambari-infra-assembly/src/main/resources/solr @@ -49,6 +49,9 @@ SOLR_SCRIPT="$0" verbose=false THIS_OS=`uname -s` +# What version of Java is required to run this version of Solr. +JAVA_VER_REQ="1.8" + stop_all=false # for now, we don't support running this script from cygwin due to problems @@ -84,6 +87,7 @@ if [ -z "$SOLR_INCLUDE" ]; then /etc/default/solr.in.sh \ /opt/solr/solr.in.sh; do if [ -r "$include" ]; then + SOLR_INCLUDE="$include" . "$include" break fi @@ -116,16 +120,44 @@ else JAVA=java fi -# test that Java exists and is executable on this server -"$JAVA" -version >/dev/null 2>&1 || { +if [ -z "$SOLR_STOP_WAIT" ]; then + SOLR_STOP_WAIT=180 +fi +# test that Java exists, is executable and correct version +JAVA_VER=$("$JAVA" -version 2>&1) +if [[ $? -ne 0 ]] ; then echo >&2 "Java not found, or an error was encountered when running java." - echo >&2 "A working Java 7 or later is required to run Solr!" - echo >&2 "Please install Java or fix JAVA_HOME before running this script." - echo >&2 "Command that we tried: '${JAVA} -version'" + echo >&2 "A working Java $JAVA_VER_REQ JRE is required to run Solr!" + echo >&2 "Please install latest version of Java $JAVA_VER_REQ or set JAVA_HOME properly." + echo >&2 "Command that we tried: '${JAVA} -version', with response:" + echo >&2 "${JAVA_VER}" + echo >&2 + echo >&2 "Debug information:" + echo >&2 "JAVA_HOME: ${JAVA_HOME:-N/A}" echo >&2 "Active Path:" echo >&2 "${PATH}" exit 1 -} +else + JAVA_VER_NUM=$(echo $JAVA_VER | head -1 | awk -F '"' '/version/ {print $2}') + if [[ "$JAVA_VER_NUM" < "$JAVA_VER_REQ" ]] ; then + echo >&2 "Your current version of Java is too old to run this version of Solr" + echo >&2 "We found version $JAVA_VER_NUM, using command '${JAVA} -version', with response:" + echo >&2 "${JAVA_VER}" + echo >&2 + echo >&2 "Please install latest version of Java $JAVA_VER_REQ or set JAVA_HOME properly." + echo >&2 + echo >&2 "Debug information:" + echo >&2 "JAVA_HOME: ${JAVA_HOME:-N/A}" + echo >&2 "Active Path:" + echo >&2 "${PATH}" + exit 1 + fi + JAVA_VENDOR="Oracle" + if [ "`echo $JAVA_VER | grep -i "IBM J9"`" != "" ]; then + JAVA_VENDOR="IBM J9" + fi +fi + # Select HTTP OR HTTPS related configurations SOLR_URL_SCHEME=http @@ -134,30 +166,109 @@ SOLR_SSL_OPTS="" if [ -n "$SOLR_SSL_KEY_STORE" ]; then SOLR_JETTY_CONFIG+=("--module=https") SOLR_URL_SCHEME=https - SOLR_SSL_OPTS=" -Dsolr.jetty.keystore=$SOLR_SSL_KEY_STORE \ - -Dsolr.jetty.keystore.password=$SOLR_SSL_KEY_STORE_PASSWORD \ - -Dsolr.jetty.truststore=$SOLR_SSL_TRUST_STORE \ - -Dsolr.jetty.truststore.password=$SOLR_SSL_TRUST_STORE_PASSWORD \ - -Dsolr.jetty.ssl.needClientAuth=$SOLR_SSL_NEED_CLIENT_AUTH \ - -Dsolr.jetty.ssl.wantClientAuth=$SOLR_SSL_WANT_CLIENT_AUTH" + SOLR_SSL_OPTS+=" -Dsolr.jetty.keystore=$SOLR_SSL_KEY_STORE" + if [ -n "$SOLR_SSL_KEY_STORE_PASSWORD" ]; then + SOLR_SSL_OPTS+=" -Dsolr.jetty.keystore.password=$SOLR_SSL_KEY_STORE_PASSWORD" + fi + if [ -n "$SOLR_SSL_KEY_STORE_TYPE" ]; then + SOLR_SSL_OPTS+=" -Dsolr.jetty.keystore.type=$SOLR_SSL_KEY_STORE_TYPE" + fi + + if [ -n "$SOLR_SSL_TRUST_STORE" ]; then + SOLR_SSL_OPTS+=" -Dsolr.jetty.truststore=$SOLR_SSL_TRUST_STORE" + fi + if [ -n "$SOLR_SSL_TRUST_STORE_PASSWORD" ]; then + SOLR_SSL_OPTS+=" -Dsolr.jetty.truststore.password=$SOLR_SSL_TRUST_STORE_PASSWORD" + fi + if [ -n "$SOLR_SSL_TRUST_STORE_TYPE" ]; then + SOLR_SSL_OPTS+=" -Dsolr.jetty.truststore.type=$SOLR_SSL_TRUST_STORE_TYPE" + fi + + if [ -n "$SOLR_SSL_NEED_CLIENT_AUTH" ]; then + SOLR_SSL_OPTS+=" -Dsolr.jetty.ssl.needClientAuth=$SOLR_SSL_NEED_CLIENT_AUTH" + fi + if [ -n "$SOLR_SSL_WANT_CLIENT_AUTH" ]; then + SOLR_SSL_OPTS+=" -Dsolr.jetty.ssl.wantClientAuth=$SOLR_SSL_WANT_CLIENT_AUTH" + fi + if [ -n "$SOLR_SSL_CLIENT_KEY_STORE" ]; then - SOLR_SSL_OPTS+=" -Djavax.net.ssl.keyStore=$SOLR_SSL_CLIENT_KEY_STORE \ - -Djavax.net.ssl.keyStorePassword=$SOLR_SSL_CLIENT_KEY_STORE_PASSWORD \ - -Djavax.net.ssl.trustStore=$SOLR_SSL_CLIENT_TRUST_STORE \ - -Djavax.net.ssl.trustStorePassword=$SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD" + SOLR_SSL_OPTS+=" -Djavax.net.ssl.keyStore=$SOLR_SSL_CLIENT_KEY_STORE" + + if [ -n "$SOLR_SSL_CLIENT_KEY_STORE_PASSWORD" ]; then + SOLR_SSL_OPTS+=" -Djavax.net.ssl.keyStorePassword=$SOLR_SSL_CLIENT_KEY_STORE_PASSWORD" + fi + if [ -n "$SOLR_SSL_CLIENT_KEY_STORE_TYPE" ]; then + SOLR_SSL_OPTS+=" -Djavax.net.ssl.keyStoreType=$SOLR_SSL_CLIENT_KEY_STORE_TYPE" + fi + else + if [ -n "$SOLR_SSL_KEY_STORE" ]; then + SOLR_SSL_OPTS+=" -Djavax.net.ssl.keyStore=$SOLR_SSL_KEY_STORE" + fi + if [ -n "$SOLR_SSL_KEY_STORE_PASSWORD" ]; then + SOLR_SSL_OPTS+=" -Djavax.net.ssl.keyStorePassword=$SOLR_SSL_KEY_STORE_PASSWORD" + fi + if [ -n "$SOLR_SSL_KEY_STORE_TYPE" ]; then + SOLR_SSL_OPTS+=" -Djavax.net.ssl.keyStoreType=$SOLR_SSL_KEYSTORE_TYPE" + fi + fi + + if [ -n "$SOLR_SSL_CLIENT_TRUST_STORE" ]; then + SOLR_SSL_OPTS+=" -Djavax.net.ssl.trustStore=$SOLR_SSL_CLIENT_TRUST_STORE" + + if [ -n "$SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD" ]; then + SOLR_SSL_OPTS+=" -Djavax.net.ssl.trustStorePassword=$SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD" + fi + + if [ -n "$SOLR_SSL_CLIENT_TRUST_STORE_TYPE" ]; then + SOLR_SSL_OPTS+=" -Djavax.net.ssl.trustStoreType=$SOLR_SSL_CLIENT_TRUST_STORE_TYPE" + fi else - SOLR_SSL_OPTS+=" -Djavax.net.ssl.keyStore=$SOLR_SSL_KEY_STORE \ - -Djavax.net.ssl.keyStorePassword=$SOLR_SSL_KEY_STORE_PASSWORD \ - -Djavax.net.ssl.trustStore=$SOLR_SSL_TRUST_STORE \ - -Djavax.net.ssl.trustStorePassword=$SOLR_SSL_TRUST_STORE_PASSWORD" + if [ -n "$SOLR_SSL_TRUST_STORE" ]; then + SOLR_SSL_OPTS+=" -Djavax.net.ssl.trustStore=$SOLR_SSL_TRUST_STORE" + fi + + if [ -n "$SOLR_SSL_TRUST_STORE_PASSWORD" ]; then + SOLR_SSL_OPTS+=" -Djavax.net.ssl.trustStorePassword=$SOLR_SSL_TRUST_STORE_PASSWORD" + fi + + if [ -n "$SOLR_SSL_TRUST_STORE_TYPE" ]; then + SOLR_SSL_OPTS+=" -Djavax.net.ssl.trustStoreType=$SOLR_SSL_TRUST_STORE_TYPE" + fi fi else SOLR_JETTY_CONFIG+=("--module=http") fi # Authentication options +if [ -z "$SOLR_AUTH_TYPE" ] && [ -n "$SOLR_AUTHENTICATION_OPTS" ]; then + echo "WARNING: SOLR_AUTHENTICATION_OPTS environment variable configured without associated SOLR_AUTH_TYPE variable" + echo " Please configure SOLR_AUTH_TYPE environment variable with the authentication type to be used." + echo " Currently supported authentication types are [kerberos, basic]" +fi + +if [ -n "$SOLR_AUTH_TYPE" ] && [ -n "$SOLR_AUTHENTICATION_CLIENT_CONFIGURER" ]; then + echo "WARNING: SOLR_AUTHENTICATION_CLIENT_CONFIGURER and SOLR_AUTH_TYPE environment variables are configured together." + echo " Use SOLR_AUTH_TYPE environment variable to configure authentication type to be used. " + echo " Currently supported authentication types are [kerberos, basic]" + echo " The value of SOLR_AUTHENTICATION_CLIENT_CONFIGURER environment variable will be ignored" +fi + +if [ -n "$SOLR_AUTH_TYPE" ]; then + case "$(echo $SOLR_AUTH_TYPE | awk '{print tolower($0)}')" in + basic) + SOLR_AUTHENTICATION_CLIENT_CONFIGURER="org.apache.solr.client.solrj.impl.PreemptiveBasicAuthConfigurer" + ;; + kerberos) + SOLR_AUTHENTICATION_CLIENT_CONFIGURER="org.apache.solr.client.solrj.impl.Krb5HttpClientConfigurer" + ;; + *) + echo "ERROR: Value specified for SOLR_AUTH_TYPE environment variable is invalid." + exit 1 + esac +fi + if [ "$SOLR_AUTHENTICATION_CLIENT_CONFIGURER" != "" ]; then - AUTHC_CLIENT_CONFIGURER_ARG="-Dsolr.authentication.httpclient.configurer=$SOLR_AUTHENTICATION_CLIENT_CONFIGURER" + AUTHC_CLIENT_CONFIGURER_ARG="-Dsolr.httpclient.builder.factory=$SOLR_AUTHENTICATION_CLIENT_CONFIGURER" fi AUTHC_OPTS="$AUTHC_CLIENT_CONFIGURER_ARG $SOLR_AUTHENTICATION_OPTS" @@ -179,7 +290,7 @@ function print_usage() { if [ -z "$CMD" ]; then echo "" echo "Usage: solr COMMAND OPTIONS" - echo " where COMMAND is one of: start, stop, restart, status, healthcheck, create, create_core, create_collection, delete, version, zk" + echo " where COMMAND is one of: start, stop, restart, status, healthcheck, create, create_core, create_collection, delete, version, zk, auth" echo "" echo " Standalone server example (start Solr running in the background on port 8984):" echo "" @@ -206,7 +317,7 @@ function print_usage() { echo "" echo " -p Specify the port to start the Solr HTTP listener on; default is 8983" echo " The specified port (SOLR_PORT) will also be used to determine the stop port" - echo " STOP_PORT=(\$SOLR_PORT-1000) and JMX RMI listen port RMI_PORT=(1\$SOLR_PORT). " + echo " STOP_PORT=(\$SOLR_PORT-1000) and JMX RMI listen port RMI_PORT=(\$SOLR_PORT+10000). " echo " For instance, if you set -p 8985, then the STOP_PORT=7985 and RMI_PORT=18985" echo "" echo " -d Specify the Solr server directory; defaults to server" @@ -237,7 +348,9 @@ function print_usage() { echo "" echo " -noprompt Don't prompt for input; accept all defaults when running examples that accept user input" echo "" - echo " -V Verbose messages from this script" + echo " -v and -q Verbose (-v) or quiet (-q) logging. Sets default log level to DEBUG or WARN instead of INFO" + echo "" + echo " -V or -verbose Verbose messages from this script" echo "" elif [ "$CMD" == "stop" ]; then echo "" @@ -354,28 +467,132 @@ function print_usage() { echo " Solr instance and will use the port of the first server it finds." echo "" elif [ "$CMD" == "zk" ]; then - echo "Usage: solr zk [-upconfig|-downconfig] [-d confdir] [-n configName] [-z zkHost]" + print_short_zk_usage "" + echo " Be sure to check the Solr logs in case of errors." + echo "" + echo " -z zkHost Optional Zookeeper connection string for all commands. If specified it" + echo " overrides the 'ZK_HOST=...'' defined in solr.in.sh." + echo "" + echo " upconfig uploads a configset from the local machine to Zookeeper. (Backcompat: -upconfig)" + echo "" + echo " downconfig downloads a configset from Zookeeper to the local machine. (Backcompat: -downconfig)" + echo "" + echo " -n configName   Name of the configset in Zookeeper that will be the destination of" + echo " 'upconfig' and the source for 'downconfig'." + echo "" + echo " -d confdir      The local directory the configuration will be uploaded from for" + echo " 'upconfig' or downloaded to for 'downconfig'. If 'confdir' is a child of" + echo " ...solr/server/solr/configsets' then the configs will be copied from/to" + echo " that directory. Otherwise it is interpreted as a simple local path." + echo "" + echo " cp copies files or folders to/from Zookeeper or Zokeeper -> Zookeeper" + echo " -r   Recursively copy to . Command will fail if has children and " + echo " -r is not specified. Optional" + echo "" + echo " , : [file:][/]path/to/local/file or zk:/path/to/zk/node" + echo " NOTE: and may both be Zookeeper resources prefixed by 'zk:'" + echo " When is a zk resource, may be '.'" + echo " If ends with '/', then will be a local folder or parent znode and the last" + echo " element of the path will be appended unless also ends in a slash. " + echo " may be zk:, which may be useful when using the cp -r form to backup/restore " + echo " the entire zk state." + echo " You must enclose local paths that end in a wildcard in quotes or just" + echo " end the local path in a slash. That is," + echo " 'bin/solr zk cp -r /some/dir/ zk:/ -z localhost:2181' is equivalent to" + echo " 'bin/solr zk cp -r \"/some/dir/*\" zk:/ -z localhost:2181'" + echo " but 'bin/solr zk cp -r /some/dir/* zk:/ -z localhost:2181' will throw an error" + echo "" + echo " here's an example of backup/restore for a ZK configuration:" + echo " to copy to local: 'bin/solr zk cp -r zk:/ /some/dir -z localhost:2181'" + echo " to restore to ZK: 'bin/solr zk cp -r /some/dir/ zk:/ -z localhost:2181'" + echo "" + echo " The 'file:' prefix is stripped, thus 'file:/wherever' specifies an absolute local path and" + echo " 'file:somewhere' specifies a relative local path. All paths on Zookeeper are absolute." + echo "" + echo " Zookeeper nodes CAN have data, so moving a single file to a parent znode" + echo " will overlay the data on the parent Znode so specifying the trailing slash" + echo " can be important." + echo "" + echo " Wildcards are supported when copying from local, trailing only and must be quoted." + echo "" + echo " rm deletes files or folders on Zookeeper" + echo " -r     Recursively delete if is a directory. Command will fail if " + echo " has children and -r is not specified. Optional" + echo "  : [zk:]/path/to/zk/node. may not be the root ('/')" + echo "" + echo " mv moves (renames) znodes on Zookeeper" + echo " , : Zookeeper nodes, the 'zk:' prefix is optional." + echo " If ends with '/', then will be a parent znode" + echo " and the last element of the path will be appended." + echo " Zookeeper nodes CAN have data, so moving a single file to a parent znode" + echo " will overlay the data on the parent Znode so specifying the trailing slash" + echo " is important." + echo "" + echo " ls lists the znodes on Zookeeper" + echo " -r recursively descends the path listing all znodes. Optional" + echo " : The Zookeeper path to use as the root." + echo "" + echo " Only the node names are listed, not data" + echo "" + echo " mkroot makes a znode on Zookeeper with no data. Can be used to make a path of arbitrary" + echo " depth but primarily intended to create a 'chroot'." echo "" - echo " -upconfig to move a configset from the local machine to Zookeeper." + echo " : The Zookeeper path to create. Leading slash is assumed if not present." + echo " Intermediate nodes are created as needed if not present." echo "" - echo " -downconfig to move a configset from Zookeeper to the local machine." + elif [ "$CMD" == "auth" ]; then echo "" - echo " -n configName Name of the configset in Zookeeper that will be the destinatino of" - echo " 'upconfig' and the source for 'downconfig'." + echo "Usage: solr auth enable [-type basicAuth] -credentials user:pass [-blockUnknown ] [-updateIncludeFileOnly ]" + echo " solr auth enable [-type basicAuth] -prompt [-blockUnknown ] [-updateIncludeFileOnly ]" + echo " solr auth disable [-updateIncludeFileOnly ]" echo "" - echo " -d confdir The local directory the configuration will be uploaded from for" - echo " 'upconfig' or downloaded to for 'downconfig'. For 'upconfig', this" - echo " can be one of the example configsets, basic_configs, data_driven_schema_configs or" - echo " sample_techproducts_configs or an arbitrary directory." + echo " -type The authentication mechanism to enable. Defaults to 'basicAuth'." echo "" - echo " -z zkHost Zookeeper connection string." + echo " -credentials The username and password of the initial user" + echo " Note: only one of -prompt or -credentials must be provided" echo "" - echo " NOTE: Solr must have been started least once (or have it running) before using this command." - echo " This initialized Zookeeper for Solr" + echo " -prompt Prompts the user to provide the credentials" + echo " Note: only one of -prompt or -credentials must be provided" + echo "" + echo " -blockUnknown When true, this blocks out access to unauthenticated users. When not provided," + echo " this defaults to false (i.e. unauthenticated users can access all endpoints, except the" + echo " operations like collection-edit, security-edit, core-admin-edit etc.). Check the reference" + echo " guide for Basic Authentication for more details." + echo "" + echo " -updateIncludeFileOnly Only update the solr.in.sh or solr.in.cmd file, and skip actual enabling/disabling" + echo " authentication (i.e. don't update security.json)" + echo "" + echo " -z zkHost Zookeeper connection string" + echo "" + echo " -d Specify the Solr server directory" + echo "" + echo " -s Specify the Solr home directory. This is where any credentials or authentication" + echo " configuration files (e.g. basicAuth.conf) would be placed." echo "" fi } # end print_usage +function print_short_zk_usage() { + + if [ "$1" != "" ]; then + echo -e "\nERROR: $1\n" + fi + + echo " Usage: solr zk upconfig|downconfig -d -n [-z zkHost]" + echo " solr zk cp [-r] [-z zkHost]" + echo " solr zk rm [-r] [-z zkHost]" + echo " solr zk mv [-z zkHost]" + echo " solr zk ls [-r] [-z zkHost]" + echo " solr zk mkroot [-z zkHost]" + echo "" + + if [ "$1" == "" ]; then + echo "Type bin/solr zk -help for full usage help" + else + exit 1 + fi +} + # used to show the script is still alive when waiting on work to complete function spinner() { local pid=$1 @@ -407,7 +624,7 @@ function solr_pid_by_port() { # extract the value of the -Djetty.port parameter from a running Solr process function jetty_port() { SOLR_PID="$1" - SOLR_PROC=`ps auxww | grep -w $SOLR_PID | grep start\.jar | grep jetty.port` + SOLR_PROC=`ps auxww | grep -w $SOLR_PID | grep start\.jar | grep jetty\.port` IFS=' ' read -a proc_args <<< "$SOLR_PROC" for arg in "${proc_args[@]}" do @@ -455,10 +672,10 @@ function get_info() { done < <(find "$SOLR_PID_DIR" -name "solr-*.pid" -type f) else # no pid files but check using ps just to be sure - numSolrs=`ps auxww | grep start\.jar | grep solr.solr.home | grep -v grep | wc -l | sed -e 's/^[ \t]*//'` + numSolrs=`ps auxww | grep start\.jar | grep solr\.solr\.home | grep -v grep | wc -l | sed -e 's/^[ \t]*//'` if [ "$numSolrs" != "0" ]; then echo -e "\nFound $numSolrs Solr nodes: " - PROCESSES=$(ps auxww | grep start\.jar | grep solr.solr.home | grep -v grep | awk '{print $2}' | sort -r) + PROCESSES=$(ps auxww | grep start\.jar | grep solr\.solr\.home | grep -v grep | awk '{print $2}' | sort -r) for ID in $PROCESSES do port=`jetty_port "$ID"` @@ -490,9 +707,24 @@ function stop_solr() { SOLR_PID="$4" if [ "$SOLR_PID" != "" ]; then - echo -e "Sending stop command to Solr running on port $SOLR_PORT ... waiting 5 seconds to allow Jetty process $SOLR_PID to stop gracefully." + echo -e "Sending stop command to Solr running on port $SOLR_PORT ... waiting up to $SOLR_STOP_WAIT seconds to allow Jetty process $SOLR_PID to stop gracefully." "$JAVA" $SOLR_SSL_OPTS $AUTHC_OPTS -jar "$DIR/start.jar" "STOP.PORT=$STOP_PORT" "STOP.KEY=$STOP_KEY" --stop || true - (sleep 5) & + (loops=0 + while true + do + CHECK_PID=`ps auxww | awk '{print $2}' | grep -w $SOLR_PID | sort -r | tr -d ' '` + if [ "$CHECK_PID" != "" ]; then + slept=$((loops * 2)) + if [ $slept -lt $SOLR_STOP_WAIT ]; then + sleep 2 + loops=$[$loops+1] + else + exit # subshell! + fi + else + exit # subshell! + fi + done) & spinner $! rm -f "$SOLR_PID_DIR/solr-$SOLR_PORT.pid" else @@ -555,6 +787,12 @@ if [ "$SCRIPT_CMD" == "status" ]; then exit fi +# assert tool +if [ "$SCRIPT_CMD" == "assert" ]; then + run_tool assert $* + exit $? +fi + # run a healthcheck and exit if requested if [ "$SCRIPT_CMD" == "healthcheck" ]; then @@ -571,7 +809,7 @@ if [ "$SCRIPT_CMD" == "healthcheck" ]; then ;; -z|-zkhost) if [[ -z "$2" || "${2:0:1}" == "-" ]]; then - print_usage "$SCRIPT_CMD" "ZooKeepeer connection string is required when using the $1 option!" + print_usage "$SCRIPT_CMD" "ZooKeeper connection string is required when using the $1 option!" exit 1 fi ZK_HOST="$2" @@ -617,6 +855,7 @@ if [[ "$SCRIPT_CMD" == "create" || "$SCRIPT_CMD" == "create_core" || "$SCRIPT_CM CREATE_NUM_SHARDS=1 CREATE_REPFACT=1 + FORCE=false if [ $# -gt 0 ]; then while true; do @@ -669,6 +908,10 @@ if [[ "$SCRIPT_CMD" == "create" || "$SCRIPT_CMD" == "create_core" || "$SCRIPT_CM CREATE_PORT="$2" shift 2 ;; + -force) + FORCE=true + shift + ;; -help|-usage) print_usage "$SCRIPT_CMD" exit 0 @@ -726,6 +969,11 @@ if [[ "$SCRIPT_CMD" == "create" || "$SCRIPT_CMD" == "create_core" || "$SCRIPT_CM exit 1 fi + if [[ "$(whoami)" == "root" ]] && [[ "$FORCE" == "false" ]] ; then + echo "WARNING: Creating cores as the root user can cause Solr to fail and is not advisable. Exiting." + echo " If you started Solr as root (not advisable either), force core creation by adding argument -force" + exit 1 + fi if [ "$SCRIPT_CMD" == "create_core" ]; then run_tool create_core -name "$CREATE_NAME" -solrUrl "$SOLR_URL_SCHEME://$SOLR_TOOL_HOST:$CREATE_PORT/solr" \ -confdir "$CREATE_CONFDIR" -configsetsDir "$SOLR_TIP/server/solr/configsets" @@ -821,105 +1069,285 @@ if [[ "$SCRIPT_CMD" == "delete" ]]; then exit $? fi -# Upload or download a configset to Zookeeper +ZK_RECURSE=false +# Zookeeper file maintenance (upconfig, downconfig, files up/down etc.) +# It's a little clumsy to have the parsing go round and round for upconfig and downconfig, but that's +# necessary for back-compat if [[ "$SCRIPT_CMD" == "zk" ]]; then if [ $# -gt 0 ]; then while true; do case "$1" in - -z|-zkhost) - if [[ -z "$2" || "${2:0:1}" == "-" ]]; then - print_usage "$SCRIPT_CMD" "ZooKeepeer connection string is required when using the $1 option!" - exit 1 - fi - ZK_HOST="$2" - shift 2 - ;; - -n|-confname) - if [[ -z "$2" || "${2:0:1}" == "-" ]]; then - print_usage "$SCRIPT_CMD" "Configuration name is required when using the $1 option!" - exit 1 - fi - CONFIGSET_CONFNAME="$2" - shift 2 - ;; - -d|-confdir) - if [[ -z "$2" || "${2:0:1}" == "-" ]]; then - print_usage "$SCRIPT_CMD" "Configuration directory is required when using the $1 option!" - exit 1 - fi - CONFIGSET_CONFDIR="$2" - shift 2 - ;; - -upconfig) - ZK_OP="upconfig" - shift 1 - ;; - -downconfig) - ZK_OP="downconfig" - shift 1 - ;; - -help|-usage|-h) - print_usage "$SCRIPT_CMD" - exit 0 - ;; - --) - shift - break - ;; - *) - if [ "$1" != "" ]; then - print_usage "$SCRIPT_CMD" "Unrecognized or misplaced argument: $1!" - exit 1 + -upconfig|upconfig|-downconfig|downconfig|cp|rm|mv|ls|mkroot) + if [ "${1:0:1}" == "-" ]; then + ZK_OP=${1:1} + else + ZK_OP=$1 + fi + shift 1 + ;; + -z|-zkhost) + if [[ -z "$2" || "${2:0:1}" == "-" ]]; then + print_short_zk_usage "$SCRIPT_CMD" "ZooKeeper connection string is required when using the $1 option!" + fi + ZK_HOST="$2" + shift 2 + ;; + -n|-confname) + if [[ -z "$2" || "${2:0:1}" == "-" ]]; then + print_short_zk_usage "$SCRIPT_CMD" "Configuration name is required when using the $1 option!" + fi + CONFIGSET_CONFNAME="$2" + shift 2 + ;; + -d|-confdir) + if [[ -z "$2" || "${2:0:1}" == "-" ]]; then + print_short_zk_usage "$SCRIPT_CMD" "Configuration directory is required when using the $1 option!" + fi + CONFIGSET_CONFDIR="$2" + shift 2 + ;; + -r) + ZK_RECURSE="true" + shift + ;; + -help|-usage|-h) + print_usage "$SCRIPT_CMD" + exit 0 + ;; + --) + shift + break + ;; + *) # Pick up or params for rm, ls, cp, mv, mkroot. + if [ "$1" == "" ]; then + break # out-of-args, stop looping + fi + if [ -z "$ZK_SRC" ]; then + ZK_SRC=$1 + else + if [ -z "$ZK_DST" ]; then + ZK_DST=$1 else - break # out-of-args, stop looping + print_short_zk_usage "Unrecognized or misplaced command $1. 'cp' with trailing asterisk requires quoting, see help text." fi - ;; + fi + shift + ;; esac done fi if [ -z "$ZK_OP" ]; then - echo "Zookeeper operation (one of '-upconfig' or '-downconfig') is required!" - print_usage "$SCRIPT_CMD" - exit 1 + print_short_zk_usage "Zookeeper operation (one of 'upconfig', 'downconfig', 'rm', 'mv', 'cp', 'ls', 'mkroot') is required!" fi if [ -z "$ZK_HOST" ]; then - echo "Zookeeper address (-z) argument is required!" - print_usage "$SCRIPT_CMD" - exit 1 + print_short_zk_usage "Zookeeper address (-z) argument is required or ZK_HOST must be specified in the solr.in.sh file." fi - if [ -z "$CONFIGSET_CONFDIR" ]; then - echo "Local directory of the configset (-d) argument is required!" - print_usage "$SCRIPT_CMD" - exit 1 + if [[ "$ZK_OP" == "upconfig" || "$ZK_OP" == "downconfig" ]]; then + if [ -z "$CONFIGSET_CONFDIR" ]; then + print_short_zk_usage "Local directory of the configset (-d) argument is required!" + fi + + if [ -z "$CONFIGSET_CONFNAME" ]; then + print_short_zk_usage "Configset name on Zookeeper (-n) argument is required!" + fi fi - if [ -z "$CONFIGSET_CONFNAME" ]; then - echo "Configset name on Zookeeper (-n) argument is required!" - print_usage "$SCRIPT_CMD" - exit 1 + if [[ "$ZK_OP" == "cp" || "$ZK_OP" == "mv" ]]; then + if [[ -z "$ZK_SRC" || -z "$ZK_DST" ]]; then + print_short_zk_usage " and must be specified when using either the 'mv' or 'cp' commands." + fi + if [[ "$ZK_OP" == "cp" && "${ZK_SRC:0:3}" != "zk:" && "${ZK_DST:0:3}" != "zk:" ]]; then + print_short_zk_usage "One of the source or desintation paths must be prefixed by 'zk:' for the 'cp' command." + fi fi - if [ "$ZK_OP" == "upconfig" ]; then - run_tool "$ZK_OP" -confname "$CONFIGSET_CONFNAME" -confdir "$CONFIGSET_CONFDIR" -zkHost "$ZK_HOST" -configsetsDir "$SOLR_TIP/server/solr/configsets" + if [[ "$ZK_OP" == "mkroot" ]]; then + if [[ -z "$ZK_SRC" ]]; then + print_short_zk_usage " must be specified when using the 'mkroot' command." + fi + fi + + + case "$ZK_OP" in + upconfig) + run_tool "$ZK_OP" -confname "$CONFIGSET_CONFNAME" -confdir "$CONFIGSET_CONFDIR" -zkHost "$ZK_HOST" -configsetsDir "$SOLR_TIP/server/solr/configsets" + ;; + downconfig) + run_tool "$ZK_OP" -confname "$CONFIGSET_CONFNAME" -confdir "$CONFIGSET_CONFDIR" -zkHost "$ZK_HOST" + ;; + rm) + if [ -z "$ZK_SRC" ]; then + print_short_zk_usage "Zookeeper path to remove must be specified when using the 'rm' command" + fi + run_tool "$ZK_OP" -path "$ZK_SRC" -zkHost "$ZK_HOST" -recurse "$ZK_RECURSE" + ;; + mv) + run_tool "$ZK_OP" -src "$ZK_SRC" -dst "$ZK_DST" -zkHost "$ZK_HOST" + ;; + cp) + run_tool "$ZK_OP" -src "$ZK_SRC" -dst "$ZK_DST" -zkHost "$ZK_HOST" -recurse "$ZK_RECURSE" + ;; + ls) + if [ -z "$ZK_SRC" ]; then + print_short_zk_usage "Zookeeper path to list must be specified when using the 'ls' command" + fi + run_tool "$ZK_OP" -path "$ZK_SRC" -recurse "$ZK_RECURSE" -zkHost "$ZK_HOST" + ;; + mkroot) + if [ -z "$ZK_SRC" ]; then + print_short_zk_usage "Zookeeper path to list must be specified when using the 'mkroot' command" + fi + run_tool "$ZK_OP" -path "$ZK_SRC" -zkHost "$ZK_HOST" + ;; + *) + print_short_zk_usage "Unrecognized Zookeeper operation $ZK_OP" + ;; + esac + + exit $? +fi + +if [[ "$SCRIPT_CMD" == "auth" ]]; then + declare -a AUTH_PARAMS + if [ $# -gt 0 ]; then + while true; do + case "$1" in + enable|disable) + AUTH_OP=$1 + AUTH_PARAMS=("${AUTH_PARAMS[@]}" "$AUTH_OP") + shift + ;; + -z|-zkhost|zkHost) + ZK_HOST="$2" + AUTH_PARAMS=("${AUTH_PARAMS[@]}" "-zkHost" "$ZK_HOST") + shift 2 + ;; + -t|-type) + AUTH_TYPE="$2" + AUTH_PARAMS=("${AUTH_PARAMS[@]}" "-type" "$AUTH_TYPE") + shift 2 + ;; + -credentials) + AUTH_CREDENTIALS="$2" + AUTH_PARAMS=("${AUTH_PARAMS[@]}" "-credentials" "$AUTH_CREDENTIALS") + shift 2 + ;; + -solrIncludeFile) + SOLR_INCLUDE="$2" + shift 2 + ;; + -prompt) + AUTH_PARAMS=("${AUTH_PARAMS[@]}" "-prompt" "$2") + shift + ;; + -blockUnknown) + AUTH_PARAMS=("${AUTH_PARAMS[@]}" "-blockUnknown" "$2") + shift + break + ;; + -updateIncludeFileOnly) + AUTH_PARAMS=("${AUTH_PARAMS[@]}" "-updateIncludeFileOnly" "$2") + shift + break + ;; + -d|-dir) + if [[ -z "$2" || "${2:0:1}" == "-" ]]; then + print_usage "$SCRIPT_CMD" "Server directory is required when using the $1 option!" + exit 1 + fi + + if [[ "$2" == "." || "$2" == "./" || "$2" == ".." || "$2" == "../" ]]; then + SOLR_SERVER_DIR="$(pwd)/$2" + else + # see if the arg value is relative to the tip vs full path + if [[ "$2" != /* ]] && [[ -d "$SOLR_TIP/$2" ]]; then + SOLR_SERVER_DIR="$SOLR_TIP/$2" + else + SOLR_SERVER_DIR="$2" + fi + fi + # resolve it to an absolute path + SOLR_SERVER_DIR="$(cd "$SOLR_SERVER_DIR"; pwd)" + shift 2 + ;; + -s|-solr.home) + if [[ -z "$2" || "${2:0:1}" == "-" ]]; then + print_usage "$SCRIPT_CMD" "Solr home directory is required when using the $1 option!" + exit 1 + fi + + SOLR_HOME="$2" + shift 2 + ;; + -help|-usage|-h) + print_usage "$SCRIPT_CMD" + exit 0 + ;; + --) + shift + break + ;; + *) + shift + break + ;; + esac + done + fi + + if [ -z "$SOLR_SERVER_DIR" ]; then + SOLR_SERVER_DIR="$DEFAULT_SERVER_DIR" + fi + if [ ! -e "$SOLR_SERVER_DIR" ]; then + echo -e "\nSolr server directory $SOLR_SERVER_DIR not found!\n" + exit 1 + fi + if [ -z "$SOLR_HOME" ]; then + SOLR_HOME="$SOLR_SERVER_DIR/solr" else - run_tool "$ZK_OP" -confname "$CONFIGSET_CONFNAME" -confdir "$CONFIGSET_CONFDIR" -zkHost "$ZK_HOST" + if [[ $SOLR_HOME != /* ]] && [[ -d "$SOLR_SERVER_DIR/$SOLR_HOME" ]]; then + SOLR_HOME="$SOLR_SERVER_DIR/$SOLR_HOME" + SOLR_PID_DIR="$SOLR_HOME" + elif [[ $SOLR_HOME != /* ]] && [[ -d "`pwd`/$SOLR_HOME" ]]; then + SOLR_HOME="$(pwd)/$SOLR_HOME" + fi fi + if [ -z "$AUTH_OP" ]; then + print_usage "$SCRIPT_CMD" + exit 0 + fi + + AUTH_PARAMS=("${AUTH_PARAMS[@]}" "-solrIncludeFile" "$SOLR_INCLUDE") + + if [ -z "$AUTH_PORT" ]; then + for ID in `ps auxww | grep java | grep start\.jar | awk '{print $2}' | sort -r` + do + port=`jetty_port "$ID"` + if [ "$port" != "" ]; then + AUTH_PORT=$port + break + fi + done + fi + run_tool auth ${AUTH_PARAMS[@]} -solrUrl "$SOLR_URL_SCHEME://$SOLR_TOOL_HOST:$AUTH_PORT/solr" -authConfDir "$SOLR_HOME" exit $? fi + # verify the command given is supported -if [ "$SCRIPT_CMD" != "stop" ] && [ "$SCRIPT_CMD" != "start" ] && [ "$SCRIPT_CMD" != "restart" ] && [ "$SCRIPT_CMD" != "status" ]; then +if [ "$SCRIPT_CMD" != "stop" ] && [ "$SCRIPT_CMD" != "start" ] && [ "$SCRIPT_CMD" != "restart" ] && [ "$SCRIPT_CMD" != "status" ] && [ "$SCRIPT_CMD" != "assert" ]; then print_usage "" "$SCRIPT_CMD is not a valid command!" exit 1 fi # Run in foreground (default is to run in the background) FG="false" +FORCE=false noprompt=false SOLR_OPTS=($SOLR_OPTS) PASS_TO_RUN_EXAMPLE= @@ -1033,10 +1461,22 @@ if [ $# -gt 0 ]; then PASS_TO_RUN_EXAMPLE+=" --verbose" shift ;; + -v) + SOLR_LOG_LEVEL=DEBUG + shift + ;; + -q) + SOLR_LOG_LEVEL=WARN + shift + ;; -all) stop_all=true shift ;; + -force) + FORCE=true + shift + ;; --) shift break @@ -1060,6 +1500,10 @@ if [ $# -gt 0 ]; then done fi +if [[ $SOLR_LOG_LEVEL ]] ; then + SOLR_LOG_LEVEL_OPT="-Dsolr.log.level=$SOLR_LOG_LEVEL" +fi + if [ -z "$SOLR_SERVER_DIR" ]; then SOLR_SERVER_DIR="$DEFAULT_SERVER_DIR" fi @@ -1157,13 +1601,21 @@ if [ -z "$STOP_PORT" ]; then STOP_PORT=`expr $SOLR_PORT - 1000` fi +if [ "$SCRIPT_CMD" == "start" ] || [ "$SCRIPT_CMD" == "restart" ] ; then + if [[ "$(whoami)" == "root" ]] && [[ "$FORCE" == "false" ]] ; then + echo "WARNING: Starting Solr as the root user is a security risk and not considered best practice. Exiting." + echo " Please consult the Reference Guide. To override this check, start with argument '-force'" + exit 1 + fi +fi + if [[ "$SCRIPT_CMD" == "start" ]]; then # see if Solr is already running SOLR_PID=`solr_pid_by_port "$SOLR_PORT"` if [ -z "$SOLR_PID" ]; then # not found using the pid file ... but use ps to ensure not found - SOLR_PID=`ps auxww | grep start\.jar | grep -w $SOLR_PORT | grep -v grep | awk '{print $2}' | sort -r` + SOLR_PID=`ps auxww | grep start\.jar | grep -w "\-Djetty\.port=$SOLR_PORT" | grep -v grep | awk '{print $2}' | sort -r` fi if [ "$SOLR_PID" != "" ]; then @@ -1176,7 +1628,7 @@ else SOLR_PID=`solr_pid_by_port "$SOLR_PORT"` if [ -z "$SOLR_PID" ]; then # not found using the pid file ... but use ps to ensure not found - SOLR_PID=`ps auxww | grep start\.jar | grep -w $SOLR_PORT | grep -v grep | awk '{print $2}' | sort -r` + SOLR_PID=`ps auxww | grep start\.jar | grep -w "\-Djetty\.port=$SOLR_PORT" | grep -v grep | awk '{print $2}' | sort -r` fi if [ "$SOLR_PID" != "" ]; then stop_solr "$SOLR_SERVER_DIR" "$SOLR_PORT" "$STOP_KEY" "$SOLR_PID" @@ -1226,38 +1678,49 @@ if [ ! -e "$SOLR_HOME" ]; then echo -e "\nSolr home directory $SOLR_HOME not found!\n" exit 1 fi - -# backup the log files before starting -if [ -f "$SOLR_LOGS_DIR/solr.log" ]; then - if $verbose ; then - echo "Backing up $SOLR_LOGS_DIR/solr.log" - fi - mv "$SOLR_LOGS_DIR/solr.log" "$SOLR_LOGS_DIR/solr_log_$(date +"%Y%m%d_%H%M")" +if $verbose ; then + q="" +else + q="-q" fi - -if [ -f "$SOLR_LOGS_DIR/solr_gc.log" ]; then - if $verbose ; then - echo "Backing up $SOLR_LOGS_DIR/solr_gc.log" - fi - mv "$SOLR_LOGS_DIR/solr_gc.log" "$SOLR_LOGS_DIR/solr_gc_log_$(date +"%Y%m%d_%H%M")" +if [ "${SOLR_LOG_PRESTART_ROTATION:=true}" == "true" ]; then + run_tool utils -s "$DEFAULT_SERVER_DIR" -l "$SOLR_LOGS_DIR" $q -remove_old_solr_logs 7 || echo "Failed removing old solr logs" + run_tool utils -s "$DEFAULT_SERVER_DIR" -l "$SOLR_LOGS_DIR" $q -archive_gc_logs $q || echo "Failed archiving old GC logs" + run_tool utils -s "$DEFAULT_SERVER_DIR" -l "$SOLR_LOGS_DIR" $q -archive_console_logs || echo "Failed archiving old console logs" + run_tool utils -s "$DEFAULT_SERVER_DIR" -l "$SOLR_LOGS_DIR" $q -rotate_solr_logs 9 || echo "Failed rotating old solr logs" fi -java_ver_out=`echo "$("$JAVA" -version 2>&1)"` -JAVA_VERSION=`echo $java_ver_out | grep "java version" | awk '{ print substr($3, 2, length($3)-2); }'` -JAVA_VENDOR="Oracle" -if [ "`echo $java_ver_out | grep -i "IBM J9"`" != "" ]; then - JAVA_VENDOR="IBM J9" +# Establish default GC logging opts if no env var set (otherwise init to sensible default) +if [ -z ${GC_LOG_OPTS+x} ]; then + if [[ "$JAVA_VER_NUM" < "9" ]] ; then + GC_LOG_OPTS=('-verbose:gc' '-XX:+PrintHeapAtGC' '-XX:+PrintGCDetails' \ + '-XX:+PrintGCDateStamps' '-XX:+PrintGCTimeStamps' '-XX:+PrintTenuringDistribution' \ + '-XX:+PrintGCApplicationStoppedTime') + else + GC_LOG_OPTS=('-Xlog:gc*') + fi +else + GC_LOG_OPTS=($GC_LOG_OPTS) fi -# if verbose gc logging enabled, setup the location of the log file +# if verbose gc logging enabled, setup the location of the log file and rotation if [ "$GC_LOG_OPTS" != "" ]; then - gc_log_flag="-Xloggc" - if [ "$JAVA_VENDOR" == "IBM J9" ]; then - gc_log_flag="-Xverbosegclog" + if [[ "$JAVA_VER_NUM" < "9" ]] ; then + gc_log_flag="-Xloggc" + if [ "$JAVA_VENDOR" == "IBM J9" ]; then + gc_log_flag="-Xverbosegclog" + fi + GC_LOG_OPTS+=("$gc_log_flag:$SOLR_LOGS_DIR/solr_gc.log" '-XX:+UseGCLogFileRotation' '-XX:NumberOfGCLogFiles=9' '-XX:GCLogFileSize=20M') + else + # http://openjdk.java.net/jeps/158 + for i in "${!GC_LOG_OPTS[@]}"; + do + # for simplicity, we only look at the prefix '-Xlog:gc' + # (if 'all' or multiple tags are used starting with anything other then 'gc' the user is on their own) + # if a single additional ':' exists in param, then there is already an explicit output specifier + GC_LOG_OPTS[$i]=$(echo ${GC_LOG_OPTS[$i]} | sed "s|^\(-Xlog:gc[^:]*$\)|\1:file=$SOLR_LOGS_DIR/solr_gc.log:time,uptime:filecount=9,filesize=20000|") + done fi - GC_LOG_OPTS=($GC_LOG_OPTS "$gc_log_flag:$SOLR_LOGS_DIR/solr_gc.log") -else - GC_LOG_OPTS=() fi # If ZK_HOST is defined, the assume SolrCloud mode @@ -1298,7 +1761,11 @@ fi if [ "$ENABLE_REMOTE_JMX_OPTS" == "true" ]; then if [ -z "$RMI_PORT" ]; then - RMI_PORT="1$SOLR_PORT" + RMI_PORT=`expr $SOLR_PORT + 10000` + if [ $RMI_PORT -gt 65535 ]; then + echo -e "\nRMI_PORT is $RMI_PORT, which is invalid!\n" + exit 1 + fi fi REMOTE_JMX_OPTS=('-Dcom.sun.management.jmxremote' \ @@ -1324,6 +1791,12 @@ else JAVA_MEM_OPTS=("-Xms$SOLR_HEAP" "-Xmx$SOLR_HEAP") fi +# Pick default for Java thread stack size, and then add to SOLR_OPTS +if [ -z ${SOLR_JAVA_STACK_SIZE+x} ]; then + SOLR_JAVA_STACK_SIZE='-Xss256k' +fi +SOLR_OPTS+=($SOLR_JAVA_STACK_SIZE) + if [ -z "$SOLR_TIMEZONE" ]; then SOLR_TIMEZONE='UTC' fi @@ -1336,20 +1809,28 @@ function launch_solr() { SOLR_ADDL_ARGS="$2" - GC_TUNE=($GC_TUNE) - # deal with Java version specific GC and other flags - if [ "${JAVA_VERSION:0:3}" == "1.7" ]; then - # Specific Java version hacking - GC_TUNE+=('-XX:CMSFullGCsBeforeCompaction=1' '-XX:CMSTriggerPermRatio=80') - if [ "$JAVA_VENDOR" != "IBM J9" ]; then - JAVA_MINOR_VERSION=${JAVA_VERSION:(-2)} - if [[ $JAVA_MINOR_VERSION -ge 40 && $JAVA_MINOR_VERSION -le 51 ]]; then - GC_TUNE+=('-XX:-UseSuperWord') - echo -e "\nWARNING: Java version $JAVA_VERSION has known bugs with Lucene and requires the -XX:-UseSuperWord flag. Please consider upgrading your JVM.\n" - fi - fi + # define default GC_TUNE + if [ -z ${GC_TUNE+x} ]; then + GC_TUNE=('-XX:NewRatio=3' \ + '-XX:SurvivorRatio=4' \ + '-XX:TargetSurvivorRatio=90' \ + '-XX:MaxTenuringThreshold=8' \ + '-XX:+UseConcMarkSweepGC' \ + '-XX:+UseParNewGC' \ + '-XX:ConcGCThreads=4' '-XX:ParallelGCThreads=4' \ + '-XX:+CMSScavengeBeforeRemark' \ + '-XX:PretenureSizeThreshold=64m' \ + '-XX:+UseCMSInitiatingOccupancyOnly' \ + '-XX:CMSInitiatingOccupancyFraction=50' \ + '-XX:CMSMaxAbortablePrecleanTime=6000' \ + '-XX:+CMSParallelRemarkEnabled' \ + '-XX:+ParallelRefProcEnabled' \ + '-XX:-OmitStackTraceInFastThrow') + else + GC_TUNE=($GC_TUNE) fi + # If SSL-related system props are set, add them to SOLR_OPTS if [ -n "$SOLR_SSL_OPTS" ]; then # If using SSL and solr.jetty.https.port not set explicitly, use the jetty.port @@ -1380,17 +1861,22 @@ function launch_solr() { fi if [ "$SOLR_OPTS" != "" ]; then - echo -e " SOLR_OPTS = ${SOLR_OPTS[@]}" + echo -e " SOLR_OPTS = ${SOLR_OPTS[@]}" fi if [ "$SOLR_ADDL_ARGS" != "" ]; then - echo -e " SOLR_ADDL_ARGS = $SOLR_ADDL_ARGS" + echo -e " SOLR_ADDL_ARGS = $SOLR_ADDL_ARGS" fi if [ "$ENABLE_REMOTE_JMX_OPTS" == "true" ]; then echo -e " RMI_PORT = $RMI_PORT" echo -e " REMOTE_JMX_OPTS = ${REMOTE_JMX_OPTS[@]}" fi + + if [ "$SOLR_LOG_LEVEL" != "" ]; then + echo -e " SOLR_LOG_LEVEL = $SOLR_LOG_LEVEL" + fi + echo -e "\n" fi @@ -1403,7 +1889,7 @@ function launch_solr() { fi SOLR_START_OPTS=('-server' "${JAVA_MEM_OPTS[@]}" "${GC_TUNE[@]}" "${GC_LOG_OPTS[@]}" \ - "${REMOTE_JMX_OPTS[@]}" "${CLOUD_MODE_OPTS[@]}" \ + "${REMOTE_JMX_OPTS[@]}" "${CLOUD_MODE_OPTS[@]}" $SOLR_LOG_LEVEL_OPT -Dsolr.log.dir="$SOLR_LOGS_DIR" \ "-Djetty.port=$SOLR_PORT" "-DSTOP.PORT=$stop_port" "-DSTOP.KEY=$STOP_KEY" \ "${SOLR_HOST_ARG[@]}" "-Duser.timezone=$SOLR_TIMEZONE" \ "-Djetty.home=$SOLR_SERVER_DIR" "-Dsolr.solr.home=$SOLR_HOME" "-Dsolr.install.dir=$SOLR_TIP" \ @@ -1413,37 +1899,57 @@ function launch_solr() { IN_CLOUD_MODE=" in SolrCloud mode" fi - mkdir -p "$SOLR_LOGS_DIR" + mkdir -p "$SOLR_LOGS_DIR" 2>/dev/null + if [ $? -ne 0 ]; then + echo -e "\nERROR: Logs directory $SOLR_LOGS_DIR could not be created. Exiting" + exit 1 + fi + if [ ! -w "$SOLR_LOGS_DIR" ]; then + echo -e "\nERROR: Logs directory $SOLR_LOGS_DIR is not writable. Exiting" + exit 1 + fi + case "$SOLR_LOGS_DIR" in + contexts|etc|lib|modules|resources|scripts|solr|solr-webapp) + echo -e "\nERROR: Logs directory $SOLR_LOGS_DIR is invalid. Reserved for the system. Exiting" + exit 1 + ;; + esac if [ "$run_in_foreground" == "true" ]; then - echo -e "\nStarting Solr$IN_CLOUD_MODE on port $SOLR_PORT from $SOLR_SERVER_DIR\n" - exec "$JAVA" "${SOLR_START_OPTS[@]}" $SOLR_ADDL_ARGS -jar start.jar "${SOLR_JETTY_CONFIG[@]}" + exec "$JAVA" "${SOLR_START_OPTS[@]}" $SOLR_ADDL_ARGS -Dsolr.kerberos.name.rules="$SOLR_KERB_NAME_RULES" -jar start.jar "${SOLR_JETTY_CONFIG[@]}" else # run Solr in the background - nohup "$JAVA" "${SOLR_START_OPTS[@]}" $SOLR_ADDL_ARGS -Dsolr.kerberos.name.rules="$SOLR_KERB_NAME_RULES" \ + nohup "$JAVA" "${SOLR_START_OPTS[@]}" $SOLR_ADDL_ARGS -Dsolr.kerberos.name.rules="$SOLR_KERB_NAME_RULES" -Dsolr.log.muteconsole \ "-XX:OnOutOfMemoryError=$SOLR_TIP/bin/oom_solr.sh $SOLR_PORT $SOLR_LOGS_DIR" \ -jar start.jar "${SOLR_JETTY_CONFIG[@]}" \ 1>"$SOLR_LOGS_DIR/solr-$SOLR_PORT-console.log" 2>&1 & echo $! > "$SOLR_PID_DIR/solr-$SOLR_PORT.pid" + # check if /proc/sys/kernel/random/entropy_avail exists then check output of cat /proc/sys/kernel/random/entropy_avail to see if less than 300 + if [[ -f /proc/sys/kernel/random/entropy_avail ]] && (( `cat /proc/sys/kernel/random/entropy_avail` < 300)); then + echo "Warning: Available entropy is low. As a result, use of the UUIDField, SSL, or any other features that require" + echo "RNG might not work properly. To check for the amount of available entropy, use 'cat /proc/sys/kernel/random/entropy_avail'." + echo "" + fi # no lsof on cygwin though if hash lsof 2>/dev/null ; then # hash returns true if lsof is on the path - echo -n "Waiting up to 30 seconds to see Solr running on port $SOLR_PORT" + echo -n "Waiting up to $SOLR_STOP_WAIT seconds to see Solr running on port $SOLR_PORT" # Launch in a subshell to show the spinner (loops=0 while true do running=`lsof -PniTCP:$SOLR_PORT -sTCP:LISTEN` if [ -z "$running" ]; then - if [ $loops -lt 6 ]; then - sleep 5 + slept=$((loops * 2)) + if [ $slept -lt $SOLR_STOP_WAIT ]; then + sleep 2 loops=$[$loops+1] else - echo -e "Still not seeing Solr listening on $SOLR_PORT after 30 seconds!" + echo -e "Still not seeing Solr listening on $SOLR_PORT after $SOLR_STOP_WAIT seconds!" tail -30 "$SOLR_LOGS_DIR/solr.log" exit # subshell! fi else - SOLR_PID=`ps auxww | grep start\.jar | grep -w $SOLR_PORT | grep -v grep | awk '{print $2}' | sort -r` + SOLR_PID=`ps auxww | grep start\.jar | grep -w "\-Djetty\.port=$SOLR_PORT" | grep -v grep | awk '{print $2}' | sort -r` echo -e "\nStarted Solr server on port $SOLR_PORT (pid=$SOLR_PID). Happy searching!\n" exit # subshell! fi @@ -1452,7 +1958,7 @@ function launch_solr() { else echo -e "NOTE: Please install lsof as this script needs it to determine if Solr is listening on port $SOLR_PORT." sleep 10 - SOLR_PID=`ps auxww | grep start\.jar | grep -w $SOLR_PORT | grep -v grep | awk '{print $2}' | sort -r` + SOLR_PID=`ps auxww | grep start\.jar | grep -w "\-Djetty\.port=$SOLR_PORT" | grep -v grep | awk '{print $2}' | sort -r` echo -e "\nStarted Solr server on port $SOLR_PORT (pid=$SOLR_PID). Happy searching!\n" return; fi http://git-wip-us.apache.org/repos/asf/ambari/blob/a795f38c/ambari-infra/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/AmbariSolrCloudCLI.java ---------------------------------------------------------------------- diff --git a/ambari-infra/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/AmbariSolrCloudCLI.java b/ambari-infra/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/AmbariSolrCloudCLI.java index e3a1e79..70bc232 100644 --- a/ambari-infra/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/AmbariSolrCloudCLI.java +++ b/ambari-infra/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/AmbariSolrCloudCLI.java @@ -52,12 +52,14 @@ public class AmbariSolrCloudCLI { private static final String UNSECURE_ZNODE_COMMAND = "unsecure-znode"; private static final String SECURE_SOLR_ZNODE_COMMAND = "secure-solr-znode"; private static final String SECURITY_JSON_LOCATION = "security-json-location"; + private static final String REMOVE_ADMIN_HANDLERS = "remove-admin-handlers"; private static final String CMD_LINE_SYNTAX = "\n./solrCloudCli.sh --create-collection -z host1:2181,host2:2181/ambari-solr -c collection -cs conf_set" + "\n./solrCloudCli.sh --upload-config -z host1:2181,host2:2181/ambari-solr -d /tmp/myconfig_dir -cs config_set" + "\n./solrCloudCli.sh --download-config -z host1:2181,host2:2181/ambari-solr -cs config_set -d /tmp/myonfig_dir" + "\n./solrCloudCli.sh --check-config -z host1:2181,host2:2181/ambari-solr -cs config_set" + "\n./solrCloudCli.sh --create-shard -z host1:2181,host2:2181/ambari-solr -c collection -sn myshard" + + "\n./solrCloudCli.sh --remove-admin-handlers -z host1:2181,host2:2181/ambari-solr -c collection" + "\n./solrCloudCli.sh --create-znode -z host1:2181,host2:2181 -zn /ambari-solr" + "\n./solrCloudCli.sh --check-znode -z host1:2181,host2:2181 -zn /ambari-solr" + "\n./solrCloudCli.sh --cluster-prop -z host1:2181,host2:2181/ambari-solr -cpn urlScheme -cpn http" @@ -137,6 +139,11 @@ public class AmbariSolrCloudCLI { .desc("Disable security for znode") .build(); + final Option removeAdminHandlerOption = Option.builder("rah") + .longOpt(REMOVE_ADMIN_HANDLERS) + .desc("Remove AdminHandlers request handler from solrconfig.xml") + .build(); + final Option shardNameOption = Option.builder("sn") .longOpt("shard-name") .desc("Name of the shard for create-shard command") @@ -328,6 +335,7 @@ public class AmbariSolrCloudCLI { options.addOption(helpOption); options.addOption(retryOption); + options.addOption(removeAdminHandlerOption); options.addOption(intervalOption); options.addOption(zkConnectStringOption); options.addOption(configSetOption); @@ -414,6 +422,9 @@ public class AmbariSolrCloudCLI { } else if (cli.hasOption("uz")) { command = UNSECURE_ZNODE_COMMAND; validateRequiredOptions(cli, command, zkConnectStringOption, znodeOption, jaasFileOption); + } else if (cli.hasOption("rah")) { + command = REMOVE_ADMIN_HANDLERS; + validateRequiredOptions(cli, command, zkConnectStringOption, collectionOption); } else { List commands = Arrays.asList(CREATE_COLLECTION_COMMAND, CREATE_SHARD_COMMAND, UPLOAD_CONFIG_COMMAND, DOWNLOAD_CONFIG_COMMAND, CONFIG_CHECK_COMMAND, SET_CLUSTER_PROP, CREATE_ZNODE, SECURE_ZNODE_COMMAND, UNSECURE_ZNODE_COMMAND, @@ -539,6 +550,9 @@ public class AmbariSolrCloudCLI { case SECURE_SOLR_ZNODE_COMMAND: solrCloudClient = clientBuilder.build(); solrCloudClient.secureSolrZnode(); + case REMOVE_ADMIN_HANDLERS: + solrCloudClient = clientBuilder.build(); + solrCloudClient.removeAdminHandlerFromCollectionConfig(); break; default: throw new AmbariSolrCloudClientException(String.format("Not found command: '%s'", command)); http://git-wip-us.apache.org/repos/asf/ambari/blob/a795f38c/ambari-infra/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/AmbariSolrCloudClient.java ---------------------------------------------------------------------- diff --git a/ambari-infra/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/AmbariSolrCloudClient.java b/ambari-infra/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/AmbariSolrCloudClient.java index 9479679..96c07a3 100644 --- a/ambari-infra/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/AmbariSolrCloudClient.java +++ b/ambari-infra/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/AmbariSolrCloudClient.java @@ -27,6 +27,7 @@ import org.apache.ambari.infra.solr.commands.EnableKerberosPluginSolrZkCommand; import org.apache.ambari.infra.solr.commands.GetShardsCommand; import org.apache.ambari.infra.solr.commands.GetSolrHostsCommand; import org.apache.ambari.infra.solr.commands.ListCollectionCommand; +import org.apache.ambari.infra.solr.commands.RemoveAdminHandlersCommand; import org.apache.ambari.infra.solr.commands.SecureSolrZNodeZkCommand; import org.apache.ambari.infra.solr.commands.SecureZNodeZkCommand; import org.apache.ambari.infra.solr.commands.SetClusterPropertyZkCommand; @@ -257,6 +258,13 @@ public class AmbariSolrCloudClient { return new GetSolrHostsCommand(getRetryTimes(), getInterval()).run(this); } + /** + * Remove solr.admin.AdminHandlers requestHandler from solrconfi.xml + */ + public boolean removeAdminHandlerFromCollectionConfig() throws Exception { + return new RemoveAdminHandlersCommand(getRetryTimes(), getInterval()).run(this); + } + public String getZkConnectString() { return zkConnectString; } http://git-wip-us.apache.org/repos/asf/ambari/blob/a795f38c/ambari-infra/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/commands/RemoveAdminHandlersCommand.java ---------------------------------------------------------------------- diff --git a/ambari-infra/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/commands/RemoveAdminHandlersCommand.java b/ambari-infra/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/commands/RemoveAdminHandlersCommand.java new file mode 100644 index 0000000..32fae7b --- /dev/null +++ b/ambari-infra/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/commands/RemoveAdminHandlersCommand.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.infra.solr.commands; + +import org.apache.ambari.infra.solr.AmbariSolrCloudClient; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.cloud.SolrZooKeeper; +import org.apache.zookeeper.data.Stat; + +public class RemoveAdminHandlersCommand extends AbstractZookeeperRetryCommand { + + public RemoveAdminHandlersCommand(int maxRetries, int interval) { + super(maxRetries, interval); + } + + @Override + protected Boolean executeZkCommand(AmbariSolrCloudClient client, SolrZkClient zkClient, SolrZooKeeper solrZooKeeper) throws Exception { + String solrConfigXmlPath = String.format("/configs/%s/solrconfig.xml", client.getCollection()); + if (zkClient.exists(solrConfigXmlPath, true)) { + Stat stat = new Stat(); + byte[] solrConfigXmlBytes = zkClient.getData(solrConfigXmlPath, null, stat, true); + String solrConfigStr = new String(solrConfigXmlBytes); + if (solrConfigStr.contains("class=\"solr.admin.AdminHandlers\"")) { + byte[] newSolrConfigXmlBytes = new String(solrConfigXmlBytes).replaceAll("(?s)", "").getBytes(); + zkClient.setData(solrConfigXmlPath, newSolrConfigXmlBytes, stat.getVersion() + 1, true); + } + } + return true; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/a795f38c/ambari-infra/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/commands/SetClusterPropertyZkCommand.java ---------------------------------------------------------------------- diff --git a/ambari-infra/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/commands/SetClusterPropertyZkCommand.java b/ambari-infra/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/commands/SetClusterPropertyZkCommand.java index 34597c6..e79773e 100644 --- a/ambari-infra/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/commands/SetClusterPropertyZkCommand.java +++ b/ambari-infra/ambari-infra-solr-client/src/main/java/org/apache/ambari/infra/solr/commands/SetClusterPropertyZkCommand.java @@ -19,9 +19,9 @@ package org.apache.ambari.infra.solr.commands; import org.apache.ambari.infra.solr.AmbariSolrCloudClient; +import org.apache.solr.common.cloud.ClusterProperties; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.SolrZooKeeper; -import org.apache.solr.common.cloud.ZkStateReader; public class SetClusterPropertyZkCommand extends AbstractZookeeperRetryCommand{ @@ -33,8 +33,8 @@ public class SetClusterPropertyZkCommand extends AbstractZookeeperRetryCommand 1.8 - 5.5.2 + 6.6.0 UTF-8 python >= 2.6 python (>= 2.6) http://git-wip-us.apache.org/repos/asf/ambari/blob/a795f38c/ambari-logsearch/ambari-logsearch-server/src/main/configsets/audit_logs/conf/solrconfig.xml ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/configsets/audit_logs/conf/solrconfig.xml b/ambari-logsearch/ambari-logsearch-server/src/main/configsets/audit_logs/conf/solrconfig.xml index 7af91df..b1290a4 100644 --- a/ambari-logsearch/ambari-logsearch-server/src/main/configsets/audit_logs/conf/solrconfig.xml +++ b/ambari-logsearch/ambari-logsearch-server/src/main/configsets/audit_logs/conf/solrconfig.xml @@ -1063,8 +1063,7 @@ Admin Handlers - This will register all the standard admin RequestHandlers. --> - + - + - + - + - +