kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From granthe...@apache.org
Subject [kudu] branch master updated: [build] Add a script to publish the kudu-binary jars
Date Fri, 01 Feb 2019 21:25:41 GMT
This is an automated email from the ASF dual-hosted git repository.

granthenke pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git


The following commit(s) were added to refs/heads/master by this push:
     new 990bb4d  [build] Add a script to publish the kudu-binary jars
990bb4d is described below

commit 990bb4d134c8fd9bd4621cd2fb9827d47f623db7
Author: Grant Henke <granthenke@apache.org>
AuthorDate: Wed Jan 30 23:12:48 2019 -0600

    [build] Add a script to publish the kudu-binary jars
    
    Adds publish_mini_cluster_binaries.sh to publish the
    kudu-binary jars. This script can install to the local
    maven repository or deploy to a remote repsitory.
    
    I also moved the files into a mini-cluster directory and
    made sure the naming of mini-cluster was consistent.
    
    Change-Id: I83fe6b81ef70ec75b10f66fa8582a861daf285e4
    Reviewed-on: http://gerrit.cloudera.org:8080/12325
    Tested-by: Kudu Jenkins
    Reviewed-by: Adar Dembo <adar@cloudera.com>
---
 .../build_mini_cluster_binaries.sh                 |   8 +-
 .../mini-cluster/publish_mini_cluster_binaries.sh  | 212 +++++++++++++++++++++
 .../relocate_binaries_for_mini_cluster.py          |   6 +-
 build-support/mini-cluster/settings.xml            |  34 ++++
 4 files changed, 254 insertions(+), 6 deletions(-)

diff --git a/build-support/build_mini_cluster_binaries.sh b/build-support/mini-cluster/build_mini_cluster_binaries.sh
similarity index 96%
rename from build-support/build_mini_cluster_binaries.sh
rename to build-support/mini-cluster/build_mini_cluster_binaries.sh
index c1d9962..80cf041 100755
--- a/build-support/build_mini_cluster_binaries.sh
+++ b/build-support/mini-cluster/build_mini_cluster_binaries.sh
@@ -18,7 +18,7 @@
 # under the License.
 ################################################################################
 #
-# Minicluster test binaries
+# Mini cluster test binaries
 # -------------------------
 #
 # This script generates optimized, dynamically-linked, stripped, fully
@@ -67,8 +67,8 @@
 ################################################################################
 set -e
 
-SOURCE_ROOT=$(cd $(dirname $0)/..; pwd)
-BUILD_ROOT=$SOURCE_ROOT/build/minicluster
+SOURCE_ROOT=$(cd $(dirname $0)/../..; pwd)
+BUILD_ROOT=$SOURCE_ROOT/build/mini-cluster
 TARGETS="kudu kudu-tserver kudu-master"
 
 cd $SOURCE_ROOT
@@ -113,7 +113,7 @@ NUM_PROCS=$(getconf _NPROCESSORS_ONLN)
 make -j$NUM_PROCS $TARGETS
 
 # Relocate the binaries.
-$SOURCE_ROOT/build-support/relocate_binaries_for_mini_cluster.py $BUILD_ROOT $TARGETS
+$SOURCE_ROOT/build-support/mini-cluster/relocate_binaries_for_mini_cluster.py $BUILD_ROOT
$TARGETS
 
 ARTIFACT_NAME=$(ls -d kudu-binary* | sed 's#/##' | head -1)
 
diff --git a/build-support/mini-cluster/publish_mini_cluster_binaries.sh b/build-support/mini-cluster/publish_mini_cluster_binaries.sh
new file mode 100755
index 0000000..326b6cb
--- /dev/null
+++ b/build-support/mini-cluster/publish_mini_cluster_binaries.sh
@@ -0,0 +1,212 @@
+#!/bin/bash
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+################################################################################
+#
+# This script publishes the Kudu binary jars generated by the
+# build_mini_cluster_binaries.sh script to a local or remote
+# repository.
+#
+# Parameters:
+#
+#   -a, --action: (default "install")
+#     Either install or deploy. If install, the jars will be
+#     installed into the local Maven repository. If deploy, the jars
+#     will be deployed to the remote Maven repository defined by MVN_REPO_URL.
+#
+#   -j, --jars: (default "build/mini-cluster")
+#    Defines the Maven repository url to deploy to.
+#     Only used when the --action is deploy.
+#
+#   -r, --repo: (default "https://repository.apache.org/content/repositories/snapshots")
+#     Defines the Maven repository url to deploy to.
+#     Only used when the --action is deploy.
+#
+#   -u, --username:
+#     The username to the maven repository.
+#     Only used when the --action is deploy.
+#
+#   -p, --password:
+#     The password to the maven repository.
+#     Only used when the --action is deploy.
+#
+#   --skipPom:
+#     If this flag is passed, the pom is not published.
+#
+# Example:
+#   publish_mini_cluster_binaries.sh -a=deploy -u="foo-user" -p="foo-pass"
+#
+################################################################################
+set -e
+
+SOURCE_ROOT=$(cd $(dirname $0)/../..; pwd)
+BUILD_ROOT="$SOURCE_ROOT/build/mini-cluster"
+VERSION=$(cat ${SOURCE_ROOT}/version.txt)
+
+# Set the parameter defaults.
+MVN_ACTION="install"
+JAR_PATH="$BUILD_ROOT"
+MVN_REPO_URL="https://repository.apache.org/content/repositories/snapshots"
+PUBLISH_POM=1
+
+# Parse the command line parameters.
+for i in "$@"; do
+  case ${i} in
+      -a=*|--action=*)
+      MVN_ACTION="${i#*=}"
+      shift
+      ;;
+      -j=*|--jars=*)
+      JAR_PATH="${i#*=}"
+      shift
+      ;;
+      -r=*|--repo=*)
+      MVN_REPO_URL="${i#*=}"
+      shift
+      ;;
+      -u=*|--username=*)
+      MVN_USERNAME="${i#*=}"
+      shift
+      ;;
+      -p=*|--password=*)
+      MVN_PASSWORD="${i#*=}"
+      shift
+      ;;
+      --skipPom)
+      PUBLISH_POM=0
+      shift
+      ;;
+      *)  # unknown option
+      ;;
+  esac
+done
+
+# Validate the passed parameters.
+if [[ "$MVN_ACTION" != "deploy" && "$MVN_ACTION" != "install" ]]; then
+  echo "--action must be install or deploy"
+  exit 1
+fi
+if [[ "$MVN_ACTION" == "deploy" ]]; then
+  if [[ -z "$MVN_USERNAME" ]]; then
+    echo "--username must be set when --action=deploy"
+    exit 1
+  fi
+  if [[ -z "$MVN_PASSWORD" ]]; then
+    echo "--password must be set when --action=deploy"
+    exit 1
+  fi
+fi
+
+# Validate Maven is installed.
+if [[ -z $(which mvn) ]]; then
+  echo "'mvn' was not found. Maven is required to publish the jars."
+  exit 1
+fi
+
+# Static variables.
+PROP_FILE="META-INF/apache-kudu-test-binary.properties"
+ARTIFACT_GROUP="org.apache.kudu"
+ARTIFACT_ID="kudu-binary"
+
+# Common Maven arguments.
+COMMON_MVN_ARGS="-DgeneratePom=false"
+COMMON_MVN_ARGS="$COMMON_MVN_ARGS -DgroupId=$ARTIFACT_GROUP"
+COMMON_MVN_ARGS="$COMMON_MVN_ARGS -DartifactId=$ARTIFACT_ID"
+COMMON_MVN_ARGS="$COMMON_MVN_ARGS -Dversion=$VERSION"
+
+# Attempts to read a property from the mini cluster properties file
+# in the passed jar. If the property is not found, it prints an error
+# and exits.
+function read_prop_or_die() {
+  local JAR=$1
+  local KEY=$2
+  if [[ ! -f "$JAR" ]]; then
+    echo "Jar file not found: $JAR"
+    exit 1
+  fi
+  local PROP=$(unzip -q -c ${JAR} ${PROP_FILE} | grep "$KEY" | cut -d'=' -f2)
+  if [[ -z ${PROP} ]]; then
+    echo "$KEY property not found in $JAR/$PROP_FILE"
+    exit 1
+  fi
+  echo ${PROP}
+}
+
+# Executes the appropriate maven action based on MVN_ACTION with the
+# passed parameters as arguments. When the action is 'deploy' additional
+# arguments are added to handle the repository, username, and password.
+function maven_action() {
+  if [[ "$MVN_ACTION" == "install" ]]; then
+    mvn install:install-file ${@}
+  elif [[ "$MVN_ACTION" == "deploy" ]]; then
+    ARGS=${@}
+    ARGS="$ARGS -Durl=$MVN_REPO_URL"
+    # Configure maven to pass the username and password with a custom settings.xml.
+    ARGS="$ARGS --settings $SOURCE_ROOT/build-support/mini-cluster/settings.xml"
+    ARGS="$ARGS -DrepositoryId=apache"
+    ARGS="$ARGS -Drepo.username=$MVN_USERNAME"
+    ARGS="$ARGS -Drepo.password=$MVN_PASSWORD"
+    mvn deploy:deploy-file ${ARGS}
+  fi
+}
+
+# Publish a generated pom.
+if [[ "${PUBLISH_POM}" == "1" ]]; then
+  # Create a minimal POM file.
+  POM_FILE=${BUILD_ROOT}/pom.xml
+  cat <<EOF > ${POM_FILE}
+<?xml version="1.0" encoding="UTF-8"?>
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>${ARTIFACT_GROUP}</groupId>
+  <artifactId>${ARTIFACT_ID}</artifactId>
+  <version>${VERSION}</version>
+  <description>An archive of Kudu binaries for use in a "mini cluster" environment
for TESTING ONLY.</description>
+  <url>https://kudu.apache.org/</url>
+  <licenses>
+    <license>
+      <name>The Apache Software License, Version 2.0</name>
+      <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+      <distribution>repo</distribution>
+    </license>
+  </licenses>
+</project>
+EOF
+  POM_MVN_ARGS="$COMMON_MVN_ARGS"
+  POM_MVN_ARGS="$POM_MVN_ARGS -Dfile=$POM_FILE"
+  POM_MVN_ARGS="$POM_MVN_ARGS -Dpackaging=pom"
+  maven_action ${POM_MVN_ARGS}
+fi
+
+# Publish the generated jars that have the correct version.
+for JAR in ${JAR_PATH}/${ARTIFACT_ID}-${VERSION}-*.jar; do
+  JAR_ARCH=$(read_prop_or_die "$JAR" "artifact.arch")
+  JAR_OS=$(read_prop_or_die "$JAR" "artifact.os")
+  JAR_VERSION=$(read_prop_or_die "$JAR" "artifact.version")
+  if [[ "$JAR_VERSION" != "$VERSION" ]]; then
+    echo "The version ($VERSION) doesn't match the jar's artifact.version property ($JAR_VERSION)"
+    exit 1
+  fi
+  JAR_CLASSIFIER="$JAR_OS-$JAR_ARCH"
+  JAR_MVN_ARGS="$COMMON_MVN_ARGS"
+  JAR_MVN_ARGS="$JAR_MVN_ARGS -Dclassifier=$JAR_CLASSIFIER"
+  JAR_MVN_ARGS="$JAR_MVN_ARGS -Dfile=$JAR"
+  JAR_MVN_ARGS="$JAR_MVN_ARGS -Dpackaging=jar"
+  maven_action ${JAR_MVN_ARGS}
+done
diff --git a/build-support/relocate_binaries_for_mini_cluster.py b/build-support/mini-cluster/relocate_binaries_for_mini_cluster.py
similarity index 98%
rename from build-support/relocate_binaries_for_mini_cluster.py
rename to build-support/mini-cluster/relocate_binaries_for_mini_cluster.py
index 182ae8b..c6764f9 100755
--- a/build-support/relocate_binaries_for_mini_cluster.py
+++ b/build-support/mini-cluster/relocate_binaries_for_mini_cluster.py
@@ -34,6 +34,10 @@ import shutil
 import subprocess
 import sys
 
+SOURCE_ROOT = os.path.join(os.path.dirname(__file__), "../..")
+# Add the build-support dir to the system path so we can import kudu-util.
+sys.path.append(os.path.join(SOURCE_ROOT, "build-support"))
+
 from kudu_util import check_output, Colors, init_logging
 from dep_extract import DependencyExtractor
 
@@ -66,8 +70,6 @@ ARTIFACT_ROOT = 'artifact_root'
 ARTIFACT_BIN_DIR = 'artifact_bin_dir'
 ARTIFACT_LIB_DIR = 'artifact_lib_dir'
 
-SOURCE_ROOT = os.path.join(os.path.dirname(__file__), "..")
-
 IS_MACOS = os.uname()[0] == "Darwin"
 IS_LINUX = os.uname()[0] == "Linux"
 
diff --git a/build-support/mini-cluster/settings.xml b/build-support/mini-cluster/settings.xml
new file mode 100644
index 0000000..ec8491f
--- /dev/null
+++ b/build-support/mini-cluster/settings.xml
@@ -0,0 +1,34 @@
+<!--
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+-->
+
+<!--
+A custom Maven Settings XML used in publish_mini_cluster_binaries.sh
+to allow passwords to be passed via command line. This means our password
+process doesn't need to be different than the Gradle build and there is no
+need to store passwords anywhere insecurely.
+-->
+<settings>
+  <servers>
+    <server>
+      <id>apache</id>
+      <username>${repo.username}</username>
+      <password>${repo.password}</password>
+    </server>
+  </servers>
+</settings>
\ No newline at end of file


Mime
View raw message