asterixdb-notifications mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Xikui Wang (Code Review)" <do-not-re...@asterixdb.incubator.apache.org>
Subject Change in asterixdb[master]: Add install/uninstall UDF to Ansible
Date Fri, 14 Apr 2017 23:21:30 GMT
Xikui Wang has submitted this change and it was merged.

Change subject: Add install/uninstall UDF to Ansible
......................................................................


Add install/uninstall UDF to Ansible

1. Add udf.sh for handling install/uninstall UDF packages to the cluster.
2. Remove unnecessary path check in TestLibrarian.

Change-Id: If7ea9640a06e2b691c19a8a819307d84b55a679e
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1682
Sonar-Qube: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Yingyi Bu <buyingyi@gmail.com>
BAD: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
---
M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java
M asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/TestLibrarian.java
A asterixdb/asterix-server/src/main/opt/ansible/bin/udf.sh
A asterixdb/asterix-server/src/main/opt/ansible/yaml/deploy_udf.yml
A asterixdb/asterix-server/src/main/opt/ansible/yaml/destroy_udf.yml
5 files changed, 155 insertions(+), 20 deletions(-)

Approvals:
  Yingyi Bu: Looks good to me, approved
  Jenkins: Verified; No violations found; No violations found; Verified



diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java
index b4e1c75..5f86c28 100755
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java
@@ -50,11 +50,11 @@
 import org.apache.asterix.metadata.entities.Library;
 import org.apache.asterix.metadata.utils.MetadataUtil;
 import org.apache.asterix.runtime.formats.NonTaggedDataFormat;
-import org.apache.hyracks.control.common.controllers.ControllerConfig;
 
 public class ExternalLibraryUtils {
 
     private static final Logger LOGGER = Logger.getLogger(ExternalLibraryUtils.class.getName());
+    private static final FilenameFilter nonHiddenFileNameFilter = (dir, name) -> !name.startsWith(".");
 
     private ExternalLibraryUtils() {
     }
@@ -72,19 +72,15 @@
         // directory exists?
         if (installLibDir.exists()) {
             // get the list of files in the directory
-            for (String dataverse : installLibDir.list()) {
-                File dataverseDir = new File(installLibDir, dataverse);
-                String[] libraries = dataverseDir.list();
-                for (String library : libraries) {
+            for (File dataverseDir : installLibDir.listFiles(File::isDirectory)) {
+                for (File libraryDir : dataverseDir.listFiles(File::isDirectory)) {
                     // for each file (library), register library
-                    registerLibrary(externalLibraryManager, dataverse, library);
+                    registerLibrary(externalLibraryManager, dataverseDir.getName(), libraryDir.getName());
                     // is metadata node?
                     if (isMetadataNode) {
                         // get library file
-                        File libraryDir = new File(installLibDir.getAbsolutePath() + File.separator
+ dataverse
-                                + File.separator + library);
                         // install if needed (i,e, add the functions, adapters, datasources,
parsers to the metadata)
-                        installLibraryIfNeeded(dataverse, libraryDir, uninstalledLibs);
+                        installLibraryIfNeeded(dataverseDir.getName(), libraryDir, uninstalledLibs);
                     }
                 }
             }
@@ -105,7 +101,7 @@
         // directory exists?
         if (uninstallLibDir.exists()) {
             // list files
-            uninstallLibNames = uninstallLibDir.list();
+            uninstallLibNames = uninstallLibDir.list(nonHiddenFileNameFilter);
             for (String uninstallLibName : uninstallLibNames) {
                 // Get the <dataverse name - library name> pair
                 String[] components = uninstallLibName.split("\\.");
@@ -295,7 +291,6 @@
      *
      * @param dataverse
      * @param libraryName
-     * @param installLibDir
      * @throws Exception
      */
     protected static void registerLibrary(ILibraryManager externalLibraryManager, String
dataverse, String libraryName)
@@ -395,16 +390,26 @@
      * @return the directory "$(ControllerConfig.defaultDir)/library": This needs to be improved
      */
     protected static File getLibraryInstallDir() {
-        String workingDir = System.getProperty("user.dir");
-        return new File(workingDir, "library");
+        // Check managix directory first. If not exists, check app home.
+        File installDir = new File(System.getProperty("user.dir"), "library");
+        if (!installDir.exists()) {
+            installDir = new File(System.getProperty("app.home", System.getProperty("user.home"))
+                    + File.separator + "lib" + File.separator + "udfs");
+        }
+        return installDir;
     }
 
     /**
      * @return the directory "$(ControllerConfig.defaultDir)/uninstall": This needs to be
improved
      */
     protected static File getLibraryUninstallDir() {
-        String workingDir = System.getProperty("user.dir");
-        return new File(workingDir, "uninstall");
+        // Check managix directory first. If not exists, check app home.
+        File uninstallDir = new File(System.getProperty("user.dir"), "uninstall");
+        if (!uninstallDir.exists()) {
+            uninstallDir = new File(System.getProperty("app.home", System.getProperty("user.home"))
+                    + File.separator + "lib" + File.separator + "udfs" + File.separator +
"uninstall");
+        }
+        return uninstallDir;
     }
 
 }
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/TestLibrarian.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/TestLibrarian.java
index e92b5e8..c67d26c 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/TestLibrarian.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/TestLibrarian.java
@@ -35,8 +35,6 @@
 
 public class TestLibrarian implements ITestLibrarian {
 
-    public static final String LIBRARY_DIR_NAME = "library";
-
     // The following list includes a library manager for the CC
     // and library managers for NCs (one-per-NC).
     private final List<ILibraryManager> libraryManagers;
@@ -84,9 +82,6 @@
 
     public static void removeLibraryDir() throws IOException {
         File installLibDir = ExternalLibraryUtils.getLibraryInstallDir();
-        if (!installLibDir.getAbsolutePath().endsWith(LIBRARY_DIR_NAME)) {
-            throw new HyracksDataException("Invalid library directory");
-        }
         FileUtils.deleteQuietly(installLibDir);
     }
 
diff --git a/asterixdb/asterix-server/src/main/opt/ansible/bin/udf.sh b/asterixdb/asterix-server/src/main/opt/ansible/bin/udf.sh
new file mode 100755
index 0000000..2f8b6fe
--- /dev/null
+++ b/asterixdb/asterix-server/src/main/opt/ansible/bin/udf.sh
@@ -0,0 +1,72 @@
+#!/bin/bash
+# ------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------
+
+# Get options
+usage() { echo "./udf.sh -m [i|u] -d DATAVERSE_NAME -l LIBRARY_NAME [-p UDF_PACKAGE_PATH]"
1>&2; exit 1; }
+
+while getopts ":d:l:p:m:" o; do
+  case "${o}" in
+    m)
+        mode=${OPTARG}
+        ;;
+    d)
+        dname=${OPTARG}
+        ;;
+    l)
+        libname=${OPTARG}
+        ;;
+    p)
+        tarpath=${OPTARG}
+        ;;
+    *)
+        echo $o
+        usage
+        ;;
+  esac
+done
+shift $((OPTIND-1))
+
+if [[ -z $dname ]] || [[ -z $libname ]]; then
+    echo "Dataverse name or Library name is missing"
+fi
+
+# Gets the absolute path so that the script can work no matter where it is invoked.
+pushd `dirname $0` > /dev/null
+SCRIPT_PATH=`pwd -P`
+popd > /dev/null
+ANSB_PATH=`dirname "${SCRIPT_PATH}"`
+
+INVENTORY=$ANSB_PATH/conf/inventory
+
+# Deploy/Destroy the UDF package on all nodes.
+export ANSIBLE_HOST_KEY_CHECKING=false
+if [[ $mode = "i" ]]; then
+    if [[ -z $tarpath ]]; then
+        echo "UDF package path is undefined"
+    else
+        echo "Install library to $dname.$libname"
+        ansible-playbook -i $INVENTORY $ANSB_PATH/yaml/deploy_udf.yml --extra-vars "dataverse=$dname
libname=$libname package_path=$tarpath"
+    fi
+elif [[ $mode == "u" ]]; then
+    echo "Uninstall library in $dname.$libname"
+    ansible-playbook -i $INVENTORY $ANSB_PATH/yaml/destroy_udf.yml --extra-vars "dataverse=$dname
libname=$libname"
+else
+    echo "Wrong mode"
+fi
\ No newline at end of file
diff --git a/asterixdb/asterix-server/src/main/opt/ansible/yaml/deploy_udf.yml b/asterixdb/asterix-server/src/main/opt/ansible/yaml/deploy_udf.yml
new file mode 100644
index 0000000..e9300ea
--- /dev/null
+++ b/asterixdb/asterix-server/src/main/opt/ansible/yaml/deploy_udf.yml
@@ -0,0 +1,30 @@
+# ------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------
+
+- name: Deploy the UDF library {{ dataverse }}.{{ libname }} to the cluster
+  hosts: all
+  tasks:
+    - include_vars: ../conf/instance_settings.yml
+    - file:
+        path: "{{ binarydir }}/lib/udfs/{{ dataverse }}/{{ libname }}"
+        state: directory
+        mode: 0755
+    - unarchive:
+        src: "{{ package_path }}"
+        dest: "{{ binarydir }}/lib/udfs/{{ dataverse }}/{{ libname }}"
diff --git a/asterixdb/asterix-server/src/main/opt/ansible/yaml/destroy_udf.yml b/asterixdb/asterix-server/src/main/opt/ansible/yaml/destroy_udf.yml
new file mode 100644
index 0000000..cc4add3
--- /dev/null
+++ b/asterixdb/asterix-server/src/main/opt/ansible/yaml/destroy_udf.yml
@@ -0,0 +1,33 @@
+# ------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------
+
+- name: Destroy the library {{ libname }} in dataverse {{ dataverse }}
+  hosts: all
+  tasks:
+    - include_vars: ../conf/instance_settings.yml
+    - file:
+        path: "{{ binarydir }}/lib/udfs/uninstall/"
+        state: directory
+        mode: 0755
+    - file:
+        state: touch
+        path: "{{ binarydir }}/lib/udfs/uninstall/{{ dataverse }}.{{ libname }}"
+    - file:
+        path: "{{ binarydir }}/lib/udfs/{{ dataverse }}/{{ libname }}"
+        state: absent

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1682
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: merged
Gerrit-Change-Id: If7ea9640a06e2b691c19a8a819307d84b55a679e
Gerrit-PatchSet: 3
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang <xkkwww@gmail.com>
Gerrit-Reviewer: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mblow@apache.org>
Gerrit-Reviewer: Xikui Wang <xkkwww@gmail.com>
Gerrit-Reviewer: Yingyi Bu <buyingyi@gmail.com>

Mime
View raw message