ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jlun...@apache.org
Subject [6/8] ambari git commit: AMBARI-18385: Add HDF management pack (jluniya)
Date Wed, 14 Sep 2016 05:26:53 GMT
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi.py
new file mode 100644
index 0000000..362a40b
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi.py
@@ -0,0 +1,231 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys, nifi_ca_util, os, pwd, grp, signal, time, glob, socket, json
+from resource_management.core import sudo
+from resource_management import *
+from subprocess import call
+from setup_ranger_nifi import setup_ranger_nifi
+
+reload(sys)
+sys.setdefaultencoding('utf8')
+
+class Master(Script):
+  def install(self, env):
+
+    import params
+    import status_params
+
+    self.install_packages(env)
+
+    Directory([params.nifi_node_dir],
+            owner=params.nifi_user,
+            group=params.nifi_group,
+            create_parents=True,
+            recursive_ownership=True
+    )
+
+    #update the configs specified by user
+    self.configure(env, True)
+
+    Execute('touch ' +  params.nifi_node_log_file, user=params.nifi_user)
+
+  def configure(self, env, isInstall=False, is_starting = False):
+    import params
+    import status_params
+    env.set_params(params)
+    env.set_params(status_params)
+
+    #create the log, pid, conf dirs if not already present
+    Directory([status_params.nifi_pid_dir, params.nifi_node_log_dir, params.nifi_internal_dir, params.nifi_database_dir, params.nifi_flowfile_repo_dir, params.nifi_content_repo_dir_default, params.nifi_provenance_repo_dir_default, params.nifi_config_dir, params.nifi_flow_config_dir, params.nifi_state_dir, params.lib_dir],
+            owner=params.nifi_user,
+            group=params.nifi_group,
+            create_parents=True,
+            recursive_ownership=True
+    )
+
+    # On some OS this folder may not exist, so we will create it before pushing files there
+    Directory(params.limits_conf_dir,
+              create_parents = True,
+              owner='root',
+              group='root'
+    )
+
+    File(os.path.join(params.limits_conf_dir, 'nifi.conf'),
+         owner='root',
+         group='root',
+         mode=0644,
+         content=Template("nifi.conf.j2")
+    )
+
+    
+    ca_client_script = nifi_ca_util.get_toolkit_script('tls-toolkit.sh')
+    File(ca_client_script, mode=0755)
+
+
+    if params.nifi_ca_host and params.nifi_ssl_enabled:
+      ca_client_json = os.path.realpath(os.path.join(params.nifi_config_dir, 'nifi-certificate-authority-client.json'))
+      File(ca_client_json,
+           owner = params.nifi_user,
+           group = params.nifi_group,
+           mode = 0600)
+      ca_client_dict = nifi_ca_util.load(ca_client_json)
+      if is_starting:
+        if params.nifi_toolkit_tls_regenerate:
+          nifi_ca_util.move_keystore_truststore(ca_client_dict)
+          ca_client_dict = {}
+        else:
+          nifi_ca_util.move_keystore_truststore_if_necessary(ca_client_dict, params.nifi_ca_client_config)
+      nifi_ca_util.overlay(ca_client_dict, params.nifi_ca_client_config)
+      nifi_ca_util.dump(ca_client_json, ca_client_dict)
+      if is_starting:
+        Execute('JAVA_HOME='+params.jdk64_home+' '+ca_client_script+' client -F -f '+ca_client_json, user=params.nifi_user)
+      nifi_ca_util.update_nifi_properties(nifi_ca_util.load(ca_client_json), params.nifi_properties)
+
+    #write out nifi.properties
+    PropertiesFile(params.nifi_config_dir + '/nifi.properties',
+                   properties = params.nifi_properties,
+                   mode = 0400,
+                   owner = params.nifi_user,
+                   group = params.nifi_group)
+
+    #write out boostrap.conf
+    bootstrap_content=InlineTemplate(params.nifi_boostrap_content)
+    File(format("{params.nifi_config_dir}/bootstrap.conf"), content=bootstrap_content, owner=params.nifi_user, group=params.nifi_group, mode=0400)
+
+    #write out logback.xml
+    logback_content=InlineTemplate(params.nifi_node_logback_content)
+    File(format("{params.nifi_config_dir}/logback.xml"), content=logback_content, owner=params.nifi_user, group=params.nifi_group, mode=0400)
+
+    #write out state-management.xml
+    statemgmt_content=InlineTemplate(params.nifi_state_management_content)
+    File(format("{params.nifi_config_dir}/state-management.xml"), content=statemgmt_content, owner=params.nifi_user, group=params.nifi_group, mode=0400)
+
+    #write out authorizers file
+    authorizers_content=InlineTemplate(params.nifi_authorizers_content)
+    File(format("{params.nifi_config_dir}/authorizers.xml"), content=authorizers_content, owner=params.nifi_user, group=params.nifi_group, mode=0400)
+
+    #write out login-identity-providers.xml
+    login_identity_providers_content=InlineTemplate(params.nifi_login_identity_providers_content)
+    File(format("{params.nifi_config_dir}/login-identity-providers.xml"), content=login_identity_providers_content, owner=params.nifi_user, group=params.nifi_group, mode=0400)
+
+    #write out nifi-env in bin as 0755 (see BUG-61769)
+    env_content=InlineTemplate(params.nifi_env_content)
+    File(format("{params.bin_dir}/nifi-env.sh"), content=env_content, owner=params.nifi_user, group=params.nifi_group, mode=0755) 
+    
+    #write out bootstrap-notification-services.xml
+    boostrap_notification_content=InlineTemplate(params.nifi_boostrap_notification_content)
+    File(format("{params.nifi_config_dir}/bootstrap-notification-services.xml"), content=boostrap_notification_content, owner=params.nifi_user, group=params.nifi_group, mode=0400) 
+
+  def stop(self, env):
+    import params
+    import status_params
+
+    Execute ('export JAVA_HOME='+params.jdk64_home+';'+params.bin_dir+'/nifi.sh stop >> ' + params.nifi_node_log_file, user=params.nifi_user)
+    if os.path.isfile(status_params.nifi_node_pid_file):
+      sudo.unlink(status_params.nifi_node_pid_file)
+
+  def start(self, env):
+    import params
+    import status_params
+    self.configure(env, is_starting = True)
+    setup_ranger_nifi(upgrade_type=None)
+
+    # Write out flow.xml.gz to internal dir only if AMS installed (must be writable by Nifi)
+    # only during first install. It is used to automate setup of Ambari metrics reporting task in Nifi
+    if params.metrics_collector_host and params.nifi_ambari_reporting_enabled and self.check_is_fresh_install(self):
+      Execute('echo "First time setup so generating flow.xml.gz" >> ' + params.nifi_node_log_file, user=params.nifi_user)
+      flow_content=InlineTemplate(params.nifi_flow_content)
+      File(format("{params.nifi_flow_config_dir}/flow.xml"), content=flow_content, owner=params.nifi_user, group=params.nifi_group, mode=0600)
+      Execute(format("cd {params.nifi_flow_config_dir}; mv flow.xml.gz flow_$(date +%d-%m-%Y).xml.gz ;"),user=params.nifi_user,ignore_failures=True)
+      Execute(format("cd {params.nifi_flow_config_dir}; gzip flow.xml;"), user=params.nifi_user)
+
+
+    Execute ('export JAVA_HOME='+params.jdk64_home+';'+params.bin_dir+'/nifi.sh start >> ' + params.nifi_node_log_file, user=params.nifi_user)
+    #If nifi pid file not created yet, wait a bit
+    if not os.path.isfile(status_params.nifi_pid_dir+'/nifi.pid'):
+      Execute ('sleep 5')
+
+
+  def status(self, env):
+    import status_params
+    check_process_status(status_params.nifi_node_pid_file)
+
+
+  def check_is_fresh_install(self, env):
+    """
+    Checks if fresh nifi install by checking if zk dir exists
+    :return:
+    """
+    import params, re
+    from resource_management.core import shell
+    from resource_management.core.exceptions import Fail
+    from resource_management.core.logger import Logger
+
+    ZK_CONNECT_ERROR = "ConnectionLoss"
+    ZK_NODE_NOT_EXIST = "Node does not exist"
+
+    zookeeper_queried = False
+    is_fresh_nifi_install = True
+
+    # For every zk server try to find nifi zk dir
+    zookeeper_server_list = params.config['clusterHostInfo']['zookeeper_hosts']
+    for zookeeper_server in zookeeper_server_list:
+      # Determine where the zkCli.sh shell script is
+      zk_command_location = os.path.join(params.stack_root, "current", "zookeeper-client", "bin", "zkCli.sh")
+      if params.stack_version_buildnum is not None:
+        zk_command_location = os.path.join(params.stack_root, params.stack_version_buildnum, "zookeeper", "bin", "zkCli.sh")
+
+      # create the ZooKeeper query command e.g.
+      # /usr/hdf/current/zookeeper-client/bin/zkCli.sh -server node:2181 ls /nifi
+      command = "{0} -server {1}:{2} ls {3}".format(
+        zk_command_location, zookeeper_server, params.zookeeper_port, params.nifi_znode)
+              
+      # echo 'ls /nifi' | /usr/hdf/current/zookeeper-client/bin/zkCli.sh -server node:2181
+      #command = "echo 'ls {3}' | {0} -server {1}:{2}".format(
+      #  zk_command_location, zookeeper_server, params.zookeeper_port, params.nifi_znode)
+
+      Logger.info("Running command: " + command)
+
+      code, out = shell.call(command, logoutput=True, quiet=False, timeout=20)
+      if not out or re.search(ZK_CONNECT_ERROR, out):
+        Logger.info("Unable to query Zookeeper: " + zookeeper_server + ". Skipping and trying next ZK server")
+        continue
+      elif re.search(ZK_NODE_NOT_EXIST, out):
+        Logger.info("Nifi ZNode does not exist, so must be fresh install of Nifi: " + params.nifi_znode)
+        zookeeper_queried = True
+        is_fresh_nifi_install = True
+        break
+      else:
+        Logger.info("Nifi ZNode already exists, so must not be a fresh install of Nifi: " + params.nifi_znode)
+        zookeeper_queried = True
+        is_fresh_nifi_install = False
+        break
+
+    # fail if the ZK data could not be queried
+    if not zookeeper_queried:
+      raise Fail("Unable to query for znode on on any of the following ZooKeeper hosts: {0}. Please ensure Zookeepers are started and retry".format(
+        zookeeper_server_list))
+    else:
+      return is_fresh_nifi_install    
+            
+
+if __name__ == "__main__":
+  Master().execute()

http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi_ca.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi_ca.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi_ca.py
new file mode 100644
index 0000000..bc343e4
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi_ca.py
@@ -0,0 +1,114 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import nifi_ca_util, os, time
+
+from resource_management.core.exceptions import ComponentIsNotRunning
+from resource_management.core.resources.system import Directory, Execute
+from resource_management.core.sudo import kill, read_file, path_isfile, unlink
+from resource_management.libraries.functions.check_process_status import check_process_status
+from resource_management.libraries.script.script import Script
+from resource_management.core.resources import File
+from signal import SIGTERM, SIGKILL
+
+class CertificateAuthority(Script):
+  def install(self, env):
+    import params
+    import status_params
+
+    self.install_packages(env)
+
+    #Be sure ca script is in cache
+    nifi_ca_util.get_toolkit_script('tls-toolkit.sh')
+
+  def configure(self, env):
+    import params
+    import status_params
+    env.set_params(params)
+    env.set_params(status_params)
+
+    #create the log, pid, conf dirs if not already present
+    Directory([status_params.nifi_pid_dir, params.nifi_node_log_dir, params.nifi_config_dir],
+      owner=params.nifi_user,
+      group=params.nifi_group,
+      create_parents=True
+    )
+
+    ca_json = os.path.join(params.nifi_config_dir, 'nifi-certificate-authority.json')
+    ca_dict = nifi_ca_util.load(ca_json)
+    nifi_ca_util.overlay(ca_dict, params.nifi_ca_config)
+    nifi_ca_util.dump(ca_json, ca_dict)
+
+    Directory([params.nifi_config_dir],
+        owner=params.nifi_user,
+        group=params.nifi_group,
+        create_parents=True,
+        recursive_ownership=True
+    )
+
+  def invalidate_ca_server(self, env):
+    import params
+    ca_json = os.path.join(params.nifi_config_dir, 'nifi-certificate-authority.json')
+    nifi_ca_util.move_store(nifi_ca_util.load(ca_json), 'keyStore')
+    unlink(ca_json)
+    
+  def status(self, env):
+    import status_params
+    check_process_status(status_params.nifi_ca_pid_file)
+
+  def start(self, env):
+    import params
+    import status_params
+
+    self.configure(env)
+    ca_server_script = nifi_ca_util.get_toolkit_script('tls-toolkit.sh')
+    run_ca_script = os.path.join(os.path.dirname(__file__), 'run_ca.sh')
+    Directory([params.nifi_config_dir],
+        owner=params.nifi_user,
+        group=params.nifi_group,
+        create_parents=True,
+        recursive_ownership=True
+    )
+
+    File(ca_server_script, mode=0755)
+    File(run_ca_script, mode=0755) 
+    Execute((run_ca_script, params.jdk64_home, ca_server_script, params.nifi_config_dir + '/nifi-certificate-authority.json', params.nifi_ca_log_file_stdout, params.nifi_ca_log_file_stderr, status_params.nifi_ca_pid_file), user=params.nifi_user)
+    if not os.path.isfile(status_params.nifi_ca_pid_file):
+      raise Exception('Expected pid file to exist')
+
+  def stop(self, env):
+    import status_params
+
+    if path_isfile(status_params.nifi_ca_pid_file):
+      try:
+        self.status(env)
+        pid = int(read_file(status_params.nifi_ca_pid_file))
+        for i in range(25):
+          kill(pid, SIGTERM)
+          time.sleep(1)
+          self.status(env)
+        kill(pid, SIGKILL)
+        time.sleep(5)
+        self.status(env)
+      except ComponentIsNotRunning:
+        unlink(status_params.nifi_ca_pid_file)
+
+if __name__ == "__main__":
+  CertificateAuthority().execute()

http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi_ca_util.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi_ca_util.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi_ca_util.py
new file mode 100644
index 0000000..6807491
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi_ca_util.py
@@ -0,0 +1,110 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import json, nifi_constants, os
+from resource_management.core import sudo
+from resource_management.core.resources.system import File
+
+script_dir = os.path.dirname(__file__)
+files_dir = os.path.realpath(os.path.join(os.path.dirname(script_dir), 'files'))
+
+def load(config_json):
+  if sudo.path_isfile(config_json):
+    contents = sudo.read_file(config_json)
+    if len(contents) > 0:
+      return json.loads(contents)
+  return {}
+
+def dump(config_json, config_dict):
+  import params
+  File(config_json,
+    owner=params.nifi_user,
+    group=params.nifi_group,
+    mode=0600,
+    content=json.dumps(config_dict, sort_keys=True, indent=4)
+  ) 
+
+def overlay(config_dict, overlay_dict):
+  for k, v in overlay_dict.iteritems():
+    if v or k not in config_dict:
+      config_dict[k] = v
+
+def get_toolkit_script(scriptName, scriptDir = files_dir):
+  nifiToolkitDir = None
+  for dir in os.listdir(scriptDir):
+    if dir.startswith('nifi-toolkit-'):
+      nifiToolkitDir = os.path.join(scriptDir, dir)
+
+  if nifiToolkitDir is None:
+    raise Exception("Couldn't find nifi toolkit directory in " + scriptDir)
+  result = nifiToolkitDir + '/bin/' + scriptName
+  if not sudo.path_isfile(result):
+    raise Exception("Couldn't find file " + result)
+  return result
+
+def update_nifi_properties(client_dict, nifi_properties):
+  nifi_properties[nifi_constants.NIFI_SECURITY_KEYSTORE_TYPE] = client_dict['keyStoreType']
+  nifi_properties[nifi_constants.NIFI_SECURITY_KEYSTORE_PASSWD] = client_dict['keyStorePassword']
+  nifi_properties[nifi_constants.NIFI_SECURITY_KEY_PASSWD] = client_dict['keyPassword']
+  nifi_properties[nifi_constants.NIFI_SECURITY_TRUSTSTORE_TYPE] = client_dict['trustStoreType']
+  nifi_properties[nifi_constants.NIFI_SECURITY_TRUSTSTORE_PASSWD] = client_dict['trustStorePassword']
+
+def store_exists(client_dict, key):
+  if key not in client_dict:
+    return False
+  return sudo.path_isfile(client_dict[key])
+
+def different(one, two, key):
+  if key not in one:
+    return False
+  if len(one[key]) == 0:
+    return False
+  if key not in two:
+    return False
+  if len(two[key]) == 0:
+    return False
+  return one[key] != two[key]
+
+def move_keystore_truststore_if_necessary(orig_client_dict, new_client_dict):
+  if not (store_exists(new_client_dict, 'keyStore') or store_exists(new_client_dict, 'trustStore')):
+    return
+  if different(orig_client_dict, new_client_dict, 'keyStoreType'):
+    move_keystore_truststore(new_client_dict)
+  elif different(orig_client_dict, new_client_dict, 'keyStorePassword'):
+    move_keystore_truststore(new_client_dict)
+  elif different(orig_client_dict, new_client_dict, 'keyPassword'):
+    move_keystore_truststore(new_client_dict)
+  elif different(orig_client_dict, new_client_dict, 'trustStoreType'):
+    move_keystore_truststore(new_client_dict)
+  elif different(orig_client_dict, new_client_dict, 'trustStorePassword'):
+    move_keystore_truststore(new_client_dict)
+
+def move_keystore_truststore(client_dict):
+  move_store(client_dict, 'keyStore')
+  move_store(client_dict, 'trustStore')
+
+def move_store(client_dict, key):
+  if store_exists(client_dict, key):
+    num = 0
+    name = client_dict[key]
+    while sudo.path_isfile(name + '.bak.' + str(num)):
+      num += 1
+    sudo.copy(name, name + '.bak.' + str(num))
+    sudo.unlink(name)

http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi_constants.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi_constants.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi_constants.py
new file mode 100644
index 0000000..632a473
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi_constants.py
@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+NIFI_SECURITY_KEY_PASSWD = 'nifi.security.keyPasswd'
+NIFI_SECURITY_KEYSTORE_PASSWD = 'nifi.security.keystorePasswd'
+NIFI_SECURITY_KEYSTORE_TYPE = 'nifi.security.keystoreType'
+NIFI_SECURITY_TRUSTSTORE_PASSWD = 'nifi.security.truststorePasswd'
+NIFI_SECURITY_TRUSTSTORE_TYPE = 'nifi.security.truststoreType'

http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/params.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/params.py
new file mode 100644
index 0000000..087cef4
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/params.py
@@ -0,0 +1,445 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+from resource_management.libraries.script.script import Script
+import sys, os, glob, socket, re
+from resource_management.libraries.functions import format
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions.version import format_stack_version
+from resource_management.libraries.functions import StackFeature
+from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.resources.hdfs_resource import HdfsResource
+from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions import get_kinit_path
+from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources
+import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set
+    
+# server configurations
+config = Script.get_config()
+stack_root = Script.get_stack_root()
+stack_version_buildnum = default("/commandParams/version", None)
+
+#nifi_install_dir = '/usr/hdf/current/nifi'
+nifi_install_dir = os.path.join(stack_root, "current", "nifi")
+if stack_version_buildnum is not None:
+  nifi_install_dir = os.path.join(stack_root, stack_version_buildnum, "nifi")
+        
+# params from nifi-ambari-config
+nifi_initial_mem = config['configurations']['nifi-ambari-config']['nifi.initial_mem']
+nifi_max_mem = config['configurations']['nifi-ambari-config']['nifi.max_mem']
+nifi_ambari_reporting_frequency = config['configurations']['nifi-ambari-config']['nifi.ambari_reporting_frequency']
+nifi_ambari_reporting_enabled = config['configurations']['nifi-ambari-config']['nifi.ambari_reporting_enabled']
+
+# note: nifi.node.port and nifi.node.ssl.port must be defined in same xml file for quicklinks to work
+nifi_node_port = config['configurations']['nifi-ambari-config']['nifi.node.port']
+nifi_node_ssl_port = config['configurations']['nifi-ambari-config']['nifi.node.ssl.port']
+nifi_node_protocol_port = config['configurations']['nifi-ambari-config']['nifi.node.protocol.port']
+
+nifi_znode = config['configurations']['nifi-ambari-config']['nifi.nifi_znode']
+
+nifi_internal_dir=config['configurations']['nifi-ambari-config']['nifi.internal.dir']
+nifi_state_dir=config['configurations']['nifi-ambari-config']['nifi.state.dir']
+nifi_database_dir=config['configurations']['nifi-ambari-config']['nifi.database.dir']
+nifi_flowfile_repo_dir=config['configurations']['nifi-ambari-config']['nifi.flowfile.repository.dir']
+nifi_content_repo_dir_default=config['configurations']['nifi-ambari-config']['nifi.content.repository.dir.default']
+nifi_provenance_repo_dir_default=config['configurations']['nifi-ambari-config']['nifi.provenance.repository.dir.default']
+nifi_config_dir = config['configurations']['nifi-ambari-config']['nifi.config.dir']
+nifi_flow_config_dir = config['configurations']['nifi-ambari-config']['nifi.flow.config.dir']
+nifi_sensitive_props_key = config['configurations']['nifi-ambari-config']['nifi.sensitive.props.key']
+
+
+nifi_flow_config_dir = nifi_flow_config_dir.replace('{nifi_internal_dir}',nifi_internal_dir)
+nifi_state_dir = nifi_state_dir.replace('{nifi_internal_dir}',nifi_internal_dir)
+nifi_config_dir = nifi_config_dir.replace('{nifi_install_dir}',nifi_install_dir)
+
+master_configs = config['clusterHostInfo']
+nifi_master_hosts = master_configs['nifi_master_hosts']
+
+# detect if running in single (sandbox) box
+nifi_num_nodes = len(master_configs['nifi_master_hosts'])
+#if nifi_num_nodes > 1:
+#  nifi_is_node='true'
+#else:
+#  nifi_is_node='false'
+#nifi_node_hosts = ",".join(master_configs['nifi_master_hosts'])
+
+# In sandbox scenario, Ambari should still setup nifi in clustered mode for now
+nifi_is_node='true'
+
+nifi_node_dir=nifi_install_dir
+bin_dir = os.path.join(*[nifi_node_dir,'bin'])
+lib_dir = os.path.join(*[nifi_node_dir,'lib'])
+
+nifi_ca_host = None
+if 'nifi_ca_hosts' in master_configs:
+  nifi_ca_hosts = master_configs['nifi_ca_hosts']
+  if len(nifi_ca_hosts) > 0:
+    nifi_ca_host = nifi_ca_hosts[0]
+
+# params from nifi-ambari-ssl-config
+
+nifi_ssl_enabled = config['configurations']['nifi-ambari-ssl-config']['nifi.node.ssl.isenabled']
+nifi_keystore = config['configurations']['nifi-ambari-ssl-config']['nifi.security.keystore']
+nifi_keystoreType = config['configurations']['nifi-ambari-ssl-config']['nifi.security.keystoreType']
+nifi_keystorePasswd = config['configurations']['nifi-ambari-ssl-config']['nifi.security.keystorePasswd']
+nifi_keyPasswd = config['configurations']['nifi-ambari-ssl-config']['nifi.security.keyPasswd']
+nifi_truststore = config['configurations']['nifi-ambari-ssl-config']['nifi.security.truststore']
+nifi_truststoreType = config['configurations']['nifi-ambari-ssl-config']['nifi.security.truststoreType']
+nifi_truststorePasswd = config['configurations']['nifi-ambari-ssl-config']['nifi.security.truststorePasswd']
+nifi_needClientAuth = config['configurations']['nifi-ambari-ssl-config']['nifi.security.needClientAuth']
+nifi_initial_admin_id = config['configurations']['nifi-ambari-ssl-config']['nifi.initial.admin.identity']
+nifi_ssl_config_content = config['configurations']['nifi-ambari-ssl-config']['content']
+
+#property that is set to hostname regardless of whether SSL enabled
+nifi_node_host = socket.getfqdn()
+
+nifi_truststore = nifi_truststore.replace('{nifi_node_ssl_host}',nifi_node_host)
+nifi_keystore = nifi_keystore.replace('{nifi_node_ssl_host}',nifi_node_host)
+
+#populate properties whose values depend on whether SSL enabled
+nifi_keystore = nifi_keystore.replace('{{nifi_config_dir}}',nifi_config_dir)
+nifi_truststore = nifi_truststore.replace('{{nifi_config_dir}}',nifi_config_dir)
+
+if nifi_ssl_enabled:
+  nifi_node_ssl_host = nifi_node_host
+  nifi_node_port = ""
+else:
+  nifi_node_nonssl_host = nifi_node_host
+  nifi_node_ssl_port = ""
+
+nifi_ca_parent_config = config['configurations']['nifi-ambari-ssl-config']
+nifi_use_ca = nifi_ca_parent_config['nifi.toolkit.tls.token']
+nifi_toolkit_dn_prefix = nifi_ca_parent_config['nifi.toolkit.dn.prefix']
+nifi_toolkit_dn_suffix = nifi_ca_parent_config['nifi.toolkit.dn.suffix']
+nifi_toolkit_tls_regenerate = nifi_ca_parent_config['nifi.toolkit.tls.regenerate']
+nifi_ca_log_file_stdout = config['configurations']['nifi-env']['nifi_node_log_dir'] + '/nifi-ca.stdout'
+nifi_ca_log_file_stderr = config['configurations']['nifi-env']['nifi_node_log_dir'] + '/nifi-ca.stderr'
+
+nifi_ca_config = { 
+  "days" : int(nifi_ca_parent_config['nifi.toolkit.tls.helper.days']),
+  "keyStore" : nifi_config_dir + '/nifi-certificate-authority-keystore.jks',
+  "token" : nifi_ca_parent_config['nifi.toolkit.tls.token'],
+  "caHostname" : nifi_ca_host,
+  "port" : int(nifi_ca_parent_config['nifi.toolkit.tls.port'])
+}
+
+if nifi_ca_host:
+  nifi_ca_config['dn'] = nifi_toolkit_dn_prefix + nifi_ca_host + nifi_toolkit_dn_suffix
+
+nifi_ca_client_config = { 
+  "days" : int(nifi_ca_parent_config['nifi.toolkit.tls.helper.days']),
+  "keyStore" : nifi_keystore,
+  "keyStoreType" : nifi_keystoreType,
+  "keyStorePassword" : nifi_keystorePasswd,
+  "keyPassword" : nifi_keyPasswd,
+  "token" : nifi_ca_parent_config['nifi.toolkit.tls.token'],
+  "dn" : nifi_toolkit_dn_prefix + nifi_node_host + nifi_toolkit_dn_suffix,
+  "port" : int(nifi_ca_parent_config['nifi.toolkit.tls.port']),
+  "caHostname" : nifi_ca_host,
+  "trustStore" : nifi_truststore,
+  "trustStoreType" : nifi_truststoreType,
+  "trustStorePassword": nifi_truststorePasswd
+}
+ 
+# params from nifi-env
+nifi_user = config['configurations']['nifi-env']['nifi_user']
+nifi_group = config['configurations']['nifi-env']['nifi_group']
+
+nifi_node_log_dir = config['configurations']['nifi-env']['nifi_node_log_dir']
+nifi_node_log_file = os.path.join(nifi_node_log_dir,'nifi-setup.log')
+
+# limits related params
+limits_conf_dir = '/etc/security/limits.d'
+nifi_user_nofile_limit = config['configurations']['nifi-env']['nifi_user_nofile_limit']
+nifi_user_nproc_limit = config['configurations']['nifi-env']['nifi_user_nproc_limit']
+
+# params from nifi-boostrap
+nifi_env_content = config['configurations']['nifi-env']['content']
+
+
+# params from nifi-logback
+nifi_master_logback_content = config['configurations']['nifi-master-logback-env']['content']
+nifi_node_logback_content = config['configurations']['nifi-node-logback-env']['content']
+
+# params from nifi-properties-env
+nifi_master_properties_content = config['configurations']['nifi-master-properties-env']['content']
+nifi_properties = config['configurations']['nifi-properties'].copy()
+
+#kerberos params
+nifi_kerberos_authentication_expiration = config['configurations']['nifi-properties']['nifi.kerberos.spnego.authentication.expiration']
+nifi_kerberos_realm = default("/configurations/kerberos-env/realm", None)
+
+# params from nifi-flow
+nifi_flow_content = config['configurations']['nifi-flow-env']['content']
+
+# params from nifi-state-management-env
+nifi_state_management_content = config['configurations']['nifi-state-management-env']['content']
+
+# params from nifi-authorizers-env
+nifi_authorizers_content = config['configurations']['nifi-authorizers-env']['content']
+
+# params from nifi-login-identity-providers-env
+nifi_login_identity_providers_content = config['configurations']['nifi-login-identity-providers-env']['content']
+
+# params from nifi-boostrap
+nifi_boostrap_content = config['configurations']['nifi-bootstrap-env']['content']
+
+# params from nifi-bootstrap-notification-services-env
+nifi_boostrap_notification_content = config['configurations']['nifi-bootstrap-notification-services-env']['content']
+
+#autodetect jdk home
+jdk64_home=config['hostLevelParams']['java_home']
+
+#autodetect ambari server for metrics
+if 'metrics_collector_hosts' in config['clusterHostInfo']:
+  metrics_collector_host = str(config['clusterHostInfo']['metrics_collector_hosts'][0])
+  metrics_collector_port = str(get_port_from_url(config['configurations']['ams-site']['timeline.metrics.service.webapp.address']))
+else:
+  metrics_collector_host = ''
+  metrics_collector_port = ''
+
+
+#detect zookeeper_quorum
+zookeeper_port=default('/configurations/zoo.cfg/clientPort', None)
+#get comma separated list of zookeeper hosts from clusterHostInfo
+index = 0
+zookeeper_quorum=""
+for host in config['clusterHostInfo']['zookeeper_hosts']:
+  zookeeper_quorum += host + ":"+str(zookeeper_port)
+  index += 1
+  if index < len(config['clusterHostInfo']['zookeeper_hosts']):
+    zookeeper_quorum += ","
+
+
+#setup ranger configuration
+
+retryAble = default("/commandParams/command_retry_enabled", False)
+version = default("/commandParams/version", None)
+namenode_hosts = default("/clusterHostInfo/namenode_host", None)
+
+if type(namenode_hosts) is list:
+  namenode_host = namenode_hosts[0]
+else:
+  namenode_host = namenode_hosts
+
+has_namenode = not namenode_host == None
+
+
+nifi_authorizer = 'file-provider'
+
+nifi_host_name = config['hostname']
+nifi_host_port = config['configurations']['nifi-ambari-config']['nifi.node.port']
+java_home = config['hostLevelParams']['java_home']
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+smokeuser = config['configurations']['cluster-env']['smokeuser']
+smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name']
+smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
+kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
+
+if security_enabled:
+  _hostname_lowercase = nifi_host_name.lower()
+  nifi_properties['nifi.kerberos.service.principal'] = nifi_properties['nifi.kerberos.service.principal'].replace('_HOST',_hostname_lowercase)
+  nifi_properties['nifi.kerberos.spnego.principal'] = nifi_properties['nifi.kerberos.spnego.principal'].replace('_HOST',_hostname_lowercase)
+
+# ranger host
+# E.g., 2.3
+stack_version_unformatted = config['hostLevelParams']['stack_version']
+stack_version_formatted = format_stack_version(stack_version_unformatted)
+stack_supports_ranger_kerberos = stack_version_formatted and check_stack_feature(StackFeature.RANGER_KERBEROS_SUPPORT, stack_version_formatted)
+stack_supports_ranger_audit_db = stack_version_formatted and check_stack_feature(StackFeature.RANGER_AUDIT_DB_SUPPORT, stack_version_formatted)
+
+ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", [])
+has_ranger_admin = not len(ranger_admin_hosts) == 0
+xml_configurations_supported = config['configurations']['ranger-env']['xml_configurations_supported']
+
+ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
+
+# ranger nifi properties
+policymgr_mgr_url = config['configurations']['admin-properties']['policymgr_external_url']
+
+if 'admin-properties' in config['configurations'] and 'policymgr_external_url' in config['configurations']['admin-properties'] and policymgr_mgr_url.endswith('/'):
+  policymgr_mgr_url = policymgr_mgr_url.rstrip('/')
+
+xa_audit_db_name = config['configurations']['admin-properties']['audit_db_name']
+xa_audit_db_user = config['configurations']['admin-properties']['audit_db_user']
+xa_db_host = config['configurations']['admin-properties']['db_host']
+repo_name = str(config['clusterName']) + '_nifi'
+
+repo_config_username = config['configurations']['ranger-nifi-plugin-properties']['REPOSITORY_CONFIG_USERNAME']
+
+ranger_env = config['configurations']['ranger-env']
+ranger_plugin_properties = config['configurations']['ranger-nifi-plugin-properties']
+policy_user = config['configurations']['ranger-nifi-plugin-properties']['policy_user']
+
+#For curl command in ranger plugin to get db connector
+jdk_location = config['hostLevelParams']['jdk_location']
+java_share_dir = '/usr/share/java'
+
+if has_ranger_admin:
+  enable_ranger_nifi = (config['configurations']['ranger-nifi-plugin-properties']['ranger-nifi-plugin-enabled'].lower() == 'yes')
+  xa_audit_db_password = unicode(config['configurations']['admin-properties']['audit_db_password']) if stack_supports_ranger_audit_db else None
+  repo_config_password = unicode(config['configurations']['ranger-nifi-plugin-properties']['REPOSITORY_CONFIG_PASSWORD'])
+  xa_audit_db_flavor = (config['configurations']['admin-properties']['DB_FLAVOR']).lower()
+  previous_jdbc_jar_name= None
+
+  if stack_supports_ranger_audit_db:
+    if xa_audit_db_flavor == 'mysql':
+      jdbc_jar_name = default("/hostLevelParams/custom_mysql_jdbc_name", None)
+      previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mysql_jdbc_name", None)
+      audit_jdbc_url = format('jdbc:mysql://{xa_db_host}/{xa_audit_db_name}')
+      jdbc_driver = "com.mysql.jdbc.Driver"
+    elif xa_audit_db_flavor == 'oracle':
+      jdbc_jar_name = default("/hostLevelParams/custom_oracle_jdbc_name", None)
+      previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_oracle_jdbc_name", None)
+      colon_count = xa_db_host.count(':')
+      if colon_count == 2 or colon_count == 0:
+        audit_jdbc_url = format('jdbc:oracle:thin:@{xa_db_host}')
+      else:
+        audit_jdbc_url = format('jdbc:oracle:thin:@//{xa_db_host}')
+      jdbc_driver = "oracle.jdbc.OracleDriver"
+    elif xa_audit_db_flavor == 'postgres':
+      jdbc_jar_name = default("/hostLevelParams/custom_postgres_jdbc_name", None)
+      previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_postgres_jdbc_name", None)
+      audit_jdbc_url = format('jdbc:postgresql://{xa_db_host}/{xa_audit_db_name}')
+      jdbc_driver = "org.postgresql.Driver"
+    elif xa_audit_db_flavor == 'mssql':
+      jdbc_jar_name = default("/hostLevelParams/custom_mssql_jdbc_name", None)
+      previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mssql_jdbc_name", None)
+      audit_jdbc_url = format('jdbc:sqlserver://{xa_db_host};databaseName={xa_audit_db_name}')
+      jdbc_driver = "com.microsoft.sqlserver.jdbc.SQLServerDriver"
+    elif xa_audit_db_flavor == 'sqla':
+      jdbc_jar_name = default("/hostLevelParams/custom_sqlanywhere_jdbc_name", None)
+      previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_sqlanywhere_jdbc_name", None)
+      audit_jdbc_url = format('jdbc:sqlanywhere:database={xa_audit_db_name};host={xa_db_host}')
+      jdbc_driver = "sap.jdbc4.sqlanywhere.IDriver"
+
+  downloaded_custom_connector = format("{tmp_dir}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None
+  driver_curl_source = format("{jdk_location}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None
+
+  driver_curl_target = format("{stack_root}/current/nifi/ext/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None
+  previous_jdbc_jar = format("{stack_root}/current/nifi/ext/{previous_jdbc_jar_name}") if stack_supports_ranger_audit_db else None
+  sql_connector_jar = ''
+
+  ssl_keystore_password = unicode(config['configurations']['ranger-nifi-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password']) if xml_configurations_supported else None
+  ssl_truststore_password = unicode(config['configurations']['ranger-nifi-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password']) if xml_configurations_supported else None
+  credential_file = format('/etc/ranger/{repo_name}/cred.jceks') if xml_configurations_supported else None
+  credential_file_type = 'jceks'
+  ranger_admin_username = config['configurations']['ranger-env']['ranger_admin_username']
+  ranger_admin_password = config['configurations']['ranger-env']['ranger_admin_password']
+
+  #create ranger service's nifi client properties
+  nifi_authentication = config['configurations']['ranger-nifi-plugin-properties']['nifi.authentication']
+  ranger_id_owner_for_certificate = config['configurations']['ranger-nifi-plugin-properties']['owner.for.certificate']
+  nifi_id_owner_for_certificate = config['configurations']['ranger-nifi-policymgr-ssl']['owner.for.certificate']
+  regex = r"(CN)=([a-zA-Z0-9\.\-\* ]*)"
+  match = re.search(regex, nifi_id_owner_for_certificate)
+  common_name_for_certificate = match.group(2) if match else 'NONE'
+
+  if nifi_authentication == 'SSL':
+
+    nifi_ranger_plugin_config = {
+      'nifi.authentication': nifi_authentication,
+      'nifi.url': format("https://{nifi_host_name}:{nifi_node_ssl_port}/nifi-api/resources"),
+      'nifi.ssl.keystore': config['configurations']['ranger-nifi-plugin-properties']['nifi.ssl.keystore'],
+      'nifi.ssl.keystoreType':config['configurations']['ranger-nifi-plugin-properties']['nifi.ssl.keystoreType'],
+      'nifi.ssl.keystorePassword': config['configurations']['ranger-nifi-plugin-properties']['nifi.ssl.keystorePassword'],
+      'nifi.ssl.truststore': config['configurations']['ranger-nifi-plugin-properties']['nifi.ssl.truststore'],
+      'nifi.ssl.truststoreType': config['configurations']['ranger-nifi-plugin-properties']['nifi.ssl.truststoreType'],
+      'nifi.ssl.truststorePassword': config['configurations']['ranger-nifi-plugin-properties']['nifi.ssl.truststorePassword'],
+      'commonNameForCertificate': common_name_for_certificate
+    }
+  else:
+    nifi_ranger_plugin_config = {
+      'nifi.authentication': nifi_authentication,
+      'nifi.url': format("https://{nifi_host_name}:{nifi_host_port}/nifi-api/resources"),
+      'commonNameForCertificate': common_name_for_certificate
+    }
+
+  nifi_ranger_plugin_repo = {
+    'isActive': 'true',
+    'config': json.dumps(nifi_ranger_plugin_config),
+    'description': 'nifi repo',
+    'name': repo_name,
+    'repositoryType': 'nifi',
+    'assetType': '5'
+  }
+
+  # used in nifi authorizers
+  ranger_admin_identity = ranger_id_owner_for_certificate
+
+  if stack_supports_ranger_kerberos and security_enabled:
+    nifi_ranger_plugin_config['policy.download.auth.users'] = nifi_user
+    nifi_ranger_plugin_config['tag.download.auth.users'] = nifi_user
+    ranger_nifi_principal = config['configurations']['nifi-properties']['nifi.kerberos.service.principal'].replace('_HOST',_hostname_lowercase)
+    ranger_nifi_keytab = config['configurations']['nifi-properties']['nifi.kerberos.service.keytab.location']
+
+  if stack_supports_ranger_kerberos:
+    nifi_ranger_plugin_config['ambari.service.check.user'] = policy_user
+
+    nifi_ranger_plugin_repo = {
+      'isEnabled': 'true',
+      'configs': nifi_ranger_plugin_config,
+      'description': 'nifi repo',
+      'name': repo_name,
+      'type': 'nifi'
+    }
+
+  xa_audit_db_is_enabled = False
+  ranger_audit_solr_urls = config['configurations']['ranger-admin-site']['ranger.audit.solr.urls']
+
+  if xml_configurations_supported and stack_supports_ranger_audit_db:
+    xa_audit_db_is_enabled = config['configurations']['ranger-nifi-audit']['xasecure.audit.destination.db']
+
+  xa_audit_hdfs_is_enabled =  default('/configurations/ranger-nifi-audit/xasecure.audit.destination.hdfs', False)
+
+
+  #For SQLA explicitly disable audit to DB for Ranger
+  if xa_audit_db_flavor == 'sqla':
+    xa_audit_db_is_enabled = False
+
+  nifi_authorizer = 'ranger-provider'
+
+hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] if has_namenode else None
+hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] if has_namenode else None
+hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] if has_namenode else None
+hdfs_site = config['configurations']['hdfs-site'] if has_namenode else None
+default_fs = config['configurations']['core-site']['fs.defaultFS'] if has_namenode else None
+hadoop_bin_dir = stack_select.get_hadoop_dir("bin") if has_namenode else None
+hadoop_conf_dir = conf_select.get_hadoop_conf_dir() if has_namenode else None
+
+import functools
+#create partial functions with common arguments for every HdfsResource call
+#to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code
+HdfsResource = functools.partial(
+  HdfsResource,
+  user=hdfs_user,
+  hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore",
+  security_enabled = security_enabled,
+  keytab = hdfs_user_keytab,
+  kinit_path_local = kinit_path_local,
+  hadoop_bin_dir = hadoop_bin_dir,
+  hadoop_conf_dir = hadoop_conf_dir,
+  principal_name = hdfs_principal_name,
+  hdfs_site = hdfs_site,
+  default_fs = default_fs,
+  immutable_paths = get_not_managed_resources()
+)

http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/run_ca.sh
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/run_ca.sh b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/run_ca.sh
new file mode 100644
index 0000000..c5863fa
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/run_ca.sh
@@ -0,0 +1,63 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Launches NiFi CA server
+# $1 -> JAVA_HOME
+# $2 -> tls-toolkit.sh path
+# $3 -> config json
+# $4 -> stdout log
+# $5 -> stderr log
+# $6 -> pid file
+
+die() {
+  echo "$1"
+  exit 1
+}
+
+read -r -d '' WAIT_FOR_LOG << 'EOF'
+  STDOUT_LOG="$0"
+  until [ -f "$STDOUT_LOG" ]; do
+    echo "Waiting for $STDOUT_LOG to exist"
+    sleep 1;
+  done
+EOF
+
+read -r -d '' WAIT_FOR_CHILD << 'EOF'
+  SHELL_PID="$0"
+  PID_FILE="$1"
+  CHILD_PROCESS="$(pgrep -P "$SHELL_PID" java)"
+  while [ -z "$CHILD_PROCESS" ]; do
+    echo "Waiting for child java process to exist"
+    sleep 1;
+    CHILD_PROCESS="$(pgrep -P "$SHELL_PID" java)"
+  done
+  echo "$CHILD_PROCESS" > "$PID_FILE"
+EOF
+
+JAVA_HOME="$1" nohup "$2" server -F -f "$3" > "$4" 2> "$5" < /dev/null &
+SHELL_PID="$!"
+
+timeout 30 bash -c "$WAIT_FOR_LOG" "$4" || die "Timed out while waiting for $4"
+
+timeout 30 bash -c "$WAIT_FOR_CHILD" "$SHELL_PID" "$6" || die "Timed out while waiting for child java process to exist"
+
+#Want to wait until Jetty starts
+#See http://superuser.com/questions/270529/monitoring-a-file-until-a-string-is-found#answer-900134
+( tail -f -n +1 "$4" & ) | timeout 180 grep -q "Server Started" || die "Timed out while waiting for CA server to start"

http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/service_check.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/service_check.py
new file mode 100644
index 0000000..535592f
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/service_check.py
@@ -0,0 +1,75 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import os
+import urllib2
+import httplib
+
+from resource_management.core.logger import Logger
+from resource_management.libraries.functions.format import format
+from resource_management.libraries.script.script import Script
+
+from ambari_commons.inet_utils import openurl
+from ambari_commons.exceptions import TimeoutError
+from resource_management.core.exceptions import Fail
+from resource_management.libraries.functions.decorator import retry
+
+class NifiServiceCheck(Script):
+  def service_check(self, env):
+    import params
+    Logger.info("Running Nifi service check")
+    for nifi_master_host in params.nifi_master_hosts:
+      url = ""
+      if params.nifi_ssl_enabled:
+        url = "https://{0}:{1}/nifi".format(nifi_master_host, params.nifi_node_ssl_port)
+      else:
+        url = "http://{0}:{1}/nifi".format(nifi_master_host, params.nifi_node_port)
+      Logger.info("Checking Nifi portal {0} status".format(url))
+      NifiServiceCheck.check_nifi_portal(url)
+
+  @staticmethod
+  @retry(times=15, sleep_time=5, max_sleep_time=20, backoff_factor=2, err_class=Fail)
+  def check_nifi_portal(url):
+    try:
+      request = urllib2.Request(url)
+      result = openurl(request, timeout=20)
+      response_code = result.getcode()
+      if response_code == 200 or response_code == 401:
+        Logger.info("Nifi portal {0} is up. Response code {1}".format(url, response_code))
+      else:
+        raise Fail("Error connecting to {0}. Response code {1}".format(url, response_code))
+    except urllib2.URLError, e:
+      if isinstance(e, urllib2.HTTPError):
+        if e.code == 401:
+          Logger.info("Nifi portal {0} is up. Response code {1}".format(url, e.code))
+        else:
+          raise Fail("Error connecting to {0}. Http status code - {1}. \n {2}".format(url, e.code, e.read()))
+      elif e.reason and "violation of protocol" in str(e.reason):
+        Logger.info("Ignore certificate validation error - {0}".format(e.reason))
+        pass
+      else:
+        raise Fail("Error connecting to {0}. Reason - {1}.".format(url, e.reason))
+    except httplib.BadStatusLine:
+      raise Fail("Error connecting to {0}. Reason - Not Reachable".format(url))
+    except TimeoutError:
+      raise Fail("Error connecting to {0}. Reason - Timeout".format(url))
+
+if __name__ == "__main__":
+  NifiServiceCheck().execute()
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/setup_ranger_nifi.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/setup_ranger_nifi.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/setup_ranger_nifi.py
new file mode 100644
index 0000000..91a6885
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/setup_ranger_nifi.py
@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+from resource_management.core.logger import Logger
+from resource_management.core.resources import File
+
+def setup_ranger_nifi(upgrade_type=None):
+    import params, os
+
+    if params.has_ranger_admin and params.enable_ranger_nifi:
+
+        stack_version = None
+        if upgrade_type is not None:
+            stack_version = params.version
+
+        if params.retryAble:
+            Logger.info("nifi: Setup ranger: command retry enables thus retrying if ranger admin is down !")
+        else:
+            Logger.info("nifi: Setup ranger: command retry not enabled thus skipping if ranger admin is down !")
+
+
+        api_version=None
+        if params.stack_supports_ranger_kerberos:
+            api_version='v2'
+        from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin
+        setup_ranger_plugin('nifi', 'nifi', params.previous_jdbc_jar,
+                            params.downloaded_custom_connector, params.driver_curl_source,
+                            params.driver_curl_target, params.java_home,
+                            params.repo_name, params.nifi_ranger_plugin_repo,
+                            params.ranger_env, params.ranger_plugin_properties,
+                            params.policy_user, params.policymgr_mgr_url,
+                            params.enable_ranger_nifi, conf_dict=params.nifi_config_dir,
+                            component_user=params.nifi_user, component_group=params.nifi_group, cache_service_list=['nifi'],
+                            plugin_audit_properties=params.config['configurations']['ranger-nifi-audit'], plugin_audit_attributes=params.config['configuration_attributes']['ranger-nifi-audit'],
+                            plugin_security_properties=params.config['configurations']['ranger-nifi-security'], plugin_security_attributes=params.config['configuration_attributes']['ranger-nifi-security'],
+                            plugin_policymgr_ssl_properties=params.config['configurations']['ranger-nifi-policymgr-ssl'], plugin_policymgr_ssl_attributes=params.config['configuration_attributes']['ranger-nifi-policymgr-ssl'],
+                            component_list=[], audit_db_is_enabled=params.xa_audit_db_is_enabled,
+                            credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password,
+                            ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password,
+                            stack_version_override = stack_version, skip_if_rangeradmin_down= not params.retryAble,api_version=api_version,
+                            is_security_enabled = params.security_enabled,
+                            is_stack_supports_ranger_kerberos = params.stack_supports_ranger_kerberos,
+                            component_user_principal=params.ranger_nifi_principal if params.security_enabled else None,
+                            component_user_keytab=params.ranger_nifi_keytab if params.security_enabled else None)
+                            
+        #change permissions of ranger xml that were written to 0400
+        File(os.path.join(params.nifi_config_dir, 'ranger-nifi-audit.xml'), owner=params.nifi_user, group=params.nifi_group, mode=0400)
+        File(os.path.join(params.nifi_config_dir, 'ranger-nifi-security.xml'), owner=params.nifi_user, group=params.nifi_group, mode=0400)
+        File(os.path.join(params.nifi_config_dir, 'ranger-policymgr-ssl.xml'), owner=params.nifi_user, group=params.nifi_group, mode=0400)        
+
+    else:
+        Logger.info('Ranger admin not installed')
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/status_params.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/status_params.py
new file mode 100755
index 0000000..9748661
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/status_params.py
@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+import sys, os
+
+config = Script.get_config()
+
+nifi_pid_dir=config['configurations']['nifi-env']['nifi_pid_dir']
+#nifi_master_pid_file=nifi_pid_dir + '/nifi-master.pid'
+nifi_node_pid_file=nifi_pid_dir + '/nifi.pid'
+nifi_ca_pid_file=nifi_pid_dir + '/nifi-ca.pid'

http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/templates/nifi.conf.j2
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/templates/nifi.conf.j2 b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/templates/nifi.conf.j2
new file mode 100644
index 0000000..97f5711
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/templates/nifi.conf.j2
@@ -0,0 +1,36 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{{nifi_user}}   - nofile   {{nifi_user_nofile_limit}}
+{{nifi_user}}   - nproc    {{nifi_user_nproc_limit}}
+

http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/quicklinks/quicklinks.json
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/quicklinks/quicklinks.json b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/quicklinks/quicklinks.json
new file mode 100644
index 0000000..bd27705
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/quicklinks/quicklinks.json
@@ -0,0 +1,35 @@
+{
+  "name": "default",
+  "description": "default quick links configuration",
+  "configuration": {
+    "protocol":
+    {
+      "type":"https",
+      "checks":[
+        {
+          "property":"nifi.node.ssl.isenabled",
+          "desired":"true",
+          "site":"nifi-ambari-ssl-config"
+        }
+      ]
+    },
+
+    "links": [
+      {
+        "name": "nifi_ui",
+        "label": "Nifi UI",
+        "requires_user_name": "false",
+        "component_name": "NIFI_MASTER",
+        "url":"%@://%@:%@/nifi",
+        "port":{
+          "http_property": "nifi.node.port",
+          "http_default_port": "9090",
+          "https_property": "nifi.node.ssl.port",
+          "https_default_port": "9091",
+          "regex": "^(\\d+)$",
+          "site": "nifi-ambari-config"
+        }
+      }
+    ]
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/widgets.json
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/widgets.json b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/widgets.json
new file mode 100644
index 0000000..cd6791b
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/widgets.json
@@ -0,0 +1,463 @@
+{
+  "layouts": [
+    {
+      "layout_name": "default_nifi_dashboard",
+      "display_name": "Standard NiFi Dashboard",
+      "section_name": "NIFI_SUMMARY",
+      "widgetLayoutInfo": [
+        {
+          "widget_name": "FlowFiles Received Last 5 mins",
+          "description": "The number of FlowFiles received in the last 5 minutes.",
+          "widget_type": "GRAPH",
+          "is_visible": true,
+          "metrics": [
+            {
+              "name": "FlowFilesReceivedLast5Minutes",
+              "metric_path": "metrics/nifi/FlowFilesReceivedLast5Minutes",
+              "service_name": "NIFI",
+              "component_name": "NIFI_MASTER"
+            }
+          ],
+          "values": [
+            {
+              "name": "FlowFiles Received",
+              "value": "${FlowFilesReceivedLast5Minutes}"
+            }
+          ],
+          "properties": {
+            "graph_type": "LINE",
+            "time_range": "1"
+          }
+        },
+        {
+          "widget_name": "MBs Received Last 5 mins",
+          "description": "The number of MBs received in the last 5 minutes.",
+          "widget_type": "GRAPH",
+          "is_visible": true,
+          "metrics": [
+            {
+              "name": "BytesReceivedLast5Minutes",
+              "metric_path": "metrics/nifi/BytesReceivedLast5Minutes",
+              "service_name": "NIFI",
+              "component_name": "NIFI_MASTER"
+            }
+          ],
+          "values": [
+            {
+              "name": "MBs Received",
+              "value": "${BytesReceivedLast5Minutes/1024000}"
+            }
+          ],
+          "properties": {
+            "graph_type": "LINE",
+            "time_range": "1"
+          }
+        },
+		 {
+          "widget_name": "FlowFiles Sent Last 5 mins",
+          "description": "The number of FlowFiles sent in the last 5 minutes.",
+          "widget_type": "GRAPH",
+          "is_visible": true,
+          "metrics": [
+            {
+              "name": "FlowFilesSentLast5Minutes",
+              "metric_path": "metrics/nifi/FlowFilesSentLast5Minutes",
+              "service_name": "NIFI",
+              "component_name": "NIFI_MASTER"
+            }
+          ],
+          "values": [
+            {
+              "name": "FlowFiles Sent",
+              "value": "${FlowFilesSentLast5Minutes}"
+            }
+          ],
+          "properties": {
+            "graph_type": "LINE",
+            "time_range": "1"
+          }
+        },
+		 {
+          "widget_name": "MBs Sent Last 5 mins",
+          "description": "The number of MBs sent in the last 5 minutes.",
+          "widget_type": "GRAPH",
+          "is_visible": true,
+          "metrics": [
+            {
+              "name": "BytesSentLast5Minutes",
+              "metric_path": "metrics/nifi/BytesSentLast5Minutes",
+              "service_name": "NIFI",
+              "component_name": "NIFI_MASTER"
+            }
+          ],
+          "values": [
+            {
+              "name": "MBs Sent",
+              "value": "${BytesSentLast5Minutes/1024000}"
+            }
+          ],
+          "properties": {
+            "graph_type": "LINE",
+            "time_range": "1"
+          }
+        },
+		 {
+          "widget_name": "FlowFiles Queued",
+          "description": "The number of FlowFiles queued.",
+          "widget_type": "GRAPH",
+          "is_visible": true,
+          "metrics": [
+            {
+              "name": "FlowFilesQueued",
+              "metric_path": "metrics/nifi/FlowFilesQueued",
+              "service_name": "NIFI",
+              "component_name": "NIFI_MASTER"
+            }
+          ],
+          "values": [
+            {
+              "name": "FlowFiles Queued",
+              "value": "${FlowFilesQueued}"
+            }
+          ],
+          "properties": {
+            "graph_type": "LINE",
+            "time_range": "1"
+          }
+        },
+		 {
+          "widget_name": "MBs Queued",
+          "description": "The number of MBs queued.",
+          "widget_type": "GRAPH",
+          "is_visible": true,
+          "metrics": [
+            {
+              "name": "BytesQueued",
+              "metric_path": "metrics/nifi/BytesQueued",
+              "service_name": "NIFI",
+              "component_name": "NIFI_MASTER"
+            }
+          ],
+          "values": [
+            {
+              "name": "MBs Queued",
+              "value": "${BytesQueued/1024000}"
+            }
+          ],
+          "properties": {
+            "graph_type": "LINE",
+            "time_range": "1"
+          }
+        },
+		 {
+          "widget_name": "MBs Read Last 5 mins",
+          "description": "The number of MBs read in the last 5 minutes.",
+          "widget_type": "GRAPH",
+          "is_visible": true,
+          "metrics": [
+            {
+              "name": "BytesReadLast5Minutes",
+              "metric_path": "metrics/nifi/BytesReadLast5Minutes",
+              "service_name": "NIFI",
+              "component_name": "NIFI_MASTER"
+            }
+          ],
+          "values": [
+            {
+              "name": "MBs Read",
+              "value": "${BytesReadLast5Minutes/1024000}"
+            }
+          ],
+          "properties": {
+            "graph_type": "LINE",
+            "time_range": "1"
+          }
+        },
+		 {
+          "widget_name": "MBs Written Last 5 mins",
+          "description": "The number of MBs written in the last 5 minutes.",
+          "widget_type": "GRAPH",
+          "is_visible": true,
+          "metrics": [
+            {
+              "name": "BytesWrittenLast5Minutes",
+              "metric_path": "metrics/nifi/BytesWrittenLast5Minutes",
+              "service_name": "NIFI",
+              "component_name": "NIFI_MASTER"
+            }
+          ],
+          "values": [
+            {
+              "name": "MBs Written",
+              "value": "${BytesWrittenLast5Minutes/1024000}"
+            }
+          ],
+          "properties": {
+            "graph_type": "LINE",
+            "time_range": "1"
+          }
+        },
+		 {
+          "widget_name": "Active Threads",
+          "description": "The number of active threads.",
+          "widget_type": "GRAPH",
+          "is_visible": true,
+          "metrics": [
+            {
+              "name": "ActiveThreads",
+              "metric_path": "metrics/nifi/ActiveThreads",
+              "service_name": "NIFI",
+              "component_name": "NIFI_MASTER"
+            }
+          ],
+          "values": [
+            {
+              "name": "Active Threads",
+              "value": "${ActiveThreads}"
+            }
+          ],
+          "properties": {
+            "graph_type": "LINE",
+            "time_range": "1"
+          }
+        },
+		 {
+          "widget_name": "Total Task Duration Milliseconds",
+          "description": "The total task duration in Milliseconds.",
+          "widget_type": "GRAPH",
+          "is_visible": true,
+          "metrics": [
+            {
+              "name": "TotalTaskDurationNanoSeconds",
+              "metric_path": "metrics/nifi/TotalTaskDurationNanoSeconds",
+              "service_name": "NIFI",
+              "component_name": "NIFI_MASTER"
+            }
+          ],
+          "values": [
+            {
+              "name": "Total Task Duration Milliseconds",
+              "value": "${TotalTaskDurationNanoSeconds/1000}"
+            }
+          ],
+          "properties": {
+            "graph_type": "LINE",
+            "time_range": "1"
+          }
+        },
+        {
+             "widget_name": "JVM Heap Used (MBs)",
+             "description": "Number of MBs being used on the JVM Heap.",
+             "widget_type": "NUMBER",
+             "is_visible": true,
+             "metrics": [
+               {
+                 "name": "jvm.heap_used",
+                 "metric_path": "metrics/jvm/heap_used",
+                 "service_name": "NIFI",
+                 "component_name": "NIFI_MASTER"
+               }
+             ],
+             "values": [
+               {
+                 "name": "JVM Heap Used",
+                 "value": "${jvm.heap_used/1024/1024}"
+               }
+             ],
+             "properties": {
+                "warning_threshold": "100000",
+                "error_threshold": "200000"
+             }
+        },
+        {
+             "widget_name": "JVM Heap Usage",
+             "description": "Precentage of JVM heap being used.",
+             "widget_type": "GAUGE",
+             "is_visible": true,
+             "metrics": [
+               {
+                 "name": "jvm.heap_usage",
+                 "metric_path": "metrics/jvm/heap_usage",
+                 "service_name": "NIFI",
+                 "component_name": "NIFI_MASTER"
+               }
+             ],
+             "values": [
+               {
+                 "name": "JVM Heap Usage",
+                 "value": "${jvm.heap_usage}"
+               }
+             ],
+             "properties": {
+               "warning_threshold": "0.8",
+               "error_threshold": "0.9"
+             }
+           },
+         {
+              "widget_name": "JVM File Descriptor Usage",
+              "description": "Precentage of file descriptors being used.",
+              "widget_type": "GAUGE",
+              "is_visible": true,
+              "metrics": [
+                {
+                  "name": "jvm.file_descriptor_usage",
+                  "metric_path": "metrics/jvm/file_descriptor_usage",
+                  "service_name": "NIFI",
+                  "component_name": "NIFI_MASTER"
+                }
+              ],
+              "values": [
+                {
+                  "name": "JVM File Descriptor Usage",
+                  "value": "${jvm.file_descriptor_usage}"
+                }
+              ],
+              "properties": {
+                "warning_threshold": "0.8",
+                "error_threshold": "0.9"
+              }
+            },
+           {
+                "widget_name": "Thread Count",
+                "description": "Number of threads in the JVM.",
+                "widget_type": "NUMBER",
+                "is_visible": true,
+                "metrics": [
+                  {
+                    "name": "jvm.thread_count",
+                    "metric_path": "metrics/jvm/thread_count",
+                    "service_name": "NIFI",
+                    "component_name": "NIFI_MASTER"
+                  }
+                ],
+                "values": [
+                  {
+                    "name": "Thread Count",
+                    "value": "${jvm.thread_count}"
+                  }
+                ],
+                "properties": {
+                }
+           },
+           {
+                "widget_name": "Daemon Thread Count",
+                "description": "Number of daemon threads in the JVM.",
+                "widget_type": "NUMBER",
+                "is_visible": true,
+                "metrics": [
+                  {
+                    "name": "jvm.daemon_thread_count",
+                    "metric_path": "metrics/jvm/daemon_thread_count",
+                    "service_name": "NIFI",
+                    "component_name": "NIFI_MASTER"
+                  }
+                ],
+                "values": [
+                  {
+                    "name": "Daemon Thread Count",
+                    "value": "${jvm.daemon_thread_count}"
+                  }
+                ],
+                "properties": {
+                }
+           },
+            {
+                 "widget_name": "Runnable Threads",
+                 "description": "Number of runnable threads in the JVM.",
+                 "widget_type": "GRAPH",
+                 "is_visible": false,
+                 "metrics": [
+                   {
+                     "name": "jvm.thread_states.runnable",
+                     "metric_path": "metrics/jvm/thread_states/runnable",
+                     "service_name": "NIFI",
+                     "component_name": "NIFI_MASTER"
+                   }
+                 ],
+                 "values": [
+                   {
+                     "name": "Runnable Threads",
+                     "value": "${jvm.thread_states.runnable}"
+                   }
+                 ],
+                 "properties": {
+                   "graph_type": "LINE",
+                   "time_range": "1"
+                 }
+            },
+            {
+                 "widget_name": "Blocked Threads",
+                 "description": "Number of blocked threads in the JVM.",
+                 "widget_type": "GRAPH",
+                 "is_visible": false,
+                 "metrics": [
+                   {
+                     "name": "jvm.thread_states.blocked",
+                     "metric_path": "metrics/jvm/thread_states/blocked",
+                     "service_name": "NIFI",
+                     "component_name": "NIFI_MASTER"
+                   }
+                 ],
+                 "values": [
+                   {
+                     "name": "Blocked Threads",
+                     "value": "${jvm.thread_states.blocked}"
+                   }
+                 ],
+                 "properties": {
+                   "graph_type": "LINE",
+                   "time_range": "1"
+                 }
+            },
+            {
+                 "widget_name": "Timed-Waiting Threads",
+                 "description": "Number of timed-waiting threads in the JVM.",
+                 "widget_type": "GRAPH",
+                 "is_visible": false,
+                 "metrics": [
+                   {
+                     "name": "jvm.thread_states.timed_waiting",
+                     "metric_path": "metrics/jvm/thread_states/timed_waiting",
+                     "service_name": "NIFI",
+                     "component_name": "NIFI_MASTER"
+                   }
+                 ],
+                 "values": [
+                   {
+                     "name": "Timed-Waiting Threads",
+                     "value": "${jvm.thread_states.timed_waiting}"
+                   }
+                 ],
+                 "properties": {
+                   "graph_type": "LINE",
+                   "time_range": "1"
+                 }
+            },
+            {
+                 "widget_name": "Terminated Threads",
+                 "description": "Number of terminated threads in the JVM.",
+                 "widget_type": "GRAPH",
+                 "is_visible": false,
+                 "metrics": [
+                   {
+                     "name": "jvm.thread_states.terminated",
+                     "metric_path": "metrics/jvm/thread_states/terminated",
+                     "service_name": "NIFI",
+                     "component_name": "NIFI_MASTER"
+                   }
+                 ],
+                 "values": [
+                   {
+                     "name": "Terminated Threads",
+                     "value": "${jvm.thread_states.terminated}"
+                   }
+                 ],
+                 "properties": {
+                   "graph_type": "LINE",
+                   "time_range": "1"
+                 }
+            }
+      ]
+    }
+  ]
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/hooks/after_install.py
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/hooks/after_install.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/hooks/after_install.py
new file mode 100644
index 0000000..8f0743b
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/hooks/after_install.py
@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import sys
+import os
+from ambari_server.serverConfiguration import get_ambari_properties, get_resources_location
+from resource_management.core import sudo
+
+def main():
+  properties = get_ambari_properties()
+  if properties == -1:
+    print >> sys.stderr, "Error getting ambari properties"
+    return -1
+
+  resources_location = get_resources_location(properties)
+  views_dir = os.path.join(resources_location, "views")
+
+  for file in os.listdir(views_dir):
+    path = os.path.join(views_dir, file)
+    if os.path.isfile(path):
+      if "ambari-admin" in path or "storm-view" in path:
+        print "Keeping views jar : " + path
+      else:
+        print "Deleting views jar : " + path
+        sudo.unlink(path)
+    else:
+      print "Deleting views directory : " + path
+      sudo.rmtree(path)
+  return 0
+
+if __name__ == "__main__":
+    exit (main())

http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/mpack.json
----------------------------------------------------------------------
diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/mpack.json b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/mpack.json
new file mode 100644
index 0000000..865561e
--- /dev/null
+++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/mpack.json
@@ -0,0 +1,30 @@
+{
+  "type" : "full-release",
+  "name" : "hdf-ambari-mpack",
+  "version": "${project.version}",
+  "description" : "HDF Ambari Management Pack",
+  "prerequisites": {
+    "min-ambari-version" : "${minAmbariVersion}",
+    "max-ambari-version" : "${maxAmbariVersion}"
+  },
+  "hooks": [
+    {
+      "name": "after-install",
+      "type": "python",
+      "script": "hooks/after_install.py"
+    }
+  ],
+  "artifacts": [
+    {
+      "name" : "hdf-service-definitions",
+      "type" : "service-definitions",
+      "source_dir": "common-services"
+    },
+    {
+      "name" : "hdf-stack-definitions",
+      "type" : "stack-definitions",
+      "source_dir": "stacks"
+    }
+  ]
+}
+


Mime
View raw message