Return-Path: X-Original-To: apmail-ambari-commits-archive@www.apache.org Delivered-To: apmail-ambari-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 12FBD1764C for ; Tue, 7 Oct 2014 22:52:31 +0000 (UTC) Received: (qmail 63103 invoked by uid 500); 7 Oct 2014 22:52:30 -0000 Delivered-To: apmail-ambari-commits-archive@ambari.apache.org Received: (qmail 63005 invoked by uid 500); 7 Oct 2014 22:52:30 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 62204 invoked by uid 99); 7 Oct 2014 22:52:30 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 07 Oct 2014 22:52:30 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 338F3905CB7; Tue, 7 Oct 2014 22:52:30 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: yusaku@apache.org To: commits@ambari.apache.org Date: Tue, 07 Oct 2014 22:52:44 -0000 Message-Id: In-Reply-To: <478d9103dc2f4d60aae0f04f68dae6a1@git.apache.org> References: <478d9103dc2f4d60aae0f04f68dae6a1@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [16/51] [partial] AMBARI-7621. Import initial contribution for Ambari support on Windows to branch-windows-dev. (Jayush Luniya and Florian Barca via yusaku) http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-common/src/main/python/resource_management/libraries/functions/check_process_status.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/check_process_status.py b/ambari-common/src/main/python/resource_management/libraries/functions/check_process_status.py deleted file mode 100644 index 7fdecdc..0000000 --- a/ambari-common/src/main/python/resource_management/libraries/functions/check_process_status.py +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" - -from resource_management.core.exceptions import ComponentIsNotRunning -from resource_management.core.logger import Logger -__all__ = ["check_process_status"] - -import os - -def check_process_status(pid_file): - """ - Function checks whether process is running. - Process is considered running, if pid file exists, and process with - a pid, mentioned in pid file is running - If process is not running, will throw ComponentIsNotRunning exception - - @param pid_file: path to service pid file - """ - if not pid_file or not os.path.isfile(pid_file): - raise ComponentIsNotRunning() - with open(pid_file, "r") as f: - try: - pid = int(f.read()) - except: - Logger.debug("Pid file {0} does not exist".format(pid_file)) - raise ComponentIsNotRunning() - try: - # Kill will not actually kill the process - # From the doc: - # If sig is 0, then no signal is sent, but error checking is still - # performed; this can be used to check for the existence of a - # process ID or process group ID. - os.kill(pid, 0) - except OSError: - Logger.debug("Process with pid {0} is not running. Stale pid file" - " at {1}".format(pid, pid_file)) - raise ComponentIsNotRunning() - pass http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-common/src/main/python/resource_management/libraries/functions/default.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/default.py b/ambari-common/src/main/python/resource_management/libraries/functions/default.py deleted file mode 100644 index 733c03a..0000000 --- a/ambari-common/src/main/python/resource_management/libraries/functions/default.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" - -__all__ = ["default"] -from resource_management.libraries.script import Script -from resource_management.libraries.script.config_dictionary import UnknownConfiguration -from resource_management.core.logger import Logger - -def default(name, default_value): - subdicts = filter(None, name.split('/')) - - curr_dict = Script.get_config() - for x in subdicts: - if x in curr_dict: - curr_dict = curr_dict[x] - else: - if not isinstance(default_value, UnknownConfiguration): - Logger.debug("Cannot find configuration: '%s'. Using '%s' value as default" % (name, default_value)) - return default_value - - return curr_dict \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-common/src/main/python/resource_management/libraries/functions/dfs_datanode_helper.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/dfs_datanode_helper.py b/ambari-common/src/main/python/resource_management/libraries/functions/dfs_datanode_helper.py deleted file mode 100644 index ca9fb27..0000000 --- a/ambari-common/src/main/python/resource_management/libraries/functions/dfs_datanode_helper.py +++ /dev/null @@ -1,149 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" -__all__ = ["handle_dfs_data_dir", ] -import os - -from resource_management.libraries.functions.file_system import get_mount_point_for_dir, get_and_cache_mount_points -from resource_management.core.logger import Logger - - -def _write_data_dir_to_mount_in_file(new_data_dir_to_mount_point): - """ - :param new_data_dir_to_mount_point: Dictionary to write to the data_dir_mount_file file, where - the key is each DFS data dir, and the value is its current mount point. - :return: Returns True on success, False otherwise. - """ - import params - - # Overwrite the existing file, or create it if doesn't exist - if params.data_dir_mount_file: - try: - with open(str(params.data_dir_mount_file), "w") as f: - f.write("# This file keeps track of the last known mount-point for each DFS data dir.\n") - f.write("# It is safe to delete, since it will get regenerated the next time that the DataNode starts.\n") - f.write("# However, it is not advised to delete this file since Ambari may \n") - f.write("# re-create a DFS data dir that used to be mounted on a drive but is now mounted on the root.\n") - f.write("# Comments begin with a hash (#) symbol\n") - f.write("# data_dir,mount_point\n") - for kv in new_data_dir_to_mount_point.iteritems(): - f.write(kv[0] + "," + kv[1] + "\n") - except Exception, e: - Logger.error("Encountered error while attempting to save DFS data dir mount mount values to file %s" % - str(params.data_dir_mount_file)) - return False - return True - - -def _get_data_dir_to_mount_from_file(): - """ - :return: Returns a dictionary by parsing the data_dir_mount_file file, - where the key is each DFS data dir, and the value is its last known mount point. - """ - import params - data_dir_to_mount = {} - - if params.data_dir_mount_file is not None and os.path.exists(str(params.data_dir_mount_file)): - try: - with open(str(params.data_dir_mount_file), "r") as f: - for line in f: - # Ignore comments - if line and len(line) > 0 and line[0] == "#": - continue - line = line.strip().lower() - line_array = line.split(",") - if line_array and len(line_array) == 2: - data_dir_to_mount[line_array[0]] = line_array[1] - except Exception, e: - Logger.error("Encountered error while attempting to read DFS data dir mount mount values from file %s" % - str(params.data_dir_mount_file)) - return data_dir_to_mount - - -def handle_dfs_data_dir(func, params): - """ - This function determine which DFS data dir paths can be created. - There are 2 uses cases: - 1. Customers that have many DFS data dirs, each one on a separate mount point that corresponds to a different drive. - 2. Developers that are using a sandbox VM and all DFS data dirs are mounted on the root. - - The goal is to avoid forcefully creating a DFS data dir when a user's drive fails. In this scenario, the - mount point for a DFS data dir changes from something like /hadoop/hdfs/data/data1 to / - If Ambari forcefully creates the directory when it doesn't exist and drive became unmounted, then Ambari will soon - fill up the root drive, which is bad. Instead, we should not create the directory and let HDFS handle the failure - based on its tolerance of missing directories. - - This function relies on the dfs.datanode.data.dir.mount.file parameter to parse a file that contains - a mapping from a DFS data dir, and its last known mount point. - After determining which DFS data dirs can be created if they don't exist, it recalculates the mount points and - writes to the file again. - :param func: Function that will be called if a directory will be created. This function - will be called as func(data_dir, params) - :param params: parameters to pass to function pointer - """ - prev_data_dir_to_mount_point = _get_data_dir_to_mount_from_file() - - allowed_to_create_any_dir = params.data_dir_mount_file is None or not os.path.exists(params.data_dir_mount_file) - - valid_data_dirs = [] - for data_dir in params.dfs_data_dir.split(","): - if data_dir is None or data_dir.strip() == "": - continue - - data_dir = data_dir.strip().lower() - valid_data_dirs.append(data_dir) - - if not os.path.isdir(data_dir): - create_this_dir = allowed_to_create_any_dir - # Determine if should be allowed to create the data_dir directory - if not create_this_dir: - last_mount_point_for_dir = prev_data_dir_to_mount_point[data_dir] if data_dir in prev_data_dir_to_mount_point else None - if last_mount_point_for_dir is None: - # Couldn't retrieve any information about where this dir used to be mounted, so allow creating the directory - # to be safe. - create_this_dir = True - else: - curr_mount_point = get_mount_point_for_dir(data_dir) - - # This means that create_this_dir will stay false if the directory became unmounted. - if last_mount_point_for_dir == "/" or (curr_mount_point is not None and curr_mount_point != "/"): - create_this_dir = True - - if create_this_dir: - Logger.info("Forcefully creating directory: %s" % str(data_dir)) - - # Call the function - func(data_dir, params) - else: - Logger.warning("Directory %s does not exist and became unmounted." % str(data_dir)) - - # Refresh the known mount points - get_and_cache_mount_points(refresh=True) - - new_data_dir_to_mount_point = {} - for data_dir in valid_data_dirs: - # At this point, the directory may or may not exist - if os.path.isdir(data_dir): - curr_mount_point = get_mount_point_for_dir(data_dir) - new_data_dir_to_mount_point[data_dir] = curr_mount_point - - # Save back to the file - _write_data_dir_to_mount_in_file(new_data_dir_to_mount_point) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-common/src/main/python/resource_management/libraries/functions/file_system.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/file_system.py b/ambari-common/src/main/python/resource_management/libraries/functions/file_system.py deleted file mode 100644 index ced4cd3..0000000 --- a/ambari-common/src/main/python/resource_management/libraries/functions/file_system.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" -__all__ = ["get_and_cache_mount_points", "get_mount_point_for_dir"] -import os -from resource_management.core.logger import Logger -from resource_management.core.providers import mount - -# Global variable -mounts = None - - -def get_and_cache_mount_points(refresh=False): - """ - :param refresh: Boolean flag indicating whether to refresh the "mounts" variable if already cached. - :return: Returns the "mounts" variable. Calculates and caches it the first time if it is None or the "refresh" param - is set to True. - """ - if mounts is not None and not refresh: - return mounts - else: - global mounts - mounts = mount.get_mounted() - for m in mounts: - if m["mount_point"] is not None: - m["mount_point"] = m["mount_point"].rstrip() - Logger.info("Host contains mounts: %s." % str([m["mount_point"] for m in mounts])) - return mounts - - -def get_mount_point_for_dir(dir): - """ - :param dir: Directory to check, even if it doesn't exist. - :return: Returns the closest mount point as a string for the directory. if the "dir" variable is None, will return None. - If the directory does not exist, will return "/". - """ - best_mount_found = None - if dir: - dir = dir.strip().lower() - - cached_mounts = get_and_cache_mount_points() - - # If the path is "/hadoop/hdfs/data", then possible matches for mounts could be - # "/", "/hadoop/hdfs", and "/hadoop/hdfs/data". - # So take the one with the greatest number of segments. - for m in cached_mounts: - if dir.startswith(m['mount_point']): - if best_mount_found is None: - best_mount_found = m["mount_point"] - elif best_mount_found.count(os.path.sep) < m["mount_point"].count(os.path.sep): - best_mount_found = m["mount_point"] - - Logger.info("Mount point for directory %s is %s" % (str(dir), str(best_mount_found))) - return best_mount_found http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-common/src/main/python/resource_management/libraries/functions/format.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/format.py b/ambari-common/src/main/python/resource_management/libraries/functions/format.py deleted file mode 100644 index 0dfc45d..0000000 --- a/ambari-common/src/main/python/resource_management/libraries/functions/format.py +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" - -__all__ = ["format"] -import sys -from string import Formatter -from resource_management.core.exceptions import Fail -from resource_management.core.utils import checked_unite -from resource_management.core.environment import Environment -from resource_management.core.logger import Logger -from resource_management.core.shell import quote_bash_args - - -class ConfigurationFormatter(Formatter): - """ - Flags: - !e - escape bash properties flag - !h - hide sensitive information from the logs - !p - password flag, !p=!s+!e. Has both !e, !h effect - """ - def format(self, format_string, *args, **kwargs): - env = Environment.get_instance() - variables = kwargs - params = env.config.params - all_params = checked_unite(variables, params) - - self.convert_field = self.convert_field_protected - result_protected = self.vformat(format_string, args, all_params) - - self.convert_field = self.convert_field_unprotected - result_unprotected = self.vformat(format_string, args, all_params) - - if result_protected != result_unprotected: - Logger.sensitive_strings[result_unprotected] = result_protected - - return result_unprotected - - def convert_field_unprotected(self, value, conversion): - return self._convert_field(value, conversion, False) - - def convert_field_protected(self, value, conversion): - """ - Enable masking sensitive information like - passwords from logs via !p (password) format flag. - """ - return self._convert_field(value, conversion, True) - - def _convert_field(self, value, conversion, is_protected): - if conversion == 'e': - return quote_bash_args(unicode(value)) - elif conversion == 'h': - return "[PROTECTED]" if is_protected else value - elif conversion == 'p': - return "[PROTECTED]" if is_protected else self._convert_field(value, 'e', is_protected) - - return super(ConfigurationFormatter, self).convert_field(value, conversion) - - -def format(format_string, *args, **kwargs): - variables = sys._getframe(1).f_locals - - result = checked_unite(kwargs, variables) - result.pop("self", None) # self kwarg would result in an error - return ConfigurationFormatter().format(format_string, args, **result) http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-common/src/main/python/resource_management/libraries/functions/get_kinit_path.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/get_kinit_path.py b/ambari-common/src/main/python/resource_management/libraries/functions/get_kinit_path.py deleted file mode 100644 index 74d331d..0000000 --- a/ambari-common/src/main/python/resource_management/libraries/functions/get_kinit_path.py +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" - -__all__ = ["get_kinit_path"] -import os - -def get_kinit_path(pathes_list): - """ - @param pathes: comma separated list - """ - kinit_path = "" - - for x in pathes_list: - if not x: - continue - - path = os.path.join(x,"kinit") - - if os.path.isfile(path): - kinit_path = path - break - - return kinit_path http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-common/src/main/python/resource_management/libraries/functions/get_port_from_url.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/get_port_from_url.py b/ambari-common/src/main/python/resource_management/libraries/functions/get_port_from_url.py deleted file mode 100644 index 70bd2d7..0000000 --- a/ambari-common/src/main/python/resource_management/libraries/functions/get_port_from_url.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" - -from resource_management import * -from resource_management.libraries.functions.is_empty import * -from resource_management.core.exceptions import Fail -import re - -def get_port_from_url(address): - """ - Return port from URL. If address is UnknownConfiguration, - UnknownConfiguration will be returned. If no port was found, Fail will be - raised. - """ - if not is_empty(address): - port = re.findall(":([\d]{1,5})(?=/|$)", address) - if port: - return port[0] - raise Fail("No port in URL:{0}".format(address)) - else: - return address \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-common/src/main/python/resource_management/libraries/functions/get_unique_id_and_date.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/get_unique_id_and_date.py b/ambari-common/src/main/python/resource_management/libraries/functions/get_unique_id_and_date.py deleted file mode 100644 index a79a1e5..0000000 --- a/ambari-common/src/main/python/resource_management/libraries/functions/get_unique_id_and_date.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" - -__all__ = ["get_unique_id_and_date"] -import datetime -from resource_management.core import shell - -def get_unique_id_and_date(): - out = shell.checked_call("hostid")[1].split('\n')[-1] # bugfix: take the lastline (stdin is not tty part cut) - id = out.strip() - - now = datetime.datetime.now() - date = now.strftime("%M%d%y") - - return "id{id}_date{date}".format(id=id, date=date) http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-common/src/main/python/resource_management/libraries/functions/is_empty.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/is_empty.py b/ambari-common/src/main/python/resource_management/libraries/functions/is_empty.py deleted file mode 100644 index f920d02..0000000 --- a/ambari-common/src/main/python/resource_management/libraries/functions/is_empty.py +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" -from resource_management.libraries.script.config_dictionary import UnknownConfiguration - -def is_empty(var): - """ - Check if certain configuration sent from the server has been received. - """ - return isinstance(var, UnknownConfiguration) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-common/src/main/python/resource_management/libraries/functions/substitute_vars.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/substitute_vars.py b/ambari-common/src/main/python/resource_management/libraries/functions/substitute_vars.py deleted file mode 100644 index 2036208..0000000 --- a/ambari-common/src/main/python/resource_management/libraries/functions/substitute_vars.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" -import re - -_MAX_SUBST = 20 - -def substitute_vars(raw, config): - """ - @param raw: str (e.g '${hbase.tmp.dir}/local') - @param config: dict (e.g {'hbase.tmp.dir': '/hadoop/hbase'}) - """ - result = raw - - pattern = re.compile("\$\{[^\}\$\x0020]+\}") - - for depth in range(0, _MAX_SUBST - 1): - match = pattern.search(result) - - if match: - start = match.start() - end = match.end() - - name = result[start + 2 : end - 1] - - try: - value = config[name] - except KeyError: - return result - - result = result[:start] + value + result[end:] - else: - break - - return result http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-common/src/main/python/resource_management/libraries/providers/__init__.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/__init__.py b/ambari-common/src/main/python/resource_management/libraries/providers/__init__.py deleted file mode 100644 index 5ca7bd9..0000000 --- a/ambari-common/src/main/python/resource_management/libraries/providers/__init__.py +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" - -PROVIDERS = dict( - redhat=dict( - Repository="resource_management.libraries.providers.repository.RhelSuseRepositoryProvider", - ), - suse=dict( - Repository="resource_management.libraries.providers.repository.RhelSuseRepositoryProvider", - ), - ubuntu=dict( - Repository="resource_management.libraries.providers.repository.UbuntuRepositoryProvider", - ), - default=dict( - ExecuteHadoop="resource_management.libraries.providers.execute_hadoop.ExecuteHadoopProvider", - TemplateConfig="resource_management.libraries.providers.template_config.TemplateConfigProvider", - XmlConfig="resource_management.libraries.providers.xml_config.XmlConfigProvider", - PropertiesFile="resource_management.libraries.providers.properties_file.PropertiesFileProvider", - MonitorWebserver="resource_management.libraries.providers.monitor_webserver.MonitorWebserverProvider", - HdfsDirectory="resource_management.libraries.providers.hdfs_directory.HdfsDirectoryProvider", - CopyFromLocal="resource_management.libraries.providers.copy_from_local.CopyFromLocalProvider" - ), -) http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-common/src/main/python/resource_management/libraries/providers/copy_from_local.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/copy_from_local.py b/ambari-common/src/main/python/resource_management/libraries/providers/copy_from_local.py deleted file mode 100644 index 19f4669..0000000 --- a/ambari-common/src/main/python/resource_management/libraries/providers/copy_from_local.py +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" - -import os -from resource_management import * - -class CopyFromLocalProvider(Provider): - def action_run(self): - path = self.resource.path - dest_dir = self.resource.dest_dir - dest_file = self.resource.dest_file - kinnit_if_needed = self.resource.kinnit_if_needed - owner = self.resource.owner - group = self.resource.group - mode = self.resource.mode - hdfs_usr=self.resource.hdfs_user - hadoop_conf_path = self.resource.hadoop_conf_dir - bin_dir = self.resource.hadoop_bin_dir - - - if dest_file: - copy_cmd = format("fs -copyFromLocal {path} {dest_dir}/{dest_file}") - dest_path = dest_dir + dest_file if dest_dir.endswith(os.sep) else dest_dir + os.sep + dest_file - else: - dest_file_name = os.path.split(path)[1] - copy_cmd = format("fs -copyFromLocal {path} {dest_dir}") - dest_path = dest_dir + os.sep + dest_file_name - # Need to run unless as resource user - su_cmd = 'su - {0} -c'.format(owner) - unless_cmd = format("{su_cmd} '{kinnit_if_needed} export PATH=$PATH:{bin_dir} ; hadoop fs -ls {dest_path}' >/dev/null 2>&1") - - ExecuteHadoop(copy_cmd, - not_if=unless_cmd, - user=owner, - bin_dir=bin_dir, - conf_dir=hadoop_conf_path - ) - - if not owner: - chown = None - else: - if not group: - chown = owner - else: - chown = format('{owner}:{group}') - - if chown: - chown_cmd = format("fs -chown {chown} {dest_path}") - - ExecuteHadoop(chown_cmd, - user=hdfs_usr, - bin_dir=bin_dir, - conf_dir=hadoop_conf_path) - pass - - if mode: - dir_mode = oct(mode)[1:] - chmod_cmd = format('fs -chmod {dir_mode} {dest_path}') - - ExecuteHadoop(chmod_cmd, - user=hdfs_usr, - bin_dir=bin_dir, - conf_dir=hadoop_conf_path) - pass http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-common/src/main/python/resource_management/libraries/providers/execute_hadoop.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/execute_hadoop.py b/ambari-common/src/main/python/resource_management/libraries/providers/execute_hadoop.py deleted file mode 100644 index efba0a0..0000000 --- a/ambari-common/src/main/python/resource_management/libraries/providers/execute_hadoop.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" -import os - -from resource_management import * - -class ExecuteHadoopProvider(Provider): - def action_run(self): - kinit__path_local = self.resource.kinit_path_local - keytab = self.resource.keytab - conf_dir = self.resource.conf_dir - command = self.resource.command - principal = self.resource.principal - - if isinstance(command, (list, tuple)): - command = ' '.join(quote_bash_args(x) for x in command) - - with Environment.get_instance_copy() as env: - if self.resource.security_enabled and not self.resource.kinit_override: - Execute (format("{kinit__path_local} -kt {keytab} {principal}"), - path = ['/bin'], - user = self.resource.user - ) - - - Execute (format("hadoop --config {conf_dir} {command}"), - user = self.resource.user, - tries = self.resource.tries, - try_sleep = self.resource.try_sleep, - logoutput = self.resource.logoutput, - path = self.resource.bin_dir - ) http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_directory.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_directory.py b/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_directory.py deleted file mode 100644 index d14968e..0000000 --- a/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_directory.py +++ /dev/null @@ -1,113 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" -import os - -from resource_management import * -directories_list = [] #direcotries list for mkdir -chmod_map = {} #(mode,recursive):dir_list map -chown_map = {} #(owner,group,recursive):dir_list map -class HdfsDirectoryProvider(Provider): - def action_create_delayed(self): - global delayed_directories - global chmod_map - global chown_map - - if not self.resource.dir_name: - return - - dir_name = self.resource.dir_name - dir_owner = self.resource.owner - dir_group = self.resource.group - dir_mode = oct(self.resource.mode)[1:] if self.resource.mode else None - directories_list.append(self.resource.dir_name) - - recursive_chown_str = "-R" if self.resource.recursive_chown else "" - recursive_chmod_str = "-R" if self.resource.recursive_chmod else "" - # grouping directories by mode/owner/group to modify them in one 'chXXX' call - if dir_mode: - chmod_key = (dir_mode,recursive_chmod_str) - if chmod_map.has_key(chmod_key): - chmod_map[chmod_key].append(dir_name) - else: - chmod_map[chmod_key] = [dir_name] - - if dir_owner: - owner_key = (dir_owner,dir_group,recursive_chown_str) - if chown_map.has_key(owner_key): - chown_map[owner_key].append(dir_name) - else: - chown_map[owner_key] = [dir_name] - - def action_create(self): - global delayed_directories - global chmod_map - global chown_map - - self.action_create_delayed() - - hdp_conf_dir = self.resource.conf_dir - hdp_hdfs_user = self.resource.hdfs_user - secured = self.resource.security_enabled - keytab_file = self.resource.keytab - kinit_path = self.resource.kinit_path_local - bin_dir = self.resource.bin_dir - - chmod_commands = [] - chown_commands = [] - - for chmod_key, chmod_dirs in chmod_map.items(): - mode = chmod_key[0] - recursive = chmod_key[1] - chmod_dirs_str = ' '.join(chmod_dirs) - chmod_commands.append(format("hadoop --config {hdp_conf_dir} fs -chmod {recursive} {mode} {chmod_dirs_str}")) - - for chown_key, chown_dirs in chown_map.items(): - owner = chown_key[0] - group = chown_key[1] - recursive = chown_key[2] - chown_dirs_str = ' '.join(chown_dirs) - if owner: - chown = owner - if group: - chown = format("{owner}:{group}") - chown_commands.append(format("hadoop --config {hdp_conf_dir} fs -chown {recursive} {chown} {chown_dirs_str}")) - - if secured: - Execute(format("{kinit_path} -kt {keytab_file} {hdfs_principal_name}"), - user=hdp_hdfs_user) - #create all directories in one 'mkdir' call - dir_list_str = ' '.join(directories_list) - #for hadoop 2 we need to specify -p to create directories recursively - parent_flag = '`rpm -q hadoop | grep -q "hadoop-1" || echo "-p"`' - - Execute(format('hadoop --config {hdp_conf_dir} fs -mkdir {parent_flag} {dir_list_str} && {chmod_cmd} && {chown_cmd}', - chmod_cmd=' && '.join(chmod_commands), - chown_cmd=' && '.join(chown_commands)), - user=hdp_hdfs_user, - path=bin_dir, - not_if=format("su - {hdp_hdfs_user} -c 'export PATH=$PATH:{bin_dir} ; " - "hadoop --config {hdp_conf_dir} fs -ls {dir_list_str}'") - ) - - directories_list[:] = [] - chmod_map.clear() - chown_map.clear() http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-common/src/main/python/resource_management/libraries/providers/monitor_webserver.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/monitor_webserver.py b/ambari-common/src/main/python/resource_management/libraries/providers/monitor_webserver.py deleted file mode 100644 index 3a053ab..0000000 --- a/ambari-common/src/main/python/resource_management/libraries/providers/monitor_webserver.py +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" - -from resource_management import * -from resource_management.core.system import System - - -class MonitorWebserverProvider(Provider): - def action_start(self): - self.get_serivice_params() - self.enable_keep_alive() - service_name = self.service_name - Execute(format("/etc/init.d/{service_name} start")) - - def action_stop(self): - self.get_serivice_params() - service_name = self.service_name - Execute(format("/etc/init.d/{service_name} stop")) - - def action_restart(self): - self.action_stop() - self.action_start() - - def get_serivice_params(self): - self.system = System.get_instance() - if self.system.os_family in ["suse","ubuntu"]: - self.service_name = "apache2" - self.httpd_conf_dir = '/etc/apache2' - else: - self.service_name = "httpd" - self.httpd_conf_dir = '/etc/httpd/conf' - - def enable_keep_alive(self): - httpd_conf_dir = self.httpd_conf_dir - Execute(format( - "grep -E 'KeepAlive (On|Off)' {httpd_conf_dir}/httpd.conf && sed -i 's/KeepAlive Off/KeepAlive On/' {httpd_conf_dir}/httpd.conf || echo 'KeepAlive On' >> {httpd_conf_dir}/httpd.conf")) http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-common/src/main/python/resource_management/libraries/providers/properties_file.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/properties_file.py b/ambari-common/src/main/python/resource_management/libraries/providers/properties_file.py deleted file mode 100644 index 94a51ff..0000000 --- a/ambari-common/src/main/python/resource_management/libraries/providers/properties_file.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" - -import time -import os -from resource_management import * - -class PropertiesFileProvider(Provider): - def action_create(self): - filename = self.resource.filename - dir = self.resource.dir - if dir == None: - filepath = filename - else: - filepath = os.path.join(dir, filename) - - config_content = InlineTemplate('''# Generated by Apache Ambari. {{time.asctime(time.localtime())}} - {% for key, value in properties_dict|dictsort %} -{{key}}={{value}}{% endfor %} - ''', extra_imports=[time], properties_dict=self.resource.properties) - - Logger.info(format("Generating properties file: {filepath}")) - - with Environment.get_instance_copy() as env: - File (format("{filepath}"), - content = config_content, - owner = self.resource.owner, - group = self.resource.group, - mode = self.resource.mode - ) http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-common/src/main/python/resource_management/libraries/providers/repository.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/repository.py b/ambari-common/src/main/python/resource_management/libraries/providers/repository.py deleted file mode 100644 index fc95fb3..0000000 --- a/ambari-common/src/main/python/resource_management/libraries/providers/repository.py +++ /dev/null @@ -1,98 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" - -import os -import filecmp -import tempfile -from ambari_commons import OSCheck -from resource_management import * - -class RhelSuseRepositoryProvider(Provider): - def action_create(self): - with Environment.get_instance_copy() as env: - repo_file_name = self.resource.repo_file_name - repo_dir = repos_dirs[env.system.os_family] - repo_template = self.resource.repo_template - File(format("{repo_dir}/{repo_file_name}.repo"), - content = Template(repo_template, repo_id=self.resource.repo_id, repo_file_name=self.resource.repo_file_name, base_url=self.resource.base_url, mirror_list=self.resource.mirror_list) - ) - - def action_remove(self): - with Environment.get_instance_copy() as env: - repo_file_name = self.resource.repo_file_name - repo_dir = repos_dirs[env.system.os_family] - - File(format("{repo_dir}/{repo_file_name}.repo"), - action = "delete") - - -repos_dirs = { - 'redhat': '/etc/yum.repos.d', - 'suse': '/etc/zypp/repos.d' -} - - -class UbuntuRepositoryProvider(Provider): - package_type = "deb" - repo_dir = "/etc/apt/sources.list.d" - update_cmd = 'apt-get update -qq -o Dir::Etc::sourcelist="sources.list.d/{repo_file_name}" -o APT::Get::List-Cleanup="0"' - missing_pkey_regex = "The following signatures couldn't be verified because the public key is not available: NO_PUBKEY (.+)" - add_pkey_cmd = "apt-key adv --recv-keys --keyserver keyserver.ubuntu.com {pkey}" - - def action_create(self): - with Environment.get_instance_copy() as env: - with tempfile.NamedTemporaryFile() as tmpf: - File(tmpf.name, - content = Template(self.resource.repo_template, - package_type=self.package_type, base_url=self.resource.base_url, components=' '.join(self.resource.components)) - ) - - repo_file_name = format("{repo_file_name}.list",repo_file_name = self.resource.repo_file_name) - repo_file_path = format("{repo_dir}/{repo_file_name}", repo_dir = self.repo_dir) - - if not os.path.isfile(repo_file_path) or not filecmp.cmp(tmpf.name, repo_file_path): - File(repo_file_path, - content = StaticFile(tmpf.name) - ) - - # this is time expensive - retcode, out = checked_call(format(self.update_cmd)) - - # add public keys for new repos - missing_pkeys = set(re.findall(self.missing_pkey_regex, out)) - for pkey in missing_pkeys: - Execute(format(self.add_pkey_cmd), - timeout = 15, # in case we are on the host w/o internet (using localrepo), we should ignore hanging - ignore_failures = True - ) - - def action_remove(self): - with Environment.get_instance_copy() as env: - repo_file_name = format("{repo_file_name}.list",repo_file_name = self.resource.repo_file_name) - repo_file_path = format("{repo_dir}/{repo_file_name}", repo_dir = self.repo_dir) - - if os.path.isfile(repo_file_path): - File(repo_file_path, - action = "delete") - - # this is time expensive - Execute(format(self.update_cmd)) http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-common/src/main/python/resource_management/libraries/providers/template_config.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/template_config.py b/ambari-common/src/main/python/resource_management/libraries/providers/template_config.py deleted file mode 100644 index 4972797..0000000 --- a/ambari-common/src/main/python/resource_management/libraries/providers/template_config.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" - -import os -from resource_management import * - -class TemplateConfigProvider(Provider): - def action_create(self): - template_tag = self.resource.template_tag - qualified_file_name = self.resource.name - file_name = os.path.basename(qualified_file_name) - - if not template_tag: - template_name = format("{file_name}.j2") - else: - template_name = format("{file_name}-{template_tag}.j2") - - with Environment.get_instance_copy() as env: - File( qualified_file_name, - owner = self.resource.owner, - group = self.resource.group, - mode = self.resource.mode, - content = Template(template_name, extra_imports=self.resource.extra_imports) - ) http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-common/src/main/python/resource_management/libraries/providers/xml_config.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/xml_config.py b/ambari-common/src/main/python/resource_management/libraries/providers/xml_config.py deleted file mode 100644 index 87fc657..0000000 --- a/ambari-common/src/main/python/resource_management/libraries/providers/xml_config.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" - -import time -from resource_management import * - -class XmlConfigProvider(Provider): - def action_create(self): - filename = self.resource.filename - xml_config_provider_config_dir = self.resource.conf_dir - - # |e - for html-like escaping of <,>,'," - config_content = InlineTemplate(''' - - {% for key, value in configurations_dict|dictsort %} - - {{ key|e }} - {{ value|e }} - {%- if not configuration_attrs is none -%} - {%- for attrib_name, attrib_occurances in configuration_attrs.items() -%} - {%- for property_name, attrib_value in attrib_occurances.items() -%} - {% if property_name == key and attrib_name %} - <{{attrib_name|e}}>{{attrib_value|e}} - {%- endif -%} - {%- endfor -%} - {%- endfor -%} - {%- endif %} - - {% endfor %} - ''', extra_imports=[time], configurations_dict=self.resource.configurations, - configuration_attrs=self.resource.configuration_attributes) - - - Logger.info(format("Generating config: {xml_config_provider_config_dir}/{filename}")) - - with Environment.get_instance_copy() as env: - File (format("{xml_config_provider_config_dir}/{filename}"), - content = config_content, - owner = self.resource.owner, - group = self.resource.group, - mode = self.resource.mode, - encoding = self.resource.encoding - ) http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-common/src/main/python/resource_management/libraries/resources/__init__.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/resources/__init__.py b/ambari-common/src/main/python/resource_management/libraries/resources/__init__.py deleted file mode 100644 index 24b497c..0000000 --- a/ambari-common/src/main/python/resource_management/libraries/resources/__init__.py +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" - -from resource_management.libraries.resources.execute_hadoop import * -from resource_management.libraries.resources.template_config import * -from resource_management.libraries.resources.xml_config import * -from resource_management.libraries.resources.properties_file import * -from resource_management.libraries.resources.repository import * -from resource_management.libraries.resources.monitor_webserver import * -from resource_management.libraries.resources.hdfs_directory import * -from resource_management.libraries.resources.copy_from_local import * \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-common/src/main/python/resource_management/libraries/resources/copy_from_local.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/resources/copy_from_local.py b/ambari-common/src/main/python/resource_management/libraries/resources/copy_from_local.py deleted file mode 100644 index 54d003e..0000000 --- a/ambari-common/src/main/python/resource_management/libraries/resources/copy_from_local.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" - -_all__ = ["CopyFromLocal"] -from resource_management.core.base import Resource, ForcedListArgument, ResourceArgument, BooleanArgument - -class CopyFromLocal(Resource): - action = ForcedListArgument(default="run") - - path = ResourceArgument(default=lambda obj: obj.name) - dest_dir = ResourceArgument(required=True) - dest_file = ResourceArgument() - owner = ResourceArgument(required=True) - group = ResourceArgument() - mode = ResourceArgument() - kinnit_if_needed = ResourceArgument(default='') - hadoop_conf_dir = ResourceArgument(default='/etc/hadoop/conf') - hdfs_user = ResourceArgument(default='hdfs') - hadoop_bin_dir = ResourceArgument(default='/usr/bin') - - actions = Resource.actions + ["run"] http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-common/src/main/python/resource_management/libraries/resources/execute_hadoop.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/resources/execute_hadoop.py b/ambari-common/src/main/python/resource_management/libraries/resources/execute_hadoop.py deleted file mode 100644 index ca66ce4..0000000 --- a/ambari-common/src/main/python/resource_management/libraries/resources/execute_hadoop.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" - -_all__ = ["ExecuteHadoop"] -from resource_management.core.base import Resource, ForcedListArgument, ResourceArgument, BooleanArgument - -class ExecuteHadoop(Resource): - action = ForcedListArgument(default="run") - command = ResourceArgument(default=lambda obj: obj.name) - kinit_override = BooleanArgument(default=False) - tries = ResourceArgument(default=1) - try_sleep = ResourceArgument(default=0) # seconds - user = ResourceArgument() - logoutput = BooleanArgument(default=False) - principal = ResourceArgument(default=lambda obj: obj.user) - bin_dir = ResourceArgument(default=[]) # appended to $PATH - - conf_dir = ResourceArgument() - - security_enabled = BooleanArgument(default=False) - keytab = ResourceArgument() - kinit_path_local = ResourceArgument() - - actions = Resource.actions + ["run"] - http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-common/src/main/python/resource_management/libraries/resources/hdfs_directory.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/resources/hdfs_directory.py b/ambari-common/src/main/python/resource_management/libraries/resources/hdfs_directory.py deleted file mode 100644 index 7888cd8..0000000 --- a/ambari-common/src/main/python/resource_management/libraries/resources/hdfs_directory.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" - -_all__ = ["HdfsDirectory"] -from resource_management.core.base import Resource, ForcedListArgument, ResourceArgument, BooleanArgument - -class HdfsDirectory(Resource): - action = ForcedListArgument() - - dir_name = ResourceArgument(default=lambda obj: obj.name) - owner = ResourceArgument() - group = ResourceArgument() - mode = ResourceArgument() - recursive_chown = BooleanArgument(default=False) - recursive_chmod = BooleanArgument(default=False) - - conf_dir = ResourceArgument() - security_enabled = BooleanArgument(default=False) - keytab = ResourceArgument() - kinit_path_local = ResourceArgument() - hdfs_user = ResourceArgument() - bin_dir = ResourceArgument(default="") - - #action 'create' immediately creates all pending directory in efficient manner - #action 'create_delayed' add directory to list of pending directories - actions = Resource.actions + ["create","create_delayed"] http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-common/src/main/python/resource_management/libraries/resources/monitor_webserver.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/resources/monitor_webserver.py b/ambari-common/src/main/python/resource_management/libraries/resources/monitor_webserver.py deleted file mode 100644 index dfd1174..0000000 --- a/ambari-common/src/main/python/resource_management/libraries/resources/monitor_webserver.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" - -_all__ = ["MonitorWebserver"] -from resource_management.core.base import Resource, ForcedListArgument, ResourceArgument, BooleanArgument - - -class MonitorWebserver(Resource): - action = ForcedListArgument(default=lambda obj: [obj.name]) - actions = Resource.actions + ["start", "stop", "restart"] http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-common/src/main/python/resource_management/libraries/resources/properties_file.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/resources/properties_file.py b/ambari-common/src/main/python/resource_management/libraries/resources/properties_file.py deleted file mode 100644 index 0e5afb4..0000000 --- a/ambari-common/src/main/python/resource_management/libraries/resources/properties_file.py +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" - -_all__ = ["PropertiesFile"] -from resource_management.core.base import Resource, ForcedListArgument, ResourceArgument, BooleanArgument - -class PropertiesFile(Resource): - action = ForcedListArgument(default="create") - filename = ResourceArgument(default=lambda obj: obj.name) - - properties = ResourceArgument() - dir = ResourceArgument() - - mode = ResourceArgument() - owner = ResourceArgument() - group = ResourceArgument() - - actions = Resource.actions + ["create"] http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-common/src/main/python/resource_management/libraries/resources/repository.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/resources/repository.py b/ambari-common/src/main/python/resource_management/libraries/resources/repository.py deleted file mode 100644 index 48a347a..0000000 --- a/ambari-common/src/main/python/resource_management/libraries/resources/repository.py +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" - -_all__ = ["Repository"] - -from resource_management.core.base import Resource, ForcedListArgument, ResourceArgument, BooleanArgument - -class Repository(Resource): - action = ForcedListArgument(default="create") - repo_id = ResourceArgument(default=lambda obj: obj.name) - base_url = ResourceArgument() - mirror_list = ResourceArgument() - repo_file_name = ResourceArgument() - repo_template = ResourceArgument() - components = ForcedListArgument(default=[]) # ubuntu specific - - actions = Resource.actions + ["create","remove"] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-common/src/main/python/resource_management/libraries/resources/template_config.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/resources/template_config.py b/ambari-common/src/main/python/resource_management/libraries/resources/template_config.py deleted file mode 100644 index 8ce2a00..0000000 --- a/ambari-common/src/main/python/resource_management/libraries/resources/template_config.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" - -_all__ = ["TemplateConfig"] -from resource_management.core.base import Resource, ForcedListArgument, ResourceArgument, BooleanArgument - -class TemplateConfig(Resource): - action = ForcedListArgument(default="create") - path = ResourceArgument(default=lambda obj: obj.name) - mode = ResourceArgument() - owner = ResourceArgument() - group = ResourceArgument() - template_tag = ResourceArgument() - extra_imports = ResourceArgument(default=[]) - - actions = Resource.actions + ["create"] http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-common/src/main/python/resource_management/libraries/resources/xml_config.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/resources/xml_config.py b/ambari-common/src/main/python/resource_management/libraries/resources/xml_config.py deleted file mode 100644 index 882d6d5..0000000 --- a/ambari-common/src/main/python/resource_management/libraries/resources/xml_config.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" - -_all__ = ["XmlConfig"] -from resource_management.core.base import Resource, ForcedListArgument, ResourceArgument, BooleanArgument - -class XmlConfig(Resource): - action = ForcedListArgument(default="create") - filename = ResourceArgument(default=lambda obj: obj.name) - - configurations = ResourceArgument() - configuration_attributes = ResourceArgument() - conf_dir = ResourceArgument() - - mode = ResourceArgument() - owner = ResourceArgument() - group = ResourceArgument() - - encoding = ResourceArgument(default="UTF-8") - - actions = Resource.actions + ["create"] http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-common/src/main/python/resource_management/libraries/script/__init__.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/script/__init__.py b/ambari-common/src/main/python/resource_management/libraries/script/__init__.py deleted file mode 100644 index 72d3aaf..0000000 --- a/ambari-common/src/main/python/resource_management/libraries/script/__init__.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Ambari Agent - -""" - -from resource_management.libraries.script.script import * -from resource_management.libraries.script.hook import * -from resource_management.libraries.script.config_dictionary import * \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-common/src/main/python/resource_management/libraries/script/config_dictionary.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/script/config_dictionary.py b/ambari-common/src/main/python/resource_management/libraries/script/config_dictionary.py deleted file mode 100644 index 453c546..0000000 --- a/ambari-common/src/main/python/resource_management/libraries/script/config_dictionary.py +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env python - -''' -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -''' -from resource_management.core.exceptions import Fail - -class ConfigDictionary(dict): - """ - Immutable config dictionary - """ - - def __init__(self, dictionary): - """ - Recursively turn dict to ConfigDictionary - """ - for k, v in dictionary.iteritems(): - if isinstance(v, dict): - dictionary[k] = ConfigDictionary(v) - - super(ConfigDictionary, self).__init__(dictionary) - - def __setitem__(self, name, value): - raise Fail("Configuration dictionary is immutable!") - - def __getitem__(self, name): - """ - - use Python types - - enable lazy failure for unknown configs. - """ - try: - value = super(ConfigDictionary, self).__getitem__(name) - except KeyError: - return UnknownConfiguration(name) - - - if value == "true": - value = True - elif value == "false": - value = False - else: - try: - value = int(value) - except (ValueError, TypeError): - try: - value = float(value) - except (ValueError, TypeError): - pass - - return value - - -class UnknownConfiguration(): - """ - Lazy failing for unknown configs. - """ - def __init__(self, name): - self.name = name - - def __getattr__(self, name): - raise Fail("Configuration parameter '"+self.name+"' was not found in configurations dictionary!") - - def __getitem__(self, name): - """ - Allow [] - """ - return self \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-common/src/main/python/resource_management/libraries/script/hook.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/script/hook.py b/ambari-common/src/main/python/resource_management/libraries/script/hook.py deleted file mode 100644 index 5c8eafc..0000000 --- a/ambari-common/src/main/python/resource_management/libraries/script/hook.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env ambari-python-wrap - -''' -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -''' - -__all__ = ["Hook"] - -from resource_management.libraries.script import Script -import subprocess -import sys - -class Hook(Script): - """ - Executes a hook for acommand for custom service. stdout and stderr are written to - tmpoutfile and to tmperrfile respectively. - """ - - HOOK_METHOD_NAME = "hook" # This method is always executed at hooks - - - def choose_method_to_execute(self, command_name): - """ - Changes logics of resolving method name - """ - return super(Hook, self).choose_method_to_execute(self.HOOK_METHOD_NAME) - - - def run_custom_hook(self, command): - """ - Runs custom hook - """ - args = sys.argv - - #Hook script to run - args[0] = args[0].replace('before-'+args[1], command) - args[0] = args[0].replace('after-'+args[1], command) - - #Hook script base directory - args[3] = args[3].replace('before-'+args[1], command) - args[3] = args[3].replace('after-'+args[1], command) - - args[1] = command.split("-")[1] - - - cmd = [sys.executable] - cmd.extend(args) - - if subprocess.call(cmd) != 0: - self.fail_with_error("Error: Unable to run the custom hook script " + - cmd.__str__()) -