Return-Path: X-Original-To: apmail-ambari-commits-archive@www.apache.org Delivered-To: apmail-ambari-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 38BC018231 for ; Wed, 10 Feb 2016 16:12:59 +0000 (UTC) Received: (qmail 10823 invoked by uid 500); 10 Feb 2016 16:12:58 -0000 Delivered-To: apmail-ambari-commits-archive@ambari.apache.org Received: (qmail 10771 invoked by uid 500); 10 Feb 2016 16:12:58 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 10460 invoked by uid 99); 10 Feb 2016 16:12:58 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 10 Feb 2016 16:12:58 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id AA4D6E699F; Wed, 10 Feb 2016 16:12:58 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: ncole@apache.org To: commits@ambari.apache.org Date: Wed, 10 Feb 2016 16:13:16 -0000 Message-Id: <5c65d19264cb46cdbc59581b84ae2f95@git.apache.org> In-Reply-To: <7a17f3b21f9c4ec69a34428c06b0ca33@git.apache.org> References: <7a17f3b21f9c4ec69a34428c06b0ca33@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [19/24] ambari git commit: AMBARI-14962 : Auto configure AMS UI to use AMS Datasource (avijayan) AMBARI-14962 : Auto configure AMS UI to use AMS Datasource (avijayan) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/14396f2b Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/14396f2b Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/14396f2b Branch: refs/heads/branch-dev-patch-upgrade Commit: 14396f2b74115744d6f4c93b97a9911cd1e58ff1 Parents: 38a5225 Author: Aravindan Vijayan Authored: Mon Feb 8 13:22:15 2016 -0800 Committer: Aravindan Vijayan Committed: Tue Feb 9 08:18:51 2016 -0800 ---------------------------------------------------------------------- .../0.1.0/package/scripts/metrics_grafana.py | 2 + .../package/scripts/metrics_grafana_util.py | 131 +++++++++++++++++++ .../metrics_grafana_datasource.json.j2 | 15 +++ .../AMBARI_METRICS/test_metrics_grafana.py | 19 ++- 4 files changed, 163 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/14396f2b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana.py index f876036..d96309c 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana.py +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana.py @@ -22,6 +22,7 @@ from resource_management import Script, Execute from resource_management.libraries.functions import format from status import check_service_status from ams import ams +from metrics_grafana_util import create_ams_datasource class AmsGrafana(Script): def install(self, env): @@ -46,6 +47,7 @@ class AmsGrafana(Script): Execute(start_cmd, user=params.ams_user ) + create_ams_datasource() def stop(self, env): import params http://git-wip-us.apache.org/repos/asf/ambari/blob/14396f2b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py new file mode 100644 index 0000000..c4a91e1 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +from resource_management.core.logger import Logger +from resource_management.core.base import Fail +from resource_management import Template + +import httplib +import time +import socket +import json + +def create_ams_datasource(): + + import params + GRAFANA_CONNECT_TRIES = 5 + GRAFANA_CONNECT_TIMEOUT = 15 + GRAFANA_URL = "/api/datasources" + METRICS_GRAFANA_DATASOURCE_NAME = "AMBARI_METRICS" + + headers = {"Content-type": "application/json"} + + Logger.info("Checking if AMS Grafana datasource already exists") + Logger.info("Connecting (GET) to %s:%s%s" % (params.hostname, + params.ams_grafana_port, + GRAFANA_URL)) + + conn = httplib.HTTPConnection(params.hostname, + int(params.ams_grafana_port)) + + conn.request("GET", GRAFANA_URL) + response = conn.getresponse() + Logger.info("Http response: %s %s" % (response.status, response.reason)) + + if(response.status == 200): + datasources = response.read() + datasources_json = json.loads(datasources) + for i in xrange(0, len(datasources_json)): + datasource_name = datasources_json[i]["name"] + if(datasource_name == METRICS_GRAFANA_DATASOURCE_NAME): + + Logger.info("Ambari Metrics Grafana datasource already present. Checking Metrics Collector URL") + datasource_url = datasources_json[i]["url"] + + if datasource_url == (params.metric_collector_host + ":" + params.metric_collector_port + + "/ws/v1/timeline/metrics") : + Logger.info("Metrics Collector URL validation succeeded. Skipping datasource creation") + GRAFANA_CONNECT_TRIES = 0 # No need to create datasource again + + else: # Metrics datasource present, but collector host is wrong. + + Logger.info("Metrics Collector URL validation failed.") + datasource_id = datasources_json[i]["id"] + Logger.info("Deleting obselete Metrics datasource.") + conn = httplib.HTTPConnection(params.hostname, int(params.ams_grafana_port)) + conn.request("DELETE", GRAFANA_URL + "/" + str(datasource_id)) + response = conn.getresponse() + Logger.info("Http response: %s %s" % (response.status, response.reason)) + + break + else: + Logger.info("Error checking for Ambari Metrics Grafana datasource. Will attempt to create.") + + if GRAFANA_CONNECT_TRIES > 0: + Logger.info("Attempting to create Ambari Metrics Grafana datasource") + + for i in xrange(0, GRAFANA_CONNECT_TRIES): + try: + ams_datasource_json = Template('metrics_grafana_datasource.json.j2', + ams_datasource_name=METRICS_GRAFANA_DATASOURCE_NAME, + ams_collector_host=params.metric_collector_host, + ams_collector_port=params.metric_collector_port).get_content() + + Logger.info("Generated datasource:\n%s" % ams_datasource_json) + + Logger.info("Connecting (POST) to %s:%s%s" % (params.hostname, + params.ams_grafana_port, + GRAFANA_URL)) + conn = httplib.HTTPConnection(params.hostname, + int(params.ams_grafana_port)) + conn.request("POST", GRAFANA_URL, ams_datasource_json, headers) + + response = conn.getresponse() + Logger.info("Http response: %s %s" % (response.status, response.reason)) + except (httplib.HTTPException, socket.error) as ex: + if i < GRAFANA_CONNECT_TRIES - 1: + time.sleep(GRAFANA_CONNECT_TIMEOUT) + Logger.info("Connection to Grafana failed. Next retry in %s seconds." + % (GRAFANA_CONNECT_TIMEOUT)) + continue + else: + raise Fail("Ambari Metrics Grafana datasource not created") + + data = response.read() + Logger.info("Http data: %s" % data) + conn.close() + + if response.status == 200: + Logger.info("Ambari Metrics Grafana data source created.") + break + elif response.status == 500: + Logger.info("Ambari Metrics Grafana data source creation failed. Not retrying.") + raise Fail("Ambari Metrics Grafana data source creation failed. POST request status: %s %s \n%s" % + (response.status, response.reason, data)) + else: + Logger.info("Ambari Metrics Grafana data source creation failed.") + if i < GRAFANA_CONNECT_TRIES - 1: + time.sleep(GRAFANA_CONNECT_TIMEOUT) + Logger.info("Next retry in %s seconds." + % (GRAFANA_CONNECT_TIMEOUT)) + else: + raise Fail("Ambari Metrics Grafana data source creation failed. POST request status: %s %s \n%s" % + (response.status, response.reason, data)) + + http://git-wip-us.apache.org/repos/asf/ambari/blob/14396f2b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metrics_grafana_datasource.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metrics_grafana_datasource.json.j2 b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metrics_grafana_datasource.json.j2 new file mode 100644 index 0000000..3edc6fb --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metrics_grafana_datasource.json.j2 @@ -0,0 +1,15 @@ +{ + "name": "{{ams_datasource_name}}", + "type": "ambarimetrics", + "access": "proxy", + "url": "{{ams_collector_host}}:{{ams_collector_port}}/ws/v1/timeline/metrics", + "password": "", + "user": "", + "database": "", + "basicAuth": false, + "basicAuthUser": "", + "basicAuthPassword": "", + "withCredentials": false, + "isDefault": false, + "jsonData": {} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/14396f2b/ambari-server/src/test/python/stacks/2.0.6/AMBARI_METRICS/test_metrics_grafana.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/AMBARI_METRICS/test_metrics_grafana.py b/ambari-server/src/test/python/stacks/2.0.6/AMBARI_METRICS/test_metrics_grafana.py index 5bc1412..755bb4f 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/AMBARI_METRICS/test_metrics_grafana.py +++ b/ambari-server/src/test/python/stacks/2.0.6/AMBARI_METRICS/test_metrics_grafana.py @@ -18,17 +18,27 @@ See the License for the specific language governing permissions and limitations under the License. ''' -from mock.mock import MagicMock, patch +from mock.mock import MagicMock, patch, call from stacks.utils.RMFTestCase import * +import os, sys @patch("os.path.exists", new = MagicMock(return_value=True)) @patch("platform.linux_distribution", new = MagicMock(return_value="Linux")) class TestMetricsGrafana(RMFTestCase): - COMMON_SERVICES_PACKAGE_DIR = "AMBARI_METRICS/0.1.0/package" + COMMON_SERVICES_PACKAGE_DIR = "AMBARI_METRICS/0.1.0/package/scripts" STACK_VERSION = "2.0.6" - def test_start(self): - self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/metrics_grafana.py", + file_path = os.path.dirname(os.path.abspath(__file__)) + file_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(file_path))))) + file_path = os.path.join(file_path, "main", "resources", "common-services", COMMON_SERVICES_PACKAGE_DIR) + + sys.path.append(file_path) + global metrics_grafana_util + import metrics_grafana_util + + @patch("metrics_grafana_util.create_ams_datasource") + def test_start(self, create_ams_datasource_mock): + self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/metrics_grafana.py", classname = "AmsGrafana", command = "start", config_file="default.json", @@ -43,6 +53,7 @@ class TestMetricsGrafana(RMFTestCase): self.assertResourceCalled('Execute', '/usr/sbin/ambari-metrics-grafana start', user = 'ams' ) + create_ams_datasource_mock.assertCalled() self.assertNoMoreResources() def assert_configure(self):